mirror of
https://github.com/kikootwo/ReadMeABook.git
synced 2026-06-02 20:30:10 +00:00
Add multi-source ebook search & processing
Refactor ebook flow to support multiple sources (Anna's Archive direct downloads + Prowlarr indexer search) and unify handling with existing audiobook processors. Key changes: - search-ebook.processor: rewritten to try Anna's Archive first then fall back to indexer search, add Prowlarr grouping, ranking (rankEbookTorrents), and handlers to route results to direct-download or download-torrent flows. - organize-files.processor: enriches audiobook/ebook metadata from AudibleCache (year, narrator), treats indexer downloads specially (seed retention), adds optional NZB cleanup/archive logic, and improves retryable error detection. - file-organizer: organizeEbook now accepts additional metadata and an isIndexerDownload flag and supports directories vs single-file paths. - API/UI: include request.type in admin requests API and remove the “coming soon” notice from Ebook settings tab. - fetch-ebook route: removed blocking error for indexer-only mode so the flow can proceed when indexer search is enabled. - Documentation: update TOC, ebook-sidecar, settings-pages, and ranking-algorithm docs to describe indexer search, unified ebook ranking, configuration, and flows. These changes enable indexer-based ebook discovery, ranking, and downloads while preserving existing Anna's Archive behavior and reusing audiobook download processors where possible.
This commit is contained in:
kikootwo
@@ -67,36 +67,53 @@ export async function processOrganizeFiles(payload: OrganizeFilesPayload): Promi
|
||||
|
||||
logger.info(`Organizing: ${audiobook.title} by ${audiobook.author}`);
|
||||
|
||||
// Fetch year from multiple sources (priority order)
|
||||
// Fetch missing metadata from AudibleCache if needed
|
||||
// Year and narrator can both be part of path templates
|
||||
let year = audiobook.year || undefined;
|
||||
logger.info(`Initial year from audiobook record: ${year || 'null'}`);
|
||||
let narrator = audiobook.narrator || undefined;
|
||||
|
||||
if (!year && audiobook.audibleAsin) {
|
||||
logger.info(`No year in audiobook record, attempting to fetch from AudibleCache for ASIN: ${audiobook.audibleAsin}`);
|
||||
logger.info(`Initial metadata from audiobook record: year=${year || 'null'}, narrator=${narrator || 'null'}`);
|
||||
|
||||
// Try to enrich missing metadata from AudibleCache
|
||||
if (audiobook.audibleAsin && (!year || !narrator)) {
|
||||
logger.info(`Missing metadata, attempting to fetch from AudibleCache for ASIN: ${audiobook.audibleAsin}`);
|
||||
|
||||
// Try AudibleCache (for popular/new releases)
|
||||
const audibleCache = await prisma.audibleCache.findUnique({
|
||||
where: { asin: audiobook.audibleAsin },
|
||||
select: { releaseDate: true },
|
||||
select: { releaseDate: true, narrator: true },
|
||||
});
|
||||
|
||||
if (audibleCache?.releaseDate) {
|
||||
logger.info(`Found AudibleCache entry with releaseDate: ${audibleCache.releaseDate}`);
|
||||
year = new Date(audibleCache.releaseDate).getFullYear();
|
||||
logger.info(`Extracted year ${year} from AudibleCache releaseDate`);
|
||||
if (audibleCache) {
|
||||
const updates: { year?: number; narrator?: string } = {};
|
||||
|
||||
// Update audiobook record with year for future use
|
||||
await prisma.audiobook.update({
|
||||
where: { id: audiobookId },
|
||||
data: { year },
|
||||
});
|
||||
logger.info(`Updated audiobook record with year ${year}`);
|
||||
// Extract year from releaseDate if missing
|
||||
if (!year && audibleCache.releaseDate) {
|
||||
year = new Date(audibleCache.releaseDate).getFullYear();
|
||||
updates.year = year;
|
||||
logger.info(`Extracted year ${year} from AudibleCache releaseDate`);
|
||||
}
|
||||
|
||||
// Get narrator if missing
|
||||
if (!narrator && audibleCache.narrator) {
|
||||
narrator = audibleCache.narrator;
|
||||
updates.narrator = narrator;
|
||||
logger.info(`Got narrator "${narrator}" from AudibleCache`);
|
||||
}
|
||||
|
||||
// Update audiobook record with enriched data for future use
|
||||
if (Object.keys(updates).length > 0) {
|
||||
await prisma.audiobook.update({
|
||||
where: { id: audiobookId },
|
||||
data: updates,
|
||||
});
|
||||
logger.info(`Updated audiobook record with enriched metadata`);
|
||||
}
|
||||
} else {
|
||||
logger.info(`No year found in AudibleCache for ASIN ${audiobook.audibleAsin}`);
|
||||
logger.info(`No AudibleCache entry found for ASIN ${audiobook.audibleAsin}`);
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Final year value for path organization: ${year || 'null (year will be omitted from path)'}`)
|
||||
logger.info(`Final metadata for path organization: year=${year || 'null'}, narrator=${narrator || 'null'}`)
|
||||
|
||||
// Get file organizer (reads media_dir from database config)
|
||||
const organizer = await getFileOrganizer();
|
||||
@@ -113,7 +130,7 @@ export async function processOrganizeFiles(payload: OrganizeFilesPayload): Promi
|
||||
{
|
||||
title: audiobook.title,
|
||||
author: audiobook.author,
|
||||
narrator: audiobook.narrator || undefined,
|
||||
narrator,
|
||||
coverArtUrl: audiobook.coverArtUrl || undefined,
|
||||
asin: audiobook.audibleAsin || undefined,
|
||||
year,
|
||||
@@ -329,8 +346,10 @@ export async function processOrganizeFiles(payload: OrganizeFilesPayload): Promi
|
||||
const errorMessage = error instanceof Error ? error.message : 'File organization failed';
|
||||
|
||||
// Check if this is a retryable error (transient filesystem issues or no files found)
|
||||
// These errors may resolve on retry (e.g., files still being extracted, permissions being set)
|
||||
const isRetryableError =
|
||||
errorMessage.includes('No audiobook files found') ||
|
||||
errorMessage.includes('No ebook files found') || // Ebook equivalent of above
|
||||
errorMessage.includes('ENOENT') || // File/directory not found
|
||||
errorMessage.includes('no such file or directory') ||
|
||||
errorMessage.includes('EACCES') || // Permission denied (might be temporary)
|
||||
@@ -501,6 +520,64 @@ async function processEbookOrganization(
|
||||
|
||||
logger.info(`Organizing ebook: ${book.title} by ${book.author}`);
|
||||
|
||||
// Fetch missing metadata from AudibleCache (same pattern as audiobooks)
|
||||
// Year, narrator, series, seriesPart can all be part of path templates
|
||||
let year = book.year || undefined;
|
||||
let narrator = book.narrator || undefined;
|
||||
let series = book.series || undefined;
|
||||
let seriesPart = book.seriesPart || undefined;
|
||||
|
||||
logger.info(`Initial metadata from book record: year=${year || 'null'}, narrator=${narrator || 'null'}, series=${series || 'null'}`);
|
||||
|
||||
// Try to enrich missing metadata from AudibleCache
|
||||
if (book.audibleAsin && (!year || !narrator)) {
|
||||
logger.info(`Missing metadata, attempting to fetch from AudibleCache for ASIN: ${book.audibleAsin}`);
|
||||
|
||||
const audibleCache = await prisma.audibleCache.findUnique({
|
||||
where: { asin: book.audibleAsin },
|
||||
select: { releaseDate: true, narrator: true, },
|
||||
});
|
||||
|
||||
if (audibleCache) {
|
||||
const updates: { year?: number; narrator?: string } = {};
|
||||
|
||||
// Extract year from releaseDate if missing
|
||||
if (!year && audibleCache.releaseDate) {
|
||||
year = new Date(audibleCache.releaseDate).getFullYear();
|
||||
updates.year = year;
|
||||
logger.info(`Extracted year ${year} from AudibleCache releaseDate`);
|
||||
}
|
||||
|
||||
// Get narrator if missing
|
||||
if (!narrator && audibleCache.narrator) {
|
||||
narrator = audibleCache.narrator;
|
||||
updates.narrator = narrator;
|
||||
logger.info(`Got narrator "${narrator}" from AudibleCache`);
|
||||
}
|
||||
|
||||
// Update book record with enriched data for future use
|
||||
if (Object.keys(updates).length > 0) {
|
||||
await prisma.audiobook.update({
|
||||
where: { id: audiobookId },
|
||||
data: updates,
|
||||
});
|
||||
logger.info(`Updated book record with enriched metadata`);
|
||||
}
|
||||
} else {
|
||||
logger.info(`No AudibleCache entry found for ASIN ${book.audibleAsin}`);
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Final metadata for path organization: year=${year || 'null'}, narrator=${narrator || 'null'}, series=${series || 'null'}, seriesPart=${seriesPart || 'null'}`);
|
||||
|
||||
// Check if this is an indexer download (needs to keep source for seeding)
|
||||
const downloadHistory = await prisma.downloadHistory.findFirst({
|
||||
where: { requestId },
|
||||
orderBy: { createdAt: 'desc' },
|
||||
});
|
||||
const isIndexerDownload = downloadHistory?.downloadClient !== 'direct';
|
||||
logger.info(`Download source: ${downloadHistory?.downloadClient || 'unknown'} (indexer download: ${isIndexerDownload})`);
|
||||
|
||||
// Get file organizer and template
|
||||
const organizer = await getFileOrganizer();
|
||||
const templateConfig = await prisma.configuration.findUnique({
|
||||
@@ -509,16 +586,21 @@ async function processEbookOrganization(
|
||||
const template = templateConfig?.value || '{author}/{title} {asin}';
|
||||
|
||||
// Organize ebook files (organizer will detect ebook type and skip audio-specific processing)
|
||||
// Pass all metadata that could be used in path templates (same as audiobooks)
|
||||
const result = await organizer.organizeEbook(
|
||||
downloadPath,
|
||||
{
|
||||
title: book.title,
|
||||
author: book.author,
|
||||
narrator,
|
||||
asin: book.audibleAsin || undefined,
|
||||
year: book.year || undefined,
|
||||
year,
|
||||
series,
|
||||
seriesPart,
|
||||
},
|
||||
template,
|
||||
jobId ? { jobId, context: 'FileOrganizer.Ebook' } : undefined
|
||||
jobId ? { jobId, context: 'FileOrganizer.Ebook' } : undefined,
|
||||
isIndexerDownload
|
||||
);
|
||||
|
||||
if (!result.success) {
|
||||
@@ -595,6 +677,88 @@ async function processEbookOrganization(
|
||||
logger.debug(`Ebook library scan disabled (scanEnabled=${scanEnabled})`);
|
||||
}
|
||||
|
||||
// Cleanup Usenet downloads if configured (same logic as audiobooks)
|
||||
try {
|
||||
logger.info('Checking if cleanup is needed for ebook download');
|
||||
|
||||
// downloadHistory was already fetched earlier in this function
|
||||
logger.info(`Download history found: ${downloadHistory ? 'yes' : 'no'}`, {
|
||||
hasNzbId: !!downloadHistory?.nzbId,
|
||||
hasIndexerId: !!downloadHistory?.indexerId,
|
||||
nzbId: downloadHistory?.nzbId || 'none',
|
||||
indexerId: downloadHistory?.indexerId || 'none',
|
||||
});
|
||||
|
||||
if (downloadHistory?.nzbId && downloadHistory?.indexerId) {
|
||||
// Get indexer configuration
|
||||
const indexersConfig = await configService.get('prowlarr_indexers');
|
||||
logger.info(`Indexers config found: ${indexersConfig ? 'yes' : 'no'}`);
|
||||
|
||||
if (indexersConfig) {
|
||||
const indexers: Array<{ id: number; protocol: string; removeAfterProcessing?: boolean }> = JSON.parse(indexersConfig);
|
||||
const indexer = indexers.find(idx => idx.id === downloadHistory.indexerId);
|
||||
|
||||
logger.info(`Indexer found in config: ${indexer ? 'yes' : 'no'}`, {
|
||||
indexerId: downloadHistory.indexerId,
|
||||
protocol: indexer?.protocol || 'none',
|
||||
removeAfterProcessing: indexer?.removeAfterProcessing ?? 'undefined',
|
||||
});
|
||||
|
||||
// Check if this is a Usenet indexer with cleanup enabled
|
||||
if (indexer && indexer.protocol?.toLowerCase() !== 'torrent' && indexer.removeAfterProcessing) {
|
||||
logger.info(`Cleaning up NZB ${downloadHistory.nzbId} (cleanup enabled for indexer ${indexer.id})`);
|
||||
|
||||
// First, manually delete files from filesystem
|
||||
if (downloadPath) {
|
||||
logger.info(`Removing download files from filesystem: ${downloadPath}`);
|
||||
|
||||
const fs = await import('fs/promises');
|
||||
|
||||
try {
|
||||
// Check if it's a file or directory
|
||||
const stats = await fs.stat(downloadPath);
|
||||
|
||||
if (stats.isDirectory()) {
|
||||
// Remove directory and all contents
|
||||
await fs.rm(downloadPath, { recursive: true, force: true });
|
||||
logger.info(`Removed directory: ${downloadPath}`);
|
||||
} else {
|
||||
// Remove single file
|
||||
await fs.unlink(downloadPath);
|
||||
logger.info(`Removed file: ${downloadPath}`);
|
||||
}
|
||||
} catch (fsError) {
|
||||
// File/directory might already be deleted or not exist
|
||||
if ((fsError as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
logger.info(`Download path already deleted: ${downloadPath}`);
|
||||
} else {
|
||||
throw fsError;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.warn(`No download path available, skipping filesystem deletion`);
|
||||
}
|
||||
|
||||
// Then archive from SABnzbd history (hides from UI but preserves for troubleshooting)
|
||||
const { getSABnzbdService } = await import('../integrations/sabnzbd.service');
|
||||
const sabnzbd = await getSABnzbdService();
|
||||
|
||||
await sabnzbd.archiveCompletedNZB(downloadHistory.nzbId);
|
||||
|
||||
logger.info(`Successfully archived NZB ${downloadHistory.nzbId} and removed files`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Log error but don't fail the job - cleanup is optional
|
||||
logger.warn(
|
||||
`Failed to cleanup NZB download: ${error instanceof Error ? error.message : 'Unknown error'}`,
|
||||
{
|
||||
error: error instanceof Error ? error.stack : undefined,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: 'Ebook organized successfully',
|
||||
@@ -638,13 +802,7 @@ async function createEbookRequestIfEnabled(
|
||||
return;
|
||||
}
|
||||
|
||||
// If only indexer search is enabled (not yet implemented), log and skip
|
||||
if (!isAnnasArchiveEnabled && isIndexerSearchEnabled) {
|
||||
logger.info('Ebook indexer search is enabled but not yet implemented, skipping ebook request creation');
|
||||
return;
|
||||
}
|
||||
|
||||
// Anna's Archive is enabled - proceed with ebook request creation
|
||||
// At least one source is enabled - proceed with ebook request creation
|
||||
|
||||
// Check if an ebook request already exists for this parent
|
||||
const existingEbookRequest = await prisma.request.findFirst({
|
||||
|
||||
@@ -2,16 +2,20 @@
|
||||
* Component: Search Ebook Job Processor
|
||||
* Documentation: documentation/integrations/ebook-sidecar.md
|
||||
*
|
||||
* Searches Anna's Archive for ebook downloads.
|
||||
* Part of the first-class ebook request flow.
|
||||
* Searches for ebook downloads using multiple sources:
|
||||
* 1. Anna's Archive (if enabled) - direct HTTP downloads
|
||||
* 2. Indexer Search (if enabled) - via Prowlarr with ebook categories
|
||||
*/
|
||||
|
||||
import { SearchEbookPayload, EbookSearchResult, getJobQueueService } from '../services/job-queue.service';
|
||||
import { prisma } from '../db';
|
||||
import { getConfigService } from '../services/config.service';
|
||||
import { RMABLogger } from '../utils/logger';
|
||||
import { getProwlarrService } from '../integrations/prowlarr.service';
|
||||
import { rankEbookTorrents, RankedEbookTorrent } from '../utils/ranking-algorithm';
|
||||
import { groupIndexersByCategories, getGroupDescription } from '../utils/indexer-grouping';
|
||||
|
||||
// Import ebook scraper functions (we'll refactor these to be reusable)
|
||||
// Import ebook scraper functions for Anna's Archive
|
||||
import {
|
||||
searchByAsin,
|
||||
searchByTitle,
|
||||
@@ -20,7 +24,7 @@ import {
|
||||
|
||||
/**
|
||||
* Process search ebook job
|
||||
* Searches Anna's Archive for ebook matching the audiobook
|
||||
* Searches Anna's Archive first (if enabled), then falls back to indexer search (if enabled)
|
||||
*/
|
||||
export async function processSearchEbook(payload: SearchEbookPayload): Promise<any> {
|
||||
const { requestId, audiobook, preferredFormat: payloadFormat, jobId } = payload;
|
||||
@@ -43,49 +47,58 @@ export async function processSearchEbook(payload: SearchEbookPayload): Promise<a
|
||||
// Get ebook configuration
|
||||
const configService = getConfigService();
|
||||
const preferredFormat = payloadFormat || await configService.get('ebook_sidecar_preferred_format') || 'epub';
|
||||
const baseUrl = await configService.get('ebook_sidecar_base_url') || 'https://annas-archive.li';
|
||||
const flaresolverrUrl = await configService.get('ebook_sidecar_flaresolverr_url') || undefined;
|
||||
const annasArchiveEnabled = await configService.get('ebook_annas_archive_enabled') === 'true';
|
||||
const indexerSearchEnabled = await configService.get('ebook_indexer_search_enabled') === 'true';
|
||||
|
||||
if (flaresolverrUrl) {
|
||||
logger.info(`Using FlareSolverr at ${flaresolverrUrl}`);
|
||||
}
|
||||
logger.info(`Sources: Anna's Archive=${annasArchiveEnabled}, Indexer Search=${indexerSearchEnabled}`);
|
||||
logger.info(`Preferred format: ${preferredFormat}`);
|
||||
|
||||
let md5: string | null = null;
|
||||
let searchMethod: 'asin' | 'title' = 'title';
|
||||
// Track whether we found a result
|
||||
let annasArchiveResult: EbookSearchResult | null = null;
|
||||
let indexerResult: RankedEbookTorrent | null = null;
|
||||
|
||||
// Step 1: Try ASIN search (exact match - best)
|
||||
if (audiobook.asin) {
|
||||
logger.info(`Searching by ASIN: ${audiobook.asin} (format: ${preferredFormat})...`);
|
||||
md5 = await searchByAsin(audiobook.asin, preferredFormat, baseUrl, logger, flaresolverrUrl);
|
||||
// ========== STEP 1: Try Anna's Archive (if enabled) ==========
|
||||
if (annasArchiveEnabled) {
|
||||
logger.info(`Searching Anna's Archive...`);
|
||||
annasArchiveResult = await searchAnnasArchive(audiobook, preferredFormat, logger);
|
||||
|
||||
if (md5) {
|
||||
logger.info(`Found via ASIN: ${md5}`);
|
||||
searchMethod = 'asin';
|
||||
if (annasArchiveResult) {
|
||||
logger.info(`Found ebook via Anna's Archive (score: ${annasArchiveResult.score})`);
|
||||
} else {
|
||||
logger.info(`No results for ASIN, falling back to title + author search...`);
|
||||
logger.info(`No results from Anna's Archive`);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: Fallback to title + author search
|
||||
if (!md5) {
|
||||
logger.info(`Searching by title + author: "${audiobook.title}" by ${audiobook.author}...`);
|
||||
md5 = await searchByTitle(audiobook.title, audiobook.author, preferredFormat, baseUrl, logger, flaresolverrUrl);
|
||||
// ========== STEP 2: Try Indexer Search (if enabled and no Anna's Archive result) ==========
|
||||
if (!annasArchiveResult && indexerSearchEnabled) {
|
||||
logger.info(`Searching indexers...`);
|
||||
indexerResult = await searchIndexers(requestId, audiobook, preferredFormat, logger);
|
||||
|
||||
if (md5) {
|
||||
logger.info(`Found via title search: ${md5}`);
|
||||
searchMethod = 'title';
|
||||
if (indexerResult) {
|
||||
logger.info(`Found ebook via indexer search (score: ${indexerResult.finalScore.toFixed(1)})`);
|
||||
} else {
|
||||
logger.info(`No results from indexer search`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!md5) {
|
||||
// No results found - queue for re-search instead of failing
|
||||
// ========== STEP 3: Handle Results ==========
|
||||
if (!annasArchiveResult && !indexerResult) {
|
||||
// No results found from any source
|
||||
const enabledSources = [];
|
||||
if (annasArchiveEnabled) enabledSources.push("Anna's Archive");
|
||||
if (indexerSearchEnabled) enabledSources.push("Indexer Search");
|
||||
|
||||
const message = enabledSources.length > 0
|
||||
? `No ebook found on ${enabledSources.join(' or ')}. Will retry automatically.`
|
||||
: 'No ebook sources enabled. Enable Anna\'s Archive or Indexer Search in settings.';
|
||||
|
||||
logger.warn(`No ebook found for request ${requestId}, marking as awaiting_search`);
|
||||
|
||||
await prisma.request.update({
|
||||
where: { id: requestId },
|
||||
data: {
|
||||
status: 'awaiting_search',
|
||||
errorMessage: 'No ebook found on Anna\'s Archive. Will retry automatically.',
|
||||
errorMessage: message,
|
||||
lastSearchAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
@@ -98,107 +111,18 @@ export async function processSearchEbook(payload: SearchEbookPayload): Promise<a
|
||||
};
|
||||
}
|
||||
|
||||
logger.info(`Found MD5: ${md5}`);
|
||||
|
||||
// Step 3: Get slow download links
|
||||
const slowLinks = await getSlowDownloadLinks(md5, baseUrl, logger, flaresolverrUrl);
|
||||
|
||||
if (slowLinks.length === 0) {
|
||||
logger.warn(`No download links available for MD5: ${md5}`);
|
||||
|
||||
await prisma.request.update({
|
||||
where: { id: requestId },
|
||||
data: {
|
||||
status: 'awaiting_search',
|
||||
errorMessage: 'Found ebook but no download links available. Will retry automatically.',
|
||||
lastSearchAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
message: 'No download links available, queued for re-search',
|
||||
requestId,
|
||||
};
|
||||
// ========== STEP 4: Route to Appropriate Download ==========
|
||||
if (annasArchiveResult) {
|
||||
// Anna's Archive result → Direct download
|
||||
return await handleAnnasArchiveDownload(requestId, audiobook, annasArchiveResult, preferredFormat, logger);
|
||||
} else if (indexerResult) {
|
||||
// Indexer result → Torrent/NZB download (reuse audiobook processor)
|
||||
return await handleIndexerDownload(requestId, audiobook, indexerResult, preferredFormat, logger);
|
||||
}
|
||||
|
||||
logger.info(`Found ${slowLinks.length} download link(s)`);
|
||||
// This should never be reached
|
||||
throw new Error('Unexpected state: no result to process');
|
||||
|
||||
// Create ebook search result
|
||||
// Note: For future multi-source ranking, this would be one of many results
|
||||
const searchResult: EbookSearchResult = {
|
||||
md5,
|
||||
title: audiobook.title,
|
||||
author: audiobook.author,
|
||||
format: preferredFormat,
|
||||
downloadUrls: slowLinks,
|
||||
source: 'annas_archive',
|
||||
score: searchMethod === 'asin' ? 100 : 80, // ASIN match = higher confidence
|
||||
};
|
||||
|
||||
// TODO: Future enhancement - when indexer support is added for ebooks:
|
||||
// 1. Search Prowlarr for ebook results (filtered to ebook categories)
|
||||
// 2. Rank results using rankEbookResults() with inverted size scoring
|
||||
// 3. Anna's Archive results should get priority bonus to come out on top
|
||||
// For now, Anna's Archive is the only source and always wins.
|
||||
|
||||
logger.info(`==================== EBOOK SEARCH RESULT ====================`);
|
||||
logger.info(`Title: "${audiobook.title}"`);
|
||||
logger.info(`Author: "${audiobook.author}"`);
|
||||
logger.info(`Match Method: ${searchMethod === 'asin' ? 'ASIN (exact)' : 'Title + Author (fuzzy)'}`);
|
||||
logger.info(`Format: ${preferredFormat}`);
|
||||
logger.info(`MD5: ${md5}`);
|
||||
logger.info(`Download Links: ${slowLinks.length}`);
|
||||
logger.info(`Score: ${searchResult.score}/100`);
|
||||
logger.info(`==============================================================`);
|
||||
|
||||
// Create download history record
|
||||
const downloadHistory = await prisma.downloadHistory.create({
|
||||
data: {
|
||||
requestId,
|
||||
indexerName: 'Anna\'s Archive',
|
||||
torrentName: `${audiobook.title} - ${audiobook.author}.${preferredFormat}`,
|
||||
torrentSizeBytes: null, // Unknown until download starts
|
||||
qualityScore: searchResult.score,
|
||||
selected: true,
|
||||
downloadClient: 'direct', // Direct HTTP download
|
||||
downloadStatus: 'queued',
|
||||
},
|
||||
});
|
||||
|
||||
// Trigger direct download job with the best (only) result
|
||||
const jobQueue = getJobQueueService();
|
||||
|
||||
// The first slow link will be tried; if it fails, the processor will try others
|
||||
await jobQueue.addStartDirectDownloadJob(
|
||||
requestId,
|
||||
downloadHistory.id,
|
||||
slowLinks[0], // Start with first link
|
||||
`${audiobook.title} - ${audiobook.author}.${preferredFormat}`,
|
||||
undefined // Size unknown
|
||||
);
|
||||
|
||||
// Store all download URLs in download history for retry purposes
|
||||
await prisma.downloadHistory.update({
|
||||
where: { id: downloadHistory.id },
|
||||
data: {
|
||||
// Store additional URLs in torrentUrl field (JSON array)
|
||||
torrentUrl: JSON.stringify(slowLinks),
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: `Found ebook via ${searchMethod === 'asin' ? 'ASIN' : 'title search'}, starting download`,
|
||||
requestId,
|
||||
searchResult: {
|
||||
md5: searchResult.md5,
|
||||
format: searchResult.format,
|
||||
score: searchResult.score,
|
||||
downloadLinksCount: slowLinks.length,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
|
||||
@@ -214,3 +138,367 @@ export async function processSearchEbook(payload: SearchEbookPayload): Promise<a
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search Anna's Archive for ebook
|
||||
*/
|
||||
async function searchAnnasArchive(
|
||||
audiobook: { title: string; author: string; asin?: string },
|
||||
preferredFormat: string,
|
||||
logger: RMABLogger
|
||||
): Promise<EbookSearchResult | null> {
|
||||
const configService = getConfigService();
|
||||
const baseUrl = await configService.get('ebook_sidecar_base_url') || 'https://annas-archive.li';
|
||||
const flaresolverrUrl = await configService.get('ebook_sidecar_flaresolverr_url') || undefined;
|
||||
|
||||
if (flaresolverrUrl) {
|
||||
logger.info(`Using FlareSolverr at ${flaresolverrUrl}`);
|
||||
}
|
||||
|
||||
let md5: string | null = null;
|
||||
let searchMethod: 'asin' | 'title' = 'title';
|
||||
|
||||
// Try ASIN search first (exact match - best)
|
||||
if (audiobook.asin) {
|
||||
logger.info(`Searching Anna's Archive by ASIN: ${audiobook.asin} (format: ${preferredFormat})...`);
|
||||
md5 = await searchByAsin(audiobook.asin, preferredFormat, baseUrl, logger, flaresolverrUrl);
|
||||
|
||||
if (md5) {
|
||||
logger.info(`Found via ASIN: ${md5}`);
|
||||
searchMethod = 'asin';
|
||||
} else {
|
||||
logger.info(`No ASIN results, trying title + author...`);
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to title + author search
|
||||
if (!md5) {
|
||||
logger.info(`Searching Anna's Archive by title + author: "${audiobook.title}" by ${audiobook.author}...`);
|
||||
md5 = await searchByTitle(audiobook.title, audiobook.author, preferredFormat, baseUrl, logger, flaresolverrUrl);
|
||||
|
||||
if (md5) {
|
||||
logger.info(`Found via title search: ${md5}`);
|
||||
searchMethod = 'title';
|
||||
}
|
||||
}
|
||||
|
||||
if (!md5) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Get slow download links
|
||||
const slowLinks = await getSlowDownloadLinks(md5, baseUrl, logger, flaresolverrUrl);
|
||||
|
||||
if (slowLinks.length === 0) {
|
||||
logger.warn(`Found MD5 ${md5} but no download links available`);
|
||||
return null;
|
||||
}
|
||||
|
||||
logger.info(`Found ${slowLinks.length} download link(s) for MD5 ${md5}`);
|
||||
|
||||
return {
|
||||
md5,
|
||||
title: audiobook.title,
|
||||
author: audiobook.author,
|
||||
format: preferredFormat,
|
||||
downloadUrls: slowLinks,
|
||||
source: 'annas_archive',
|
||||
score: searchMethod === 'asin' ? 100 : 80,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Search indexers for ebook torrents/NZBs
|
||||
*/
|
||||
async function searchIndexers(
|
||||
requestId: string,
|
||||
audiobook: { title: string; author: string },
|
||||
preferredFormat: string,
|
||||
logger: RMABLogger
|
||||
): Promise<RankedEbookTorrent | null> {
|
||||
const configService = getConfigService();
|
||||
|
||||
// Get enabled indexers from configuration
|
||||
const indexersConfigStr = await configService.get('prowlarr_indexers');
|
||||
|
||||
if (!indexersConfigStr) {
|
||||
logger.warn('No indexers configured');
|
||||
return null;
|
||||
}
|
||||
|
||||
const indexersConfig = JSON.parse(indexersConfigStr);
|
||||
|
||||
if (indexersConfig.length === 0) {
|
||||
logger.warn('No indexers enabled');
|
||||
return null;
|
||||
}
|
||||
|
||||
// Build indexer priorities map (indexerId -> priority 1-25, default 10)
|
||||
const indexerPriorities = new Map<number, number>(
|
||||
indexersConfig.map((indexer: any) => [indexer.id, indexer.priority ?? 10])
|
||||
);
|
||||
|
||||
// Get flag configurations
|
||||
const flagConfigStr = await configService.get('indexer_flag_config');
|
||||
const flagConfigs = flagConfigStr ? JSON.parse(flagConfigStr) : [];
|
||||
|
||||
// Group indexers by their EBOOK category configuration
|
||||
const groups = groupIndexersByCategories(indexersConfig, 'ebook');
|
||||
|
||||
logger.info(`Searching ${indexersConfig.length} enabled indexers in ${groups.length} group${groups.length > 1 ? 's' : ''}`);
|
||||
|
||||
// Log each group for transparency
|
||||
groups.forEach((group, index) => {
|
||||
logger.info(`Group ${index + 1}: ${getGroupDescription(group)}`);
|
||||
});
|
||||
|
||||
// Get Prowlarr service
|
||||
const prowlarr = await getProwlarrService();
|
||||
|
||||
// Build search query (title only - cast wide net, let ranking filter)
|
||||
const searchQuery = audiobook.title;
|
||||
|
||||
logger.info(`Searching for: "${searchQuery}"`);
|
||||
|
||||
// Search Prowlarr for each group and combine results
|
||||
const allResults = [];
|
||||
|
||||
for (let i = 0; i < groups.length; i++) {
|
||||
const group = groups[i];
|
||||
logger.info(`Searching group ${i + 1}/${groups.length}: ${getGroupDescription(group)}`);
|
||||
|
||||
try {
|
||||
const groupResults = await prowlarr.search(searchQuery, {
|
||||
categories: group.categories,
|
||||
indexerIds: group.indexerIds,
|
||||
minSeeders: 0, // Ebooks may have fewer seeders
|
||||
maxResults: 100,
|
||||
});
|
||||
|
||||
logger.info(`Group ${i + 1} returned ${groupResults.length} results`);
|
||||
allResults.push(...groupResults);
|
||||
} catch (error) {
|
||||
logger.error(`Group ${i + 1} search failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
||||
// Continue with other groups even if one fails
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`Found ${allResults.length} total results from ${groups.length} group${groups.length > 1 ? 's' : ''}`);
|
||||
|
||||
if (allResults.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Log filter info (ebooks > 20MB will be filtered)
|
||||
const preFilterCount = allResults.length;
|
||||
const aboveThreshold = allResults.filter(r => (r.size / (1024 * 1024)) > 20);
|
||||
if (aboveThreshold.length > 0) {
|
||||
logger.info(`Will filter ${aboveThreshold.length} results > 20 MB (too large for ebooks)`);
|
||||
}
|
||||
|
||||
// Rank results with ebook-specific scoring
|
||||
// This filters out > 20MB and uses inverted size scoring
|
||||
const rankedResults = rankEbookTorrents(allResults, {
|
||||
title: audiobook.title,
|
||||
author: audiobook.author,
|
||||
preferredFormat,
|
||||
}, {
|
||||
indexerPriorities,
|
||||
flagConfigs,
|
||||
requireAuthor: true, // Automatic mode - prevent wrong authors
|
||||
});
|
||||
|
||||
// Log filter results
|
||||
const postFilterCount = rankedResults.length;
|
||||
if (postFilterCount < preFilterCount) {
|
||||
logger.info(`Filtered out ${preFilterCount - postFilterCount} results > 20 MB`);
|
||||
}
|
||||
|
||||
// Dual threshold filtering (same as audiobooks)
|
||||
const filteredResults = rankedResults.filter(result =>
|
||||
result.score >= 50 && result.finalScore >= 50
|
||||
);
|
||||
|
||||
const disqualifiedByNegativeBonus = rankedResults.filter(result =>
|
||||
result.score >= 50 && result.finalScore < 50
|
||||
).length;
|
||||
|
||||
logger.info(`Ranked ${rankedResults.length} results, ${filteredResults.length} above threshold (50/100 base + final)`);
|
||||
if (disqualifiedByNegativeBonus > 0) {
|
||||
logger.info(`${disqualifiedByNegativeBonus} ebooks disqualified by negative flag bonuses`);
|
||||
}
|
||||
|
||||
if (filteredResults.length === 0) {
|
||||
logger.warn(`No quality matches found (all below 50/100)`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Select best result
|
||||
const bestResult = filteredResults[0];
|
||||
|
||||
// Log top 3 results with detailed breakdown
|
||||
const top3 = filteredResults.slice(0, 3);
|
||||
logger.info(`==================== EBOOK RANKING DEBUG ====================`);
|
||||
logger.info(`Requested Title: "${audiobook.title}"`);
|
||||
logger.info(`Requested Author: "${audiobook.author}"`);
|
||||
logger.info(`Preferred Format: ${preferredFormat}`);
|
||||
logger.info(`Top ${top3.length} results (out of ${filteredResults.length} above threshold):`);
|
||||
logger.info(`--------------------------------------------------------------`);
|
||||
for (let i = 0; i < top3.length; i++) {
|
||||
const result = top3[i];
|
||||
const sizeMB = (result.size / (1024 * 1024)).toFixed(1);
|
||||
|
||||
logger.info(`${i + 1}. "${result.title}"`);
|
||||
logger.info(` Indexer: ${result.indexer}${result.indexerId ? ` (ID: ${result.indexerId})` : ''}`);
|
||||
logger.info(``);
|
||||
logger.info(` Base Score: ${result.score.toFixed(1)}/100`);
|
||||
logger.info(` - Title/Author Match: ${result.breakdown.matchScore.toFixed(1)}/60`);
|
||||
logger.info(` - Format Match: ${result.breakdown.formatScore.toFixed(1)}/10`);
|
||||
logger.info(` - Size Quality: ${result.breakdown.sizeScore.toFixed(1)}/15 (${sizeMB} MB)`);
|
||||
logger.info(` - Seeder Count: ${result.breakdown.seederScore.toFixed(1)}/15 (${result.seeders !== undefined ? result.seeders + ' seeders' : 'N/A for Usenet'})`);
|
||||
logger.info(``);
|
||||
logger.info(` Bonus Points: +${result.bonusPoints.toFixed(1)}`);
|
||||
if (result.bonusModifiers.length > 0) {
|
||||
for (const mod of result.bonusModifiers) {
|
||||
logger.info(` - ${mod.reason}: +${mod.points.toFixed(1)}`);
|
||||
}
|
||||
}
|
||||
logger.info(``);
|
||||
logger.info(` Final Score: ${result.finalScore.toFixed(1)}`);
|
||||
if (result.breakdown.notes.length > 0) {
|
||||
logger.info(` Notes: ${result.breakdown.notes.join(', ')}`);
|
||||
}
|
||||
if (i < top3.length - 1) {
|
||||
logger.info(`--------------------------------------------------------------`);
|
||||
}
|
||||
}
|
||||
logger.info(`==============================================================`);
|
||||
logger.info(`Selected best result: ${bestResult.title} (final score: ${bestResult.finalScore.toFixed(1)})`);
|
||||
|
||||
return bestResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle Anna's Archive download (direct HTTP)
|
||||
*/
|
||||
async function handleAnnasArchiveDownload(
|
||||
requestId: string,
|
||||
audiobook: { title: string; author: string },
|
||||
result: EbookSearchResult,
|
||||
preferredFormat: string,
|
||||
logger: RMABLogger
|
||||
): Promise<any> {
|
||||
logger.info(`==================== EBOOK SEARCH RESULT ====================`);
|
||||
logger.info(`Source: Anna's Archive`);
|
||||
logger.info(`Title: "${audiobook.title}"`);
|
||||
logger.info(`Author: "${audiobook.author}"`);
|
||||
logger.info(`Format: ${preferredFormat}`);
|
||||
logger.info(`MD5: ${result.md5}`);
|
||||
logger.info(`Download Links: ${result.downloadUrls.length}`);
|
||||
logger.info(`Score: ${result.score}/100`);
|
||||
logger.info(`==============================================================`);
|
||||
|
||||
// Create download history record
|
||||
const downloadHistory = await prisma.downloadHistory.create({
|
||||
data: {
|
||||
requestId,
|
||||
indexerName: "Anna's Archive",
|
||||
torrentName: `${audiobook.title} - ${audiobook.author}.${preferredFormat}`,
|
||||
torrentSizeBytes: null, // Unknown until download starts
|
||||
qualityScore: result.score,
|
||||
selected: true,
|
||||
downloadClient: 'direct', // Direct HTTP download
|
||||
downloadStatus: 'queued',
|
||||
},
|
||||
});
|
||||
|
||||
// Trigger direct download job
|
||||
const jobQueue = getJobQueueService();
|
||||
await jobQueue.addStartDirectDownloadJob(
|
||||
requestId,
|
||||
downloadHistory.id,
|
||||
result.downloadUrls[0], // Start with first link
|
||||
`${audiobook.title} - ${audiobook.author}.${preferredFormat}`,
|
||||
undefined // Size unknown
|
||||
);
|
||||
|
||||
// Store all download URLs for retry purposes
|
||||
await prisma.downloadHistory.update({
|
||||
where: { id: downloadHistory.id },
|
||||
data: {
|
||||
torrentUrl: JSON.stringify(result.downloadUrls),
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: `Found ebook via Anna's Archive, starting download`,
|
||||
requestId,
|
||||
source: 'annas_archive',
|
||||
searchResult: {
|
||||
md5: result.md5,
|
||||
format: result.format,
|
||||
score: result.score,
|
||||
downloadLinksCount: result.downloadUrls.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle indexer download (torrent/NZB via download-torrent processor)
|
||||
*/
|
||||
async function handleIndexerDownload(
|
||||
requestId: string,
|
||||
audiobook: { title: string; author: string },
|
||||
result: RankedEbookTorrent,
|
||||
preferredFormat: string,
|
||||
logger: RMABLogger
|
||||
): Promise<any> {
|
||||
logger.info(`==================== EBOOK SEARCH RESULT ====================`);
|
||||
logger.info(`Source: Indexer (${result.indexer})`);
|
||||
logger.info(`Title: "${audiobook.title}"`);
|
||||
logger.info(`Author: "${audiobook.author}"`);
|
||||
logger.info(`Torrent: "${result.title}"`);
|
||||
logger.info(`Size: ${(result.size / (1024 * 1024)).toFixed(1)} MB`);
|
||||
logger.info(`Seeders: ${result.seeders !== undefined ? result.seeders : 'N/A'}`);
|
||||
logger.info(`Final Score: ${result.finalScore.toFixed(1)}/100`);
|
||||
logger.info(`==============================================================`);
|
||||
|
||||
// Trigger download job using the SAME processor as audiobooks
|
||||
// The download-torrent processor is already generic and handles both torrent and NZB
|
||||
const jobQueue = getJobQueueService();
|
||||
|
||||
// Fetch the request to get the parent audiobook ID for the download job
|
||||
const request = await prisma.request.findUnique({
|
||||
where: { id: requestId },
|
||||
include: { parentRequest: true },
|
||||
});
|
||||
|
||||
if (!request) {
|
||||
throw new Error(`Request ${requestId} not found`);
|
||||
}
|
||||
|
||||
// Use the parent audiobook's ID for the download job, or fall back to request ID
|
||||
const audiobookId = request.parentRequest?.id || request.id;
|
||||
|
||||
await jobQueue.addDownloadJob(requestId, {
|
||||
id: audiobookId,
|
||||
title: audiobook.title,
|
||||
author: audiobook.author,
|
||||
}, result);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: `Found ebook via indexer search, starting download`,
|
||||
requestId,
|
||||
source: 'prowlarr',
|
||||
resultsCount: 1,
|
||||
selectedTorrent: {
|
||||
title: result.title,
|
||||
score: result.score,
|
||||
finalScore: result.finalScore,
|
||||
seeders: result.seeders || 0,
|
||||
size: result.size,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user