From 425129e8c96e64ea2562414cad93235342a7c9c4 Mon Sep 17 00:00:00 2001 From: kikootwo Date: Tue, 18 Aug 2026 16:09:36 -0400 Subject: [PATCH] fix: try next release after transient grab failure (#275) --- AGENTS.md | 26 ++ documentation/backend/services/jobs.md | 8 + documentation/phase3/README.md | 2 + documentation/phase3/qbittorrent.md | 8 +- src/lib/integrations/deluge.service.ts | 26 +- src/lib/integrations/nzbget.service.ts | 8 +- src/lib/integrations/qbittorrent.service.ts | 31 ++- src/lib/integrations/sabnzbd.service.ts | 8 +- src/lib/integrations/transmission.service.ts | 22 +- .../interfaces/download-client.interface.ts | 27 +++ .../processors/download-torrent.processor.ts | 222 +++++++++++------- .../processors/search-indexers.processor.ts | 2 +- src/lib/services/job-queue.service.ts | 6 +- .../integrations/qbittorrent.service.test.ts | 27 +++ .../download-torrent.processor.test.ts | 104 ++++++++ .../search-indexers.processor.test.ts | 20 +- tests/services/job-queue.service.test.ts | 13 +- 17 files changed, 455 insertions(+), 105 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..ec97152 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,26 @@ +# ReadMeABook Agent Instructions + +These are the repository-wide instructions for all agents. Also follow the project standards and approval workflow in `CLAUDE.md`. + +## Shared Developer Docker Compose + +- The canonical developer Compose file is `C:\GIT\ReadMeABook\docker-compose.yml`. +- It is intentionally local-only, marked `skip-worktree`, and contains private machine configuration. Never print its full contents, expose its credentials, or include it in a commit. +- The active data bind mounts are hard-coded to `C:\GIT\ReadMeABook\config`, `cache`, `bookdrop`, `pgdata`, and `redis`, so every worktree uses the same developer data. Keep these mounts absolute. +- From any ReadMeABook worktree, pass its Git root as `--project-directory`. This keeps `build.context: .` pointed at that worktree's source code while the absolute data mounts continue to use the canonical root directories: + +```powershell +$rmabWorktree = (git rev-parse --show-toplevel).Trim() +docker compose --project-directory $rmabWorktree -f C:\GIT\ReadMeABook\docker-compose.yml build readmeabook +docker compose --project-directory $rmabWorktree -f C:\GIT\ReadMeABook\docker-compose.yml up -d +``` + +- To build and start in one command: + +```powershell +$rmabWorktree = (git rev-parse --show-toplevel).Trim() +docker compose --project-directory $rmabWorktree -f C:\GIT\ReadMeABook\docker-compose.yml up -d --build +``` + +- Do not use a plain `docker compose build readmeabook` unless the active worktree's Compose file has first been verified to contain a `build` section. The tracked production Compose normally references a prebuilt registry image, making that command a no-op. +- The developer Compose uses the fixed container name `readmeabook-test`. Before starting it, inspect any existing container with that name. Do not remove or replace an existing container without user approval. diff --git a/documentation/backend/services/jobs.md b/documentation/backend/services/jobs.md index 8ccc016..7e38857 100644 --- a/documentation/backend/services/jobs.md +++ b/documentation/backend/services/jobs.md @@ -33,6 +33,14 @@ Manages background job queue using Bull (Redis-backed) for async tasks: searchin 5. **plex_recently_added_check** - Lightweight polling of recently added items, match all non-terminal requests 6. **match_plex** - Fuzzy match to Plex item (deprecated - now handled by scan_plex) +## Download Grab Fallback + +- Automatic audiobook search passes every above-threshold release to `download_torrent` in ranked order. +- HTTP `429` and `5xx` responses while fetching a `.torrent`/`.nzb` from Prowlarr are source-grab failures, not download-client failures. +- `download_torrent` immediately tries the next ranked release; the successful candidate is the only one written to download history. +- If every ranked grab is temporarily unavailable, the job remains retryable under Bull's exponential backoff. The global failed handler marks the request failed only after all job attempts are exhausted. +- Manual selections carry no alternates and retain their explicit-release behavior. + ## Special Behaviors **monitor_download:** diff --git a/documentation/phase3/README.md b/documentation/phase3/README.md index e189d94..b464473 100644 --- a/documentation/phase3/README.md +++ b/documentation/phase3/README.md @@ -20,6 +20,8 @@ Request → search_indexers → rank_results → download_torrent 5. **process_audiobook** - Organize files to media directory 6. **update_plex** - Trigger scan, fuzzy match +Automatic searches keep the ranked candidate list. If Prowlarr returns HTTP `429` or `5xx` while grabbing the selected release, `download_torrent` attributes the error to the indexer/source and falls through to the next release. When all candidates have temporary grab failures, Bull retries the job with backoff. + ## Integration Points **Indexers:** Prowlarr (primary), Jackett (fallback) diff --git a/documentation/phase3/qbittorrent.md b/documentation/phase3/qbittorrent.md index 3c7f648..f2f92e0 100644 --- a/documentation/phase3/qbittorrent.md +++ b/documentation/phase3/qbittorrent.md @@ -247,7 +247,13 @@ type TorrentState = - `pausedUP` → `seeding` (unified) / `completed` (legacy) — triggers completion in monitor - `stoppedUP` → `seeding` (unified) / `completed` (legacy) — same fix for qBittorrent v5.x - `pausedDL`/`stoppedDL` remain `paused` — download phase genuinely paused - - Key insight: any `*UP` state is post-download; any `*DL` state is pre-completion +- Key insight: any `*UP` state is post-download; any `*DL` state is pre-completion + +**17. Prowlarr grab errors blamed on qBittorrent** - Source HTTP errors were collapsed into `Failed to add torrent to qBittorrent`, so a rate-limited indexer permanently failed the request. Fixed by: +- Forwarding Prowlarr source headers when fetching `.torrent` files +- Preserving the upstream HTTP status as a structured source-grab error +- Trying the remaining ranked releases on HTTP `429`/`5xx` +- Reporting the indexer, release, and upstream status instead of blaming a healthy qBittorrent instance ## Tech Stack diff --git a/src/lib/integrations/deluge.service.ts b/src/lib/integrations/deluge.service.ts index 1ca4ea5..7fa69f5 100644 --- a/src/lib/integrations/deluge.service.ts +++ b/src/lib/integrations/deluge.service.ts @@ -14,6 +14,7 @@ import { PathMapper, PathMappingConfig } from '../utils/path-mapper'; import { IDownloadClient, DownloadClientType, ProtocolType, DownloadInfo, DownloadStatus, AddDownloadOptions, ConnectionTestResult, + DownloadSourceError, } from '../interfaces/download-client.interface'; const parseTorrent = (parseTorrentModule as any).default || parseTorrentModule; @@ -193,7 +194,7 @@ export class DelugeService implements IDownloadClient { torrentResponse = await axios.get(torrentUrl, { responseType: 'arraybuffer', maxRedirects: 0, validateStatus: (s) => s >= 200 && s < 300, timeout: DOWNLOAD_CLIENT_TIMEOUT, - headers: { 'User-Agent': RMAB_USER_AGENT }, + headers: { 'User-Agent': RMAB_USER_AGENT, ...options?.sourceHeaders }, }); if (torrentResponse.data.length > 0) { const magnetMatch = torrentResponse.data.toString().match(/^magnet:\?[^\s]+$/); @@ -206,10 +207,27 @@ export class DelugeService implements IDownloadClient { const loc = error.response.headers['location']; if (loc?.startsWith('magnet:')) return this.addMagnetLink(loc, category, options); if (loc?.startsWith('http://') || loc?.startsWith('https://')) { - try { torrentResponse = await axios.get(loc, { responseType: 'arraybuffer', timeout: DOWNLOAD_CLIENT_TIMEOUT, maxRedirects: 5, headers: { 'User-Agent': RMAB_USER_AGENT } }); } - catch { throw new Error('Failed to download torrent file after redirect'); } + try { torrentResponse = await axios.get(loc, { responseType: 'arraybuffer', timeout: DOWNLOAD_CLIENT_TIMEOUT, maxRedirects: 5, headers: { 'User-Agent': RMAB_USER_AGENT, ...options?.sourceHeaders } }); } + catch (redirectError) { + if (axios.isAxiosError(redirectError) && redirectError.response?.status) { + throw new DownloadSourceError( + `Grab failed: source returned HTTP ${redirectError.response.status}`, + redirectError.response.status, + loc, + redirectError + ); + } + throw new Error('Failed to download torrent file after redirect'); + } } else { throw new Error(`Invalid redirect location: ${loc}`); } - } else { throw new Error(`Failed to download torrent: HTTP ${status}`); } + } else { + throw new DownloadSourceError( + `Grab failed: source returned HTTP ${status}`, + status, + torrentUrl, + error + ); + } } const torrentBuffer = Buffer.from(torrentResponse.data); diff --git a/src/lib/integrations/nzbget.service.ts b/src/lib/integrations/nzbget.service.ts index 678502c..8aa0779 100644 --- a/src/lib/integrations/nzbget.service.ts +++ b/src/lib/integrations/nzbget.service.ts @@ -17,6 +17,7 @@ import { DownloadStatus, AddDownloadOptions, ConnectionTestResult, + DownloadSourceError, } from '../interfaces/download-client.interface'; const logger = RMABLogger.create('NZBGet'); @@ -255,7 +256,12 @@ export class NZBGetService implements IDownloadClient { if (axios.isAxiosError(error)) { const status = error.response?.status; if (status) { - throw new Error(`Failed to download NZB file: HTTP ${status} from source URL`); + throw new DownloadSourceError( + `Grab failed: source returned HTTP ${status}`, + status, + url, + error + ); } if (error.code === 'ECONNREFUSED') { throw new Error('Failed to download NZB file: Connection refused. Is Prowlarr running?'); diff --git a/src/lib/integrations/qbittorrent.service.ts b/src/lib/integrations/qbittorrent.service.ts index b1248b6..710b049 100644 --- a/src/lib/integrations/qbittorrent.service.ts +++ b/src/lib/integrations/qbittorrent.service.ts @@ -20,6 +20,7 @@ import { DownloadStatus, AddDownloadOptions, ConnectionTestResult, + DownloadSourceError, } from '../interfaces/download-client.interface'; // Handle both ESM and CommonJS imports @@ -33,6 +34,7 @@ export interface AddTorrentOptions { tags?: string[]; paused?: boolean; skipChecking?: boolean; + sourceHeaders?: Record; } export interface TorrentInfo { @@ -282,6 +284,11 @@ export class QBittorrentService implements IDownloadClient { return await this.addTorrentFile(url, category, options); } } catch (error) { + // Preserve source/indexer failures so the processor can try another release. + if (error instanceof DownloadSourceError) { + throw error; + } + // Try re-authenticating once if we get a 403 — only meaningful when credentials are configured. // In auth-optional mode a 403 means the server actually wants auth (e.g. IP no longer whitelisted), // so retrying login is pointless and would mask the real error. @@ -378,6 +385,10 @@ export class QBittorrentService implements IDownloadClient { maxRedirects: 0, validateStatus: (status) => status >= 200 && status < 300, // Only 2xx is success timeout: DOWNLOAD_CLIENT_TIMEOUT, + headers: { + 'User-Agent': RMAB_USER_AGENT, + ...options?.sourceHeaders, + }, }); logger.info(` Got 2xx response, size=${torrentResponse.data.length} bytes`); @@ -421,10 +432,22 @@ export class QBittorrentService implements IDownloadClient { responseType: 'arraybuffer', timeout: DOWNLOAD_CLIENT_TIMEOUT, maxRedirects: 5, + headers: { + 'User-Agent': RMAB_USER_AGENT, + ...options?.sourceHeaders, + }, }); logger.info(` After following redirect: size=${torrentResponse.data.length} bytes`); } catch (redirectError) { logger.error('Failed to follow redirect', { error: redirectError instanceof Error ? redirectError.message : String(redirectError) }); + if (axios.isAxiosError(redirectError) && redirectError.response?.status) { + throw new DownloadSourceError( + `Grab failed: source returned HTTP ${redirectError.response.status}`, + redirectError.response.status, + location, + redirectError + ); + } throw new Error('Failed to download torrent file after redirect'); } } else { @@ -433,7 +456,12 @@ export class QBittorrentService implements IDownloadClient { } else { // Non-redirect error (4xx, 5xx) logger.error(`HTTP error ${status}`, { error: error.message }); - throw new Error(`Failed to download torrent: HTTP ${status}`); + throw new DownloadSourceError( + `Grab failed: source returned HTTP ${status}`, + status, + torrentUrl, + error + ); } } @@ -1077,6 +1105,7 @@ export class QBittorrentService implements IDownloadClient { category: options?.category, paused: options?.paused, tags: ['audiobook'], + sourceHeaders: options?.sourceHeaders, }); } diff --git a/src/lib/integrations/sabnzbd.service.ts b/src/lib/integrations/sabnzbd.service.ts index 78cd6e5..bcf2573 100644 --- a/src/lib/integrations/sabnzbd.service.ts +++ b/src/lib/integrations/sabnzbd.service.ts @@ -17,6 +17,7 @@ import { DownloadStatus, AddDownloadOptions, ConnectionTestResult, + DownloadSourceError, } from '../interfaces/download-client.interface'; const logger = RMABLogger.create('SABnzbd'); @@ -519,7 +520,12 @@ export class SABnzbdService implements IDownloadClient { if (axios.isAxiosError(error)) { const status = error.response?.status; if (status) { - throw new Error(`Failed to download NZB file: HTTP ${status} from source URL`); + throw new DownloadSourceError( + `Grab failed: source returned HTTP ${status}`, + status, + url, + error + ); } if (error.code === 'ECONNREFUSED') { throw new Error('Failed to download NZB file: Connection refused. Is Prowlarr running?'); diff --git a/src/lib/integrations/transmission.service.ts b/src/lib/integrations/transmission.service.ts index 8705873..6b3d05c 100644 --- a/src/lib/integrations/transmission.service.ts +++ b/src/lib/integrations/transmission.service.ts @@ -19,6 +19,7 @@ import { DownloadStatus, AddDownloadOptions, ConnectionTestResult, + DownloadSourceError, } from '../interfaces/download-client.interface'; // Handle both ESM and CommonJS imports @@ -278,7 +279,7 @@ export class TransmissionService implements IDownloadClient { maxRedirects: 0, validateStatus: (status) => status >= 200 && status < 300, timeout: DOWNLOAD_CLIENT_TIMEOUT, - headers: { 'User-Agent': RMAB_USER_AGENT }, + headers: { 'User-Agent': RMAB_USER_AGENT, ...options?.sourceHeaders }, }); // Check if response body is a magnet link @@ -308,16 +309,29 @@ export class TransmissionService implements IDownloadClient { responseType: 'arraybuffer', timeout: DOWNLOAD_CLIENT_TIMEOUT, maxRedirects: 5, - headers: { 'User-Agent': RMAB_USER_AGENT }, + headers: { 'User-Agent': RMAB_USER_AGENT, ...options?.sourceHeaders }, }); - } catch { + } catch (redirectError) { + if (axios.isAxiosError(redirectError) && redirectError.response?.status) { + throw new DownloadSourceError( + `Grab failed: source returned HTTP ${redirectError.response.status}`, + redirectError.response.status, + location, + redirectError + ); + } throw new Error('Failed to download torrent file after redirect'); } } else { throw new Error(`Invalid redirect location: ${location}`); } } else { - throw new Error(`Failed to download torrent: HTTP ${status}`); + throw new DownloadSourceError( + `Grab failed: source returned HTTP ${status}`, + status, + torrentUrl, + error + ); } } diff --git a/src/lib/interfaces/download-client.interface.ts b/src/lib/interfaces/download-client.interface.ts index 3fd347e..d983cc3 100644 --- a/src/lib/interfaces/download-client.interface.ts +++ b/src/lib/interfaces/download-client.interface.ts @@ -106,6 +106,33 @@ export interface AddDownloadOptions { sourceHeaders?: Record; } +/** + * Failure while fetching a .torrent or .nzb from its source URL. + * Keeping this distinct from download-client API failures lets the pipeline + * try another release without blaming a healthy download client. + */ +export class DownloadSourceError extends Error { + readonly name = 'DownloadSourceError'; + + constructor( + message: string, + public readonly status: number | undefined, + public readonly sourceUrl: string, + public readonly originalError?: unknown + ) { + super(message); + } +} + +/** HTTP source failures that may clear on their own or succeed via another indexer. */ +export function isRetryableDownloadSourceError( + error: unknown +): error is DownloadSourceError { + return error instanceof DownloadSourceError + && typeof error.status === 'number' + && (error.status === 429 || (error.status >= 500 && error.status < 600)); +} + /** Result of a connection test */ export interface ConnectionTestResult { success: boolean; diff --git a/src/lib/processors/download-torrent.processor.ts b/src/lib/processors/download-torrent.processor.ts index 153da51..38cd9ab 100644 --- a/src/lib/processors/download-torrent.processor.ts +++ b/src/lib/processors/download-torrent.processor.ts @@ -10,6 +10,10 @@ import { getDownloadClientManager } from '../services/download-client-manager.se import { ProwlarrService } from '../integrations/prowlarr.service'; import { RMABLogger } from '../utils/logger'; import { isTransientConnectionError } from '../utils/connection-errors'; +import { + DownloadSourceError, + isRetryableDownloadSourceError, +} from '../interfaces/download-client.interface'; /** * Process download job @@ -17,7 +21,8 @@ import { isTransientConnectionError } from '../utils/connection-errors'; * Adds selected result to download client and starts monitoring */ export async function processDownloadTorrent(payload: DownloadTorrentPayload): Promise { - const { requestId, audiobook, torrent, jobId } = payload; + const { requestId, audiobook, torrent, alternateTorrents = [], jobId } = payload; + const candidates = [torrent, ...alternateTorrents]; const logger = RMABLogger.forJob(jobId, 'DownloadTorrent'); @@ -27,6 +32,7 @@ export async function processDownloadTorrent(payload: DownloadTorrentPayload): P seeders: torrent.seeders, format: torrent.format, indexer: torrent.indexer, + alternateCount: alternateTorrents.length, }); try { @@ -43,24 +49,9 @@ export async function processDownloadTorrent(payload: DownloadTorrentPayload): P }, }); - // Detect protocol from result and get appropriate client - const isUsenet = ProwlarrService.isNZBResult(torrent); - const protocol = isUsenet ? 'usenet' : 'torrent'; const config = await getConfigService(); const manager = getDownloadClientManager(config); - const client = await manager.getClientServiceForProtocol(protocol); - - if (!client) { - throw new Error(`No ${protocol} download client configured. Please add a ${protocol} client in Settings > Download Clients.`); - } - - // Get client config for category - const clientConfig = await manager.getClientForProtocol(protocol); - const category = clientConfig?.category || 'readmeabook'; - - logger.info(`Routing to ${client.clientType} (${client.protocol})`); - // Include Prowlarr API key as source header so NZB/torrent downloads from // Prowlarr proxy URLs are authenticated (fixes 403 for indexers like NZBFinder) const prowlarrApiKey = (await config.getMany(['prowlarr_api_key'])).prowlarr_api_key || process.env.PROWLARR_API_KEY; @@ -69,90 +60,149 @@ export async function processDownloadTorrent(payload: DownloadTorrentPayload): P sourceHeaders['X-Api-Key'] = prowlarrApiKey; } - // Add download via unified interface - const downloadClientId = await client.addDownload(torrent.downloadUrl, { - category, - priority: 'normal', - sourceHeaders, - }); + for (let candidateIndex = 0; candidateIndex < candidates.length; candidateIndex++) { + const candidate = candidates[candidateIndex]; + const isUsenet = ProwlarrService.isNZBResult(candidate); + const protocol = isUsenet ? 'usenet' : 'torrent'; + const client = await manager.getClientServiceForProtocol(protocol); - logger.info(`Download added with ID: ${downloadClientId}`); + if (!client) { + throw new Error(`No ${protocol} download client configured. Please add a ${protocol} client in Settings > Download Clients.`); + } - // Create DownloadHistory record - // Determine indexer page URL - exclude magnet links from guid fallback - const indexerPageUrl = torrent.infoUrl || (torrent.guid?.startsWith('magnet:') ? null : torrent.guid); + const clientConfig = await manager.getClientForProtocol(protocol); + const category = clientConfig?.category || 'readmeabook'; - const downloadHistory = await prisma.downloadHistory.create({ - data: { + logger.info( + `Trying release ${candidateIndex + 1}/${candidates.length}: ${candidate.title}`, + { indexer: candidate.indexer, protocol, downloadClient: client.clientType } + ); + + let downloadClientId: string; + try { + downloadClientId = await client.addDownload(candidate.downloadUrl, { + category, + priority: 'normal', + sourceHeaders, + }); + } catch (error) { + if (!(error instanceof DownloadSourceError)) { + throw error; + } + + const contextualError = new DownloadSourceError( + `Grab failed: ${candidate.indexer} returned HTTP ${error.status} while fetching "${candidate.title}" through Prowlarr`, + error.status, + error.sourceUrl, + error + ); + + if (!isRetryableDownloadSourceError(error)) { + throw contextualError; + } + + const hasNextCandidate = candidateIndex < candidates.length - 1; + + logger.warn( + `${contextualError.message}${hasNextCandidate ? '; trying next ranked release' : ''}`, + { + indexer: candidate.indexer, + upstreamStatus: error.status, + candidate: candidateIndex + 1, + candidateCount: candidates.length, + } + ); + + if (hasNextCandidate) { + continue; + } + + throw contextualError; + } + + logger.info(`Download added with ID: ${downloadClientId}`); + + // Create DownloadHistory record. Exclude magnet links from the indexer-page fallback. + const indexerPageUrl = candidate.infoUrl || (candidate.guid?.startsWith('magnet:') ? null : candidate.guid); + + const downloadHistory = await prisma.downloadHistory.create({ + data: { + requestId, + indexerName: candidate.indexer, + indexerId: candidate.indexerId, + downloadClient: client.clientType, + downloadClientId, + torrentName: candidate.title, + // Set protocol-specific ID fields for backward compatibility + torrentHash: client.protocol === 'torrent' ? (candidate.infoHash || downloadClientId) : undefined, + nzbId: client.protocol === 'usenet' ? downloadClientId : undefined, + torrentSizeBytes: candidate.size, + torrentUrl: indexerPageUrl, + magnetLink: candidate.downloadUrl, + seeders: candidate.seeders || 0, + leechers: candidate.leechers || 0, + downloadStatus: 'downloading', + selected: true, + startedAt: new Date(), + }, + }); + + logger.info(`Created download history record: ${downloadHistory.id}`); + + // Send grab notification (non-blocking — failures here don't fail the download) + const jobQueue = getJobQueueService(); + const grabMessage = `${candidate.title} via ${candidate.indexer} (${client.clientType})`; + await jobQueue.addNotificationJob( + 'request_grabbed', requestId, - indexerName: torrent.indexer, - indexerId: torrent.indexerId, - downloadClient: client.clientType, + audiobook.title, + audiobook.author, + request.user.plexUsername || 'Unknown User', + grabMessage, + request.type + ).catch((error) => { + logger.error('Failed to queue grab notification', { error: error instanceof Error ? error.message : String(error) }); + }); + + // Trigger monitor download job with initial delay + await jobQueue.addMonitorJob( + requestId, + downloadHistory.id, downloadClientId, - torrentName: torrent.title, - // Set protocol-specific ID fields for backward compatibility - torrentHash: client.protocol === 'torrent' ? (torrent.infoHash || downloadClientId) : undefined, - nzbId: client.protocol === 'usenet' ? downloadClientId : undefined, - torrentSizeBytes: torrent.size, - torrentUrl: indexerPageUrl, - magnetLink: torrent.downloadUrl, - seeders: torrent.seeders || 0, - leechers: torrent.leechers || 0, - downloadStatus: 'downloading', - selected: true, - startedAt: new Date(), - }, - }); + client.clientType, + 3 // Wait 3 seconds before first check + ); - logger.info(`Created download history record: ${downloadHistory.id}`); + logger.info(`Started monitoring job for request ${requestId} (${client.clientType}, 3s initial delay)`); - // Send grab notification (non-blocking — failures here don't fail the download) - const jobQueue = getJobQueueService(); - const grabMessage = `${torrent.title} via ${torrent.indexer} (${client.clientType})`; - await jobQueue.addNotificationJob( - 'request_grabbed', - requestId, - audiobook.title, - audiobook.author, - request.user.plexUsername || 'Unknown User', - grabMessage, - request.type - ).catch((error) => { - logger.error('Failed to queue grab notification', { error: error instanceof Error ? error.message : String(error) }); - }); + return { + success: true, + message: `Download added to ${client.clientType} and monitoring started`, + requestId, + downloadHistoryId: downloadHistory.id, + downloadClientId, + torrent: { + title: candidate.title, + size: candidate.size, + seeders: candidate.seeders || 0, + format: candidate.format, + }, + }; + } - // Trigger monitor download job with initial delay - await jobQueue.addMonitorJob( - requestId, - downloadHistory.id, - downloadClientId, - client.clientType, - 3 // Wait 3 seconds before first check - ); - - logger.info(`Started monitoring job for request ${requestId} (${client.clientType}, 3s initial delay)`); - - return { - success: true, - message: `Download added to ${client.clientType} and monitoring started`, - requestId, - downloadHistoryId: downloadHistory.id, - downloadClientId, - torrent: { - title: torrent.title, - size: torrent.size, - seeders: torrent.seeders || 0, - format: torrent.format, - }, - }; + throw new Error('No release candidates were available to download'); } catch (error) { logger.error(`Error: ${error instanceof Error ? error.message : 'Unknown error'}`); - if (isTransientConnectionError(error)) { + if (isTransientConnectionError(error) || isRetryableDownloadSourceError(error)) { // Connection error — don't mark request as failed yet. // Bull will retry this job (3 attempts with exponential backoff). // If all retries are exhausted, the global failed handler marks it failed. - logger.warn(`Download client unreachable for request ${requestId}, allowing Bull to retry`); + logger.warn( + isRetryableDownloadSourceError(error) + ? `All ranked release grabs were temporarily unavailable for request ${requestId}, allowing Bull to retry` + : `Download client unreachable for request ${requestId}, allowing Bull to retry` + ); } else { // Permanent error — mark request as failed immediately await prisma.request.update({ diff --git a/src/lib/processors/search-indexers.processor.ts b/src/lib/processors/search-indexers.processor.ts index 11917cf..ca80ec6 100644 --- a/src/lib/processors/search-indexers.processor.ts +++ b/src/lib/processors/search-indexers.processor.ts @@ -283,7 +283,7 @@ export async function processSearchIndexers(payload: SearchIndexersPayload): Pro id: audiobook.id, title: audiobook.title, author: audiobook.author, - }, bestResult); + }, bestResult, filteredResults.slice(1)); return { success: true, diff --git a/src/lib/services/job-queue.service.ts b/src/lib/services/job-queue.service.ts index 0bc6862..0d77018 100644 --- a/src/lib/services/job-queue.service.ts +++ b/src/lib/services/job-queue.service.ts @@ -58,6 +58,8 @@ export interface DownloadTorrentPayload extends JobPayload { author: string; }; torrent: TorrentResult; + /** Ranked candidates after `torrent`, used when an upstream grab is rate-limited/unavailable. */ + alternateTorrents?: TorrentResult[]; } export interface MonitorDownloadPayload extends JobPayload { @@ -599,7 +601,8 @@ export class JobQueueService { async addDownloadJob( requestId: string, audiobook: { id: string; title: string; author: string }, - torrent: TorrentResult + torrent: TorrentResult, + alternateTorrents: TorrentResult[] = [] ): Promise { return await this.addJob( 'download_torrent', @@ -607,6 +610,7 @@ export class JobQueueService { requestId, audiobook, torrent, + alternateTorrents, } as DownloadTorrentPayload, { priority: 9, // High priority - download selected torrent diff --git a/tests/integrations/qbittorrent.service.test.ts b/tests/integrations/qbittorrent.service.test.ts index f419021..b6a4f15 100644 --- a/tests/integrations/qbittorrent.service.test.ts +++ b/tests/integrations/qbittorrent.service.test.ts @@ -6,6 +6,7 @@ import path from 'path'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { QBittorrentService, getQBittorrentService, invalidateQBittorrentService } from '@/lib/integrations/qbittorrent.service'; +import { DownloadSourceError } from '@/lib/interfaces/download-client.interface'; const clientMock = vi.hoisted(() => ({ get: vi.fn(), @@ -693,6 +694,32 @@ describe('QBittorrentService', () => { ); }); + it('preserves upstream HTTP status and forwards source headers', async () => { + const service = new QBittorrentService('http://qb', 'user', 'pass'); + (service as any).cookie = 'SID=source-error'; + vi.spyOn(service as any, 'ensureCategory').mockResolvedValue(undefined); + + axiosMock.get.mockRejectedValueOnce({ + isAxiosError: true, + response: { status: 500, headers: {} }, + message: 'Request failed', + }); + + const error = await service.addDownload( + 'https://prowlarr/1/download/limited', + { sourceHeaders: { 'X-Api-Key': 'secret' } } + ).catch((caught) => caught); + + expect(error).toBeInstanceOf(DownloadSourceError); + expect(error).toMatchObject({ status: 500 }); + expect(axiosMock.get).toHaveBeenCalledWith( + 'https://prowlarr/1/download/limited', + expect.objectContaining({ + headers: expect.objectContaining({ 'X-Api-Key': 'secret' }), + }) + ); + }); + it('throws for invalid redirect locations when fetching torrents', async () => { const service = new QBittorrentService('http://qb', 'user', 'pass'); diff --git a/tests/processors/download-torrent.processor.test.ts b/tests/processors/download-torrent.processor.test.ts index 1d8d6fd..87e65e8 100644 --- a/tests/processors/download-torrent.processor.test.ts +++ b/tests/processors/download-torrent.processor.test.ts @@ -6,6 +6,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { createPrismaMock } from '../helpers/prisma'; import { createJobQueueMock } from '../helpers/job-queue'; +import { DownloadSourceError } from '@/lib/interfaces/download-client.interface'; const prismaMock = createPrismaMock(); const configMock = vi.hoisted(() => ({ @@ -214,4 +215,107 @@ describe('processDownloadTorrent', () => { expect(downloadClientManagerMock.getClientServiceForProtocol).toHaveBeenCalledWith('usenet'); }); + + it('tries the next ranked release when an indexer grab returns 500', async () => { + const fallbackTorrent = { + ...torrentPayload.torrent, + indexer: 'Healthy Indexer', + indexerId: 2, + title: 'Book - Author - Fallback', + downloadUrl: 'https://prowlarr/2/download/fallback', + guid: 'guid-fallback', + }; + const qbtClientMock = { + clientType: 'qbittorrent', + protocol: 'torrent', + addDownload: vi.fn() + .mockRejectedValueOnce(new DownloadSourceError( + 'Grab failed: source returned HTTP 500', + 500, + 'https://prowlarr/1/download/limited' + )) + .mockResolvedValueOnce('fallback-hash'), + }; + downloadClientManagerMock.getClientServiceForProtocol.mockResolvedValue(qbtClientMock); + downloadClientManagerMock.getClientForProtocol.mockResolvedValue({ + id: 'client-1', + type: 'qbittorrent', + enabled: true, + category: 'readmeabook', + }); + prismaMock.request.update.mockResolvedValue({ + type: 'audiobook', + user: { plexUsername: 'testuser' }, + }); + prismaMock.downloadHistory.create.mockResolvedValue({ id: 'dh-fallback' }); + + const { processDownloadTorrent } = await import('@/lib/processors/download-torrent.processor'); + const result = await processDownloadTorrent({ + ...torrentPayload, + torrent: { + ...torrentPayload.torrent, + downloadUrl: 'https://prowlarr/1/download/limited', + }, + alternateTorrents: [fallbackTorrent], + }); + + expect(result.success).toBe(true); + expect(result.torrent.title).toBe(fallbackTorrent.title); + expect(qbtClientMock.addDownload).toHaveBeenNthCalledWith( + 1, + 'https://prowlarr/1/download/limited', + expect.anything() + ); + expect(qbtClientMock.addDownload).toHaveBeenNthCalledWith( + 2, + fallbackTorrent.downloadUrl, + expect.anything() + ); + expect(prismaMock.downloadHistory.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + indexerName: 'Healthy Indexer', + torrentName: fallbackTorrent.title, + downloadClientId: 'fallback-hash', + }), + }) + ); + }); + + it('leaves the request retryable when every ranked grab is temporarily unavailable', async () => { + const qbtClientMock = { + clientType: 'qbittorrent', + protocol: 'torrent', + addDownload: vi.fn().mockRejectedValue(new DownloadSourceError( + 'Grab failed: source returned HTTP 429', + 429, + 'https://prowlarr/1/download/limited' + )), + }; + downloadClientManagerMock.getClientServiceForProtocol.mockResolvedValue(qbtClientMock); + downloadClientManagerMock.getClientForProtocol.mockResolvedValue({ + id: 'client-1', + type: 'qbittorrent', + enabled: true, + category: 'readmeabook', + }); + prismaMock.request.update.mockResolvedValue({ + type: 'audiobook', + user: { plexUsername: 'testuser' }, + }); + + const { processDownloadTorrent } = await import('@/lib/processors/download-torrent.processor'); + await expect(processDownloadTorrent({ + ...torrentPayload, + torrent: { + ...torrentPayload.torrent, + downloadUrl: 'https://prowlarr/1/download/limited', + }, + })).rejects.toThrow('Indexer returned HTTP 429'); + + expect(prismaMock.request.update).not.toHaveBeenCalledWith( + expect.objectContaining({ data: expect.objectContaining({ status: 'failed' }) }) + ); + expect(prismaMock.downloadHistory.create).not.toHaveBeenCalled(); + }); }); diff --git a/tests/processors/search-indexers.processor.test.ts b/tests/processors/search-indexers.processor.test.ts index fa21cb5..3b89a97 100644 --- a/tests/processors/search-indexers.processor.test.ts +++ b/tests/processors/search-indexers.processor.test.ts @@ -88,6 +88,17 @@ describe('processSearchIndexers', () => { guid: 'guid-1', format: 'M4B', }, + { + indexer: 'Fallback Indexer', + indexerId: 2, + title: 'Book - Author - Alternate', + size: 50 * 1024 * 1024, + seeders: 1, + publishDate: new Date(), + downloadUrl: 'https://prowlarr/2/download/alternate', + guid: 'guid-2', + format: 'M4B', + }, ]); prismaMock.request.update.mockResolvedValue({}); @@ -103,7 +114,8 @@ describe('processSearchIndexers', () => { expect(jobQueueMock.addDownloadJob).toHaveBeenCalledWith( 'req-2', { id: 'a2', title: 'Book', author: 'Author' }, - expect.objectContaining({ title: 'Book - Author' }) + expect.objectContaining({ title: 'Book - Author' }), + [expect.objectContaining({ title: 'Book - Author - Alternate' })] ); }); @@ -179,7 +191,8 @@ describe('processSearchIndexers', () => { expect(jobQueueMock.addDownloadJob).toHaveBeenCalledWith( 'req-filter-name', expect.objectContaining({ id: 'a-filter' }), - expect.objectContaining({ title: 'Good Release - Author' }) + expect.objectContaining({ title: 'Good Release - Author' }), + [] ); }); @@ -235,7 +248,8 @@ describe('processSearchIndexers', () => { expect(jobQueueMock.addDownloadJob).toHaveBeenCalledWith( 'req-filter-hash', expect.anything(), - expect.objectContaining({ title: 'Good Release - Author' }) + expect.objectContaining({ title: 'Good Release - Author' }), + [] ); }); diff --git a/tests/services/job-queue.service.test.ts b/tests/services/job-queue.service.test.ts index 247e8c0..96d667c 100644 --- a/tests/services/job-queue.service.test.ts +++ b/tests/services/job-queue.service.test.ts @@ -202,11 +202,20 @@ describe('JobQueueService', () => { const { JobQueueService } = await import('@/lib/services/job-queue.service'); const service = new JobQueueService(); - await service.addDownloadJob('req-1', { id: 'ab-1', title: 'Title', author: 'Author' }, { hash: 'hash' } as any); + await service.addDownloadJob( + 'req-1', + { id: 'ab-1', title: 'Title', author: 'Author' }, + { hash: 'hash' } as any, + [{ hash: 'fallback-hash' } as any] + ); expect(queueMock.add).toHaveBeenCalledWith( 'download_torrent', - expect.objectContaining({ requestId: 'req-1', jobId: 'job-2' }), + expect.objectContaining({ + requestId: 'req-1', + jobId: 'job-2', + alternateTorrents: [{ hash: 'fallback-hash' }], + }), expect.objectContaining({ priority: 9 }) ); });