From 1abaff16778ba68ce3dc2cbded083abe5781804a Mon Sep 17 00:00:00 2001 From: Orvanix Date: Sat, 14 Mar 2026 17:45:31 +0000 Subject: [PATCH 01/12] feat(audiobook): add language, format and publisher to details modal --- .../audiobooks/AudiobookDetailsModal.tsx | 24 +++++++++++++++++++ src/lib/hooks/useAudiobooks.ts | 3 +++ src/lib/integrations/audible.service.ts | 6 +++++ 3 files changed, 33 insertions(+) diff --git a/src/components/audiobooks/AudiobookDetailsModal.tsx b/src/components/audiobooks/AudiobookDetailsModal.tsx index a3a79fc..5bf23b8 100644 --- a/src/components/audiobooks/AudiobookDetailsModal.tsx +++ b/src/components/audiobooks/AudiobookDetailsModal.tsx @@ -548,6 +548,30 @@ export function AudiobookDetailsModal({ + {/* Language */} + {audiobook.language && ( +
+

Language

+

{audiobook.language.charAt(0).toUpperCase() + audiobook.language.slice(1)}

+
+ )} + + {/* Format */} + {audiobook.formatType && ( +
+

Format

+

{audiobook.formatType.charAt(0).toUpperCase() + audiobook.formatType.slice(1)}

+
+ )} + + {/* Publisher */} + {audiobook.publisherName && ( +
+

Publisher

+

{audiobook.publisherName}

+
+ )} + {/* Download Link - subtle utility, visible from any context */} {isAvailable && downloadAvailable && requestId && user?.permissions?.download !== false && (
diff --git a/src/lib/hooks/useAudiobooks.ts b/src/lib/hooks/useAudiobooks.ts index 9380f40..4aa94c5 100644 --- a/src/lib/hooks/useAudiobooks.ts +++ b/src/lib/hooks/useAudiobooks.ts @@ -34,6 +34,9 @@ export interface Audiobook { requestedByUsername?: string | null; // Username who requested (only if not current user) hasReportedIssue?: boolean; // True if an open issue exists for this audiobook isIgnored?: boolean; // True if this user has ignored this audiobook from auto-requests + language?: string; + formatType?: string; + publisherName?: string; } export function useAudiobooks(type: 'popular' | 'new-releases', limit: number = 20, page: number = 1, hideAvailable: boolean = false) { diff --git a/src/lib/integrations/audible.service.ts b/src/lib/integrations/audible.service.ts index bc1bd90..954511d 100644 --- a/src/lib/integrations/audible.service.ts +++ b/src/lib/integrations/audible.service.ts @@ -50,6 +50,9 @@ export interface AudibleAudiobook { series?: string; seriesPart?: string; seriesAsin?: string; + language?: string; + formatType?: string; + publisherName?: string; } export interface AudibleSearchResult { @@ -774,6 +777,9 @@ export class AudibleService { series: data.seriesPrimary?.name || undefined, seriesPart: data.seriesPrimary?.position || undefined, seriesAsin: data.seriesPrimary?.asin || undefined, + language: data.language || undefined, + formatType: data.formatType || undefined, + publisherName: data.publisherName || undefined, }; // Ensure cover art URL is high quality From e9d7a2359a343af8f91aa5bee29192a68c309dea Mon Sep 17 00:00:00 2001 From: xFlawless11x Date: Tue, 24 Mar 2026 11:29:26 -0400 Subject: [PATCH 02/12] feat: add book info modal to admin pending approval cards Adds an info icon button (top-right of each card) in the Requests Awaiting Approval section. Clicking it opens AudiobookDetailsModal with full book details (cover, description, narrator, series, genres, etc.) and embeds the Approve / Search / Deny action buttons so admins can review and act without navigating away from the admin panel. Implementation: - AudiobookDetailsModal: adds optional `adminActions` prop rendered as a second row inside the existing sticky action bar - admin/page.tsx: adds detailsAsin/detailsRequestId state, info button per card (conditional on audibleAsin presence), and AudiobookDetailsModal wired with admin action buttons matching the card button behaviour - Documentation updated: request-approval.md, components.md, TABLEOFCONTENTS.md Closes #157 Co-Authored-By: Claude Sonnet 4.6 --- documentation/TABLEOFCONTENTS.md | 1 + .../admin-features/request-approval.md | 3 + documentation/frontend/components.md | 1 + src/app/admin/page.tsx | 84 ++++++++++++++++++- .../audiobooks/AudiobookDetailsModal.tsx | 10 +++ 5 files changed, 98 insertions(+), 1 deletion(-) diff --git a/documentation/TABLEOFCONTENTS.md b/documentation/TABLEOFCONTENTS.md index ab1e724..c4bca05 100644 --- a/documentation/TABLEOFCONTENTS.md +++ b/documentation/TABLEOFCONTENTS.md @@ -144,6 +144,7 @@ **"How do I delete requests?"** → [admin-features/request-deletion.md](admin-features/request-deletion.md) **"How do I approve/deny user requests?"** → [admin-features/request-approval.md](admin-features/request-approval.md) **"How do I enable auto-approve for requests?"** → [admin-features/request-approval.md](admin-features/request-approval.md) +**"How does the admin book info modal work?"** → [admin-features/request-approval.md](admin-features/request-approval.md#ui-features), [frontend/components.md](frontend/components.md#component-apis) **"How do I customize audiobook folder organization?"** → [settings-pages.md](settings-pages.md#audiobook-organization-template), [phase3/file-organization.md](phase3/file-organization.md#target-structure) **"How do I deploy?"** → [deployment/docker.md](deployment/docker.md) (multi-container), [deployment/unified.md](deployment/unified.md) (all-in-one) **"How do I use the unified container?"** → [deployment/unified.md](deployment/unified.md) diff --git a/documentation/admin-features/request-approval.md b/documentation/admin-features/request-approval.md index 8104090..c40af23 100644 --- a/documentation/admin-features/request-approval.md +++ b/documentation/admin-features/request-approval.md @@ -259,8 +259,11 @@ Update user (includes autoApproveRequests field) - Title and author - User avatar and username - Request timestamp (relative: "2 hours ago") + - Info button (ⓘ, top-right corner) — opens AudiobookDetailsModal for full book details - Approve button (green, checkmark icon) + - Search button (blue, magnifier icon) — opens InteractiveTorrentSearchModal - Deny button (red, X icon) +- **Info modal:** `AudiobookDetailsModal` rendered with `adminActions` prop containing Approve/Search/Deny buttons, allowing admin to review full book details (cover, description, series, genres, narrator, etc.) without leaving the approval workflow - Auto-refreshes every 10 seconds (SWR) - Loading states on buttons during approval/denial - Success/error toast notifications diff --git a/documentation/frontend/components.md b/documentation/frontend/components.md index 62f8e6f..c51a27e 100644 --- a/documentation/frontend/components.md +++ b/documentation/frontend/components.md @@ -113,6 +113,7 @@ interface AudiobookDetailsModalProps { requestStatus?: string | null; isAvailable?: boolean; requestedByUsername?: string | null; + adminActions?: React.ReactNode; // Optional admin buttons (Approve/Search/Deny) rendered as second row in action bar } interface RequestCardProps { diff --git a/src/app/admin/page.tsx b/src/app/admin/page.tsx index a4837dc..de2b9cd 100644 --- a/src/app/admin/page.tsx +++ b/src/app/admin/page.tsx @@ -14,8 +14,10 @@ import { RecentRequestsTable } from './components/RecentRequestsTable'; import { ToastProvider, useToast } from '@/components/ui/Toast'; import { ReportedIssuesSection } from './components/ReportedIssuesSection'; import { InteractiveTorrentSearchModal } from '@/components/requests/InteractiveTorrentSearchModal'; +import { AudiobookDetailsModal } from '@/components/audiobooks/AudiobookDetailsModal'; import { BulkImportWizard } from '@/components/admin/BulkImportWizard'; import { TorrentResult } from '@/lib/utils/ranking-algorithm'; +import { InformationCircleIcon } from '@heroicons/react/24/outline'; import { formatDistanceToNow } from 'date-fns'; import { useState } from 'react'; @@ -60,11 +62,17 @@ function PendingApprovalSection({ requests }: { requests: PendingApprovalRequest const toast = useToast(); const [loadingStates, setLoadingStates] = useState>({}); const [searchModalRequestId, setSearchModalRequestId] = useState(null); + const [detailsAsin, setDetailsAsin] = useState(null); + const [detailsRequestId, setDetailsRequestId] = useState(null); const searchModalRequest = searchModalRequestId ? requests.find((r) => r.id === searchModalRequestId) : null; + const detailsRequest = detailsRequestId + ? requests.find((r) => r.id === detailsRequestId) + : null; + const handleApproveRequest = async (requestId: string) => { setLoadingStates((prev) => ({ ...prev, [requestId]: true })); @@ -170,8 +178,22 @@ function PendingApprovalSection({ requests }: { requests: PendingApprovalRequest return (
+ {/* Info Button — opens AudiobookDetailsModal */} + {request.audiobook.audibleAsin && ( + + )} + {/* Card Content */}
@@ -375,6 +397,66 @@ function PendingApprovalSection({ requests }: { requests: PendingApprovalRequest }} /> )} + + {/* Book Details Modal — opened via info button on each approval card */} + {detailsAsin && detailsRequestId && ( + { setDetailsAsin(null); setDetailsRequestId(null); }} + requestStatus="awaiting_approval" + requestedByUsername={detailsRequest?.user.plexUsername ?? null} + adminActions={(() => { + const isLoading = loadingStates[detailsRequestId] || false; + return ( + <> + + + + + ); + })()} + /> + )}
); } diff --git a/src/components/audiobooks/AudiobookDetailsModal.tsx b/src/components/audiobooks/AudiobookDetailsModal.tsx index a3a79fc..886763c 100644 --- a/src/components/audiobooks/AudiobookDetailsModal.tsx +++ b/src/components/audiobooks/AudiobookDetailsModal.tsx @@ -38,6 +38,8 @@ interface AudiobookDetailsModalProps { hideRequestActions?: boolean; hasReportedIssue?: boolean; aiReason?: string | null; + /** Optional admin action buttons (Approve / Search / Deny) rendered as a second row in the action bar */ + adminActions?: React.ReactNode; } // Status helper @@ -80,6 +82,7 @@ export function AudiobookDetailsModal({ hideRequestActions = false, hasReportedIssue = false, aiReason = null, + adminActions, }: AudiobookDetailsModalProps) { const { user } = useAuth(); const { squareCovers } = usePreferences(); @@ -739,6 +742,13 @@ export function AudiobookDetailsModal({ )}
+ + {/* Admin Actions Row (Approve / Search / Deny) — injected by admin pages */} + {adminActions && ( +
+ {adminActions} +
+ )}
)} From ade12cb82db3dc7a20a9eaaa20b3dbedfeb3d2a3 Mon Sep 17 00:00:00 2001 From: kikootwo Date: Tue, 21 Apr 2026 01:56:39 -0400 Subject: [PATCH 03/12] Add Path Mapping Helper page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new client-side Path Mapping Helper page at src/app/path-helper/page.tsx. Implements a multi-step wizard to help users configure Docker volume mappings for download clients and ReadMeABook (RMAB): select clients, enter container save paths, enter host/container volume mappings (with optional remote path mapping), and generate recommended RMAB docker-compose volume snippet. Includes utility functions to compute common roots and relative paths, UI components (step indicator, info/warning boxes, code block), and logic to derive RMAB download directory, per-client custom paths, and verification instructions. No API calls — purely client-side helper with sensible defaults for supported clients. --- src/app/path-helper/page.tsx | 932 +++++++++++++++++++++++++++++++++++ 1 file changed, 932 insertions(+) create mode 100644 src/app/path-helper/page.tsx diff --git a/src/app/path-helper/page.tsx b/src/app/path-helper/page.tsx new file mode 100644 index 0000000..cbab4e3 --- /dev/null +++ b/src/app/path-helper/page.tsx @@ -0,0 +1,932 @@ +/** + * Component: Path Mapping Helper + * Documentation: documentation/deployment/volume-mapping.md + * + * Public, unprotected page that guides users through configuring + * Docker volume mappings for their download clients and RMAB. + * Purely client-side — no API calls, no real data access. + */ + +'use client'; + +import { useState, useMemo } from 'react'; +import { Button } from '@/components/ui/Button'; +import { Input } from '@/components/ui/Input'; +import { + CLIENT_DISPLAY_NAMES, + CLIENT_PROTOCOL_MAP, + type DownloadClientType, +} from '@/lib/interfaces/download-client.interface'; + +// ========================================================================= +// TYPES +// ========================================================================= + +interface ClientConfig { + type: DownloadClientType; + /** The path inside the download client container where completed downloads land */ + savePath: string; + /** The volume mapping from the client's docker-compose (host:container) — host side */ + hostPath: string; + /** The volume mapping from the client's docker-compose (host:container) — container side */ + containerMountPath: string; + /** Whether this client needs remote path mapping */ + remotePathMapping: boolean; + /** The path as seen by the remote download client (for remote path mapping) */ + remotePath: string; +} + +type Step = 'clients' | 'save-paths' | 'host-paths' | 'results'; + +const STEPS: { key: Step; title: string }[] = [ + { key: 'clients', title: 'Clients' }, + { key: 'save-paths', title: 'Save Paths' }, + { key: 'host-paths', title: 'Volume Mapping' }, + { key: 'results', title: 'Results' }, +]; + +const ALL_CLIENTS: DownloadClientType[] = ['qbittorrent', 'transmission', 'deluge', 'sabnzbd', 'nzbget']; + +const DEFAULT_SAVE_PATHS: Record = { + qbittorrent: '/downloads', + transmission: '/downloads/complete', + deluge: '/downloads', + sabnzbd: '/downloads/complete', + nzbget: '/downloads/completed', +}; + +// ========================================================================= +// UTILITY FUNCTIONS +// ========================================================================= + +/** + * Find the longest common path prefix across multiple paths. + * Only meaningful when there are multiple DIFFERENT paths. + */ +function findCommonRoot(paths: string[]): string { + if (paths.length === 0) return ''; + if (paths.length === 1) return paths[0]; + + const unique = [...new Set(paths)]; + if (unique.length === 1) return unique[0]; + + // Split each path into segments + const segmentArrays = unique.map((p) => p.replace(/\/+$/, '').split('/').filter(Boolean)); + const minLength = Math.min(...segmentArrays.map((s) => s.length)); + + const commonSegments: string[] = []; + for (let i = 0; i < minLength; i++) { + const segment = segmentArrays[0][i]; + if (segmentArrays.every((s) => s[i] === segment)) { + commonSegments.push(segment); + } else { + break; + } + } + + if (commonSegments.length === 0) return '/'; + return '/' + commonSegments.join('/'); +} + +/** + * Get the relative path from a root to a full path. + * Returns empty string if they're the same. + */ +function getRelativePath(root: string, fullPath: string): string { + const normalizedRoot = root.replace(/\/+$/, ''); + const normalizedFull = fullPath.replace(/\/+$/, ''); + + if (normalizedRoot === normalizedFull) return ''; + + if (normalizedFull.startsWith(normalizedRoot + '/')) { + return normalizedFull.slice(normalizedRoot.length + 1); + } + + // Shouldn't happen if common root is correct, but fallback + return normalizedFull; +} + +/** + * Find the common root of the host paths to build the RMAB volume mapping. + * Maps from the host path hierarchy to the container path hierarchy. + */ +function findHostCommonRoot(configs: ClientConfig[]): string { + const hostPaths = configs.map((c) => c.hostPath); + if (hostPaths.length === 0) return ''; + if (hostPaths.length === 1) return hostPaths[0]; + + const unique = [...new Set(hostPaths)]; + if (unique.length === 1) return unique[0]; + + return findCommonRoot(hostPaths); +} + +// ========================================================================= +// COMPONENTS +// ========================================================================= + +function StepIndicator({ currentStep }: { currentStep: Step }) { + const currentIndex = STEPS.findIndex((s) => s.key === currentStep); + + return ( +
+ {STEPS.map((step, index) => ( +
+
+
+ {index < currentIndex ? ( + + + + ) : ( + index + 1 + )} +
+ + {step.title} + +
+ {index < STEPS.length - 1 && ( +
+ )} +
+ ))} +
+ ); +} + +function InfoBox({ children }: { children: React.ReactNode }) { + return ( +
+
+ + + +
{children}
+
+
+ ); +} + +function WarningBox({ children }: { children: React.ReactNode }) { + return ( +
+
+ + + +
{children}
+
+
+ ); +} + +function CodeBlock({ children, label, onCopy }: { children: string; label?: string; onCopy?: () => void }) { + const [copied, setCopied] = useState(false); + + const handleCopy = () => { + navigator.clipboard.writeText(children); + setCopied(true); + onCopy?.(); + setTimeout(() => setCopied(false), 2000); + }; + + return ( +
+ {label && ( +
{label}
+ )} +
+ 
{children}
+
+ +
+ ); +} + +// ========================================================================= +// STEP COMPONENTS +// ========================================================================= + +function ClientSelectionStep({ + selectedClients, + onToggle, + onNext, +}: { + selectedClients: Set; + onToggle: (client: DownloadClientType) => void; + onNext: () => void; +}) { + return ( +
+
+

+ Which download clients do you use? +

+

+ Select all the download clients you have configured or plan to use with ReadMeABook. +

+
+ +
+ {ALL_CLIENTS.map((client) => { + const protocol = CLIENT_PROTOCOL_MAP[client]; + const isSelected = selectedClients.has(client); + + return ( + + ); + })} +
+ +
+ +
+
+ ); +} + +function SavePathsStep({ + configs, + onUpdateConfig, + onNext, + onBack, +}: { + configs: ClientConfig[]; + onUpdateConfig: (type: DownloadClientType, field: keyof ClientConfig, value: string) => void; + onNext: () => void; + onBack: () => void; +}) { + const allFilled = configs.every((c) => c.savePath.trim() !== ''); + + return ( +
+
+

+ Download client save paths +

+

+ For each client, enter the path inside that client's container where + completed downloads are saved. This is the path you see in the client's own settings + (e.g., qBittorrent Web UI → Options → Downloads → Default Save Path). +

+
+ + +

+ This is the container path, not the host path. For example, if your + qBittorrent docker-compose has - + /mnt/data/torrents:/downloads, and qBittorrent is configured to save + to /downloads, then + enter /downloads here. +

+ + +
+ {configs.map((config) => ( +
+
+ + {CLIENT_DISPLAY_NAMES[config.type]} + + + {CLIENT_PROTOCOL_MAP[config.type]} + +
+ onUpdateConfig(config.type, 'savePath', e.target.value)} + className="font-mono" + helperText={`Default: ${DEFAULT_SAVE_PATHS[config.type]}`} + /> +
+ ))} +
+ +
+ + +
+
+ ); +} + +function HostPathsStep({ + configs, + onUpdateConfig, + onNext, + onBack, +}: { + configs: ClientConfig[]; + onUpdateConfig: (type: DownloadClientType, field: keyof ClientConfig, value: string | boolean) => void; + onNext: () => void; + onBack: () => void; +}) { + const allFilled = configs.every( + (c) => c.hostPath.trim() !== '' && c.containerMountPath.trim() !== '' && (!c.remotePathMapping || c.remotePath.trim() !== '') + ); + + return ( +
+
+

+ Docker volume mappings +

+

+ For each client, enter the volume mapping from that client's docker-compose + file. This tells us where on your host machine the downloads actually end up. +

+
+ + +

+ A Docker volume mapping looks like /host/path:/container/path in + your docker-compose.yml. We need both sides so we know how to map RMAB to the same files. +

+ + +
+ {configs.map((config) => ( +
+
+ + {CLIENT_DISPLAY_NAMES[config.type]} + + + {CLIENT_PROTOCOL_MAP[config.type]} + +
+ +
+ onUpdateConfig(config.type, 'hostPath', e.target.value)} + className="font-mono" + helperText="The real path on your server" + /> + onUpdateConfig(config.type, 'containerMountPath', e.target.value)} + className="font-mono" + helperText="The path inside the container" + /> +
+ + {config.containerMountPath && config.hostPath && ( +
+ {config.hostPath}:{config.containerMountPath} +
+ )} + + {/* Remote path mapping toggle */} +
+
+ onUpdateConfig(config.type, 'remotePathMapping', e.target.checked)} + className="mt-1 h-5 w-5 rounded border-gray-300 text-blue-600 focus:ring-blue-500" + /> +
+ +

+ Enable this if the download client is on a seedbox, separate server, or otherwise has a + different filesystem than where RMAB runs. Also enable this if the client runs on the + host (not in Docker) while RMAB runs in Docker. +

+
+
+ + {config.remotePathMapping && ( +
+ onUpdateConfig(config.type, 'remotePath', e.target.value)} + className="font-mono" + helperText="The path the download client reports when a download completes. This is often the same as the client's save path." + /> +
+ )} +
+
+ ))} +
+ +
+ + +
+
+ ); +} + +function ResultsStep({ + configs, + onBack, + onRestart, +}: { + configs: ClientConfig[]; + onBack: () => void; + onRestart: () => void; +}) { + // Determine if we need custom paths (multiple clients with different save paths) + const savePaths = configs.map((c) => c.savePath.replace(/\/+$/, '')); + const uniqueSavePaths = [...new Set(savePaths)]; + const needsCustomPaths = configs.length > 1 && uniqueSavePaths.length > 1; + + // Calculate RMAB download directory + const rmabDownloadDir = needsCustomPaths ? findCommonRoot(savePaths) : savePaths[0]; + + // Calculate custom paths per client (only if needed) + const clientCustomPaths = needsCustomPaths + ? configs.map((c) => ({ + type: c.type, + customPath: getRelativePath(rmabDownloadDir, c.savePath.replace(/\/+$/, '')), + })) + : []; + + // Calculate RMAB volume mapping + // We need the host path that corresponds to the rmabDownloadDir + // If all clients share the same save path, we use that client's host path directly. + // If multiple different paths, we find the common host root. + let rmabHostPath: string; + let rmabContainerPath: string; + + if (!needsCustomPaths) { + // Single path scenario — use the first client's host path + // But we need to consider if the container mount path differs from the save path + const config = configs[0]; + const saveRelativeToMount = getRelativePath( + config.containerMountPath.replace(/\/+$/, ''), + config.savePath.replace(/\/+$/, '') + ); + + if (saveRelativeToMount) { + // Save path is deeper than the mount: host must include that extra depth + rmabHostPath = config.hostPath.replace(/\/+$/, '') + '/' + saveRelativeToMount; + } else { + rmabHostPath = config.hostPath; + } + rmabContainerPath = rmabDownloadDir; + } else { + // Multiple different paths — we need to find the host root that covers all + // For each client, compute the host path that corresponds to the common container root + const hostRoots = configs.map((c) => { + const mountRelativeToCommon = getRelativePath( + rmabDownloadDir, + c.containerMountPath.replace(/\/+$/, '') + ); + const saveRelativeToMount = getRelativePath( + c.containerMountPath.replace(/\/+$/, ''), + c.savePath.replace(/\/+$/, '') + ); + // The host path maps to containerMountPath. We need to go up if rmabDownloadDir + // is a parent of the container mount path. + const containerMountNorm = c.containerMountPath.replace(/\/+$/, ''); + const rmabDirNorm = rmabDownloadDir.replace(/\/+$/, ''); + + if (containerMountNorm === rmabDirNorm) { + return c.hostPath.replace(/\/+$/, ''); + } else if (containerMountNorm.startsWith(rmabDirNorm + '/')) { + // Container mount is deeper than RMAB dir — we need to go up on the host side + const depth = containerMountNorm.slice(rmabDirNorm.length + 1).split('/').length; + const hostSegments = c.hostPath.replace(/\/+$/, '').split('/'); + return hostSegments.slice(0, -depth).join('/') || '/'; + } else if (rmabDirNorm.startsWith(containerMountNorm + '/')) { + // RMAB dir is deeper than container mount — append the extra to host + const extra = rmabDirNorm.slice(containerMountNorm.length + 1); + return c.hostPath.replace(/\/+$/, '') + '/' + extra; + } + return c.hostPath.replace(/\/+$/, ''); + }); + + rmabHostPath = findHostCommonRoot( + configs.map((c, i) => ({ ...c, hostPath: hostRoots[i] })) + ); + rmabContainerPath = rmabDownloadDir; + } + + // Build the RMAB compose snippet + const composeSnippet = `services: + readmeabook: + volumes: + - ${rmabHostPath}:${rmabContainerPath} + # ... your other RMAB volumes (config, media, etc.)`; + + // Build remote path mapping info + const remoteClients = configs.filter((c) => c.remotePathMapping); + + return ( +
+
+

+ Your recommended configuration +

+

+ Based on your inputs, here's how to configure ReadMeABook and your download clients. +

+
+ + {/* RMAB Download Directory */} +
+

+ 1. RMAB Download Directory Setting +

+

+ Set this in RMAB's settings under Admin → Settings → Paths → Download Directory. +

+ {rmabDownloadDir} +
+ + {/* Custom paths per client */} + {needsCustomPaths && clientCustomPaths.some((c) => c.customPath) && ( +
+

+ 2. Client Custom Paths +

+

+ Since your clients save to different locations, set these custom paths on each download client + in RMAB (Admin → Settings → Download Clients → Edit → Custom Path). +

+
+ {clientCustomPaths.map((c) => ( +
+ + {CLIENT_DISPLAY_NAMES[c.type as DownloadClientType]}: + + + {c.customPath || '(none — same as download directory)'} + +
+ ))} +
+
+ )} + + {/* RMAB Docker Compose Volume */} +
+

+ {needsCustomPaths ? '3' : '2'}. RMAB Docker Compose Volume Mapping +

+

+ Add this volume mapping to your RMAB docker-compose.yml. This ensures RMAB can see the + same files your download clients produce. +

+ {composeSnippet} +
+ + {/* Golden Rule explanation */} + +

The Golden Rule

+

+ Both your download client and RMAB must see files at the same container path. + The volume mapping above ensures that when your download client saves a file + to {configs[0]?.savePath}, + RMAB can also find it at that same path. +

+ + + {/* Verification */} +
+

+ {needsCustomPaths ? '4' : '3'}. Verify your setup +

+
+
    + {configs.map((c) => ( +
  • + • + + {CLIENT_DISPLAY_NAMES[c.type]} saves + to {c.savePath} + {' '}→ host path {c.hostPath} + {needsCustomPaths && ( + <> + {' '}→ RMAB custom + path: + {getRelativePath(rmabDownloadDir, c.savePath.replace(/\/+$/, '')) || '(none)'} + + + )} + +
    • + ))} +
  • + • + + RMAB mounts {rmabHostPath}:{rmabContainerPath} + {' '}→ download directory set + to {rmabDownloadDir} + +
    • +
+
+
+ + {/* Remote Path Mapping */} + {remoteClients.length > 0 && ( +
+

+ Remote Path Mapping +

+

+ These clients run on a different machine. Configure remote path mapping for each in + RMAB (Admin → Settings → Download Clients → Edit). +

+
+ {remoteClients.map((c) => { + const localPath = needsCustomPaths + ? rmabDownloadDir + '/' + getRelativePath(rmabDownloadDir, c.savePath.replace(/\/+$/, '')) + : rmabDownloadDir; + + return ( +
+
+ {CLIENT_DISPLAY_NAMES[c.type]} +
+
+
+ Enable Remote Path Mapping: + Yes +
+
+ Remote Path: + {c.remotePath} +
+
+ Local Path: + {localPath} +
+
+ +

+ When this client reports a file at {c.remotePath}/audiobook.m4b, + RMAB will translate it to {localPath}/audiobook.m4b. +

+ +
+ ); + })} +
+
+ )} + +
+ + +
+
+ ); +} + +// ========================================================================= +// MAIN PAGE +// ========================================================================= + +export default function PathHelperPage() { + const [step, setStep] = useState('clients'); + const [selectedClients, setSelectedClients] = useState>(new Set()); + const [clientConfigs, setClientConfigs] = useState>(new Map()); + + // Build ordered configs array from selected clients + const configs = useMemo(() => { + return ALL_CLIENTS + .filter((c) => selectedClients.has(c)) + .map((type) => { + const existing = clientConfigs.get(type); + return ( + existing || { + type, + savePath: DEFAULT_SAVE_PATHS[type], + hostPath: '', + containerMountPath: '', + remotePathMapping: false, + remotePath: '', + } + ); + }); + }, [selectedClients, clientConfigs]); + + const toggleClient = (client: DownloadClientType) => { + setSelectedClients((prev) => { + const next = new Set(prev); + if (next.has(client)) { + next.delete(client); + } else { + next.add(client); + // Initialize config if not exists + if (!clientConfigs.has(client)) { + setClientConfigs((prev) => { + const next = new Map(prev); + next.set(client, { + type: client, + savePath: DEFAULT_SAVE_PATHS[client], + hostPath: '', + containerMountPath: '', + remotePathMapping: false, + remotePath: '', + }); + return next; + }); + } + } + return next; + }); + }; + + const updateConfig = (type: DownloadClientType, field: keyof ClientConfig, value: string | boolean) => { + setClientConfigs((prev) => { + const next = new Map(prev); + const existing = next.get(type); + if (existing) { + next.set(type, { ...existing, [field]: value }); + } + return next; + }); + }; + + const goToStep = (target: Step) => setStep(target); + + const restart = () => { + setStep('clients'); + setSelectedClients(new Set()); + setClientConfigs(new Map()); + }; + + return ( +
+ {/* Header */} +
+
+

+ Path Mapping Helper +

+

+ Get your download client volume mappings configured correctly for ReadMeABook +

+
+
+ + {/* Step Indicator */} +
+
+ +
+
+ + {/* Main Content */} +
+
+ {step === 'clients' && ( + goToStep('save-paths')} + /> + )} + {step === 'save-paths' && ( + goToStep('host-paths')} + onBack={() => goToStep('clients')} + /> + )} + {step === 'host-paths' && ( + goToStep('results')} + onBack={() => goToStep('save-paths')} + /> + )} + {step === 'results' && ( + goToStep('host-paths')} + onRestart={restart} + /> + )} +
+
+
+ ); +} From f564d0a5748b4fae22f617157bfed468f60bd0b5 Mon Sep 17 00:00:00 2001 From: kikootwo Date: Tue, 21 Apr 2026 03:08:08 -0400 Subject: [PATCH 04/12] Audible: switch to JSON catalog API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move Audible catalog operations from HTML scraping to Audible's unauthenticated JSON catalog API (/1.0/catalog/*) while keeping Audnexus as the primary per‑ASIN detail source. audible.service.ts: remove cheerio parsing, add apiClient/htmlClient split, CATALOG_RESPONSE_GROUPS constant, catalog response types, stripHtml and mapCatalogProduct mappers, and paging (API is 0-indexed) + author-ASIN client-side filtering. Update search, popular, new-releases and author endpoints to call the catalog API, use apiClient for retries/backoff, and preserve htmlClient only for series-page scraping and link generation. Improve retry logic to accept an Axios client, move to jittered/exponential backoff for API/external calls, and adjust delays/AdaptivePacer usage. Documentation updated to reflect architecture, data sources, region handling, and gotchas. --- documentation/integrations/audible.md | 275 ++--- src/lib/integrations/audible.service.ts | 1258 +++++++---------------- src/lib/types/audible.ts | 11 +- 3 files changed, 547 insertions(+), 997 deletions(-) diff --git a/documentation/integrations/audible.md b/documentation/integrations/audible.md index 9711695..3ac4740 100644 --- a/documentation/integrations/audible.md +++ b/documentation/integrations/audible.md @@ -1,104 +1,120 @@ # Audible Integration -**Status:** ✅ Implemented (Audnexus API + Web Scraping) +**Status:** Implemented | Unauthenticated Audible JSON catalog API (primary) + Audnexus API (per-ASIN details) -Audiobook metadata from Audnexus API (primary) and Audible.com scraping (fallback) for discovery, search, and detail pages. +## Overview -## Detail Page Strategy +Audiobook metadata for discovery, search, and detail pages. All catalog operations (search, popular, new releases, categories, category books, author books, single-product details) now call Audible's unauthenticated public JSON catalog API (`api.audible./1.0/catalog/*`). Per-ASIN detail lookups prefer Audnexus; the catalog API is used as fallback. -**Primary: Audnexus API** -- Endpoint: `https://api.audnex.us/books/{asin}` -- Structured JSON response (no parsing needed) -- Provides: title, authors, narrators, description, duration, rating, genres, cover art -- Free, no API key required -- ~95% success rate for popular audiobooks +## Architecture -**Fallback: Audible Scraping** -- Used when Audnexus returns 404 -- Parse Audible HTML with Cheerio -- Multiple selector strategies with promotional text filtering -- Extract JSON-LD structured data when available +- **Primary data source:** Audible JSON catalog API, same endpoint used by the official Audible mobile apps. No authentication, no API key, no user credentials, no special headers. +- **Per-ASIN details:** Audnexus (`api.audnex.us/books/{asin}`) remains primary; catalog API (`/1.0/catalog/products/{asin}`) is the fallback when Audnexus returns 404. +- **HTML scraping:** Removed from `audible.service.ts`. The only remaining HTML path is `audible-series.ts` (series-page scraping, out of scope). +- **`www.audible.`:** Still used by `audible-series.ts` and by `getBaseUrl()` for "View on Audible" link generation. Not used for any catalog operation. + +## Data Sources + +All catalog operations are HTTP GET against `{apiBaseUrl}` (region-dependent, e.g. `https://api.audible.com`): + +| Operation | Endpoint | Key params | +|---|---|---| +| Search | `/1.0/catalog/products` | `keywords=` | +| Author books | `/1.0/catalog/products` | `author=` (name, NOT ASIN) | +| Popular | `/1.0/catalog/products` | `products_sort_by=BestSellers` | +| New releases | `/1.0/catalog/products` | `products_sort_by=-ReleaseDate` | +| Category books | `/1.0/catalog/products` | `category_id=&products_sort_by=BestSellers` | +| Categories listing | `/1.0/catalog/categories` | (none) | +| Single product | `/1.0/catalog/products/{asin}` | — | +| Audnexus (per-ASIN) | `https://api.audnex.us/books/{asin}` | `region={audnexusParam}` | + +All `products` endpoints share: +- `num_results` — max **50** (service constant `AUDIBLE_PAGE_SIZE = 50`) +- `page` — **0-indexed at the API** (service public interface is 1-indexed; the service subtracts 1 at the call site). See Gotchas. +- `response_groups=` + +## `response_groups` Constant + +`CATALOG_RESPONSE_GROUPS = 'contributors,product_desc,product_attrs,product_extended_attrs,media,rating,series,category_ladders,product_details'` + +Populates every `AudibleAudiobook` field. Covered: +- `contributors` → authors (with ASINs), narrators +- `product_desc` → `publisher_summary`, `merchandising_summary` +- `product_attrs` / `product_extended_attrs` / `product_details` → title, release_date, language, runtime_length_min +- `media` → `product_images` (cover URLs, uses `500` variant) +- `rating` → `overall_distribution.display_stars` +- `series` → array of `{asin, title, sequence}` +- `category_ladders` → genre names (deduped, capped at 5) + +## Gotchas + +- **`author=` takes a name, not an ASIN.** The catalog API has no ASIN-based author param. `searchByAuthorAsin()` queries by name, then filters client-side: keeps only products where `products[].authors[].asin === authorAsin`. Preserves ASIN-authoritative author identity. Also filters by `product.language` via `isAcceptedLanguage()` for the configured region. +- **Invalid ASIN returns HTTP 200 with stub body.** `/1.0/catalog/products/{asin}` responds 200 with `{product: {asin: INPUT}}` and no other fields. `fetchAudibleDetailsFromApi()` detects this via missing `product.title` and returns `null`. +- **`publisher_summary` is HTML.** Service strips tags via inline `stripHtml()` helper (regex-based, no cheerio) before populating `description`. Falls back to `merchandising_summary` (plain text) if `publisher_summary` missing. +- **Series is an array.** `products[].series[]` — a book may belong to multiple series. Service picks the first entry with non-empty `sequence`, else the first entry. `sequence` is cleaned by extracting first `/\d+(?:\.\d+)?/` match for numeric ordering. +- **Stub `product_images`:** cover URL reads from `product_images['500']`; missing keys fall back to `undefined`. +- **`page` is 0-indexed.** Despite the default value appearing to be 1, the API returns items `(page * num_results)` through `((page + 1) * num_results - 1)`. So `page=1` fetches items 51–100, not 1–50. All service methods accept a 1-indexed `page` and subtract 1 at the axios call. The symptom of getting this wrong is silent: queries whose `total_results ≤ num_results` return an empty `products` array while `total_results` is populated (e.g. author searches for small catalogues). + +## Rate Limiting & Resilience + +- 503s still possible but dramatically less frequent than the HTML surface. +- `fetchWithRetry()` — jittered exponential backoff, 5 retries, retries on 503/429/5xx. +- `AdaptivePacer` circuit-breaker preserved. +- Inter-page base delay on API paths: **500–1500ms** (down from 2000–4000ms for HTML). +- API responses include `Cache-Control: private, max-age=1800`. ## Region Configuration -**Status:** ✅ Implemented +**Status:** Implemented -Configurable Audible region for accurate metadata matching across different international Audible stores. +Configurable Audible region for accurate metadata matching across international stores. **Supported Regions:** -- United States (`us`) - `audible.com` (default, English) -- Canada (`ca`) - `audible.ca` (English) -- United Kingdom (`uk`) - `audible.co.uk` (English) -- Australia (`au`) - `audible.com.au` (English) -- India (`in`) - `audible.in` (English) -- Germany (`de`) - `audible.de` (non-English) -- Spain (`es`) - `audible.es` (non-English) -- French (`fr`) - `audible.fr` (non-English) -**`isEnglish` Flag:** -- Each region has `isEnglish: boolean` in `AudibleRegionConfig` -- Non-English regions (`isEnglish: false`) display an amber warning in all region dropdowns (setup wizard + admin settings) -- Warning text: "Many features such as search, discovery, and metadata matching are not yet fully supported for non-English regions." -- Dropdown options for non-English regions show `*` suffix (e.g., "Germany *") +| Code | Name | HTML baseUrl | apiBaseUrl | isEnglish | +|---|---|---|---|---| +| `us` | United States | `https://www.audible.com` | `https://api.audible.com` | true (default) | +| `ca` | Canada | `https://www.audible.ca` | `https://api.audible.ca` | true | +| `uk` | United Kingdom | `https://www.audible.co.uk` | `https://api.audible.co.uk` | true | +| `au` | Australia | `https://www.audible.com.au` | `https://api.audible.com.au` | true | +| `in` | India | `https://www.audible.in` | `https://api.audible.in` | true | +| `de` | Germany | `https://www.audible.de` | `https://api.audible.de` | false | +| `es` | Spain | `https://www.audible.es` | `https://api.audible.es` | false | +| `fr` | France | `https://www.audible.fr` | `https://api.audible.fr` | false | -**Why Regions Matter:** -- Each Audible region uses different ASINs for the same audiobook -- Metadata engines (Audnexus/Audible Agent) in Plex/Audiobookshelf must match RMAB's region -- Mismatched regions cause poor search results and failed metadata matching +**`AudibleRegionConfig` fields:** `code`, `name`, `baseUrl`, `apiBaseUrl`, `audnexusParam`, `language`. + +**`isEnglish` flag:** +- Non-English regions show amber warning in region dropdowns (setup wizard + admin settings): "Many features such as search, discovery, and metadata matching are not yet fully supported for non-English regions." +- Dropdown options for non-English regions show `*` suffix. + +**Why regions matter:** +- Each Audible region uses different ASINs for the same audiobook. +- Metadata engines (Audnexus / Audible Agent) in Plex / Audiobookshelf must match RMAB's region. **Configuration:** - Key: `audible.region` (stored in database) - Default: `us` - Set during: Setup wizard (Backend Selection step) or Admin Settings (Library tab) -- Help text instructs users to match their metadata engine region +- Auto-detection: Service checks config before each request and re-initializes if region changed. +- Cache clearing: Region change clears ConfigService cache and AudibleService state. +- Automatic refresh: Region change triggers `audible_refresh` job. -**Implementation:** -- `AudibleService` loads region from config on initialization -- Dynamically builds base URL: `AUDIBLE_REGIONS[region].baseUrl` -- Audnexus API calls include region parameter: `?region={code}` -- IP redirect prevention: `?ipRedirectOverride=true` on all Audible requests (region only) -- **Locale enforcement:** `?language=english` query parameter on all Audible requests (forces English content regardless of server IP geolocation) -- Configuration service helper: `getAudibleRegion()` returns configured region -- **Auto-detection of region changes**: Service checks config before each request and re-initializes if region changed -- **Cache clearing**: When region changes, ConfigService cache and AudibleService initialization are cleared -- **Automatic refresh**: Changing region automatically triggers `audible_refresh` job to fetch new data +**Per-region HTTP clients (on init):** +- `apiClient` — `baseURL=apiBaseUrl`, `Accept: application/json`, `User-Agent: ReadMeABook/1.0`, no language/ipRedirect params. +- `htmlClient` — `baseURL=baseUrl`, browser headers, default params `ipRedirectOverride=true` + `language=`. Used only by `audible-series.ts` and `getBaseUrl()`-based link generation. +- Audnexus calls include `region=`. **Files:** - Types: `src/lib/types/audible.ts` - Service: `src/lib/integrations/audible.service.ts` +- Series (HTML): `src/lib/integrations/audible-series.ts` - Config: `src/lib/services/config.service.ts` - API: `src/app/api/admin/settings/audible/route.ts` -## Discovery Strategy (Popular/New/Search) - -- Parse Audible HTML with Cheerio -- Multi-page scraping (20 items/page) -- Rate limit: max 10 req/min, 1.5s delay between pages -- Cache results in database (24hr TTL) - -## Data Sources - -URLs dynamically built based on configured region: - -1. **Best Sellers:** `{baseUrl}/adblbestsellers` -2. **New Releases:** `{baseUrl}/newreleases` -3. **Search:** `{baseUrl}/search?keywords={query}&ipRedirectOverride=true` -4. **Detail Page:** `{baseUrl}/pd/{asin}?ipRedirectOverride=true` -5. **Audnexus API:** `https://api.audnex.us/books/{asin}?region={code}` - -Where `{baseUrl}` is determined by configured region (e.g., `https://www.audible.co.uk` for UK). - -## Metadata Extracted - -- ASIN (Audible ID) -- Title, author, narrator -- Duration (minutes), release date, rating -- Description, cover art URL -- Genres/categories - ## Unified Matching (`audiobook-matcher.ts`) -**Status:** ✅ Production Ready (ASIN-Only Matching) +**Status:** Production Ready (ASIN-Only Matching) Single matching algorithm used everywhere (search, popular, new-releases, jobs). @@ -112,50 +128,42 @@ Single matching algorithm used everywhere (search, popular, new-releases, jobs). - `findPlexMatch()`: ASIN (field) → ASIN (GUID) → null - `matchAudiobook()`: ASIN → ISBN → null -**Benefits:** -- Real-time matching at query time (not pre-matched) -- 100% confidence matches only (eliminates false positives) -- O(1) indexed lookups (faster than fuzzy matching) -- Solves race condition with Audiobookshelf ASIN population -- Used by all APIs for consistency - -**Note:** Fuzzy matching (70% threshold) is preserved in `ranking-algorithm.ts` for Prowlarr torrent ranking, where it's needed to score multiple release candidates. Library availability checks require exact ASIN matches only. +**Note:** Fuzzy matching (70% threshold) is preserved in `ranking-algorithm.ts` for Prowlarr torrent ranking. Library availability checks require exact ASIN matches only. ## Database-First Approach -**Status:** ✅ Implemented +**Status:** Implemented Discovery APIs serve cached data from DB with real-time matching. **Flow:** -1. `audible_refresh` job runs daily → fetches 200 popular + 200 new releases + user-configured categories -2. Downloads and caches cover thumbnails locally (reduces Audible load) -3. Stores metadata in `audible_cache`, ranked entries in `audible_cache_categories` with reserved IDs (`__popular__`, `__new_releases__`) and user category IDs -4. Cleans up unused thumbnails after sync -5. API routes query `AudibleCacheCategory` by categoryId → join with `AudibleCache` metadata → apply real-time matching → return enriched results -6. Homepage loads instantly (no Audible API hits) +1. `audible_refresh` cron runs daily → fetches 200 popular + 200 new releases + user-configured categories via catalog API. +2. Downloads and caches cover thumbnails locally. +3. Stores metadata in `audible_cache`, ranked entries in `audible_cache_categories` with reserved IDs (`__popular__`, `__new_releases__`) and user category IDs. +4. Cleans up unused thumbnails after sync. +5. API routes query `AudibleCacheCategory` by categoryId → join with `AudibleCache` metadata → apply real-time matching → return enriched results. +6. Homepage loads instantly (no Audible API hits). ## Thumbnail Caching -**Status:** ✅ Implemented +**Status:** Implemented -Cover images cached locally to reduce external requests and improve performance. +Cover images cached locally to reduce external requests. -**Features:** -- Downloads covers during `audible_refresh` job -- Stores in `/app/cache/thumbnails` (Docker volume) -- Serves via `/api/cache/thumbnails/[filename]` -- Auto-cleanup of unused thumbnails -- Falls back to original URL if cache fails -- 24-hour browser cache headers +- Downloads covers during `audible_refresh` job. +- Stores in `/app/cache/thumbnails` (Docker volume). +- Serves via `/api/cache/thumbnails/[filename]`. +- Auto-cleanup of unused thumbnails. +- Falls back to original URL if cache fails. +- 24-hour browser cache headers. +- Filename: `{asin}.{ext}` (e.g. `B08G9PRS1K.jpg`). -**Implementation:** +**Files:** - Service: `src/lib/services/thumbnail-cache.service.ts` - API Route: `src/app/api/cache/thumbnails/[filename]/route.ts` - Storage: Docker volume `cache` mounted at `/app/cache` -- Filename: `{asin}.{ext}` (e.g., `B08G9PRS1K.jpg`) -**API Endpoints:** +## App-Level API Endpoints **GET /api/audiobooks/popular?page=1&limit=20** **GET /api/audiobooks/new-releases?page=1&limit=20** @@ -182,6 +190,7 @@ interface AudibleAudiobook { asin: string; title: string; author: string; + authorAsin?: string; narrator?: string; description?: string; coverArtUrl?: string; @@ -189,6 +198,9 @@ interface AudibleAudiobook { releaseDate?: string; rating?: number; genres?: string[]; + series?: string; + seriesPart?: string; + seriesAsin?: string; } interface EnrichedAudibleAudiobook extends AudibleAudiobook { @@ -197,48 +209,45 @@ interface EnrichedAudibleAudiobook extends AudibleAudiobook { plexGuid: string | null; dbId: string; } + +interface AudibleSearchResult { + query: string; + results: AudibleAudiobook[]; + totalResults: number; + page: number; + hasMore: boolean; +} + +interface AuthorBooksResult { + books: AudibleAudiobook[]; + hasMore: boolean; + page: number; + totalResults: number; +} ``` ## Tech Stack -- axios (HTTP) -- cheerio (HTML parsing) -- Redis (caching, optional) -- Database (PostgreSQL) -- string-similarity (matching) +- `axios` (HTTP, two clients: `apiClient` for JSON catalog, `htmlClient` for series-page scraping only) +- Audnexus API (per-ASIN details, primary) +- PostgreSQL (`audible_cache`, `audible_cache_categories`) ## Fixed Issues -**Search returning empty results (2026-01-07)** -- **Problem:** Audible changed HTML structure for search results from `.productListItem` to `.s-result-item` -- **Impact:** All search queries returned 0 results -- **Fix:** Updated `search()` method to support both `.s-result-item` (current) and `.productListItem` (legacy) -- **Selectors updated:** - - Main: `.s-result-item, .productListItem` - - Title: `h2` (new) or `h3 a` (legacy) - - Author: `a[href*="/author/"]` (new) or `.authorLabel` (legacy) - - Narrator: `a[href*="searchNarrator="]` (new) or `.narratorLabel` (legacy) - - Runtime: `span:contains("Length:")` (new) or `.runtimeLabel` (legacy) - - Rating: `.a-icon-star span` (new) or `.ratingsLabel` (legacy) -- **Location:** `src/lib/integrations/audible.service.ts:235` - -**Some audiobooks missing from search results (2026-01-07)** -- **Problem:** ASIN extraction only matched `/pd/` URLs but some audiobooks use `/ac/` URLs -- **Impact:** Books like "Beatitude" by DJ Krimmer (ASIN: B0DVH7XL36) were skipped -- **Fix:** Updated ASIN regex to match both `/pd/` and `/ac/` URL patterns: `/\/(?:pd|ac)\/[^\/]+\/([A-Z0-9]{10})/` -- **Location:** `src/lib/integrations/audible.service.ts:75, 161, 240` -- **Affects:** `getPopularAudiobooks()`, `getNewReleases()`, `search()` methods - **Audiobookshelf metadata matching not respecting configured region (2026-01-28)** -- **Problem:** `triggerABSItemMatch()` hardcoded `'audible'` provider (audible.com) instead of respecting user's configured Audible region -- **Impact:** Users with non-US regions (CA, UK, AU, IN) had incorrect metadata matching in Audiobookshelf, causing wrong ASINs and poor search results -- **Fix:** Added `mapRegionToABSProvider()` to convert RMAB region codes to AudiobookShelf provider values. US → `'audible'`, others → `'audible.{region}'` (e.g., `'audible.ca'`, `'audible.uk'`) +- **Problem:** `triggerABSItemMatch()` hardcoded `'audible'` provider (audible.com) instead of respecting user's configured Audible region. +- **Impact:** Users with non-US regions (CA, UK, AU, IN) had incorrect metadata matching in Audiobookshelf, causing wrong ASINs. +- **Fix:** Added `mapRegionToABSProvider()` to convert RMAB region codes to Audiobookshelf provider values. US → `'audible'`, others → `'audible.{region}'` (e.g. `'audible.ca'`, `'audible.uk'`). - **Location:** `src/lib/services/audiobookshelf/api.ts:14, 147` -- **Affects:** All Audiobookshelf metadata matching operations **Non-English locale pages served to users outside US (2026-02-05)** -- **Problem:** Audible uses IP geolocation to serve locale-specific pages (e.g., Spanish content for Dominican Republic IPs). `ipRedirectOverride=true` only prevents region redirects (audible.com → audible.co.uk), NOT language/locale changes. -- **Impact:** Users self-hosting from non-English-speaking countries got non-English bestsellers/new releases on their homepage. -- **Fix:** Added `language=english` query parameter to all Audible requests via axios default params. Audible respects this parameter and serves English content regardless of IP geolocation. Fails gracefully for regions where English isn't available. -- **Location:** `src/lib/integrations/audible.service.ts` — `initialize()` (axios default params) -- **Affects:** All Audible scraping: popular, new releases, search, detail pages +- **Problem:** Audible uses IP geolocation to serve locale-specific pages. `ipRedirectOverride=true` only prevents region redirects, NOT language/locale changes. +- **Impact:** Users self-hosting from non-English-speaking countries got non-English content on HTML-scraped surfaces. +- **Fix:** Added `language=` default param on `htmlClient` (axios default params). Still in effect for the remaining HTML path (`audible-series.ts`). **Not applied to `apiClient`** — the catalog JSON API is region-bound via `apiBaseUrl` and does not require the language param. +- **Location:** `src/lib/integrations/audible.service.ts` — `initialize()` (htmlClient params) + +## Related + +- [Audiobookshelf Integration](./audiobookshelf.md) +- [Plex Integration](./plex.md) +- [Ranking Algorithm](../phase3/ranking-algorithm.md) diff --git a/src/lib/integrations/audible.service.ts b/src/lib/integrations/audible.service.ts index bc1bd90..cd30487 100644 --- a/src/lib/integrations/audible.service.ts +++ b/src/lib/integrations/audible.service.ts @@ -1,40 +1,32 @@ /** - * Component: Audible Integration Service (Web Scraping) + * Component: Audible Integration Service * Documentation: documentation/integrations/audible.md */ import axios, { AxiosInstance } from 'axios'; -import * as cheerio from 'cheerio'; import { RMABLogger } from '../utils/logger'; import { getConfigService } from '../services/config.service'; import { AudibleRegion, AUDIBLE_REGIONS, DEFAULT_AUDIBLE_REGION } from '../types/audible'; import { getLanguageForRegion, - stripPrefixes, - buildContainsSelector, - extractByPatterns, isAcceptedLanguage, - type LanguageConfig, } from '../constants/language-config'; import { pickUserAgent, getBrowserHeaders, jitteredBackoff, + randomDelay, AdaptivePacer, FetchResultMeta, } from '../utils/scrape-resilience'; -import { parseRuntime as parseRuntimeUtil } from '../utils/parse-runtime'; -// Module-level logger const logger = RMABLogger.create('Audible'); -/** - * Audible supports a pageSize query parameter (default ~20). - * Using 50 significantly reduces the number of HTTP requests needed - * for bulk operations like popular/new-release refreshes and search. - */ const AUDIBLE_PAGE_SIZE = 50; +const CATALOG_RESPONSE_GROUPS = + 'contributors,product_desc,product_attrs,product_extended_attrs,media,rating,series,category_ladders,product_details'; + export interface AudibleAudiobook { asin: string; title: string; @@ -67,112 +59,219 @@ export interface AuthorBooksResult { totalResults: number; } +interface CatalogProductAuthor { + asin?: string; + name: string; +} + +interface CatalogProductNarrator { + name: string; +} + +interface CatalogProductSeries { + asin?: string; + title?: string; + sequence?: string; +} + +interface CatalogProductLadderNode { + name: string; +} + +interface CatalogProductLadder { + ladder: CatalogProductLadderNode[]; +} + +interface CatalogProduct { + asin: string; + title?: string; + authors?: CatalogProductAuthor[]; + narrators?: CatalogProductNarrator[]; + publisher_summary?: string; + merchandising_summary?: string; + product_images?: Record; + runtime_length_min?: number; + release_date?: string; + language?: string; + rating?: { + overall_distribution?: { + display_stars?: number; + }; + }; + category_ladders?: CatalogProductLadder[]; + series?: CatalogProductSeries[]; +} + +interface CatalogProductsResponse { + products: CatalogProduct[]; + total_results?: number; +} + +interface CatalogProductResponse { + product: CatalogProduct; +} + +interface CatalogCategoriesResponse { + categories?: Array<{ id: string; name: string }>; +} + +function stripHtml(html: string): string { + return html + .replace(/<[^>]+>/g, '') + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/ /g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function mapCatalogProduct(product: CatalogProduct): AudibleAudiobook { + const author = product.authors?.map((a) => a.name).join(', ') ?? ''; + const authorAsin = product.authors?.[0]?.asin ?? undefined; + const narrator = + product.narrators && product.narrators.length > 0 + ? product.narrators.map((n) => n.name).join(', ') + : undefined; + + const rawDescription = product.publisher_summary ?? product.merchandising_summary; + const description = rawDescription ? stripHtml(rawDescription) : undefined; + + const coverArtUrl = product.product_images?.['500'] ?? undefined; + + const genreNames = + product.category_ladders?.flatMap((ladder) => + ladder.ladder.map((node) => node.name), + ) ?? []; + const genres = + genreNames.length > 0 + ? [...new Set(genreNames)].slice(0, 5) + : undefined; + + let series: string | undefined; + let seriesPart: string | undefined; + let seriesAsin: string | undefined; + + if (product.series && product.series.length > 0) { + const preferred = + product.series.find((s) => s.sequence && s.sequence.trim() !== '') ?? + product.series[0]; + + series = preferred.title ?? undefined; + seriesAsin = preferred.asin ?? undefined; + + if (preferred.sequence && preferred.sequence.trim() !== '') { + const digitMatch = preferred.sequence.match(/\d+(?:\.\d+)?/); + seriesPart = digitMatch ? digitMatch[0] : preferred.sequence; + } + } + + return { + asin: product.asin, + title: product.title ?? '', + author, + authorAsin, + narrator, + description, + coverArtUrl, + durationMinutes: product.runtime_length_min ?? undefined, + releaseDate: product.release_date ?? undefined, + rating: product.rating?.overall_distribution?.display_stars ?? undefined, + genres, + series, + seriesPart, + seriesAsin, + }; +} + export class AudibleService { - private client!: AxiosInstance; + private htmlClient!: AxiosInstance; + private apiClient!: AxiosInstance; private baseUrl: string = 'https://www.audible.com'; private region: AudibleRegion = 'us'; private initialized: boolean = false; private sessionUserAgent: string = ''; private pacer: AdaptivePacer = new AdaptivePacer(); - constructor() { - // Client will be created lazily on first use - } - - /** - * Get the current Audible base URL for the configured region - */ public getBaseUrl(): string { return this.baseUrl; } - /** - * Get the current Audible region code - */ public getRegion(): AudibleRegion { return this.region; } - /** - * Public fetch wrapper for external scraping modules (e.g. audible-series.ts). - * Ensures the service is initialized and delegates to fetchWithRetry. - */ public async fetch(url: string, config: any = {}): Promise<{ data: any; meta: FetchResultMeta }> { await this.initialize(); return this.fetchWithRetry(url, config); } - /** - * Get the language config for the current region - */ - private getLangConfig(): LanguageConfig { - return getLanguageForRegion(this.region); - } - - /** - * Force re-initialization (used when region config changes) - */ public forceReinitialize(): void { logger.info('Force re-initializing AudibleService'); this.initialized = false; } - /** - * Initialize service with configured region - * Lazy initialization allows async config loading - * Automatically re-initializes if region has changed - */ private async initialize(): Promise { - // If already initialized, check if region has changed if (this.initialized) { const configService = getConfigService(); const currentRegion = await configService.getAudibleRegion(); - // If region changed, force re-initialization if (currentRegion !== this.region) { logger.info(`Region changed from ${this.region} to ${currentRegion}, re-initializing`); this.initialized = false; } else { - return; // Region unchanged, use existing initialization + return; } } try { const configService = getConfigService(); this.region = await configService.getAudibleRegion(); - this.baseUrl = AUDIBLE_REGIONS[this.region].baseUrl; + const regionConfig = AUDIBLE_REGIONS[this.region]; + this.baseUrl = regionConfig.baseUrl; this.sessionUserAgent = pickUserAgent(); this.pacer.reset(); logger.info(`Initializing Audible service with region: ${this.region} (${this.baseUrl})`); - // Get language config for the region const langConfig = getLanguageForRegion(this.region); - // Create axios client with region-specific base URL and realistic browser headers - this.client = axios.create({ - baseURL: this.baseUrl, + this.htmlClient = axios.create({ + baseURL: regionConfig.baseUrl, timeout: 15000, headers: getBrowserHeaders(this.sessionUserAgent), params: { - ipRedirectOverride: 'true', // Prevent IP-based region redirects - language: langConfig.scraping.audibleLocaleParam, // Force locale (prevents IP-based language serving) + ipRedirectOverride: 'true', + language: langConfig.scraping.audibleLocaleParam, + }, + }); + + this.apiClient = axios.create({ + baseURL: regionConfig.apiBaseUrl, + timeout: 10000, + headers: { + Accept: 'application/json', + 'User-Agent': 'ReadMeABook/1.0', }, }); this.initialized = true; } catch (error) { - logger.error('Failed to initialize AudibleService', { error: error instanceof Error ? error.message : String(error) }); - // Fallback to default region + logger.error('Failed to initialize AudibleService', { + error: error instanceof Error ? error.message : String(error), + }); this.region = DEFAULT_AUDIBLE_REGION; - this.baseUrl = AUDIBLE_REGIONS[this.region].baseUrl; + const fallbackConfig = AUDIBLE_REGIONS[this.region]; + this.baseUrl = fallbackConfig.baseUrl; this.sessionUserAgent = pickUserAgent(); this.pacer.reset(); const fallbackLangConfig = getLanguageForRegion(this.region); - this.client = axios.create({ - baseURL: this.baseUrl, + this.htmlClient = axios.create({ + baseURL: fallbackConfig.baseUrl, timeout: 15000, headers: getBrowserHeaders(this.sessionUserAgent), params: { @@ -180,18 +279,25 @@ export class AudibleService { language: fallbackLangConfig.scraping.audibleLocaleParam, }, }); + + this.apiClient = axios.create({ + baseURL: fallbackConfig.apiBaseUrl, + timeout: 10000, + headers: { + Accept: 'application/json', + 'User-Agent': 'ReadMeABook/1.0', + }, + }); + this.initialized = true; } } - /** - * Fetch with retry logic and jittered exponential backoff. - * Returns the axios response plus metadata about retries encountered. - */ private async fetchWithRetry( url: string, config: any = {}, - maxRetries: number = 5 + maxRetries: number = 5, + client: AxiosInstance = this.htmlClient, ): Promise<{ data: any; meta: FetchResultMeta }> { let lastError: Error | null = null; let retriesUsed = 0; @@ -199,7 +305,7 @@ export class AudibleService { for (let attempt = 0; attempt <= maxRetries; attempt++) { try { - const response = await this.client.get(url, config); + const response = await client.get(url, config); return { data: response, meta: { retriesUsed, encountered503 } }; } catch (error: any) { lastError = error; @@ -208,38 +314,32 @@ export class AudibleService { if (status === 503) encountered503 = true; - // Don't retry on 404, 403, etc. if (!isRetryable) { throw error; } - // Don't retry on last attempt if (attempt === maxRetries) { break; } retriesUsed++; - // Jittered exponential backoff instead of predictable doubling const backoffMs = jitteredBackoff(attempt); - logger.info(` Request failed (${status || 'network error'}), retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})...`); + logger.info( + ` Request failed (${status || 'network error'}), retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})...`, + ); await this.delay(backoffMs); } } - // All retries exhausted throw lastError || new Error('Request failed after retries'); } - /** - * External API fetch with retry logic and exponential backoff - * Used for Audnexus and other external APIs - */ private async externalFetchWithRetry( url: string, config: any = {}, - maxRetries: number = 3 + maxRetries: number = 3, ): Promise { let lastError: Error | null = null; @@ -251,12 +351,10 @@ export class AudibleService { const status = error.response?.status; const isRetryable = !status || status === 503 || status === 429 || status >= 500; - // Don't retry on 404, 403, etc. if (!isRetryable) { throw error; } - // Don't retry on deterministic 500 errors (e.g. "Release date is in the future") if (status === 500) { const message = error.response?.data?.message || ''; if (message.includes('Release date is in the future')) { @@ -265,26 +363,22 @@ export class AudibleService { } } - // Don't retry on last attempt if (attempt === maxRetries) { break; } - // Exponential backoff: 2^attempt * 1000ms (1s, 2s, 4s...) const backoffMs = Math.pow(2, attempt) * 1000; - logger.info(` External API request failed (${status || 'network error'}), retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})...`); + logger.info( + ` External API request failed (${status || 'network error'}), retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})...`, + ); await this.delay(backoffMs); } } - // All retries exhausted throw lastError || new Error('External API request failed after retries'); } - /** - * Get popular audiobooks from best sellers (with pagination support) - */ async getPopularAudiobooks(limit: number = 20): Promise { await this.initialize(); @@ -300,85 +394,49 @@ export class AudibleService { try { logger.info(` Fetching page ${page}/${maxPages}...`); - const { data: response, meta } = await this.fetchWithRetry('/adblbestsellers', { - params: { - ipRedirectOverride: 'true', // Explicitly include to prevent IP-based region redirects - pageSize: AUDIBLE_PAGE_SIZE, - ...(page > 1 ? { page } : {}), + const { data: response, meta } = await this.fetchWithRetry( + '/1.0/catalog/products', + { + params: { + products_sort_by: 'BestSellers', + num_results: AUDIBLE_PAGE_SIZE, + page: page - 1, + response_groups: CATALOG_RESPONSE_GROUPS, + }, }, - }); - const $ = cheerio.load(response.data); + 5, + this.apiClient, + ); - let foundOnPage = 0; + const envelope: CatalogProductsResponse = response.data; + const products = envelope.products ?? []; + const totalResults = envelope.total_results ?? 0; - // Parse audiobook items from best sellers page - $('.productListItem').each((index, element) => { - if (audiobooks.length >= limit) return false; - - const $el = $(element); - - // Extract ASIN from data attribute or link - handle both /pd/ and /ac/ URLs - const asin = $el.find('li').attr('data-asin') || - $el.find('a').attr('href')?.match(/\/(?:pd|ac)\/[^\/]+\/([A-Z0-9]{10})/)?.[1] || ''; - - if (!asin) return; - - // Skip duplicates - if (audiobooks.some(book => book.asin === asin)) return; - - const title = $el.find('h3 a').text().trim() || - $el.find('.bc-heading a').text().trim(); - - const authorText = $el.find('.authorLabel').text().trim() || - $el.find('.bc-size-small .bc-text-bold').first().text().trim(); - - // Extract author ASIN from author link if available - const authorHref = $el.find('a[href*="/author/"]').first().attr('href') || ''; - const authorAsinMatch = authorHref.match(/\/author\/[^\/]+\/([A-Z0-9]{10})/); - - const narratorText = $el.find('.narratorLabel').text().trim() || - $el.find('.bc-size-small .bc-text-bold').eq(1).text().trim(); - - const coverArtUrl = $el.find('img').attr('src') || ''; - - const ratingText = $el.find('.ratingsLabel').text().trim(); - const rating = ratingText ? parseFloat(ratingText.split(' ')[0]) : undefined; - - const langConfig = this.getLangConfig(); - - audiobooks.push({ - asin, - title, - author: stripPrefixes(authorText, langConfig.scraping.authorPrefixes), - authorAsin: authorAsinMatch?.[1] || undefined, - narrator: stripPrefixes(narratorText, langConfig.scraping.narratorPrefixes), - coverArtUrl: coverArtUrl.replace(/\._.*_\./, '._SL500_.'), - rating, - }); - - foundOnPage++; - }); - - logger.info(` Found ${foundOnPage} audiobooks on page ${page}`); - - // If we got significantly fewer than requested, probably no more pages - if (foundOnPage < AUDIBLE_PAGE_SIZE / 2) { - logger.info(` Reached end of available pages`); - break; + for (const product of products) { + if (audiobooks.length >= limit) break; + if (audiobooks.some((b) => b.asin === product.asin)) continue; + audiobooks.push(mapCatalogProduct(product)); } + logger.info(` Found ${products.length} audiobooks on page ${page}`); + + const hasMore = + totalResults > 0 + ? totalResults > page * AUDIBLE_PAGE_SIZE + : products.length >= AUDIBLE_PAGE_SIZE; + + if (!hasMore) break; + page++; - // Adaptive delay between pages based on retry pressure if (page <= maxPages && audiobooks.length < limit) { - await this.delay(this.pacer.reportPageResult(meta)); + await this.delay(this.apiPageDelay(meta)); } } catch (error) { logger.error(`Failed to fetch page ${page} of popular audiobooks`, { error: error instanceof Error ? error.message : String(error), - collectedSoFar: audiobooks.length + collectedSoFar: audiobooks.length, }); - // Stop pagination on error, but return what we collected break; } } @@ -387,9 +445,6 @@ export class AudibleService { return audiobooks; } - /** - * Get new release audiobooks (with pagination support) - */ async getNewReleases(limit: number = 20): Promise { await this.initialize(); @@ -405,84 +460,49 @@ export class AudibleService { try { logger.info(` Fetching page ${page}/${maxPages}...`); - const { data: response, meta } = await this.fetchWithRetry('/newreleases', { - params: { - ipRedirectOverride: 'true', // Explicitly include to prevent IP-based region redirects - pageSize: AUDIBLE_PAGE_SIZE, - ...(page > 1 ? { page } : {}), + const { data: response, meta } = await this.fetchWithRetry( + '/1.0/catalog/products', + { + params: { + products_sort_by: '-ReleaseDate', + num_results: AUDIBLE_PAGE_SIZE, + page: page - 1, + response_groups: CATALOG_RESPONSE_GROUPS, + }, }, - }); - const $ = cheerio.load(response.data); + 5, + this.apiClient, + ); - let foundOnPage = 0; + const envelope: CatalogProductsResponse = response.data; + const products = envelope.products ?? []; + const totalResults = envelope.total_results ?? 0; - // Parse audiobook items from new releases page - $('.productListItem').each((index, element) => { - if (audiobooks.length >= limit) return false; - - const $el = $(element); - - // Extract ASIN from data attribute or link - handle both /pd/ and /ac/ URLs - const asin = $el.find('li').attr('data-asin') || - $el.find('a').attr('href')?.match(/\/(?:pd|ac)\/[^\/]+\/([A-Z0-9]{10})/)?.[1] || ''; - - if (!asin) return; - - // Skip duplicates - if (audiobooks.some(book => book.asin === asin)) return; - - const title = $el.find('h3 a').text().trim() || - $el.find('.bc-heading a').text().trim(); - - const authorText = $el.find('.authorLabel').text().trim() || - $el.find('.bc-size-small .bc-text-bold').first().text().trim(); - - // Extract author ASIN from author link if available - const authorHref = $el.find('a[href*="/author/"]').first().attr('href') || ''; - const authorAsinMatch = authorHref.match(/\/author\/[^\/]+\/([A-Z0-9]{10})/); - - const narratorText = $el.find('.narratorLabel').text().trim(); - - const coverArtUrl = $el.find('img').attr('src') || ''; - - const ratingText = $el.find('.ratingsLabel').text().trim(); - const rating = ratingText ? parseFloat(ratingText.split(' ')[0]) : undefined; - - const langConfig = this.getLangConfig(); - - audiobooks.push({ - asin, - title, - author: stripPrefixes(authorText, langConfig.scraping.authorPrefixes), - authorAsin: authorAsinMatch?.[1] || undefined, - narrator: stripPrefixes(narratorText, langConfig.scraping.narratorPrefixes), - coverArtUrl: coverArtUrl.replace(/\._.*_\./, '._SL500_.'), - rating, - }); - - foundOnPage++; - }); - - logger.info(` Found ${foundOnPage} audiobooks on page ${page}`); - - // If we got significantly fewer than requested, probably no more pages - if (foundOnPage < AUDIBLE_PAGE_SIZE / 2) { - logger.info(` Reached end of available pages`); - break; + for (const product of products) { + if (audiobooks.length >= limit) break; + if (audiobooks.some((b) => b.asin === product.asin)) continue; + audiobooks.push(mapCatalogProduct(product)); } + logger.info(` Found ${products.length} audiobooks on page ${page}`); + + const hasMore = + totalResults > 0 + ? totalResults > page * AUDIBLE_PAGE_SIZE + : products.length >= AUDIBLE_PAGE_SIZE; + + if (!hasMore) break; + page++; - // Adaptive delay between pages based on retry pressure if (page <= maxPages && audiobooks.length < limit) { - await this.delay(this.pacer.reportPageResult(meta)); + await this.delay(this.apiPageDelay(meta)); } } catch (error) { logger.error(`Failed to fetch page ${page} of new releases`, { error: error instanceof Error ? error.message : String(error), - collectedSoFar: audiobooks.length + collectedSoFar: audiobooks.length, }); - // Stop pagination on error, but return what we collected break; } } @@ -491,216 +511,109 @@ export class AudibleService { return audiobooks; } - /** - * Search for audiobooks - */ async search(query: string, page: number = 1): Promise { await this.initialize(); try { logger.info(` Searching for "${query}"...`); - const { data: response } = await this.fetchWithRetry('/search', { - params: { - ipRedirectOverride: 'true', // Explicitly include to prevent IP-based region redirects - keywords: query, - pageSize: AUDIBLE_PAGE_SIZE, - page, + const { data: response } = await this.fetchWithRetry( + '/1.0/catalog/products', + { + params: { + keywords: query, + num_results: AUDIBLE_PAGE_SIZE, + page: page - 1, + response_groups: CATALOG_RESPONSE_GROUPS, + }, }, - }); + 5, + this.apiClient, + ); - const $ = cheerio.load(response.data); + const envelope: CatalogProductsResponse = response.data; + const products = envelope.products ?? []; + const totalResults = envelope.total_results ?? 0; - const audiobooks: AudibleAudiobook[] = []; + const results = products.map(mapCatalogProduct); - // Parse search results - Audible uses s-result-item for search pages - $('.s-result-item, .productListItem').each((index, element) => { - const $el = $(element); - - // Extract ASIN from product detail link - handle both /pd/ and /ac/ URLs - const asin = $el.find('li').attr('data-asin') || - $el.find('a[href*="/pd/"]').attr('href')?.match(/\/pd\/[^\/]+\/([A-Z0-9]{10})/)?.[1] || - $el.find('a[href*="/ac/"]').attr('href')?.match(/\/ac\/[^\/]+\/([A-Z0-9]{10})/)?.[1] || - $el.find('a').attr('href')?.match(/\/(?:pd|ac)\/[^\/]+\/([A-Z0-9]{10})/)?.[1] || ''; - - if (!asin) return; - - // Extract title from h2 tag (search results) or h3 (legacy) - const title = $el.find('h2').first().text().trim() || - $el.find('h3 a').text().trim() || - $el.find('.bc-heading a').text().trim(); - - // Extract author from author link - const authorLink = $el.find('a[href*="/author/"]').first(); - const authorText = authorLink.text().trim() || - $el.find('.authorLabel').text().trim() || - $el.find('.bc-size-small .bc-text-bold').first().text().trim(); - - // Extract author ASIN from author link href - const authorHref = authorLink.attr('href') || ''; - const authorAsinMatch = authorHref.match(/\/author\/[^\/]+\/([A-Z0-9]{10})/); - - // Extract narrator from narrator search link - const narratorText = $el.find('a[href*="searchNarrator="]').first().text().trim() || - $el.find('.narratorLabel').text().trim(); - - const coverArtUrl = $el.find('img').attr('src') || ''; - - const langConfig = this.getLangConfig(); - - // Extract runtime/duration - const runtimeText = $el.find('.runtimeLabel').text().trim() || - $el.find(buildContainsSelector('span', langConfig.scraping.lengthLabels)).text().trim(); - const durationMinutes = this.parseRuntime(runtimeText); - - // Extract rating - const ratingText = $el.find('.ratingsLabel').text().trim() || - $el.find('.a-icon-star span').first().text().trim(); - const rating = ratingText ? parseFloat(ratingText.split(' ')[0]) : undefined; - - audiobooks.push({ - asin, - title, - author: stripPrefixes(authorText, langConfig.scraping.authorPrefixes), - authorAsin: authorAsinMatch?.[1] || undefined, - narrator: stripPrefixes(narratorText, langConfig.scraping.narratorPrefixes), - coverArtUrl: coverArtUrl.replace(/\._.*_\./, '._SL500_.'), - durationMinutes, - rating, - }); - }); - - // Try to extract total results count - const resultsText = $('.resultsInfo').text().trim(); - const totalResults = parseInt(resultsText.match(/of ([\d,]+)/)?.[1]?.replace(/,/g, '') || '0'); - - logger.info(` Found ${audiobooks.length} results for "${query}"`); + logger.info(` Found ${results.length} results for "${query}"`); return { query, - results: audiobooks, + results, totalResults, page, - hasMore: audiobooks.length > 0 && (totalResults > 0 - ? totalResults > page * AUDIBLE_PAGE_SIZE - : audiobooks.length >= AUDIBLE_PAGE_SIZE), + hasMore: + results.length > 0 && + (totalResults > 0 + ? totalResults > page * AUDIBLE_PAGE_SIZE + : results.length >= AUDIBLE_PAGE_SIZE), }; } catch (error) { - logger.error('Search failed', { error: error instanceof Error ? error.message : String(error) }); - return { - query, - results: [], - totalResults: 0, - page, - hasMore: false, - }; + logger.error('Search failed', { + error: error instanceof Error ? error.message : String(error), + }); + return { query, results: [], totalResults: 0, page, hasMore: false }; } } /** - * Search for all books by a specific author, validated by ASIN. - * Uses Audible's searchAuthor parameter and paginates through all results. - * Filters: (1) author link must contain the target ASIN, (2) language must be English. + * The catalog API `author=` param takes an author name (not ASIN), so we filter + * client-side by checking that at least one author entry matches the target ASIN. */ - async searchByAuthorAsin(authorName: string, authorAsin: string, page: number = 1): Promise { + async searchByAuthorAsin( + authorName: string, + authorAsin: string, + page: number = 1, + ): Promise { await this.initialize(); + const langConfig = getLanguageForRegion(this.region); const books: AudibleAudiobook[] = []; - const seenAsins = new Set(); try { logger.info(`Searching books by author "${authorName}" (ASIN: ${authorAsin}), page ${page}...`); - const { data: response } = await this.fetchWithRetry('/search', { - params: { - ipRedirectOverride: 'true', - searchAuthor: authorName, - pageSize: AUDIBLE_PAGE_SIZE, - page, + const { data: response } = await this.fetchWithRetry( + '/1.0/catalog/products', + { + params: { + author: authorName, + num_results: AUDIBLE_PAGE_SIZE, + page: page - 1, + response_groups: CATALOG_RESPONSE_GROUPS, + }, }, - }); + 5, + this.apiClient, + ); - const $ = cheerio.load(response.data); + const envelope: CatalogProductsResponse = response.data; + const products = envelope.products ?? []; + const totalResults = envelope.total_results ?? 0; - // Count raw items on page before filtering (for hasMore fallback) - const pageItemCount = $('.s-result-item, .productListItem').length; + for (const product of products) { + const authorMatch = product.authors?.some((a) => a.asin === authorAsin) ?? false; + if (!authorMatch) continue; - $('.s-result-item, .productListItem').each((_index, element) => { - const $el = $(element); + const langMatch = product.language + ? isAcceptedLanguage(product.language, langConfig) + : false; + if (!langMatch) continue; - // --- Language filter: require matching language for region --- - const langConfig = this.getLangConfig(); - const langText = $el.find(buildContainsSelector('span', langConfig.scraping.languageLabels)).text().trim() || - $el.find('.languageLabel').text().trim(); - const langLabelPattern = new RegExp(`(?:${langConfig.scraping.languageLabels.map(l => l.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|')})\\s*(.+)`, 'i'); - const langMatch = langText.match(langLabelPattern); - const language = langMatch?.[1]?.trim(); - if (!language || !isAcceptedLanguage(language, langConfig)) return; + books.push(mapCatalogProduct(product)); + } - // --- Author ASIN filter: verify target ASIN in author links --- - const authorLinks = $el.find('a[href*="/author/"]'); - let hasMatchingAuthor = false; - authorLinks.each((_i, link) => { - const href = $(link).attr('href') || ''; - const asinMatch = href.match(/\/author\/[^\/]+\/([A-Z0-9]{10})/); - if (asinMatch && asinMatch[1] === authorAsin) { - hasMatchingAuthor = true; - return false; // break .each() - } - }); - if (!hasMatchingAuthor) return; + const hasMore = + books.length > 0 && + (totalResults > 0 + ? totalResults > page * AUDIBLE_PAGE_SIZE + : products.length >= AUDIBLE_PAGE_SIZE); - // --- Extract book ASIN --- - const bookAsin = $el.find('li').attr('data-asin') || - $el.find('a[href*="/pd/"]').attr('href')?.match(/\/pd\/[^\/]+\/([A-Z0-9]{10})/)?.[1] || - $el.find('a[href*="/ac/"]').attr('href')?.match(/\/ac\/[^\/]+\/([A-Z0-9]{10})/)?.[1] || - $el.find('a').attr('href')?.match(/\/(?:pd|ac)\/[^\/]+\/([A-Z0-9]{10})/)?.[1] || ''; - if (!bookAsin || seenAsins.has(bookAsin)) return; - seenAsins.add(bookAsin); - - // --- Parse book details --- - const title = $el.find('h2').first().text().trim() || - $el.find('h3 a').text().trim() || - $el.find('.bc-heading a').text().trim(); - - const authorText = $el.find('a[href*="/author/"]').first().text().trim() || - $el.find('.authorLabel').text().trim() || - $el.find('.bc-size-small .bc-text-bold').first().text().trim(); - - const narratorText = $el.find('a[href*="searchNarrator="]').first().text().trim() || - $el.find('.narratorLabel').text().trim(); - - const coverArtUrl = $el.find('img').attr('src') || ''; - - const runtimeText = $el.find('.runtimeLabel').text().trim() || - $el.find(buildContainsSelector('span', langConfig.scraping.lengthLabels)).text().trim(); - const durationMinutes = this.parseRuntime(runtimeText); - - const ratingText = $el.find('.ratingsLabel').text().trim() || - $el.find('.a-icon-star span').first().text().trim(); - const rating = ratingText ? parseFloat(ratingText.split(' ')[0]) : undefined; - - books.push({ - asin: bookAsin, - title, - author: stripPrefixes(authorText, langConfig.scraping.authorPrefixes), - authorAsin, - narrator: stripPrefixes(narratorText, langConfig.scraping.narratorPrefixes), - coverArtUrl: coverArtUrl.replace(/\._.*_\./, '._SL500_.'), - durationMinutes, - rating, - }); - }); - - // Check total results for pagination - const resultsText = $('.resultsInfo').text().trim(); - const totalResults = parseInt(resultsText.match(/of ([\d,]+)/)?.[1]?.replace(/,/g, '') || '0'); - // Use totalResults if available; otherwise fall back to whether Audible returned a full page - const hasMore = books.length > 0 && (totalResults > 0 - ? totalResults > page * AUDIBLE_PAGE_SIZE - : pageItemCount >= AUDIBLE_PAGE_SIZE); - - logger.info(`Author books page ${page}: ${books.length} valid results (${totalResults} Audible total)`); + logger.info( + `Author books page ${page}: ${books.length} valid results (${totalResults} Audible total)`, + ); return { books, hasMore, page, totalResults }; } catch (error) { logger.error(`Author books search failed for "${authorName}"`, { @@ -710,55 +623,45 @@ export class AudibleService { } } - /** - * Get detailed audiobook information - * Primary: Audnexus API (reliable, structured data) - * Fallback: Audible scraping - */ async getAudiobookDetails(asin: string): Promise { await this.initialize(); try { logger.info(` Fetching details for ASIN ${asin}...`); - // Try Audnexus first (more reliable) const audnexusData = await this.fetchFromAudnexus(asin); if (audnexusData) { logger.info(` Successfully fetched from Audnexus for "${audnexusData.title}"`); return audnexusData; } - logger.info(` Audnexus failed, falling back to Audible scraping...`); + logger.info(` Audnexus failed, falling back to Audible catalog API...`); - // Fallback to Audible scraping - return await this.scrapeAudibleDetails(asin); + return await this.fetchAudibleDetailsFromApi(asin); } catch (error) { - logger.error(`Failed to fetch details for ${asin}`, { error: error instanceof Error ? error.message : String(error) }); + logger.error(`Failed to fetch details for ${asin}`, { + error: error instanceof Error ? error.message : String(error), + }); return null; } } - /** - * Fetch audiobook details from Audnexus API - */ private async fetchFromAudnexus(asin: string): Promise { try { const audnexusRegion = AUDIBLE_REGIONS[this.region].audnexusParam; logger.debug(`Fetching ASIN from Audnexus: ${asin} (region: ${audnexusRegion})`); - const response = await this.externalFetchWithRetry(`https://api.audnex.us/books/${asin}`, { - params: { - region: audnexusRegion, // Pass region parameter to Audnexus + const response = await this.externalFetchWithRetry( + `https://api.audnex.us/books/${asin}`, + { + params: { region: audnexusRegion }, + timeout: 10000, + headers: { 'User-Agent': 'ReadMeABook/1.0' }, }, - timeout: 10000, - headers: { - 'User-Agent': 'ReadMeABook/1.0', - }, - }); + ); const data = response.data; - // Build result from Audnexus data const result: AudibleAudiobook = { asin, title: data.title || '', @@ -770,13 +673,12 @@ export class AudibleService { durationMinutes: data.runtimeLengthMin ? parseInt(data.runtimeLengthMin) : undefined, releaseDate: data.releaseDate || undefined, rating: data.rating ? parseFloat(data.rating) : undefined, - genres: data.genres?.map((g: any) => typeof g === 'string' ? g : g.name).slice(0, 5) || undefined, + genres: data.genres?.map((g: any) => (typeof g === 'string' ? g : g.name)).slice(0, 5) || undefined, series: data.seriesPrimary?.name || undefined, seriesPart: data.seriesPrimary?.position || undefined, seriesAsin: data.seriesPrimary?.asin || undefined, }; - // Ensure cover art URL is high quality if (result.coverArtUrl && !result.coverArtUrl.includes('_SL500_')) { result.coverArtUrl = result.coverArtUrl.replace(/\._.*_\./, '._SL500_.'); } @@ -791,7 +693,7 @@ export class AudibleService { genreCount: result.genres?.length || 0, series: result.series, seriesPart: result.seriesPart, - seriesAsin: result.seriesAsin + seriesAsin: result.seriesAsin, }); return result; @@ -805,367 +707,46 @@ export class AudibleService { } } - /** - * Scrape audiobook details from Audible (fallback method) - */ - private async scrapeAudibleDetails(asin: string): Promise { + private async fetchAudibleDetailsFromApi(asin: string): Promise { try { - const { data: response } = await this.fetchWithRetry(`/pd/${asin}`, { - params: { - ipRedirectOverride: 'true', // Explicitly include to prevent IP-based region redirects - }, - }); - const $ = cheerio.load(response.data); + const { data: response } = await this.fetchWithRetry( + `/1.0/catalog/products/${asin}`, + { params: { response_groups: CATALOG_RESPONSE_GROUPS } }, + 5, + this.apiClient, + ); - // Initialize result object - let result: AudibleAudiobook = { - asin, - title: '', - author: '', - narrator: '', - description: '', - coverArtUrl: '', - }; + const envelope: CatalogProductResponse = response.data; + const product = envelope.product; - // Debug: Save HTML in development - const isDev = process.env.NODE_ENV === 'development'; - if (isDev) { - const fs = require('fs'); - const path = require('path'); - const debugPath = path.join('/tmp', `audible-${asin}.html`); - fs.writeFileSync(debugPath, response.data); - logger.info(` Saved HTML to ${debugPath} for debugging`); + // The API returns HTTP 200 with a stub object for invalid ASINs; + // a missing title is the reliable signal that the ASIN is unrecognised. + if (!product?.title) { + logger.debug(`Catalog API returned stub for ASIN ${asin} (no title)`); + return null; } - // Try to extract JSON-LD structured data first - const jsonLdScripts = $('script[type="application/ld+json"]'); - logger.info(` Found ${jsonLdScripts.length} JSON-LD script tags`); - - jsonLdScripts.each((i, elem) => { - try { - const jsonData = JSON.parse($(elem).html() || '{}'); - logger.info(` JSON-LD ${i} type:`, jsonData['@type']); - - if (jsonData['@type'] === 'Book' || jsonData['@type'] === 'Audiobook' || jsonData['@type'] === 'Product') { - logger.debug('Found valid JSON-LD structured data'); - - if (jsonData.name) result.title = jsonData.name; - - if (jsonData.author) { - result.author = Array.isArray(jsonData.author) - ? jsonData.author.map((a: any) => a.name || a).join(', ') - : jsonData.author?.name || jsonData.author || ''; - } - - if (jsonData.readBy) { - result.narrator = Array.isArray(jsonData.readBy) - ? jsonData.readBy.map((n: any) => n.name || n).join(', ') - : jsonData.readBy?.name || jsonData.readBy || ''; - } - - if (jsonData.description) result.description = jsonData.description; - if (jsonData.image) result.coverArtUrl = jsonData.image; - if (jsonData.aggregateRating?.ratingValue) result.rating = jsonData.aggregateRating.ratingValue; - if (jsonData.datePublished) result.releaseDate = jsonData.datePublished; - - if (jsonData.duration) { - const durationMatch = jsonData.duration.match(/PT(\d+)H(\d+)M/); - if (durationMatch) { - result.durationMinutes = parseInt(durationMatch[1]) * 60 + parseInt(durationMatch[2]); - } - } - } - } catch (e) { - logger.debug(`JSON-LD ${i} parsing failed`, { error: e instanceof Error ? e.message : String(e) }); - } - }); - - // Fallback to HTML parsing for any missing fields - // Title - try multiple selectors - if (!result.title) { - result.title = $('h1.bc-heading').first().text().trim() || - $('h1[class*="heading"]').first().text().trim() || - $('.bc-container h1').first().text().trim() || - $('h1').first().text().trim(); - logger.info(` Title from HTML: "${result.title}"`); - } - - // Author - try multiple approaches (only in product details area) - if (!result.author) { - // Look specifically in the product details section, not the whole page - const productSection = $('.bc-section, .product-top-section, [class*="product"]').first(); - const authors: string[] = []; - - // First try labeled author sections - productSection.find('li.authorLabel a, span.authorLabel a, .authorLabel a').each((_, elem) => { - const text = $(elem).text().trim(); - if (text && text.length > 0 && text.length < 80) { - authors.push(text); - } - }); - - // If no labeled authors, look for author links near the title (first 3 only to avoid recommendations) - if (authors.length === 0) { - $('a[href*="/author/"]').slice(0, 3).each((_, elem) => { - const text = $(elem).text().trim(); - // Filter out navigation breadcrumbs and promotional text - if (text && text.length > 1 && text.length < 80 && - !text.includes('›') && !text.includes('...') && - !text.toLowerCase().includes('more') && !text.toLowerCase().includes('see all')) { - authors.push(text); - } - }); - } - - if (authors.length > 0) { - // Deduplicate and limit to max 3 authors - result.author = [...new Set(authors)].slice(0, 3).join(', '); - } - - const authorLangConfig = this.getLangConfig(); - result.author = stripPrefixes(result.author, authorLangConfig.scraping.authorPrefixes); - logger.info(` Author from HTML: "${result.author}"`); - } - - // Author ASIN - extract from the first author link - if (!result.authorAsin) { - const firstAuthorHref = $('a[href*="/author/"]').first().attr('href') || ''; - const authorAsinMatch = firstAuthorHref.match(/\/author\/[^\/]+\/([A-Z0-9]{10})/); - if (authorAsinMatch) { - result.authorAsin = authorAsinMatch[1]; - } - } - - // Narrator - try multiple approaches (only in product details area) - if (!result.narrator) { - // Look specifically in the product details section - const productSection = $('.bc-section, .product-top-section, [class*="product"]').first(); - const narrators: string[] = []; - - // First try labeled narrator sections - productSection.find('li.narratorLabel a, span.narratorLabel a, .narratorLabel a').each((_, elem) => { - const text = $(elem).text().trim(); - if (text && text.length > 0 && text.length < 80) { - narrators.push(text); - } - }); - - // If no labeled narrators, look for narrator links (first 5 only) - if (narrators.length === 0) { - $('a[href*="/narrator/"]').slice(0, 5).each((_, elem) => { - const text = $(elem).text().trim(); - if (text && text.length > 1 && text.length < 80 && - !text.includes('›') && !text.includes('...')) { - narrators.push(text); - } - }); - } - - if (narrators.length > 0) { - // Deduplicate and limit to reasonable count - result.narrator = [...new Set(narrators)].slice(0, 5).join(', '); - } - - if (result.narrator) { - const detailLangConfig = this.getLangConfig(); - result.narrator = stripPrefixes(result.narrator, detailLangConfig.scraping.narratorPrefixes); - } - logger.info(` Narrator from HTML: "${result.narrator || ''}"`); - } - - // Description - try multiple approaches with strict filtering - if (!result.description) { - const descLangConfig = this.getLangConfig(); - const excludePatterns = descLangConfig.scraping.descriptionExcludePatterns; - - const isValidDescription = (text: string): boolean => { - if (!text || text.length < 50 || text.length > 5000) return false; - // Reject if it contains promotional patterns - for (const pattern of excludePatterns) { - if (pattern.test(text)) return false; - } - return true; - }; - - // Try specific description selectors first - const candidates = [ - $('.bc-expander-content').first().text().trim(), - $('[class*="productPublisherSummary"]').first().text().trim(), - $('[data-widget="publisherSummary"]').first().text().trim(), - $('.bc-section p').first().text().trim(), - ]; - - // Find first valid candidate - for (const candidate of candidates) { - if (isValidDescription(candidate)) { - result.description = candidate; - break; - } - } - - // If still no description, search for valid paragraphs - if (!result.description) { - $('p, div[class*="description"]').each((_, elem) => { - const text = $(elem).text().trim(); - if (isValidDescription(text) && text.length > (result.description?.length || 0)) { - result.description = text; - } - }); - } - - logger.info(` Description length: ${result.description?.length || 0} chars`); - } - - // Cover art - try multiple selectors - if (!result.coverArtUrl) { - result.coverArtUrl = $('img.bc-image-inset-border').attr('src') || - $('img[class*="product-image"]').first().attr('src') || - $('img[class*="cover"]').first().attr('src') || - $('.bc-pub-detail-image img').attr('src') || - $('img[src*="images-na.ssl-images-amazon.com"]').first().attr('src') || - $('img[src*="m.media-amazon.com"]').first().attr('src') || - ''; - if (result.coverArtUrl) { - result.coverArtUrl = result.coverArtUrl.replace(/\._.*_\./, '._SL500_.'); - } - } - - // Runtime/Duration - try multiple approaches - if (!result.durationMinutes) { - const rtLangConfig = this.getLangConfig(); - - // Look for runtime text in various places - const runtimeText = - $('li.runtimeLabel span').text().trim() || - $('.runtimeLabel').text().trim() || - $(buildContainsSelector('span', rtLangConfig.scraping.lengthLabels)).parent().text().trim() || - $(buildContainsSelector('li', rtLangConfig.scraping.lengthLabels)).text().trim() || - (() => { - // Look for any text matching duration pattern - let found = ''; - $('li, span, div').each((_, elem) => { - const text = $(elem).text().trim(); - if (text.match(rtLangConfig.scraping.durationDetectionPattern) && text.length < 100) { - found = text; - return false; // break - } - }); - return found; - })(); - - result.durationMinutes = this.parseRuntime(runtimeText); - logger.info(` Duration from "${runtimeText}": ${result.durationMinutes} minutes`); - } - - // Rating - try multiple approaches - if (!result.rating) { - const ratingLangConfig = this.getLangConfig(); - const ratingText = - $('.ratingsLabel').text().trim() || - $('[class*="rating"]').first().text().trim() || - $(`span:contains("${ratingLangConfig.scraping.ratingTextSelector}")`).parent().text().trim() || - (() => { - // Look for rating pattern using language-specific patterns - let found = ''; - $('span, div').each((_, elem) => { - const text = $(elem).text().trim(); - if (text.length < 50) { - for (const pattern of ratingLangConfig.scraping.ratingPatterns) { - if (pattern.test(text)) { - found = text; - return false; - } - } - } - }); - return found; - })(); - - if (ratingText) { - let ratingValue: number | undefined; - for (const pattern of ratingLangConfig.scraping.ratingPatterns) { - const ratingMatch = ratingText.match(pattern); - if (ratingMatch) { - // Handle comma as decimal separator (e.g. "4,5" in German/Spanish) - ratingValue = parseFloat(ratingMatch[1].replace(',', '.')); - break; - } - } - result.rating = ratingValue; - } - logger.info(` Rating from "${ratingText}": ${result.rating}`); - } - - // Release date - try multiple selectors - if (!result.releaseDate) { - const rdLangConfig = this.getLangConfig(); - const releaseDateText = - $(buildContainsSelector('li', rdLangConfig.scraping.releaseDateLabels)).text().trim() || - $(buildContainsSelector('span', rdLangConfig.scraping.releaseDateLabels)).parent().text().trim() || - $('[class*="release"]').text().trim(); - - const dateMatch = extractByPatterns(releaseDateText, rdLangConfig.scraping.releaseDatePatterns) || - releaseDateText.match(/(\w+ \d{1,2},? \d{4})/)?.[1]; - if (dateMatch) { - result.releaseDate = dateMatch.trim(); - } - logger.info(` Release date from "${releaseDateText}": ${result.releaseDate}`); - } - - // Genres - try to extract categories - const genres: string[] = []; - $('a[href*="/cat/"]').each((_, el) => { - const genre = $(el).text().trim(); - if (genre && !genres.includes(genre) && genre.length < 50 && genre.length > 2) { - genres.push(genre); - } - }); - if (genres.length > 0) { - result.genres = genres.slice(0, 5); // Limit to 5 genres - logger.info(` Genres: ${result.genres.join(', ')}`); - } - - logger.info(`Successfully fetched details for "${result.title}"`); - logger.debug('Final result', { - title: result.title, - author: result.author, - narrator: result.narrator, - descLength: result.description?.length || 0, - duration: result.durationMinutes, - rating: result.rating, - genreCount: result.genres?.length || 0 - }); - - return result; + return mapCatalogProduct(product); } catch (error) { - logger.error(`Failed to fetch details for ${asin}`, { error: error instanceof Error ? error.message : String(error) }); + logger.error(`Catalog API details fetch failed for ${asin}`, { + error: error instanceof Error ? error.message : String(error), + }); return null; } } - /** - * Parse runtime text to minutes using language-specific patterns. - * Delegates to shared utility in src/lib/utils/parse-runtime.ts. - */ - private parseRuntime(runtimeText: string): number | undefined { - return parseRuntimeUtil(runtimeText, this.getLangConfig()); - } - - /** - * Get runtime (in minutes) for an audiobook by ASIN - * Lightweight method for size validation during search - * Returns null if not found or error - */ async getRuntime(asin: string): Promise { try { - // Use Audnexus API for fast, reliable runtime data const audnexusRegion = AUDIBLE_REGIONS[this.region].audnexusParam; - const response = await this.externalFetchWithRetry(`https://api.audnex.us/books/${asin}`, { - params: { region: audnexusRegion }, - timeout: 5000, // Quick timeout for search performance - headers: { 'User-Agent': 'ReadMeABook/1.0' }, - }); + const response = await this.externalFetchWithRetry( + `https://api.audnex.us/books/${asin}`, + { + params: { region: audnexusRegion }, + timeout: 5000, + headers: { 'User-Agent': 'ReadMeABook/1.0' }, + }, + ); const runtimeMin = response.data?.runtimeLengthMin; if (runtimeMin) { @@ -1181,38 +762,24 @@ export class AudibleService { } } - /** - * Get top-level categories from Audible's categories page. - * Scrapes {baseUrl}/categories and returns {id, name}[] for top-level nodes. - */ async getCategories(): Promise<{ id: string; name: string }[]> { await this.initialize(); logger.info('Fetching Audible categories...'); try { - const { data: response } = await this.fetchWithRetry('/categories', { - params: { ipRedirectOverride: 'true' }, - }); + const { data: response } = await this.fetchWithRetry( + '/1.0/catalog/categories', + {}, + 5, + this.apiClient, + ); - const $ = cheerio.load(response.data); - const categories: { id: string; name: string }[] = []; - - // Top-level category links are in the main categories grid - // They follow the pattern /cat/{name}/{nodeId} - $('a[href*="/cat/"]').each((_index, element) => { - const $el = $(element); - const href = $el.attr('href') || ''; - const match = href.match(/\/cat\/[^\/]+\/(\d+)/); - if (!match) return; - - const id = match[1]; - const name = $el.text().trim(); - - if (name && !categories.some((c) => c.id === id)) { - categories.push({ id, name }); - } - }); + const envelope: CatalogCategoriesResponse = response.data; + const categories = (envelope.categories ?? []).map((c) => ({ + id: c.id, + name: c.name, + })); logger.info(`Found ${categories.length} top-level categories`); return categories; @@ -1224,10 +791,6 @@ export class AudibleService { } } - /** - * Get audiobooks for a specific category using Audible search with node parameter. - * Scrapes {baseUrl}/search?node={categoryId}&pageSize=50, up to `limit` results. - */ async getCategoryBooks(categoryId: string, limit: number = 200): Promise { await this.initialize(); @@ -1241,81 +804,44 @@ export class AudibleService { while (audiobooks.length < limit && page <= maxPages) { try { - const { data: response, meta } = await this.fetchWithRetry('/search', { - params: { - ipRedirectOverride: 'true', - node: categoryId, - pageSize: AUDIBLE_PAGE_SIZE, - sort: 'popularity-rank', - ...(page > 1 ? { page } : {}), + const { data: response, meta } = await this.fetchWithRetry( + '/1.0/catalog/products', + { + params: { + category_id: categoryId, + products_sort_by: 'BestSellers', + num_results: AUDIBLE_PAGE_SIZE, + page: page - 1, + response_groups: CATALOG_RESPONSE_GROUPS, + }, }, - }); + 5, + this.apiClient, + ); - const $ = cheerio.load(response.data); - let foundOnPage = 0; + const envelope: CatalogProductsResponse = response.data; + const products = envelope.products ?? []; + const totalResults = envelope.total_results ?? 0; - // Parse search results — same selectors as search() - $('.s-result-item, .productListItem').each((_index, element) => { - if (audiobooks.length >= limit) return false; - const $el = $(element); + for (const product of products) { + if (audiobooks.length >= limit) break; + if (audiobooks.some((b) => b.asin === product.asin)) continue; + audiobooks.push(mapCatalogProduct(product)); + } - const asin = - $el.find('li').attr('data-asin') || - $el.find('a').attr('href')?.match(/\/(?:pd|ac)\/[^\/]+\/([A-Z0-9]{10})/)?.[1] || - ''; - if (!asin || audiobooks.some((b) => b.asin === asin)) return; + logger.info(`Category ${categoryId}: found ${products.length} books on page ${page}`); - const title = - $el.find('h2').first().text().trim() || - $el.find('h3 a').text().trim() || - $el.find('.bc-heading a').text().trim(); + const hasMore = + totalResults > 0 + ? totalResults > page * AUDIBLE_PAGE_SIZE + : products.length >= AUDIBLE_PAGE_SIZE; - const authorLink = $el.find('a[href*="/author/"]').first(); - const authorText = - authorLink.text().trim() || - $el.find('.authorLabel').text().trim(); - const authorHref = authorLink.attr('href') || ''; - const authorAsinMatch = authorHref.match(/\/author\/[^\/]+\/([A-Z0-9]{10})/); - - const narratorText = - $el.find('a[href*="searchNarrator="]').first().text().trim() || - $el.find('.narratorLabel').text().trim(); - - const coverArtUrl = $el.find('img').attr('src') || ''; - - const langConfig = this.getLangConfig(); - const runtimeText = - $el.find('.runtimeLabel').text().trim() || - $el.find(buildContainsSelector('span', langConfig.scraping.lengthLabels)).text().trim(); - const durationMinutes = this.parseRuntime(runtimeText); - - const ratingText = - $el.find('.ratingsLabel').text().trim() || - $el.find('.a-icon-star span').first().text().trim(); - const rating = ratingText ? parseFloat(ratingText.split(' ')[0]) : undefined; - - audiobooks.push({ - asin, - title, - author: stripPrefixes(authorText, langConfig.scraping.authorPrefixes), - authorAsin: authorAsinMatch?.[1] || undefined, - narrator: stripPrefixes(narratorText, langConfig.scraping.narratorPrefixes), - coverArtUrl: coverArtUrl.replace(/\._.*_\./, '._SL500_.'), - durationMinutes, - rating, - }); - - foundOnPage++; - }); - - logger.info(`Category ${categoryId}: found ${foundOnPage} books on page ${page}`); - - if (foundOnPage < AUDIBLE_PAGE_SIZE / 2) break; + if (!hasMore) break; page++; if (page <= maxPages && audiobooks.length < limit) { - await this.delay(this.pacer.reportPageResult(meta)); + await this.delay(this.apiPageDelay(meta)); } } catch (error) { logger.error(`Failed to fetch category ${categoryId} page ${page}`, { @@ -1326,19 +852,25 @@ export class AudibleService { } } - logger.info(`Category ${categoryId}: collected ${audiobooks.length} books across ${page - 1} pages`); + logger.info( + `Category ${categoryId}: collected ${audiobooks.length} books across ${page - 1} pages`, + ); return audiobooks; } - /** - * Add delay between requests to respect rate limits - */ + private apiPageDelay(meta: FetchResultMeta): number { + if (meta.retriesUsed > 0) { + return this.pacer.reportPageResult(meta); + } + this.pacer.reportPageResult(meta); + return randomDelay(500, 1500); + } + private async delay(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); + return new Promise((resolve) => setTimeout(resolve, ms)); } } -// Singleton instance let audibleService: AudibleService | null = null; export function getAudibleService(): AudibleService { diff --git a/src/lib/types/audible.ts b/src/lib/types/audible.ts index 2160d92..fa594f0 100644 --- a/src/lib/types/audible.ts +++ b/src/lib/types/audible.ts @@ -11,6 +11,7 @@ export interface AudibleRegionConfig { code: AudibleRegion; name: string; baseUrl: string; + apiBaseUrl: string; audnexusParam: string; language: SupportedLanguage; } @@ -20,6 +21,7 @@ export const AUDIBLE_REGIONS: Record = { code: 'us', name: 'United States', baseUrl: 'https://www.audible.com', + apiBaseUrl: 'https://api.audible.com', audnexusParam: 'us', language: 'en', }, @@ -27,6 +29,7 @@ export const AUDIBLE_REGIONS: Record = { code: 'ca', name: 'Canada', baseUrl: 'https://www.audible.ca', + apiBaseUrl: 'https://api.audible.ca', audnexusParam: 'ca', language: 'en', }, @@ -34,6 +37,7 @@ export const AUDIBLE_REGIONS: Record = { code: 'uk', name: 'United Kingdom', baseUrl: 'https://www.audible.co.uk', + apiBaseUrl: 'https://api.audible.co.uk', audnexusParam: 'uk', language: 'en', }, @@ -41,6 +45,7 @@ export const AUDIBLE_REGIONS: Record = { code: 'au', name: 'Australia', baseUrl: 'https://www.audible.com.au', + apiBaseUrl: 'https://api.audible.com.au', audnexusParam: 'au', language: 'en', }, @@ -48,6 +53,7 @@ export const AUDIBLE_REGIONS: Record = { code: 'in', name: 'India', baseUrl: 'https://www.audible.in', + apiBaseUrl: 'https://api.audible.in', audnexusParam: 'in', language: 'en', }, @@ -55,6 +61,7 @@ export const AUDIBLE_REGIONS: Record = { code: 'de', name: 'Germany', baseUrl: 'https://www.audible.de', + apiBaseUrl: 'https://api.audible.de', audnexusParam: 'de', language: 'de', }, @@ -62,6 +69,7 @@ export const AUDIBLE_REGIONS: Record = { code: 'es', name: 'Spain', baseUrl: 'https://www.audible.es', + apiBaseUrl: 'https://api.audible.es', audnexusParam: 'es', language: 'es', }, @@ -69,9 +77,10 @@ export const AUDIBLE_REGIONS: Record = { code: 'fr', name: 'France', baseUrl: 'https://www.audible.fr', + apiBaseUrl: 'https://api.audible.fr', audnexusParam: 'fr', language: 'fr', - } + }, }; export const DEFAULT_AUDIBLE_REGION: AudibleRegion = 'us'; From 44524667a2fca498e98d703d67a096510a82b939 Mon Sep 17 00:00:00 2001 From: kikootwo Date: Tue, 21 Apr 2026 03:08:33 -0400 Subject: [PATCH 05/12] Bump package version to 1.1.8 Update package.json version from 1.1.7 to 1.1.8 to prepare a new patch release. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 91b4444..eca9c4f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "readmeabook", - "version": "1.1.7", + "version": "1.1.8", "private": true, "scripts": { "dev": "next dev", From 5f0855b2f8d929846263524f36675a53b6a0402f Mon Sep 17 00:00:00 2001 From: kikootwo Date: Tue, 21 Apr 2026 03:21:25 -0400 Subject: [PATCH 06/12] Refactor AudibleService tests and mocks Restructure and expand tests for AudibleService: replace a single hoisted axios client mock with separate htmlClientMock and apiClientMock, update axios.create to return clients in initialization order, and remove the fs mock. Add reusable fixture helpers (makeProduct, makeProductsResponse, apiResponse) and many new/spec-complete test cases organized into describe blocks (initialization, search, mapping, series rules, author search, popular/new releases, categories, and audiobook details). Improve assertions for pagination, deduplication, field mapping, error handling, and region/config behavior; reset and clear mocks in beforeEach to ensure isolation. --- tests/integrations/audible.service.test.ts | 1446 ++++++++++++++------ 1 file changed, 1027 insertions(+), 419 deletions(-) diff --git a/tests/integrations/audible.service.test.ts b/tests/integrations/audible.service.test.ts index 59e8792..9186a47 100644 --- a/tests/integrations/audible.service.test.ts +++ b/tests/integrations/audible.service.test.ts @@ -7,12 +7,17 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { AudibleService } from '@/lib/integrations/audible.service'; import { AUDIBLE_REGIONS, DEFAULT_AUDIBLE_REGION } from '@/lib/types/audible'; -const clientMock = vi.hoisted(() => ({ - get: vi.fn(), -})); +// --------------------------------------------------------------------------- +// Hoisted mocks +// --------------------------------------------------------------------------- + +// Two separate client mocks so we can distinguish htmlClient vs apiClient calls. +const htmlClientMock = vi.hoisted(() => ({ get: vi.fn() })); +const apiClientMock = vi.hoisted(() => ({ get: vi.fn() })); const axiosMock = vi.hoisted(() => ({ - create: vi.fn(() => clientMock), + // First call → htmlClient, second call → apiClient (matches initialize() order). + create: vi.fn(), get: vi.fn(), })); @@ -20,10 +25,6 @@ const configServiceMock = vi.hoisted(() => ({ getAudibleRegion: vi.fn(), })); -const fsCoreMock = vi.hoisted(() => ({ - writeFileSync: vi.fn(), -})); - vi.mock('axios', () => ({ default: axiosMock, ...axiosMock, @@ -33,462 +34,1069 @@ vi.mock('@/lib/services/config.service', () => ({ getConfigService: () => configServiceMock, })); -vi.mock('fs', () => fsCoreMock); +// --------------------------------------------------------------------------- +// Fixture helpers +// --------------------------------------------------------------------------- + +interface ProductOverrides { + asin?: string; + title?: string; + authors?: Array<{ asin?: string; name: string }>; + narrators?: Array<{ name: string }>; + publisher_summary?: string; + merchandising_summary?: string; + product_images?: Record; + runtime_length_min?: number; + release_date?: string; + language?: string; + rating?: { overall_distribution?: { display_stars?: number } }; + category_ladders?: Array<{ ladder: Array<{ name: string }> }>; + series?: Array<{ asin?: string; title?: string; sequence?: string }>; +} + +function makeProduct(overrides: ProductOverrides = {}): ProductOverrides { + return { + asin: 'B000000001', + title: 'Test Book', + authors: [{ asin: 'A000000001', name: 'Test Author' }], + narrators: [{ name: 'Test Narrator' }], + publisher_summary: 'A plain description.', + product_images: { '500': 'https://images.example.com/cover.jpg' }, + runtime_length_min: 300, + release_date: '2024-01-01', + language: 'english', + rating: { overall_distribution: { display_stars: 4.5 } }, + ...overrides, + }; +} + +function makeProductsResponse(products: ProductOverrides[], totalResults = products.length) { + return { products, total_results: totalResults }; +} + +// Produces the value that client.get() should resolve to (the axios response object). +// fetchWithRetry captures this as `response`, then callers do `response.data` to +// unwrap the API envelope. So the mock must be shaped as: { data: }. +function apiResponse(envelope: object) { + return { data: envelope }; +} + +// --------------------------------------------------------------------------- +// Test setup +// --------------------------------------------------------------------------- describe('AudibleService', () => { beforeEach(() => { vi.clearAllMocks(); - clientMock.get.mockReset(); + htmlClientMock.get.mockReset(); + apiClientMock.get.mockReset(); axiosMock.get.mockReset(); configServiceMock.getAudibleRegion.mockReset(); - }); - const buildListHtml = (count: number, startIndex: number = 0) => - Array.from({ length: count }, (_, i) => { - const asin = `B${String(i + 1 + startIndex).padStart(9, '0')}`; - return ` -
-
  • -

    Title ${i + 1}

    - By: Author ${i + 1} - Narrated by: Narrator ${i + 1} - - 4.${i} out of 5 stars -
    - `; - }).join(''); - - it('parses search results from HTML', async () => { - const html = ` -
    -
  • -

    The Test Book

    - Author Name - Narrated by: Narrator Name - - Length: 5 hrs and 30 mins - 4.5 out of 5 stars -
    -
    1-20 of 55 results
    - `; + // Default: first create() → htmlClient, second → apiClient. + axiosMock.create + .mockReturnValueOnce(htmlClientMock) + .mockReturnValueOnce(apiClientMock); configServiceMock.getAudibleRegion.mockResolvedValue('us'); - clientMock.get.mockResolvedValueOnce({ data: html }); - - const service = new AudibleService(); - const result = await service.search('test', 1); - - expect(result.results).toHaveLength(1); - expect(result.results[0].asin).toBe('B000123456'); - expect(result.results[0].title).toBe('The Test Book'); - expect(result.results[0].author).toBe('Author Name'); - expect(result.results[0].narrator).toBe('Narrator Name'); - expect(result.results[0].durationMinutes).toBe(330); - expect(result.results[0].rating).toBe(4.5); - expect(result.results[0].coverArtUrl).toContain('_SL500_'); - expect(result.totalResults).toBe(55); - expect(result.hasMore).toBe(true); }); - it('reinitializes when the configured region changes', async () => { - const html = `
    0 results
    `; - configServiceMock.getAudibleRegion - .mockResolvedValueOnce('us') - .mockResolvedValueOnce('uk') - .mockResolvedValueOnce('uk'); - clientMock.get.mockResolvedValue({ data: html }); + // ------------------------------------------------------------------------- + // Initialization + // ------------------------------------------------------------------------- - const service = new AudibleService(); - await service.search('test', 1); - await service.search('test', 1); + describe('initialization', () => { + it('calls axios.create twice on first search (htmlClient + apiClient)', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); - expect(axiosMock.create).toHaveBeenCalledTimes(2); - expect(axiosMock.create.mock.calls[1][0].baseURL).toBe(AUDIBLE_REGIONS.uk.baseUrl); - }); + const service = new AudibleService(); + await service.search('test', 1); - it('reinitializes when forced manually', async () => { - const html = `
    0 results
    `; - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - clientMock.get.mockResolvedValue({ data: html }); - - const service = new AudibleService(); - await service.search('test', 1); - service.forceReinitialize(); - await service.search('test', 1); - - expect(axiosMock.create).toHaveBeenCalledTimes(2); - }); - - it('falls back to default region when initialization fails', async () => { - const html = `
    0 results
    `; - configServiceMock.getAudibleRegion.mockRejectedValue(new Error('config fail')); - clientMock.get.mockResolvedValue({ data: html }); - - const service = new AudibleService(); - const result = await service.search('fallback', 1); - - expect(result.totalResults).toBe(0); - expect(axiosMock.create.mock.calls[0][0].baseURL).toBe(AUDIBLE_REGIONS[DEFAULT_AUDIBLE_REGION].baseUrl); - }); - - it('paginates new releases and respects delays between pages', async () => { - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - clientMock.get - .mockResolvedValueOnce({ data: buildListHtml(50, 0) }) - .mockResolvedValueOnce({ data: buildListHtml(25, 50) }); - - const service = new AudibleService(); - const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); - const results = await service.getNewReleases(75); - - expect(results).toHaveLength(75); - expect(delaySpy).toHaveBeenCalledTimes(1); - }); - - it('parses popular audiobooks and stops early when fewer results are found', async () => { - const html = ` -
    -
  • -

    Popular One

    - By: Author One - Narrated by: Narrator One - - 4.2 out of 5 stars -
    - `; - - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - clientMock.get.mockResolvedValueOnce({ data: html }); - - const service = new AudibleService(); - const results = await service.getPopularAudiobooks(1); - - expect(results).toHaveLength(1); - expect(results[0].asin).toBe('B000111111'); - expect(results[0].title).toBe('Popular One'); - }); - - it('skips duplicate ASINs when parsing new releases', async () => { - const html = ` -
    -
  • -

    Title One

    -
    -
    -
  • -

    Title Two

    -
    - `; - - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - clientMock.get.mockResolvedValueOnce({ data: html }); - - const service = new AudibleService(); - const results = await service.getNewReleases(20); - - expect(results).toHaveLength(1); - expect(results[0].title).toBe('Title One'); - }); - - it('returns empty search results on failures', async () => { - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - // Use 404 error which is not retryable - const error: any = new Error('Not Found'); - error.response = { status: 404 }; - clientMock.get.mockRejectedValue(error); - - const service = new AudibleService(); - const result = await service.search('oops', 1); - - expect(result.results).toEqual([]); - expect(result.hasMore).toBe(false); - }); - - it('returns audiobooks from Audnexus when available', async () => { - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - axiosMock.get.mockResolvedValueOnce({ - data: { - title: 'Audnexus Book', - authors: [{ name: 'Author A' }], - narrators: [{ name: 'Narrator A' }], - description: 'Desc', - image: 'https://images.example.com/cover._SL200_.jpg', - runtimeLengthMin: '300', - genres: ['Fiction'], - rating: '4.7', - }, + expect(axiosMock.create).toHaveBeenCalledTimes(2); }); - const service = new AudibleService(); - const details = await service.getAudiobookDetails('B000AAAAAA'); + it('creates htmlClient with the region baseUrl', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); - expect(details?.title).toBe('Audnexus Book'); - expect(details?.author).toBe('Author A'); - expect(details?.durationMinutes).toBe(300); - expect(details?.coverArtUrl).toContain('_SL500_'); - }); + const service = new AudibleService(); + await service.search('test', 1); - it('scrapes details from HTML when Audnexus fails', async () => { - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - axiosMock.get.mockRejectedValueOnce({ response: { status: 500 }, message: 'boom' }); - - const html = ` - -
    -

    HTML Title

    -
  • By: HTML Author
    • -
  • Narrated by: HTML Narrator
    • -
  • Length: 2 hrs and 5 mins
    • -
  • Release date: Jan 2, 2022
    • - 4.8 out of 5 stars - -
    - This is a long description for testing the Audible HTML parsing logic. -
    - Fiction -
    - `; - - clientMock.get.mockResolvedValueOnce({ data: html }); - - const service = new AudibleService(); - const details = await service.getAudiobookDetails('B000CCCCCC'); - - expect(details?.title).toBe('HTML Title'); - expect(details?.author).toBe('HTML Author'); - expect(details?.narrator).toBe('HTML Narrator'); - expect(details?.durationMinutes).toBe(125); - expect(details?.rating).toBe(4.8); - expect(details?.releaseDate).toBe('Jan 2, 2022'); - expect(details?.coverArtUrl).toContain('_SL500_'); - expect(details?.genres).toContain('Fiction'); - }); - - it('falls back to Audible scraping when Audnexus returns 404', async () => { - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - axiosMock.get.mockRejectedValueOnce({ response: { status: 404 }, message: 'Not found' }); - - const html = ` - - `; - - clientMock.get.mockResolvedValueOnce({ data: html }); - - const service = new AudibleService(); - const details = await service.getAudiobookDetails('B000BBBBBB'); - - expect(details?.title).toBe('Fallback Book'); - expect(details?.author).toBe('Fallback Author'); - expect(details?.durationMinutes).toBe(510); - }); - - it('returns runtime from Audnexus data', async () => { - axiosMock.get.mockResolvedValue({ data: { runtimeLengthMin: '480' } }); - - const service = new AudibleService(); - const runtime = await service.getRuntime('B000123456'); - - expect(runtime).toBe(480); - }); - - it('returns null runtime when Audnexus returns 404', async () => { - axiosMock.get.mockRejectedValue({ response: { status: 404 }, message: 'Not found' }); - - const service = new AudibleService(); - const runtime = await service.getRuntime('B000404404'); - - expect(runtime).toBeNull(); - }); - - it('returns null runtime when Audnexus errors unexpectedly', async () => { - axiosMock.get.mockRejectedValue({ response: { status: 500 }, message: 'Boom' }); - - const service = new AudibleService(); - const runtime = await service.getRuntime('B000500500'); - - expect(runtime).toBeNull(); - }); - - it('parses runtime strings into minutes', () => { - const service = new AudibleService(); - const parseRuntime = (service as any).parseRuntime.bind(service); - - expect(parseRuntime('Length: 1 hr and 5 mins')).toBe(65); - expect(parseRuntime('Length: 45 mins')).toBe(45); - expect(parseRuntime('')).toBeUndefined(); - }); - - it('does not reinitialize when the region is unchanged', async () => { - const html = `
    0 results
    `; - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - clientMock.get.mockResolvedValue({ data: html }); - - const service = new AudibleService(); - await service.search('test', 1); - await service.search('test', 1); - - expect(axiosMock.create).toHaveBeenCalledTimes(1); - }); - - it('paginates popular audiobooks across pages', async () => { - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - clientMock.get - .mockResolvedValueOnce({ data: buildListHtml(50, 0) }) - .mockResolvedValueOnce({ data: buildListHtml(25, 50) }); - - const service = new AudibleService(); - const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); - const results = await service.getPopularAudiobooks(75); - - expect(results).toHaveLength(75); - expect(delaySpy).toHaveBeenCalledTimes(1); - }); - - it('returns empty popular audiobooks on errors', async () => { - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - // Use 404 error which is not retryable - const error: any = new Error('Not Found'); - error.response = { status: 404 }; - clientMock.get.mockRejectedValue(error); - - const service = new AudibleService(); - const results = await service.getPopularAudiobooks(5); - - expect(results).toEqual([]); - }); - - it('returns empty new releases on errors', async () => { - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - // Use 404 error which is not retryable - const error: any = new Error('Not Found'); - error.response = { status: 404 }; - clientMock.get.mockRejectedValue(error); - - const service = new AudibleService(); - const results = await service.getNewReleases(5); - - expect(results).toEqual([]); - }); - - it('returns null when getAudiobookDetails throws', async () => { - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - - const service = new AudibleService(); - vi.spyOn(service as any, 'fetchFromAudnexus').mockResolvedValue(null); - vi.spyOn(service as any, 'scrapeAudibleDetails').mockRejectedValue(new Error('boom')); - - const result = await service.getAudiobookDetails('B000TEST'); - - expect(result).toBeNull(); - }); - - it('writes debug HTML in development mode', async () => { - const originalEnv = process.env.NODE_ENV; - process.env.NODE_ENV = 'development'; - - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - axiosMock.get.mockRejectedValueOnce({ response: { status: 404 }, message: 'Not found' }); - clientMock.get.mockResolvedValueOnce({ - data: '

    Dev Book

    ', + expect(axiosMock.create.mock.calls[0][0].baseURL).toBe(AUDIBLE_REGIONS.us.baseUrl); }); - const service = new AudibleService(); - const details = await service.getAudiobookDetails('B000DEV'); + it('creates apiClient with the region apiBaseUrl', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); - expect(details?.title).toBe('Dev Book'); + const service = new AudibleService(); + await service.search('test', 1); - process.env.NODE_ENV = originalEnv; + expect(axiosMock.create.mock.calls[1][0].baseURL).toBe(AUDIBLE_REGIONS.us.apiBaseUrl); + }); + + it('does not reinitialize when the region is unchanged between calls', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + + const service = new AudibleService(); + await service.search('test', 1); + await service.search('test', 1); + + // Still only 2 creates total (not 4). + expect(axiosMock.create).toHaveBeenCalledTimes(2); + }); + + it('reinitializes when the configured region changes between calls', async () => { + configServiceMock.getAudibleRegion + .mockResolvedValueOnce('us') + .mockResolvedValueOnce('uk') + .mockResolvedValueOnce('uk'); + + // Prepare creates for both init cycles. + axiosMock.create.mockReset(); + axiosMock.create + .mockReturnValueOnce(htmlClientMock) // first init: htmlClient + .mockReturnValueOnce(apiClientMock) // first init: apiClient + .mockReturnValueOnce(htmlClientMock) // second init: htmlClient + .mockReturnValueOnce(apiClientMock); // second init: apiClient + + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + + const service = new AudibleService(); + await service.search('test', 1); + await service.search('test', 1); + + expect(axiosMock.create).toHaveBeenCalledTimes(4); + expect(axiosMock.create.mock.calls[2][0].baseURL).toBe(AUDIBLE_REGIONS.uk.baseUrl); + }); + + it('reinitializes after forceReinitialize() is called', async () => { + axiosMock.create.mockReset(); + axiosMock.create + .mockReturnValueOnce(htmlClientMock) + .mockReturnValueOnce(apiClientMock) + .mockReturnValueOnce(htmlClientMock) + .mockReturnValueOnce(apiClientMock); + + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + + const service = new AudibleService(); + await service.search('test', 1); + service.forceReinitialize(); + await service.search('test', 1); + + expect(axiosMock.create).toHaveBeenCalledTimes(4); + }); + + it('falls back to the default US region when config service throws', async () => { + configServiceMock.getAudibleRegion.mockRejectedValue(new Error('config fail')); + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + + const service = new AudibleService(); + await service.search('fallback', 1); + + expect(axiosMock.create.mock.calls[0][0].baseURL).toBe( + AUDIBLE_REGIONS[DEFAULT_AUDIBLE_REGION].baseUrl, + ); + }); + + it('creates both clients even when config service throws', async () => { + configServiceMock.getAudibleRegion.mockRejectedValue(new Error('config fail')); + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + + const service = new AudibleService(); + await service.search('fallback', 1); + + expect(axiosMock.create).toHaveBeenCalledTimes(2); + }); }); - it('parses JSON-LD author and narrator arrays', async () => { - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - axiosMock.get.mockRejectedValueOnce({ response: { status: 404 }, message: 'Not found' }); + // ------------------------------------------------------------------------- + // search() + // ------------------------------------------------------------------------- - const html = ` - - `; + describe('search()', () => { + it('sends correct endpoint, keywords, num_results, and response_groups to apiClient', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); - clientMock.get.mockResolvedValueOnce({ data: html }); + const service = new AudibleService(); + await service.search('fantasy', 1); - const service = new AudibleService(); - const details = await service.getAudiobookDetails('B000ARRAY'); + expect(apiClientMock.get).toHaveBeenCalledWith( + '/1.0/catalog/products', + expect.objectContaining({ + params: expect.objectContaining({ + keywords: 'fantasy', + num_results: 50, + response_groups: expect.stringContaining('contributors'), + }), + }), + ); + }); - expect(details?.author).toBe('Author One, Author Two'); - expect(details?.narrator).toBe('Narrator One, Narrator Two'); + it('subtracts 1 from public page=1 before calling the API (page offset regression)', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + const service = new AudibleService(); + + await service.search('test', 1); + expect(apiClientMock.get.mock.calls[0][1].params.page).toBe(0); + }); + + it('subtracts 1 from public page=2 before calling the API', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + const service = new AudibleService(); + + await service.search('test', 2); + expect(apiClientMock.get.mock.calls[0][1].params.page).toBe(1); + }); + + it('subtracts 1 from public page=3 before calling the API', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + const service = new AudibleService(); + + await service.search('test', 3); + expect(apiClientMock.get.mock.calls[0][1].params.page).toBe(2); + }); + + it('returns query, results, totalResults, page, and hasMore fields', async () => { + const products = [makeProduct()]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products, 1))); + + const service = new AudibleService(); + const result = await service.search('test', 1); + + expect(result).toMatchObject({ + query: 'test', + page: 1, + totalResults: 1, + hasMore: false, + }); + expect(result.results).toHaveLength(1); + }); + + it('sets hasMore=true when totalResults exceeds page * pageSize', async () => { + const products = Array.from({ length: 50 }, (_, i) => + makeProduct({ asin: `B${String(i).padStart(9, '0')}`, title: `Book ${i}` }), + ); + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products, 150))); + + const service = new AudibleService(); + const result = await service.search('test', 1); + + expect(result.hasMore).toBe(true); + }); + + it('sets hasMore=false when all results fit on the current page', async () => { + const products = [makeProduct()]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products, 1))); + + const service = new AudibleService(); + const result = await service.search('test', 1); + + expect(result.hasMore).toBe(false); + }); + + it('returns empty results on error without throwing', async () => { + const error: Error & { response?: { status: number } } = new Error('Not Found'); + error.response = { status: 404 }; + apiClientMock.get.mockRejectedValue(error); + + const service = new AudibleService(); + const result = await service.search('oops', 1); + + expect(result.results).toEqual([]); + expect(result.hasMore).toBe(false); + expect(result.totalResults).toBe(0); + }); + + it('uses apiClient (not htmlClient) for catalog requests', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + + const service = new AudibleService(); + await service.search('test', 1); + + expect(apiClientMock.get).toHaveBeenCalled(); + expect(htmlClientMock.get).not.toHaveBeenCalled(); + }); }); - it('falls back to author and narrator links when labels are missing', async () => { - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - axiosMock.get.mockRejectedValueOnce({ response: { status: 404 }, message: 'Not found' }); + // ------------------------------------------------------------------------- + // mapCatalogProduct correctness (tested via search()) + // ------------------------------------------------------------------------- - const html = ` -
    - Author One - See all - Narrator One -
    - `; + describe('mapCatalogProduct field mapping', () => { + it('maps asin and title from catalog product', async () => { + const products = [makeProduct({ asin: 'B000AAABBB', title: 'My Great Book' })]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); - clientMock.get.mockResolvedValueOnce({ data: html }); + const service = new AudibleService(); + const { results } = await service.search('test', 1); - const service = new AudibleService(); - const details = await service.getAudiobookDetails('B000LINKS'); + expect(results[0].asin).toBe('B000AAABBB'); + expect(results[0].title).toBe('My Great Book'); + }); - expect(details?.author).toBe('Author One'); - expect(details?.narrator).toBe('Narrator One'); + it('joins multiple author names with a comma and maps first author asin', async () => { + const products = [ + makeProduct({ + authors: [ + { asin: 'A111', name: 'First Author' }, + { asin: 'A222', name: 'Second Author' }, + ], + }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].author).toBe('First Author, Second Author'); + expect(results[0].authorAsin).toBe('A111'); + }); + + it('joins multiple narrator names with a comma', async () => { + const products = [ + makeProduct({ + narrators: [{ name: 'Narrator A' }, { name: 'Narrator B' }], + }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].narrator).toBe('Narrator A, Narrator B'); + }); + + it('sets narrator to undefined when narrators array is absent', async () => { + const { narrators: _n, ...base } = makeProduct(); + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([base]))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].narrator).toBeUndefined(); + }); + + it('strips HTML tags and entities from publisher_summary to produce plain text description', async () => { + const products = [ + makeProduct({ + // Use a space before
    so whitespace is preserved after tag removal. + publisher_summary: + '

    A & B book with smart text.
    More here.

    ', + }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].description).toBe('A & B book with smart text. More here.'); + }); + + it('falls back to merchandising_summary when publisher_summary is absent', async () => { + const { publisher_summary: _p, ...base } = makeProduct(); + const products = [{ ...base, merchandising_summary: 'Merchandising text.' }]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].description).toBe('Merchandising text.'); + }); + + it('sets description to undefined when both summary fields are absent', async () => { + const { publisher_summary: _p, ...base } = makeProduct(); + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([base]))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].description).toBeUndefined(); + }); + + it('maps coverArtUrl from product_images["500"]', async () => { + const products = [ + makeProduct({ product_images: { '500': 'https://images.example.com/cover500.jpg' } }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].coverArtUrl).toBe('https://images.example.com/cover500.jpg'); + }); + + it('sets coverArtUrl to undefined when product_images is absent', async () => { + const { product_images: _pi, ...base } = makeProduct(); + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([base]))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].coverArtUrl).toBeUndefined(); + }); + + it('maps durationMinutes from runtime_length_min', async () => { + const products = [makeProduct({ runtime_length_min: 480 })]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].durationMinutes).toBe(480); + }); + + it('maps releaseDate from release_date', async () => { + const products = [makeProduct({ release_date: '2023-06-15' })]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].releaseDate).toBe('2023-06-15'); + }); + + it('maps rating from rating.overall_distribution.display_stars', async () => { + const products = [ + makeProduct({ rating: { overall_distribution: { display_stars: 4.7 } } }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].rating).toBe(4.7); + }); + + it('sets rating to undefined when rating field is absent', async () => { + const { rating: _r, ...base } = makeProduct(); + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([base]))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].rating).toBeUndefined(); + }); + + it('flattens, deduplicates, and caps genres at 5 from category_ladders', async () => { + const products = [ + makeProduct({ + category_ladders: [ + { ladder: [{ name: 'Fiction' }, { name: 'Fantasy' }, { name: 'Epic Fantasy' }] }, + { ladder: [{ name: 'Fiction' }, { name: 'Adventure' }] }, // "Fiction" is a duplicate + { ladder: [{ name: 'Young Adult' }, { name: 'Coming of Age' }] }, + ], + }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + // After dedupe: Fiction, Fantasy, Epic Fantasy, Adventure, Young Adult, Coming of Age = 6 → capped at 5 + expect(results[0].genres).toHaveLength(5); + expect(results[0].genres).not.toContain('Coming of Age'); + // Duplicates removed + const genreSet = new Set(results[0].genres); + expect(genreSet.size).toBe(5); + }); }); - it('extracts descriptions from fallback paragraphs', async () => { - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - axiosMock.get.mockRejectedValueOnce({ response: { status: 404 }, message: 'Not found' }); + // ------------------------------------------------------------------------- + // Series selection rules + // ------------------------------------------------------------------------- - const html = ` -

    This description is intentionally long enough to satisfy the minimum length requirement for parsing.

    - `; + describe('series selection', () => { + it('picks the series entry that has a non-empty sequence (even if not first)', async () => { + const products = [ + makeProduct({ + series: [ + { asin: 'S000', title: 'Wrong Series', sequence: '' }, + { asin: 'S001', title: 'Right Series', sequence: '3' }, + ], + }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); - clientMock.get.mockResolvedValueOnce({ data: html }); + const service = new AudibleService(); + const { results } = await service.search('test', 1); - const service = new AudibleService(); - const details = await service.getAudiobookDetails('B000DESC'); + expect(results[0].series).toBe('Right Series'); + expect(results[0].seriesAsin).toBe('S001'); + expect(results[0].seriesPart).toBe('3'); + }); - expect(details?.description).toContain('intentionally long enough'); + it('falls back to series[0] when all sequence values are empty', async () => { + const products = [ + makeProduct({ + series: [ + { asin: 'S010', title: 'Fallback Series', sequence: '' }, + { asin: 'S011', title: 'Other Series', sequence: '' }, + ], + }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].series).toBe('Fallback Series'); + expect(results[0].seriesPart).toBeUndefined(); + }); + + it('leaves all series fields undefined when series array is absent', async () => { + const { series: _s, ...base } = makeProduct(); + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([base]))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].series).toBeUndefined(); + expect(results[0].seriesPart).toBeUndefined(); + expect(results[0].seriesAsin).toBeUndefined(); + }); + + it('extracts leading numeric part from a compound sequence string like "2, Dramatized Adaptation"', async () => { + const products = [ + makeProduct({ + series: [{ asin: 'S020', title: 'Drama Series', sequence: '2, Dramatized Adaptation' }], + }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].seriesPart).toBe('2'); + }); + + it('preserves decimal sequence values like "1.5"', async () => { + const products = [ + makeProduct({ + series: [{ asin: 'S021', title: 'Decimal Series', sequence: '1.5' }], + }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].seriesPart).toBe('1.5'); + }); + + it('keeps non-numeric sequence text as-is when there are no digits (e.g. "Prequel")', async () => { + const products = [ + makeProduct({ + series: [{ asin: 'S022', title: 'Prequel Series', sequence: 'Prequel' }], + }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].seriesPart).toBe('Prequel'); + }); }); - it('detects runtime from generic duration text', async () => { - configServiceMock.getAudibleRegion.mockResolvedValue('us'); - axiosMock.get.mockRejectedValueOnce({ response: { status: 404 }, message: 'Not found' }); + // ------------------------------------------------------------------------- + // searchByAuthorAsin() + // ------------------------------------------------------------------------- - const html = ` - 10 hr 2 min - `; + describe('searchByAuthorAsin()', () => { + it('sends author name (not ASIN) as the author param', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); - clientMock.get.mockResolvedValueOnce({ data: html }); + const service = new AudibleService(); + await service.searchByAuthorAsin('Brandon Sanderson', 'A000AUTHOR', 1); - const service = new AudibleService(); - const details = await service.getAudiobookDetails('B000TIME'); + expect(apiClientMock.get.mock.calls[0][1].params.author).toBe('Brandon Sanderson'); + }); - expect(details?.durationMinutes).toBe(602); + it('subtracts 1 from public page=1 before calling the API', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + const service = new AudibleService(); + + await service.searchByAuthorAsin('Test Author', 'AASIN', 1); + expect(apiClientMock.get.mock.calls[0][1].params.page).toBe(0); + }); + + it('subtracts 1 from public page=2 before calling the API', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + const service = new AudibleService(); + + await service.searchByAuthorAsin('Test Author', 'AASIN', 2); + expect(apiClientMock.get.mock.calls[0][1].params.page).toBe(1); + }); + + it('filters out products whose authors array does not contain the target ASIN', async () => { + const matchingAsin = 'A000AUTHOR'; + const products = [ + makeProduct({ asin: 'B001', authors: [{ asin: matchingAsin, name: 'Author' }], language: 'english' }), + makeProduct({ asin: 'B002', authors: [{ asin: 'A999OTHER', name: 'Other' }], language: 'english' }), + makeProduct({ asin: 'B003', authors: [{ asin: matchingAsin, name: 'Author' }], language: 'english' }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products, 3))); + + const service = new AudibleService(); + const result = await service.searchByAuthorAsin('Author', matchingAsin, 1); + + expect(result.books).toHaveLength(2); + expect(result.books.map((b) => b.asin)).toEqual(['B001', 'B003']); + }); + + it('filters out products whose language does not match the region accepted values', async () => { + const matchingAsin = 'A000AUTHOR'; + const products = [ + makeProduct({ asin: 'B004', authors: [{ asin: matchingAsin, name: 'Author' }], language: 'english' }), + makeProduct({ asin: 'B005', authors: [{ asin: matchingAsin, name: 'Author' }], language: 'spanish' }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products, 2))); + + const service = new AudibleService(); + // US region only accepts 'english' + const result = await service.searchByAuthorAsin('Author', matchingAsin, 1); + + expect(result.books).toHaveLength(1); + expect(result.books[0].asin).toBe('B004'); + }); + + it('applies both ASIN and language filters together (AND logic)', async () => { + const matchingAsin = 'A000AUTHOR'; + const products = [ + // passes both + makeProduct({ asin: 'B006', authors: [{ asin: matchingAsin, name: 'Author' }], language: 'english' }), + // wrong ASIN + makeProduct({ asin: 'B007', authors: [{ asin: 'A999OTHER', name: 'Other' }], language: 'english' }), + // wrong language + makeProduct({ asin: 'B008', authors: [{ asin: matchingAsin, name: 'Author' }], language: 'spanish' }), + // wrong ASIN + wrong language + makeProduct({ asin: 'B009', authors: [{ asin: 'A999OTHER', name: 'Other' }], language: 'spanish' }), + ]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products, 4))); + + const service = new AudibleService(); + const result = await service.searchByAuthorAsin('Author', matchingAsin, 1); + + expect(result.books).toHaveLength(1); + expect(result.books[0].asin).toBe('B006'); + }); + }); + + // ------------------------------------------------------------------------- + // getPopularAudiobooks() + // ------------------------------------------------------------------------- + + describe('getPopularAudiobooks()', () => { + it('uses products_sort_by: BestSellers', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + + const service = new AudibleService(); + await service.getPopularAudiobooks(1); + + expect(apiClientMock.get.mock.calls[0][1].params.products_sort_by).toBe('BestSellers'); + }); + + it('subtracts 1 from public page=1 before calling the API', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + const service = new AudibleService(); + const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); + + await service.getPopularAudiobooks(1); + expect(apiClientMock.get.mock.calls[0][1].params.page).toBe(0); + delaySpy.mockRestore(); + }); + + it('makes a second call with page=1 when paginating to page 2', async () => { + const page1Products = Array.from({ length: 50 }, (_, i) => + makeProduct({ asin: `B${String(i).padStart(9, '0')}`, title: `Book ${i}` }), + ); + const page2Products = Array.from({ length: 25 }, (_, i) => + makeProduct({ asin: `B${String(i + 50).padStart(9, '0')}`, title: `Book ${i + 50}` }), + ); + + apiClientMock.get + .mockResolvedValueOnce(apiResponse(makeProductsResponse(page1Products, 75))) + .mockResolvedValueOnce(apiResponse(makeProductsResponse(page2Products, 75))); + + const service = new AudibleService(); + const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); + + await service.getPopularAudiobooks(75); + + expect(apiClientMock.get.mock.calls[1][1].params.page).toBe(1); + delaySpy.mockRestore(); + }); + + it('paginates and returns up to the requested limit', async () => { + const page1Products = Array.from({ length: 50 }, (_, i) => + makeProduct({ asin: `B${String(i).padStart(9, '0')}`, title: `Book ${i}` }), + ); + const page2Products = Array.from({ length: 25 }, (_, i) => + makeProduct({ asin: `B${String(i + 50).padStart(9, '0')}`, title: `Book ${i + 50}` }), + ); + + apiClientMock.get + .mockResolvedValueOnce(apiResponse(makeProductsResponse(page1Products, 75))) + .mockResolvedValueOnce(apiResponse(makeProductsResponse(page2Products, 75))); + + const service = new AudibleService(); + const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); + const results = await service.getPopularAudiobooks(75); + + expect(results).toHaveLength(75); + delaySpy.mockRestore(); + }); + + it('stops early when a page returns fewer than the page size', async () => { + const products = [makeProduct()]; + apiClientMock.get.mockResolvedValueOnce(apiResponse(makeProductsResponse(products, 1))); + + const service = new AudibleService(); + const results = await service.getPopularAudiobooks(50); + + expect(results).toHaveLength(1); + expect(apiClientMock.get).toHaveBeenCalledTimes(1); + }); + + it('deduplicates by ASIN across pages', async () => { + const sharedProduct = makeProduct({ asin: 'BDUP000001', title: 'Duplicated Book' }); + const uniqueProduct = makeProduct({ asin: 'BUNIQ000001', title: 'Unique Book' }); + + apiClientMock.get + .mockResolvedValueOnce( + apiResponse(makeProductsResponse([sharedProduct], 51)), + ) + .mockResolvedValueOnce( + // page 2 returns the same ASIN plus a new one + apiResponse(makeProductsResponse([sharedProduct, uniqueProduct], 51)), + ); + + const service = new AudibleService(); + const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); + const results = await service.getPopularAudiobooks(100); + + const asins = results.map((r) => r.asin); + expect(asins.filter((a) => a === 'BDUP000001')).toHaveLength(1); + delaySpy.mockRestore(); + }); + + it('returns empty array on error without throwing', async () => { + const error: Error & { response?: { status: number } } = new Error('Not Found'); + error.response = { status: 404 }; + apiClientMock.get.mockRejectedValue(error); + + const service = new AudibleService(); + const results = await service.getPopularAudiobooks(5); + + expect(results).toEqual([]); + }); + }); + + // ------------------------------------------------------------------------- + // getNewReleases() + // ------------------------------------------------------------------------- + + describe('getNewReleases()', () => { + it('uses products_sort_by: -ReleaseDate', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + + const service = new AudibleService(); + await service.getNewReleases(1); + + expect(apiClientMock.get.mock.calls[0][1].params.products_sort_by).toBe('-ReleaseDate'); + }); + + it('subtracts 1 from public page=1 before calling the API', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + const service = new AudibleService(); + const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); + + await service.getNewReleases(1); + expect(apiClientMock.get.mock.calls[0][1].params.page).toBe(0); + delaySpy.mockRestore(); + }); + + it('subtracts 1 from public page=2 when paginating to the second page', async () => { + const page1Products = Array.from({ length: 50 }, (_, i) => + makeProduct({ asin: `B${String(i).padStart(9, '0')}` }), + ); + const page2Products = [makeProduct({ asin: 'BNEW000099' })]; + + apiClientMock.get + .mockResolvedValueOnce(apiResponse(makeProductsResponse(page1Products, 51))) + .mockResolvedValueOnce(apiResponse(makeProductsResponse(page2Products, 51))); + + const service = new AudibleService(); + const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); + + await service.getNewReleases(51); + expect(apiClientMock.get.mock.calls[1][1].params.page).toBe(1); + delaySpy.mockRestore(); + }); + + it('deduplicates by ASIN across pages', async () => { + const sharedProduct = makeProduct({ asin: 'BDUP000002' }); + apiClientMock.get + .mockResolvedValueOnce(apiResponse(makeProductsResponse([sharedProduct], 51))) + .mockResolvedValueOnce(apiResponse(makeProductsResponse([sharedProduct], 51))); + + const service = new AudibleService(); + const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); + const results = await service.getNewReleases(100); + + expect(results.filter((r) => r.asin === 'BDUP000002')).toHaveLength(1); + delaySpy.mockRestore(); + }); + + it('returns empty array on error without throwing', async () => { + const error: Error & { response?: { status: number } } = new Error('Not Found'); + error.response = { status: 404 }; + apiClientMock.get.mockRejectedValue(error); + + const service = new AudibleService(); + const results = await service.getNewReleases(5); + + expect(results).toEqual([]); + }); + }); + + // ------------------------------------------------------------------------- + // getCategoryBooks() + // ------------------------------------------------------------------------- + + describe('getCategoryBooks()', () => { + it('sends category_id and BestSellers sort param', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + + const service = new AudibleService(); + await service.getCategoryBooks('18685580011', 1); + + const params = apiClientMock.get.mock.calls[0][1].params; + expect(params.category_id).toBe('18685580011'); + expect(params.products_sort_by).toBe('BestSellers'); + }); + + it('subtracts 1 from public page=1 before calling the API', async () => { + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + const service = new AudibleService(); + const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); + + await service.getCategoryBooks('CAT001', 1); + expect(apiClientMock.get.mock.calls[0][1].params.page).toBe(0); + delaySpy.mockRestore(); + }); + + it('subtracts 1 from public page=2 when paginating to the second page', async () => { + const page1Products = Array.from({ length: 50 }, (_, i) => + makeProduct({ asin: `B${String(i).padStart(9, '0')}` }), + ); + const page2Products = [makeProduct({ asin: 'BCAT000099' })]; + + apiClientMock.get + .mockResolvedValueOnce(apiResponse(makeProductsResponse(page1Products, 51))) + .mockResolvedValueOnce(apiResponse(makeProductsResponse(page2Products, 51))); + + const service = new AudibleService(); + const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); + + await service.getCategoryBooks('CAT001', 51); + expect(apiClientMock.get.mock.calls[1][1].params.page).toBe(1); + delaySpy.mockRestore(); + }); + + it('deduplicates by ASIN across pages', async () => { + const sharedProduct = makeProduct({ asin: 'BDUP000003' }); + apiClientMock.get + .mockResolvedValueOnce(apiResponse(makeProductsResponse([sharedProduct], 51))) + .mockResolvedValueOnce(apiResponse(makeProductsResponse([sharedProduct], 51))); + + const service = new AudibleService(); + const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); + const results = await service.getCategoryBooks('CAT001', 100); + + expect(results.filter((r) => r.asin === 'BDUP000003')).toHaveLength(1); + delaySpy.mockRestore(); + }); + }); + + // ------------------------------------------------------------------------- + // getCategories() + // ------------------------------------------------------------------------- + + describe('getCategories()', () => { + it('hits /1.0/catalog/categories and maps top-level categories to id+name', async () => { + apiClientMock.get.mockResolvedValue( + apiResponse({ + categories: [ + { id: '18685580011', name: 'Science Fiction & Fantasy' }, + { id: '18685812011', name: 'Mystery, Thriller & Suspense' }, + ], + }), + ); + + const service = new AudibleService(); + const categories = await service.getCategories(); + + expect(apiClientMock.get).toHaveBeenCalledWith('/1.0/catalog/categories', expect.anything()); + expect(categories).toHaveLength(2); + expect(categories[0]).toEqual({ id: '18685580011', name: 'Science Fiction & Fantasy' }); + }); + + it('returns empty array when categories field is missing', async () => { + apiClientMock.get.mockResolvedValue(apiResponse({})); + + const service = new AudibleService(); + const categories = await service.getCategories(); + + expect(categories).toEqual([]); + }); + + it('returns empty array on error without throwing', async () => { + const error: Error & { response?: { status: number } } = new Error('Not Found'); + error.response = { status: 404 }; + apiClientMock.get.mockRejectedValue(error); + + const service = new AudibleService(); + const categories = await service.getCategories(); + + expect(categories).toEqual([]); + }); + }); + + // ------------------------------------------------------------------------- + // getAudiobookDetails() — Audnexus primary + catalog fallback + // ------------------------------------------------------------------------- + + describe('getAudiobookDetails()', () => { + it('returns Audnexus data directly when Audnexus succeeds', async () => { + axiosMock.get.mockResolvedValueOnce({ + data: { + title: 'Audnexus Book', + authors: [{ name: 'Author A', asin: 'A111' }], + narrators: [{ name: 'Narrator A' }], + description: 'A fine description.', + image: 'https://images.example.com/cover._SL500_.jpg', + runtimeLengthMin: '300', + genres: ['Fiction'], + rating: '4.7', + }, + }); + + const service = new AudibleService(); + const details = await service.getAudiobookDetails('B000AAAAAA'); + + expect(details?.title).toBe('Audnexus Book'); + expect(details?.author).toBe('Author A'); + expect(details?.durationMinutes).toBe(300); + // Catalog API should NOT be called when Audnexus succeeds. + expect(apiClientMock.get).not.toHaveBeenCalled(); + }); + + it('falls back to the catalog API when Audnexus returns 404', async () => { + axiosMock.get.mockRejectedValueOnce({ response: { status: 404 }, message: 'Not found' }); + + const product = makeProduct({ asin: 'B000BBBBBB', title: 'Catalog Book' }); + apiClientMock.get.mockResolvedValue( + apiResponse({ product }), + ); + + const service = new AudibleService(); + const details = await service.getAudiobookDetails('B000BBBBBB'); + + expect(details?.title).toBe('Catalog Book'); + expect(apiClientMock.get).toHaveBeenCalled(); + }); + + it('returns null when the catalog API returns a stub body (product.title missing)', async () => { + axiosMock.get.mockRejectedValueOnce({ response: { status: 404 }, message: 'Not found' }); + + // Stub body: asin present but no title + apiClientMock.get.mockResolvedValue( + apiResponse({ product: { asin: 'B000STUB01' } }), + ); + + const service = new AudibleService(); + const details = await service.getAudiobookDetails('B000STUB01'); + + expect(details).toBeNull(); + }); + + it('returns null when both Audnexus and the catalog API fail', async () => { + axiosMock.get.mockRejectedValueOnce({ response: { status: 404 }, message: 'Not found' }); + // Use a non-retryable 404 so the test does not incur retry delays. + const error: Error & { response?: { status: number } } = new Error('Not Found'); + error.response = { status: 404 }; + apiClientMock.get.mockRejectedValue(error); + + const service = new AudibleService(); + const details = await service.getAudiobookDetails('B000FAIL01'); + + expect(details).toBeNull(); + }); + + it('returns null when fetchAudibleDetailsFromApi throws unexpectedly', async () => { + axiosMock.get.mockRejectedValueOnce({ response: { status: 404 }, message: 'Not found' }); + + const service = new AudibleService(); + vi.spyOn(service as any, 'fetchAudibleDetailsFromApi').mockRejectedValue( + new Error('unexpected boom'), + ); + + const result = await service.getAudiobookDetails('B000TEST'); + + expect(result).toBeNull(); + }); + }); + + // ------------------------------------------------------------------------- + // getRuntime() + // ------------------------------------------------------------------------- + + describe('getRuntime()', () => { + it('returns runtime in minutes from Audnexus runtimeLengthMin', async () => { + axiosMock.get.mockResolvedValue({ data: { runtimeLengthMin: '480' } }); + + const service = new AudibleService(); + const runtime = await service.getRuntime('B000123456'); + + expect(runtime).toBe(480); + }); + + it('returns null when Audnexus returns 404', async () => { + axiosMock.get.mockRejectedValue({ response: { status: 404 }, message: 'Not found' }); + + const service = new AudibleService(); + const runtime = await service.getRuntime('B000404404'); + + expect(runtime).toBeNull(); + }); + + it('returns null when Audnexus errors unexpectedly', async () => { + axiosMock.get.mockRejectedValue({ response: { status: 500 }, message: 'Boom' }); + + const service = new AudibleService(); + // Suppress retry delays so the test runs instantly. + const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); + const runtime = await service.getRuntime('B000500500'); + + expect(runtime).toBeNull(); + delaySpy.mockRestore(); + }); + }); + + // ------------------------------------------------------------------------- + // fetch() public wrapper — must use htmlClient + // ------------------------------------------------------------------------- + + describe('fetch() public wrapper', () => { + it('routes through htmlClient so audible-series.ts callers continue to work', async () => { + htmlClientMock.get.mockResolvedValue({ data: 'test' }); + + const service = new AudibleService(); + await service.fetch('/some-path'); + + expect(htmlClientMock.get).toHaveBeenCalledWith('/some-path', expect.anything()); + expect(apiClientMock.get).not.toHaveBeenCalled(); + }); }); }); From ba1efa88f5967f67b6ce2e0f4ba28d595d095bde Mon Sep 17 00:00:00 2001 From: xFlawless11x Date: Tue, 24 Mar 2026 22:18:31 -0400 Subject: [PATCH 07/12] feat: add On Grab notification event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds request_grabbed event that fires when a torrent/NZB is successfully handed off to the configured download client, filling the gap between request_approved (pre-search) and request_available (fully imported). - Add request_grabbed to NOTIFICATION_EVENTS with titleByRequestType (Audiobook Grabbed / Ebook Grabbed), info severity, Details messageLabel - Add NotificationEventConfig interface and update getEventMeta() return type to expose messageLabel to all providers without TypeScript errors - Add messageLabel: 'Reason' to issue_reported event - Fix all 4 providers (Discord, ntfy, Pushover, Apprise) to derive message field label from meta.messageLabel ?? 'Error' instead of hardcoded isIssue ternary — prevents grab details showing as Error - Trigger request_grabbed in download-torrent.processor.ts after client.addDownload() succeeds; message carries torrent title, indexer, and download client name; requestType sourced from request.type - Update notifications.md documentation Co-Authored-By: Claude Sonnet 4.6 --- .../backend/services/notifications.md | 10 +++++- src/lib/constants/notification-events.ts | 33 +++++++++++++++++-- .../processors/download-torrent.processor.ts | 26 ++++++++++++++- .../providers/apprise.provider.ts | 5 ++- .../providers/discord.provider.ts | 2 +- .../notification/providers/ntfy.provider.ts | 5 ++- .../providers/pushover.provider.ts | 4 ++- 7 files changed, 77 insertions(+), 8 deletions(-) diff --git a/documentation/backend/services/notifications.md b/documentation/backend/services/notifications.md index aba1a9f..9ad9be9 100644 --- a/documentation/backend/services/notifications.md +++ b/documentation/backend/services/notifications.md @@ -7,7 +7,7 @@ Sends notifications for audiobook request events (pending approval, approved, av ## Key Details - **Backends:** Apprise (API), Discord (webhooks), ntfy (API), Pushover (API) -- **Events:** request_pending_approval, request_approved, request_available, request_error, issue_reported +- **Events:** request_pending_approval, request_approved, request_grabbed, request_available, request_error, issue_reported - **Encryption:** AES-256-GCM for sensitive config (webhook URLs, API keys, notification URLs) - **Delivery:** Async via Bull job queue (priority 5) - **Failure Handling:** Non-blocking, Promise.allSettled (one backend fails, others succeed) @@ -33,11 +33,14 @@ model NotificationBackend { |-------|---------|------------------------| | request_pending_approval | User creates request | Request needs admin approval | | request_approved | Admin approves OR auto-approval | Request approved (manual or auto) | +| request_grabbed | Torrent/NZB added to download client | Download handed off to configured download client (title resolves by type) | | request_available | Plex/ABS scan or ebook download completes | Request available (title resolves by type) | | request_error | Download/import fails | Request failed at any stage | | issue_reported | User reports issue | User reports problem with available audiobook | **Dynamic Titles:** Events can define `titleByRequestType` in `notification-events.ts` for type-specific titles. +- `request_grabbed` + `requestType: 'audiobook'` → "Audiobook Grabbed" +- `request_grabbed` + `requestType: 'ebook'` → "Ebook Grabbed" - `request_available` + `requestType: 'audiobook'` → "Audiobook Available" - `request_available` + `requestType: 'ebook'` → "Ebook Available" - `request_available` + no requestType → "Request Available" (fallback) @@ -66,6 +69,11 @@ model NotificationBackend { - Approve (with or without pre-selected torrent): After job triggered → request_approved - Deny: No notification +**Download Grabbed (processor: download-torrent)** +- After `client.addDownload()` succeeds and `DownloadHistory` record created → request_grabbed +- `message` field: `"${torrent.title} via ${indexer} (${clientType})"` +- `requestType`: from `request.type` (audiobook/ebook) + **Audiobook Available (processors: scan-plex, plex-recently-added)** - After `status: 'available'` update → request_available (requestType: 'audiobook') - Includes user info in query (plexUsername) diff --git a/src/lib/constants/notification-events.ts b/src/lib/constants/notification-events.ts index 51eba7c..37bfba4 100644 --- a/src/lib/constants/notification-events.ts +++ b/src/lib/constants/notification-events.ts @@ -19,6 +19,7 @@ export type NotificationPriority = 'normal' | 'high'; * - `emoji`: Emoji prefix for notification titles * - `severity`: Drives provider formatting (colors, Apprise types, ntfy tags) * - `priority`: Drives notification urgency (Pushover/ntfy priority levels) + * - `messageLabel`: Optional label for the `message` payload field (defaults to "Error" if omitted) */ export const NOTIFICATION_EVENTS = { request_pending_approval: { @@ -35,6 +36,18 @@ export const NOTIFICATION_EVENTS = { severity: 'success' as const, priority: 'normal' as const, }, + request_grabbed: { + label: 'Request Grabbed', + title: 'Download Grabbed', + titleByRequestType: { + audiobook: 'Audiobook Grabbed', + ebook: 'Ebook Grabbed', + } as Record, + emoji: '\u{1F4E5}', + severity: 'info' as const, + priority: 'normal' as const, + messageLabel: 'Details', + }, request_available: { label: 'Request Available', title: 'Request Available', @@ -59,6 +72,7 @@ export const NOTIFICATION_EVENTS = { emoji: '\u{1F6A9}', severity: 'warning' as const, priority: 'high' as const, + messageLabel: 'Reason', }, } as const; @@ -71,9 +85,24 @@ export const NOTIFICATION_EVENT_KEYS = Object.keys(NOTIFICATION_EVENTS) as [Noti /** Metadata shape for a single notification event */ export type NotificationEventMeta = (typeof NOTIFICATION_EVENTS)[NotificationEvent]; +/** + * Normalized interface for event metadata consumed by providers. + * Broadens the `as const` literal union to make optional fields accessible. + */ +export interface NotificationEventConfig { + label: string; + title: string; + titleByRequestType?: Record; + emoji: string; + severity: NotificationSeverity; + priority: NotificationPriority; + /** Label for the `message` payload field. Defaults to "Error" in providers when absent. */ + messageLabel?: string; +} + /** Helper: get event metadata by key */ -export function getEventMeta(event: NotificationEvent) { - return NOTIFICATION_EVENTS[event]; +export function getEventMeta(event: NotificationEvent): NotificationEventConfig { + return NOTIFICATION_EVENTS[event] as NotificationEventConfig; } /** diff --git a/src/lib/processors/download-torrent.processor.ts b/src/lib/processors/download-torrent.processor.ts index 83e9b09..eabadfc 100644 --- a/src/lib/processors/download-torrent.processor.ts +++ b/src/lib/processors/download-torrent.processor.ts @@ -103,8 +103,32 @@ export async function processDownloadTorrent(payload: DownloadTorrentPayload): P logger.info(`Created download history record: ${downloadHistory.id}`); - // Trigger monitor download job with initial delay + // Send grab notification + const requestWithUser = await prisma.request.findUnique({ + where: { id: requestId }, + include: { + user: { select: { plexUsername: true } }, + }, + }); + const jobQueue = getJobQueueService(); + + if (requestWithUser) { + const grabMessage = `${torrent.title} via ${torrent.indexer} (${client.clientType})`; + await jobQueue.addNotificationJob( + 'request_grabbed', + requestId, + audiobook.title, + audiobook.author, + requestWithUser.user.plexUsername || 'Unknown User', + grabMessage, + requestWithUser.type + ).catch((error) => { + logger.error('Failed to queue grab notification', { error: error instanceof Error ? error.message : String(error) }); + }); + } + + // Trigger monitor download job with initial delay await jobQueue.addMonitorJob( requestId, downloadHistory.id, diff --git a/src/lib/services/notification/providers/apprise.provider.ts b/src/lib/services/notification/providers/apprise.provider.ts index ebafce9..afb7923 100644 --- a/src/lib/services/notification/providers/apprise.provider.ts +++ b/src/lib/services/notification/providers/apprise.provider.ts @@ -127,6 +127,7 @@ export class AppriseProvider implements INotificationProvider { private formatMessage(payload: NotificationPayload): { title: string; body: string } { const { event, title, author, userName, message, requestType } = payload; + const meta = getEventMeta(event); const isIssue = event === 'issue_reported'; const messageLines = [ @@ -136,7 +137,9 @@ export class AppriseProvider implements INotificationProvider { ]; if (message) { - messageLines.push(isIssue ? `\u{1F4DD} Reason: ${message}` : `\u26A0\uFE0F Error: ${message}`); + const messageLabel = meta.messageLabel ?? 'Error'; + const msgEmoji = meta.severity === 'error' ? '\u26A0\uFE0F' : '\u{1F4DD}'; + messageLines.push(`${msgEmoji} ${messageLabel}: ${message}`); } return { diff --git a/src/lib/services/notification/providers/discord.provider.ts b/src/lib/services/notification/providers/discord.provider.ts index f1dadcc..a52631b 100644 --- a/src/lib/services/notification/providers/discord.provider.ts +++ b/src/lib/services/notification/providers/discord.provider.ts @@ -71,7 +71,7 @@ export class DiscordProvider implements INotificationProvider { ]; if (message) { - fields.push({ name: isIssue ? 'Reason' : 'Error', value: message, inline: false }); + fields.push({ name: meta.messageLabel ?? 'Error', value: message, inline: false }); } return { diff --git a/src/lib/services/notification/providers/ntfy.provider.ts b/src/lib/services/notification/providers/ntfy.provider.ts index e293df5..12648a0 100644 --- a/src/lib/services/notification/providers/ntfy.provider.ts +++ b/src/lib/services/notification/providers/ntfy.provider.ts @@ -84,6 +84,7 @@ export class NtfyProvider implements INotificationProvider { private formatMessage(payload: NotificationPayload): { title: string; message: string } { const { event, title, author, userName, message, requestType } = payload; + const meta = getEventMeta(event); const isIssue = event === 'issue_reported'; const messageLines = [ @@ -93,7 +94,9 @@ export class NtfyProvider implements INotificationProvider { ]; if (message) { - messageLines.push(isIssue ? `\u{1F4DD} Reason: ${message}` : `\u26A0\uFE0F Error: ${message}`); + const messageLabel = meta.messageLabel ?? 'Error'; + const msgEmoji = meta.severity === 'error' ? '\u26A0\uFE0F' : '\u{1F4DD}'; + messageLines.push(`${msgEmoji} ${messageLabel}: ${message}`); } return { diff --git a/src/lib/services/notification/providers/pushover.provider.ts b/src/lib/services/notification/providers/pushover.provider.ts index 19ab355..e4ccf7c 100644 --- a/src/lib/services/notification/providers/pushover.provider.ts +++ b/src/lib/services/notification/providers/pushover.provider.ts @@ -91,7 +91,9 @@ export class PushoverProvider implements INotificationProvider { ]; if (message) { - messageLines.push('', isIssue ? `\u{1F4DD} Reason: ${message}` : `\u26A0\uFE0F Error: ${message}`); + const messageLabel = meta.messageLabel ?? 'Error'; + const msgEmoji = meta.severity === 'error' ? '\u26A0\uFE0F' : '\u{1F4DD}'; + messageLines.push('', `${msgEmoji} ${messageLabel}: ${message}`); } return { From fcae3bcf098f583fcb55fcd3cf83466ed5acf00c Mon Sep 17 00:00:00 2001 From: kikootwo Date: Thu, 14 May 2026 15:23:15 -0400 Subject: [PATCH 08/12] Audible: HTML refresh, multi-narrator & works dedup Switch nightly discovery refresh to scrape Audible's curated HTML storefronts (popular, new releases, category pages) while keeping real-time user paths on the JSON catalog API. Add robust HTML resilience knobs (increased retries, capped jittered backoff, AdaptivePacer changes and per-batch cooldowns) to avoid failing nightly jobs during 503 storms. Implement multi-narrator capture via a new extractAllNarrators helper and update parsers to preserve all narrator anchors. Introduce two-pass dedup: in-memory deduplicateAndCollectGroups + collapseByExistingWorks that consults the works table, export metadataScore for consistent representative selection, and persist dedup groups (fire-and-forget). Wire collapseByExistingWorks into search/author/series routes and make defensive dedup in the refresh processor. Add HTML parsing helpers, runtime/lang-aware parsing, jitteredBackoff cap, and tests for the new behaviors. --- documentation/TABLEOFCONTENTS.md | 2 + documentation/integrations/audible.md | 104 +++- src/app/api/audiobooks/search/route.ts | 11 +- src/app/api/authors/[asin]/books/route.ts | 11 +- src/app/api/series/[asin]/route.ts | 11 +- src/lib/integrations/audible-series.ts | 7 +- src/lib/integrations/audible.service.ts | 307 ++++++++---- .../processors/audible-refresh.processor.ts | 27 +- src/lib/services/works.service.ts | 93 +++- src/lib/utils/deduplicate-audiobooks.ts | 7 +- src/lib/utils/extract-narrator.ts | 37 ++ src/lib/utils/scrape-resilience.ts | 12 +- tests/integrations/audible.service.test.ts | 459 ++++++++++++++---- .../audible-refresh.processor.test.ts | 65 +++ tests/services/works.service.test.ts | 189 ++++++++ tests/utils/extract-narrator.test.ts | 95 ++++ tests/utils/scrape-resilience.test.ts | 18 + 17 files changed, 1241 insertions(+), 214 deletions(-) create mode 100644 src/lib/utils/extract-narrator.ts create mode 100644 tests/utils/extract-narrator.test.ts diff --git a/documentation/TABLEOFCONTENTS.md b/documentation/TABLEOFCONTENTS.md index ab1e724..39cc480 100644 --- a/documentation/TABLEOFCONTENTS.md +++ b/documentation/TABLEOFCONTENTS.md @@ -45,6 +45,8 @@ - **Web scraping (popular, new releases)** → [integrations/audible.md](integrations/audible.md) - **Database caching, real-time matching** → [integrations/audible.md](integrations/audible.md) - **Book covers API for login page** → [frontend/pages/login.md](frontend/pages/login.md) +- **Dedup & works table (cross-ASIN identity)** → [integrations/audible.md](integrations/audible.md#dedup--works-table) +- **Multi-narrator capture in HTML scrapers** → [integrations/audible.md](integrations/audible.md#narrator-capture-in-html-scrapers) ## E-book Support (First-Class) - **First-class ebook requests, separate tracking** → [integrations/ebook-sidecar.md](integrations/ebook-sidecar.md) diff --git a/documentation/integrations/audible.md b/documentation/integrations/audible.md index 3ac4740..b7bac3b 100644 --- a/documentation/integrations/audible.md +++ b/documentation/integrations/audible.md @@ -1,29 +1,40 @@ # Audible Integration -**Status:** Implemented | Unauthenticated Audible JSON catalog API (primary) + Audnexus API (per-ASIN details) +**Status:** Implemented | Hybrid — curated HTML for discovery refresh + Audible JSON catalog API for user-facing real-time + Audnexus for per-ASIN details ## Overview -Audiobook metadata for discovery, search, and detail pages. All catalog operations (search, popular, new releases, categories, category books, author books, single-product details) now call Audible's unauthenticated public JSON catalog API (`api.audible./1.0/catalog/*`). Per-ASIN detail lookups prefer Audnexus; the catalog API is used as fallback. +Audiobook metadata for discovery, search, and detail pages. Split by access pattern: + +- **Nightly discovery refresh** (popular / new releases / category lists) — scraped from Audible's **curated HTML storefronts** (`www.audible./adblbestsellers`, `/newreleases`, `/search?node=`). The HTML pages reflect Audible's own editorial picks. +- **User-facing real-time** (search, author books, categories listing, per-ASIN details) — Audible's unauthenticated public **JSON catalog API** (`api.audible./1.0/catalog/*`). +- **Per-ASIN detail lookups** — Audnexus (`api.audnex.us/books/{asin}`) primary; catalog API used as fallback when Audnexus returns 404. ## Architecture -- **Primary data source:** Audible JSON catalog API, same endpoint used by the official Audible mobile apps. No authentication, no API key, no user credentials, no special headers. -- **Per-ASIN details:** Audnexus (`api.audnex.us/books/{asin}`) remains primary; catalog API (`/1.0/catalog/products/{asin}`) is the fallback when Audnexus returns 404. -- **HTML scraping:** Removed from `audible.service.ts`. The only remaining HTML path is `audible-series.ts` (series-page scraping, out of scope). -- **`www.audible.`:** Still used by `audible-series.ts` and by `getBaseUrl()` for "View on Audible" link generation. Not used for any catalog operation. +- **Curated HTML (refresh job only):** the three methods called solely by `audible-refresh.processor.ts` (`getPopularAudiobooks`, `getNewReleases`, `getCategoryBooks`) scrape Audible's storefront HTML to inherit editorial curation. Beefed-up retry/backoff knobs (12 retries, 3-min jittered cap) handle 503 storms patiently on the nightly job without slowing healthy users. +- **JSON catalog API (real-time):** `search`, `searchByAuthorAsin`, `getCategories` (categories listing), and `fetchAudibleDetailsFromApi` (per-ASIN fallback). Same endpoint used by the official Audible mobile apps. No authentication, no API key, no user credentials, no special headers. +- **Audnexus (per-ASIN):** `getAudiobookDetails` and `getRuntime` prefer Audnexus, with catalog API fallback for `getAudiobookDetails`. +- **`www.audible.`:** Used by HTML refresh scraping, by `audible-series.ts`, and by `getBaseUrl()` for "View on Audible" link generation. ## Data Sources -All catalog operations are HTTP GET against `{apiBaseUrl}` (region-dependent, e.g. `https://api.audible.com`): +### Nightly refresh (HTML — `htmlClient`, baseURL `www.audible.`) + +| Operation | Endpoint | Key params | +|---|---|---| +| Popular | `/adblbestsellers` | `pageSize=50`, `page=` (omitted on first page) | +| New releases | `/newreleases` | `pageSize=50`, `page=` (omitted on first page) | +| Category books | `/search` | `node=&pageSize=50&sort=popularity-rank&page=` | + +Parsed via cheerio. Selectors: `.productListItem` (popular/new releases), `.s-result-item, .productListItem` (categories). + +### Real-time (JSON catalog API — `apiClient`, baseURL `api.audible.`) | Operation | Endpoint | Key params | |---|---|---| | Search | `/1.0/catalog/products` | `keywords=` | | Author books | `/1.0/catalog/products` | `author=` (name, NOT ASIN) | -| Popular | `/1.0/catalog/products` | `products_sort_by=BestSellers` | -| New releases | `/1.0/catalog/products` | `products_sort_by=-ReleaseDate` | -| Category books | `/1.0/catalog/products` | `category_id=&products_sort_by=BestSellers` | | Categories listing | `/1.0/catalog/categories` | (none) | | Single product | `/1.0/catalog/products/{asin}` | — | | Audnexus (per-ASIN) | `https://api.audnex.us/books/{asin}` | `region={audnexusParam}` | @@ -48,20 +59,20 @@ Populates every `AudibleAudiobook` field. Covered: ## Gotchas +- **Catalog API cannot filter preorders or surface curated bestsellers.** The API's `BestSellers` sort is a right-now velocity rank that spikes on launch-day promos and preorder windows; the `-ReleaseDate` sort returns 100% future preorders. There is no server-side `release_time`, `released-only`, `customer_rights`, or alternate sort (`Reviewed`, `MostListened`, etc.) — every plausible variant was tested and silently ignored. This is why the nightly refresh job uses the curated HTML storefront pages instead. - **`author=` takes a name, not an ASIN.** The catalog API has no ASIN-based author param. `searchByAuthorAsin()` queries by name, then filters client-side: keeps only products where `products[].authors[].asin === authorAsin`. Preserves ASIN-authoritative author identity. Also filters by `product.language` via `isAcceptedLanguage()` for the configured region. - **Invalid ASIN returns HTTP 200 with stub body.** `/1.0/catalog/products/{asin}` responds 200 with `{product: {asin: INPUT}}` and no other fields. `fetchAudibleDetailsFromApi()` detects this via missing `product.title` and returns `null`. - **`publisher_summary` is HTML.** Service strips tags via inline `stripHtml()` helper (regex-based, no cheerio) before populating `description`. Falls back to `merchandising_summary` (plain text) if `publisher_summary` missing. - **Series is an array.** `products[].series[]` — a book may belong to multiple series. Service picks the first entry with non-empty `sequence`, else the first entry. `sequence` is cleaned by extracting first `/\d+(?:\.\d+)?/` match for numeric ordering. - **Stub `product_images`:** cover URL reads from `product_images['500']`; missing keys fall back to `undefined`. -- **`page` is 0-indexed.** Despite the default value appearing to be 1, the API returns items `(page * num_results)` through `((page + 1) * num_results - 1)`. So `page=1` fetches items 51–100, not 1–50. All service methods accept a 1-indexed `page` and subtract 1 at the axios call. The symptom of getting this wrong is silent: queries whose `total_results ≤ num_results` return an empty `products` array while `total_results` is populated (e.g. author searches for small catalogues). +- **`page` is 0-indexed (catalog API only).** Despite the default value appearing to be 1, the API returns items `(page * num_results)` through `((page + 1) * num_results - 1)`. So `page=1` fetches items 51–100, not 1–50. All catalog-API service methods accept a 1-indexed `page` and subtract 1 at the axios call. The symptom of getting this wrong is silent: queries whose `total_results ≤ num_results` return an empty `products` array while `total_results` is populated (e.g. author searches for small catalogues). HTML paths use Audible's native 1-indexed `page` query param and omit it on the first page. ## Rate Limiting & Resilience -- 503s still possible but dramatically less frequent than the HTML surface. -- `fetchWithRetry()` — jittered exponential backoff, 5 retries, retries on 503/429/5xx. -- `AdaptivePacer` circuit-breaker preserved. -- Inter-page base delay on API paths: **500–1500ms** (down from 2000–4000ms for HTML). -- API responses include `Cache-Control: private, max-age=1800`. +- **Real-time JSON API paths:** 503s are uncommon. `fetchWithRetry()` uses jittered exponential backoff, 5 retries, retries on 503/429/5xx. API responses include `Cache-Control: private, max-age=1800`. +- **Nightly HTML refresh paths:** 503s are more likely (HTML storefront is more rate-sensitive). Same `fetchWithRetry()`, but with `HTML_MAX_RETRIES=12` and `HTML_MAX_BACKOFF_MS=180_000` (3-minute cap on jittered backoff). Healthy refreshes still complete fast (per-page success on attempt 0); users hit by sustained 503 storms grind through patiently rather than abandoning the refresh. +- **`AdaptivePacer`** — inter-page delay 2–4 s baseline, scales up multiplicatively under retry pressure, with a 45–60 s circuit-breaker cooldown after 3 consecutive retry-pages. +- **Per-batch cooldowns** in `audible-refresh.processor.ts` — 15–30 s between popular/new-releases, 10–20 s between categories. ## Region Configuration @@ -101,8 +112,8 @@ Configurable Audible region for accurate metadata matching across international - Automatic refresh: Region change triggers `audible_refresh` job. **Per-region HTTP clients (on init):** -- `apiClient` — `baseURL=apiBaseUrl`, `Accept: application/json`, `User-Agent: ReadMeABook/1.0`, no language/ipRedirect params. -- `htmlClient` — `baseURL=baseUrl`, browser headers, default params `ipRedirectOverride=true` + `language=`. Used only by `audible-series.ts` and `getBaseUrl()`-based link generation. +- `apiClient` — `baseURL=apiBaseUrl`, `Accept: application/json`, `User-Agent: ReadMeABook/1.0`, no language/ipRedirect params. Used for the real-time JSON catalog operations (search, author books, categories listing, per-ASIN details fallback). +- `htmlClient` — `baseURL=baseUrl`, rotating browser headers (`pickUserAgent` + `getBrowserHeaders`), default params `ipRedirectOverride=true` + `language=`. Used by the nightly discovery refresh (`/adblbestsellers`, `/newreleases`, `/search?node=...`), by `audible-series.ts`, and by `getBaseUrl()`-based link generation. - Audnexus calls include `region=`. **Files:** @@ -130,6 +141,44 @@ Single matching algorithm used everywhere (search, popular, new-releases, jobs). **Note:** Fuzzy matching (70% threshold) is preserved in `ranking-algorithm.ts` for Prowlarr torrent ranking. Library availability checks require exact ASIN matches only. +## Dedup & Works Table + +**Status:** ✅ Implemented | Two-pass dedup on every discovery view + cross-batch identity via works table + +Discovery views (search, author books, series detail) collapse duplicate Audible listings for the same recording (publisher re-listings, regional re-issues, full-cast vs single-narrator productions) into a single card. Two passes run in sequence: + +1. **Local pass — `deduplicateAndCollectGroups()`** (`src/lib/utils/deduplicate-audiobooks.ts`) + - Stateless, in-memory. Keys books by normalized title + sorted narrator set + duration (±max(5%, 10 min) tolerance), with subtitle compatibility to keep distinct series entries separate. + - Picks a canonical representative per group by `metadataScore()` (cover + rating + duration + description + narrator + release date + genres). + - Emits `DedupGroup[]` describing every multi-ASIN collapse → handed to `persistDedupGroups()` for the works table. + +2. **Works pass — `collapseByExistingWorks()`** (`src/lib/services/works.service.ts`) + - Async DB lookup. Reads `work_asins` for every ASIN in the local-passed list and collapses any books sharing a `workId` to one representative (same `metadataScore()` ranking). + - Catches duplicates the local pass misses: source-metadata divergence (e.g. HTML scraper captured different narrators), cross-page splits (paginated series), or non-matching field shapes. + - Degrades gracefully — returns the input unchanged on DB failure (view still renders). + +### Works Table Schema +- `Work { id, title, author }` — one row per logical book +- `WorkAsin { id, workId, asin, narrator?, durationMinutes?, isCanonical, source, createdAt }` — many ASINs per Work + +### Population Layers +- **Layer 1 (auto):** `persistDedupGroups()` writes whenever the local pass finds a duplicate. Merges across pre-existing works when a new group spans them. +- **Layer 2 (seed):** `seedAsin()` writes a single-ASIN work at request creation time, ensuring every requested ASIN has an entry to grow from. + +### Read Paths +- **`collapseByExistingWorks()`** — view-level collapse (this section). +- **`getSiblingAsins()`** — library availability matching (`audiobook-matcher.ts`), request-creation duplicate prevention (`request-creator.service.ts`), ignored-audiobook expansion. Returns sibling ASINs grouped by input ASIN. + +### Narrator Capture in HTML Scrapers +- HTML scrapers (`audible-series.ts`, the two `parse*Items` parsers in `audible.service.ts`) capture **all** narrator anchors via `extractAllNarrators()` (`src/lib/utils/extract-narrator.ts`). Multi-narrator productions render each name as its own `` link; capturing only the first (prior bug) made co-narrated audiobooks fail to dedup. Order is not significant — `normalizeNarrator()` sorts before comparison. + +### Wired Routes +- `src/app/api/audiobooks/search/route.ts` +- `src/app/api/authors/[asin]/books/route.ts` +- `src/app/api/series/[asin]/route.ts` + +Watched-list background jobs (`watched-lists.service.ts`) run the local pass only — they don't render a view, and the downstream `request-creator.service.ts` already does sibling-aware dedup at request creation time. + ## Database-First Approach **Status:** Implemented @@ -137,12 +186,12 @@ Single matching algorithm used everywhere (search, popular, new-releases, jobs). Discovery APIs serve cached data from DB with real-time matching. **Flow:** -1. `audible_refresh` cron runs daily → fetches 200 popular + 200 new releases + user-configured categories via catalog API. +1. `audible_refresh` cron runs daily → fetches 200 popular + 200 new releases + user-configured categories by scraping Audible's curated HTML storefronts (`/adblbestsellers`, `/newreleases`, `/search?node=&sort=popularity-rank`). 2. Downloads and caches cover thumbnails locally. 3. Stores metadata in `audible_cache`, ranked entries in `audible_cache_categories` with reserved IDs (`__popular__`, `__new_releases__`) and user category IDs. 4. Cleans up unused thumbnails after sync. 5. API routes query `AudibleCacheCategory` by categoryId → join with `AudibleCache` metadata → apply real-time matching → return enriched results. -6. Homepage loads instantly (no Audible API hits). +6. Homepage loads instantly (no Audible HTTP hits at request time). ## Thumbnail Caching @@ -228,12 +277,25 @@ interface AuthorBooksResult { ## Tech Stack -- `axios` (HTTP, two clients: `apiClient` for JSON catalog, `htmlClient` for series-page scraping only) +- `axios` (HTTP, two clients: `apiClient` for JSON catalog API, `htmlClient` for HTML refresh + series scraping) +- `cheerio` (HTML parsing for refresh job and `audible-series.ts`) - Audnexus API (per-ASIN details, primary) - PostgreSQL (`audible_cache`, `audible_cache_categories`) ## Fixed Issues +**Series-page duplicates not collapsing across user views (2026-05-14)** +- **Problem:** Two re-listings of the same audiobook (same title, same narrator set, same duration, different ASINs) showed as two cards on series detail pages, even after the works table had already linked them via search-page dedup. +- **Root cause (two-part):** (1) HTML scrapers used `$el.find('a[href*="searchNarrator="]').first()` for multi-narrator productions, capturing only the first co-narrator. So two listings of the same recording landed in `deduplicateAndCollectGroups` with mismatched single-narrator strings and never merged. (2) `deduplicateAndCollectGroups` was stateless — it wrote to the works table but never read it back, so even when one path (e.g. search) successfully merged two ASINs and persisted the Work, every other path (series, author books) re-derived the dedup decision from scratch and split them again. +- **Fix:** (1) New `extractAllNarrators()` helper (`src/lib/utils/extract-narrator.ts`) captures every `searchNarrator=` anchor and joins them; all three HTML scrapers route through it. (2) New `collapseByExistingWorks()` consults the works table after the local pass and collapses any remaining books sharing a `workId`. Wired into the three user-facing discovery routes (search / author books / series detail). Skipped for watched-list background jobs — those feed `request-creator.service.ts` which already does sibling-aware dedup. +- **Location:** `src/lib/utils/extract-narrator.ts` (new); `src/lib/integrations/audible-series.ts` (parseSeriesBooks); `src/lib/integrations/audible.service.ts` (parseProductListItems + parseSearchResultItems); `src/lib/utils/deduplicate-audiobooks.ts` (`metadataScore` exported); `src/lib/services/works.service.ts` (`collapseByExistingWorks` added); three API routes updated. + +**Discovery refresh reverted to curated HTML scraping (2026-05-14)** +- **Problem:** After switching all catalog ops to the JSON catalog API in `f564d0a`, the nightly discovery refresh (Popular / New Releases / user-configured Categories) started serving junk: New Releases became 100% preorders out to 2027, and Popular was dominated by launch-day no-name shovelware. +- **Root cause:** `products_sort_by=BestSellers` is a right-now sales velocity rank that spikes on launch promos and preorder windows; `-ReleaseDate` returns all catalog items in date order with no released-only filter. The catalog API exposes no server-side filter to exclude preorders or sort by established popularity (verified by exhaustively testing `release_time`, `availability_status`, `customer_rights`, `Reviewed`/`MostListened`/`SalesRank` sorts — all silently ignored or rejected). Doing the curation client-side would have made RMAB the editorial curator, which Audible's storefront pages already do well. +- **Fix:** Hybrid architecture — the three refresh-only methods (`getPopularAudiobooks`, `getNewReleases`, `getCategoryBooks`) went back to scraping Audible's curated HTML storefronts (`/adblbestsellers`, `/newreleases`, `/search?node=&sort=popularity-rank`). All user-facing real-time paths (search, author books, categories listing, per-ASIN details) stayed on the JSON catalog API. To keep the higher-503-risk HTML traffic resilient on the unattended nightly job, `fetchWithRetry()` accepts an optional `maxBackoffMs` cap and HTML callers use `HTML_MAX_RETRIES=12` + `HTML_MAX_BACKOFF_MS=180_000` (3-min cap). Healthy users finish quickly; 503-blocked users grind through patiently. +- **Location:** `src/lib/integrations/audible.service.ts` (three methods + two private parsers `parseProductListItems` / `parseSearchResultItems`); `src/lib/utils/scrape-resilience.ts` (`jitteredBackoff` cap parameter). + **Audiobookshelf metadata matching not respecting configured region (2026-01-28)** - **Problem:** `triggerABSItemMatch()` hardcoded `'audible'` provider (audible.com) instead of respecting user's configured Audible region. - **Impact:** Users with non-US regions (CA, UK, AU, IN) had incorrect metadata matching in Audiobookshelf, causing wrong ASINs. diff --git a/src/app/api/audiobooks/search/route.ts b/src/app/api/audiobooks/search/route.ts index 1285e10..7412cb2 100644 --- a/src/app/api/audiobooks/search/route.ts +++ b/src/app/api/audiobooks/search/route.ts @@ -7,7 +7,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { getAudibleService } from '@/lib/integrations/audible.service'; import { enrichAudiobooksWithMatches } from '@/lib/utils/audiobook-matcher'; import { deduplicateAndCollectGroups } from '@/lib/utils/deduplicate-audiobooks'; -import { persistDedupGroups } from '@/lib/services/works.service'; +import { persistDedupGroups, collapseByExistingWorks } from '@/lib/services/works.service'; import { getCurrentUser } from '@/lib/middleware/auth'; import { RMABLogger } from '@/lib/utils/logger'; import { annotateWithIgnoreStatus } from '@/lib/utils/ignored-audiobooks'; @@ -41,16 +41,19 @@ export async function GET(request: NextRequest) { const currentUser = getCurrentUser(request); const userId = currentUser?.sub || undefined; - // Deduplicate before enrichment to avoid wasted DB queries on duplicate entries + // Two-pass dedup: local title/narrator/duration matching first, then collapse + // any remaining duplicates that the works table already knows are the same book + // (handles cases where source metadata diverges across paths or pages). const { books: dedupedResults, groups } = deduplicateAndCollectGroups(results.results); - // Fire-and-forget: persist dedup groups to works table for cross-ASIN matching if (groups.length > 0) { persistDedupGroups(groups).catch(() => {}); } + const collapsedResults = await collapseByExistingWorks(dedupedResults); + // Enrich search results with availability and request status information - const enrichedResults = await enrichAudiobooksWithMatches(dedupedResults, userId); + const enrichedResults = await enrichAudiobooksWithMatches(collapsedResults, userId); // Annotate with per-user ignore status const annotatedResults = await annotateWithIgnoreStatus(enrichedResults, userId); diff --git a/src/app/api/authors/[asin]/books/route.ts b/src/app/api/authors/[asin]/books/route.ts index 3a27bd5..e839547 100644 --- a/src/app/api/authors/[asin]/books/route.ts +++ b/src/app/api/authors/[asin]/books/route.ts @@ -7,7 +7,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { getAudibleService } from '@/lib/integrations/audible.service'; import { enrichAudiobooksWithMatches } from '@/lib/utils/audiobook-matcher'; import { deduplicateAndCollectGroups } from '@/lib/utils/deduplicate-audiobooks'; -import { persistDedupGroups } from '@/lib/services/works.service'; +import { persistDedupGroups, collapseByExistingWorks } from '@/lib/services/works.service'; import { getCurrentUser } from '@/lib/middleware/auth'; import { RMABLogger } from '@/lib/utils/logger'; import { annotateWithIgnoreStatus } from '@/lib/utils/ignored-audiobooks'; @@ -56,17 +56,20 @@ export async function GET( const audibleService = getAudibleService(); const result = await audibleService.searchByAuthorAsin(authorName.trim(), asin, page); - // Deduplicate before enrichment to avoid wasted DB queries on duplicate entries + // Two-pass dedup: local title/narrator/duration matching first, then collapse + // any remaining duplicates that the works table already knows are the same book + // (handles cases where source metadata diverges across paths or pages). const { books: dedupedBooks, groups } = deduplicateAndCollectGroups(result.books); - // Fire-and-forget: persist dedup groups to works table for cross-ASIN matching if (groups.length > 0) { persistDedupGroups(groups).catch(() => {}); } + const collapsedBooks = await collapseByExistingWorks(dedupedBooks); + // Enrich with library availability and request status const userId = currentUser.sub || undefined; - const enrichedBooks = await enrichAudiobooksWithMatches(dedupedBooks, userId); + const enrichedBooks = await enrichAudiobooksWithMatches(collapsedBooks, userId); // Annotate with per-user ignore status const annotatedBooks = await annotateWithIgnoreStatus(enrichedBooks, userId); diff --git a/src/app/api/series/[asin]/route.ts b/src/app/api/series/[asin]/route.ts index 2a60b6b..6b74143 100644 --- a/src/app/api/series/[asin]/route.ts +++ b/src/app/api/series/[asin]/route.ts @@ -9,7 +9,7 @@ import { RMABLogger } from '@/lib/utils/logger'; import { scrapeSeriesPage } from '@/lib/integrations/audible-series'; import { enrichAudiobooksWithMatches } from '@/lib/utils/audiobook-matcher'; import { deduplicateAndCollectGroups } from '@/lib/utils/deduplicate-audiobooks'; -import { persistDedupGroups } from '@/lib/services/works.service'; +import { persistDedupGroups, collapseByExistingWorks } from '@/lib/services/works.service'; import { annotateWithIgnoreStatus } from '@/lib/utils/ignored-audiobooks'; const logger = RMABLogger.create('API.Series.Detail'); @@ -52,17 +52,20 @@ export async function GET( ); } - // Deduplicate before enrichment to avoid wasted DB queries on duplicate entries + // Two-pass dedup: local title/narrator/duration matching first, then collapse + // any remaining duplicates that the works table already knows are the same book + // (handles cases where source metadata diverges across paths or pages). const { books: dedupedBooks, groups } = deduplicateAndCollectGroups(detail.books); - // Fire-and-forget: persist dedup groups to works table for cross-ASIN matching if (groups.length > 0) { persistDedupGroups(groups).catch(() => {}); } + const collapsedBooks = await collapseByExistingWorks(dedupedBooks); + // Enrich books with library availability and request status const userId = currentUser.sub || undefined; - const enrichedBooks = await enrichAudiobooksWithMatches(dedupedBooks, userId); + const enrichedBooks = await enrichAudiobooksWithMatches(collapsedBooks, userId); // Annotate with per-user ignore status const annotatedBooks = await annotateWithIgnoreStatus(enrichedBooks, userId); diff --git a/src/lib/integrations/audible-series.ts b/src/lib/integrations/audible-series.ts index 7cf976b..3aef01e 100644 --- a/src/lib/integrations/audible-series.ts +++ b/src/lib/integrations/audible-series.ts @@ -19,6 +19,7 @@ import { import { RMABLogger } from '../utils/logger'; import { parseRuntime } from '../utils/parse-runtime'; import { randomDelay } from '../utils/scrape-resilience'; +import { extractAllNarrators } from '../utils/extract-narrator'; const logger = RMABLogger.create('Audible.Series'); @@ -442,10 +443,8 @@ function parseSeriesBooks( const authorHref = authorLink.attr('href') || ''; const authorAsinMatch = authorHref.match(/\/author\/[^/]+\/([A-Z0-9]{10})/); - // Narrator - const narratorText = $el.find('a[href*="searchNarrator="]').first().text().trim() || - $el.find('.narratorLabel').text().trim() || - ''; + // Narrator — capture all narrator links (multi-narrator productions are common) + const narratorText = extractAllNarrators($, $el); // Cover art const coverArtUrl = $el.find('img').first().attr('src')?.replace(/\._.*_\./, '._SL500_.') || ''; diff --git a/src/lib/integrations/audible.service.ts b/src/lib/integrations/audible.service.ts index cd30487..6ba4f64 100644 --- a/src/lib/integrations/audible.service.ts +++ b/src/lib/integrations/audible.service.ts @@ -4,21 +4,26 @@ */ import axios, { AxiosInstance } from 'axios'; +import * as cheerio from 'cheerio'; import { RMABLogger } from '../utils/logger'; import { getConfigService } from '../services/config.service'; import { AudibleRegion, AUDIBLE_REGIONS, DEFAULT_AUDIBLE_REGION } from '../types/audible'; import { getLanguageForRegion, isAcceptedLanguage, + stripPrefixes, + buildContainsSelector, + type LanguageConfig, } from '../constants/language-config'; import { pickUserAgent, getBrowserHeaders, jitteredBackoff, - randomDelay, AdaptivePacer, FetchResultMeta, } from '../utils/scrape-resilience'; +import { parseRuntime as parseRuntimeUtil } from '../utils/parse-runtime'; +import { extractAllNarrators } from '../utils/extract-narrator'; const logger = RMABLogger.create('Audible'); @@ -27,6 +32,13 @@ const AUDIBLE_PAGE_SIZE = 50; const CATALOG_RESPONSE_GROUPS = 'contributors,product_desc,product_attrs,product_extended_attrs,media,rating,series,category_ladders,product_details'; +// Retry/backoff knobs for HTML scraping (nightly refresh job only). +// Healthy users still finish quickly — per-page success returns on attempt 0 +// with a 2-4s inter-page delay. Struggling users grind through 503 storms +// patiently: up to ~12 retries per request, with each backoff capped at 3 min. +const HTML_MAX_RETRIES = 12; +const HTML_MAX_BACKOFF_MS = 180_000; + export interface AudibleAudiobook { asin: string; title: string; @@ -298,6 +310,7 @@ export class AudibleService { config: any = {}, maxRetries: number = 5, client: AxiosInstance = this.htmlClient, + maxBackoffMs: number = Number.POSITIVE_INFINITY, ): Promise<{ data: any; meta: FetchResultMeta }> { let lastError: Error | null = null; let retriesUsed = 0; @@ -324,7 +337,7 @@ export class AudibleService { retriesUsed++; - const backoffMs = jitteredBackoff(attempt); + const backoffMs = jitteredBackoff(attempt, 1000, maxBackoffMs); logger.info( ` Request failed (${status || 'network error'}), retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})...`, ); @@ -379,6 +392,12 @@ export class AudibleService { throw lastError || new Error('External API request failed after retries'); } + /** + * Popular audiobooks from Audible's curated /adblbestsellers HTML page. + * Uses HTML scraping (not the catalog API) because the API's BestSellers sort + * is a right-now velocity rank that surfaces launch-day shovelware and preorders; + * the HTML page reflects Audible's editorial curation. + */ async getPopularAudiobooks(limit: number = 20): Promise { await this.initialize(); @@ -395,42 +414,36 @@ export class AudibleService { logger.info(` Fetching page ${page}/${maxPages}...`); const { data: response, meta } = await this.fetchWithRetry( - '/1.0/catalog/products', + '/adblbestsellers', { params: { - products_sort_by: 'BestSellers', - num_results: AUDIBLE_PAGE_SIZE, - page: page - 1, - response_groups: CATALOG_RESPONSE_GROUPS, + ipRedirectOverride: 'true', + pageSize: AUDIBLE_PAGE_SIZE, + ...(page > 1 ? { page } : {}), }, }, - 5, - this.apiClient, + HTML_MAX_RETRIES, + this.htmlClient, + HTML_MAX_BACKOFF_MS, ); - const envelope: CatalogProductsResponse = response.data; - const products = envelope.products ?? []; - const totalResults = envelope.total_results ?? 0; + const foundOnPage = this.parseProductListItems( + response.data, + audiobooks, + limit, + ); - for (const product of products) { - if (audiobooks.length >= limit) break; - if (audiobooks.some((b) => b.asin === product.asin)) continue; - audiobooks.push(mapCatalogProduct(product)); + logger.info(` Found ${foundOnPage} audiobooks on page ${page}`); + + if (foundOnPage < AUDIBLE_PAGE_SIZE / 2) { + logger.info(` Reached end of available pages`); + break; } - logger.info(` Found ${products.length} audiobooks on page ${page}`); - - const hasMore = - totalResults > 0 - ? totalResults > page * AUDIBLE_PAGE_SIZE - : products.length >= AUDIBLE_PAGE_SIZE; - - if (!hasMore) break; - page++; if (page <= maxPages && audiobooks.length < limit) { - await this.delay(this.apiPageDelay(meta)); + await this.delay(this.pacer.reportPageResult(meta)); } } catch (error) { logger.error(`Failed to fetch page ${page} of popular audiobooks`, { @@ -445,6 +458,11 @@ export class AudibleService { return audiobooks; } + /** + * New release audiobooks from Audible's curated /newreleases HTML page. + * Uses HTML scraping (not the catalog API) because the API's -ReleaseDate sort + * returns 100% future preorders with no released-only filter available. + */ async getNewReleases(limit: number = 20): Promise { await this.initialize(); @@ -461,42 +479,36 @@ export class AudibleService { logger.info(` Fetching page ${page}/${maxPages}...`); const { data: response, meta } = await this.fetchWithRetry( - '/1.0/catalog/products', + '/newreleases', { params: { - products_sort_by: '-ReleaseDate', - num_results: AUDIBLE_PAGE_SIZE, - page: page - 1, - response_groups: CATALOG_RESPONSE_GROUPS, + ipRedirectOverride: 'true', + pageSize: AUDIBLE_PAGE_SIZE, + ...(page > 1 ? { page } : {}), }, }, - 5, - this.apiClient, + HTML_MAX_RETRIES, + this.htmlClient, + HTML_MAX_BACKOFF_MS, ); - const envelope: CatalogProductsResponse = response.data; - const products = envelope.products ?? []; - const totalResults = envelope.total_results ?? 0; + const foundOnPage = this.parseProductListItems( + response.data, + audiobooks, + limit, + ); - for (const product of products) { - if (audiobooks.length >= limit) break; - if (audiobooks.some((b) => b.asin === product.asin)) continue; - audiobooks.push(mapCatalogProduct(product)); + logger.info(` Found ${foundOnPage} audiobooks on page ${page}`); + + if (foundOnPage < AUDIBLE_PAGE_SIZE / 2) { + logger.info(` Reached end of available pages`); + break; } - logger.info(` Found ${products.length} audiobooks on page ${page}`); - - const hasMore = - totalResults > 0 - ? totalResults > page * AUDIBLE_PAGE_SIZE - : products.length >= AUDIBLE_PAGE_SIZE; - - if (!hasMore) break; - page++; if (page <= maxPages && audiobooks.length < limit) { - await this.delay(this.apiPageDelay(meta)); + await this.delay(this.pacer.reportPageResult(meta)); } } catch (error) { logger.error(`Failed to fetch page ${page} of new releases`, { @@ -791,6 +803,11 @@ export class AudibleService { } } + /** + * Category audiobooks from Audible's HTML /search?node= page, + * sorted by popularity-rank. Uses HTML scraping (not the catalog API) so + * results match Audible's curated category-storefront ordering. + */ async getCategoryBooks(categoryId: string, limit: number = 200): Promise { await this.initialize(); @@ -805,43 +822,35 @@ export class AudibleService { while (audiobooks.length < limit && page <= maxPages) { try { const { data: response, meta } = await this.fetchWithRetry( - '/1.0/catalog/products', + '/search', { params: { - category_id: categoryId, - products_sort_by: 'BestSellers', - num_results: AUDIBLE_PAGE_SIZE, - page: page - 1, - response_groups: CATALOG_RESPONSE_GROUPS, + ipRedirectOverride: 'true', + node: categoryId, + pageSize: AUDIBLE_PAGE_SIZE, + sort: 'popularity-rank', + ...(page > 1 ? { page } : {}), }, }, - 5, - this.apiClient, + HTML_MAX_RETRIES, + this.htmlClient, + HTML_MAX_BACKOFF_MS, ); - const envelope: CatalogProductsResponse = response.data; - const products = envelope.products ?? []; - const totalResults = envelope.total_results ?? 0; + const foundOnPage = this.parseSearchResultItems( + response.data, + audiobooks, + limit, + ); - for (const product of products) { - if (audiobooks.length >= limit) break; - if (audiobooks.some((b) => b.asin === product.asin)) continue; - audiobooks.push(mapCatalogProduct(product)); - } + logger.info(`Category ${categoryId}: found ${foundOnPage} books on page ${page}`); - logger.info(`Category ${categoryId}: found ${products.length} books on page ${page}`); - - const hasMore = - totalResults > 0 - ? totalResults > page * AUDIBLE_PAGE_SIZE - : products.length >= AUDIBLE_PAGE_SIZE; - - if (!hasMore) break; + if (foundOnPage < AUDIBLE_PAGE_SIZE / 2) break; page++; if (page <= maxPages && audiobooks.length < limit) { - await this.delay(this.apiPageDelay(meta)); + await this.delay(this.pacer.reportPageResult(meta)); } } catch (error) { logger.error(`Failed to fetch category ${categoryId} page ${page}`, { @@ -858,12 +867,148 @@ export class AudibleService { return audiobooks; } - private apiPageDelay(meta: FetchResultMeta): number { - if (meta.retriesUsed > 0) { - return this.pacer.reportPageResult(meta); - } - this.pacer.reportPageResult(meta); - return randomDelay(500, 1500); + private getLangConfig(): LanguageConfig { + return getLanguageForRegion(this.region); + } + + private parseRuntime(runtimeText: string): number | undefined { + return parseRuntimeUtil(runtimeText, this.getLangConfig()); + } + + /** + * Parse the `.productListItem` blocks used by /adblbestsellers and /newreleases. + * Pushes matched books into `audiobooks` (skipping duplicates and respecting `limit`) + * and returns the count parsed from this page. + */ + private parseProductListItems( + html: string, + audiobooks: AudibleAudiobook[], + limit: number, + ): number { + const $ = cheerio.load(html); + const langConfig = this.getLangConfig(); + let foundOnPage = 0; + + $('.productListItem').each((_index, element) => { + if (audiobooks.length >= limit) return false; + + const $el = $(element); + + const asin = + $el.find('li').attr('data-asin') || + $el.find('a').attr('href')?.match(/\/(?:pd|ac)\/[^\/]+\/([A-Z0-9]{10})/)?.[1] || + ''; + if (!asin) return; + if (audiobooks.some((book) => book.asin === asin)) return; + + const title = + $el.find('h3 a').text().trim() || + $el.find('.bc-heading a').text().trim(); + + const authorText = + $el.find('.authorLabel').text().trim() || + $el.find('.bc-size-small .bc-text-bold').first().text().trim(); + + const authorHref = $el.find('a[href*="/author/"]').first().attr('href') || ''; + const authorAsinMatch = authorHref.match(/\/author\/[^\/]+\/([A-Z0-9]{10})/); + + // Narrator — capture all narrator links (multi-narrator productions are common); + // fall back to .narratorLabel text, then to the bc-text-bold sibling for layouts + // that omit both anchor links and the .narratorLabel span. + const narratorText = + extractAllNarrators($, $el) || + $el.find('.bc-size-small .bc-text-bold').eq(1).text().trim(); + + const coverArtUrl = $el.find('img').attr('src') || ''; + + const ratingText = $el.find('.ratingsLabel').text().trim(); + const rating = ratingText ? parseFloat(ratingText.split(' ')[0]) : undefined; + + audiobooks.push({ + asin, + title, + author: stripPrefixes(authorText, langConfig.scraping.authorPrefixes), + authorAsin: authorAsinMatch?.[1] || undefined, + narrator: stripPrefixes(narratorText, langConfig.scraping.narratorPrefixes), + coverArtUrl: coverArtUrl.replace(/\._.*_\./, '._SL500_.'), + rating, + }); + + foundOnPage++; + }); + + return foundOnPage; + } + + /** + * Parse the `.s-result-item` / `.productListItem` blocks used by + * /search?node=. Pushes matched books into `audiobooks` + * (skipping duplicates and respecting `limit`) and returns the count parsed + * from this page. + */ + private parseSearchResultItems( + html: string, + audiobooks: AudibleAudiobook[], + limit: number, + ): number { + const $ = cheerio.load(html); + const langConfig = this.getLangConfig(); + let foundOnPage = 0; + + $('.s-result-item, .productListItem').each((_index, element) => { + if (audiobooks.length >= limit) return false; + + const $el = $(element); + + const asin = + $el.find('li').attr('data-asin') || + $el.find('a').attr('href')?.match(/\/(?:pd|ac)\/[^\/]+\/([A-Z0-9]{10})/)?.[1] || + ''; + if (!asin) return; + if (audiobooks.some((b) => b.asin === asin)) return; + + const title = + $el.find('h2').first().text().trim() || + $el.find('h3 a').text().trim() || + $el.find('.bc-heading a').text().trim(); + + const authorLink = $el.find('a[href*="/author/"]').first(); + const authorText = + authorLink.text().trim() || + $el.find('.authorLabel').text().trim(); + const authorHref = authorLink.attr('href') || ''; + const authorAsinMatch = authorHref.match(/\/author\/[^\/]+\/([A-Z0-9]{10})/); + + // Narrator — capture all narrator links (multi-narrator productions are common) + const narratorText = extractAllNarrators($, $el); + + const coverArtUrl = $el.find('img').attr('src') || ''; + + const runtimeText = + $el.find('.runtimeLabel').text().trim() || + $el.find(buildContainsSelector('span', langConfig.scraping.lengthLabels)).text().trim(); + const durationMinutes = this.parseRuntime(runtimeText); + + const ratingText = + $el.find('.ratingsLabel').text().trim() || + $el.find('.a-icon-star span').first().text().trim(); + const rating = ratingText ? parseFloat(ratingText.split(' ')[0]) : undefined; + + audiobooks.push({ + asin, + title, + author: stripPrefixes(authorText, langConfig.scraping.authorPrefixes), + authorAsin: authorAsinMatch?.[1] || undefined, + narrator: stripPrefixes(narratorText, langConfig.scraping.narratorPrefixes), + coverArtUrl: coverArtUrl.replace(/\._.*_\./, '._SL500_.'), + durationMinutes, + rating, + }); + + foundOnPage++; + }); + + return foundOnPage; } private async delay(ms: number): Promise { diff --git a/src/lib/processors/audible-refresh.processor.ts b/src/lib/processors/audible-refresh.processor.ts index 0c15bee..b2d82ca 100644 --- a/src/lib/processors/audible-refresh.processor.ts +++ b/src/lib/processors/audible-refresh.processor.ts @@ -138,16 +138,37 @@ async function persistSectionBooks( logger: ReturnType, labelForErrors: string, ): Promise { + // Defensive dedup: the (asin, categoryId) unique constraint means a duplicate ASIN + // in `books` crashes the second .create() with P2002. The HTML parser already dedupes + // per page and across pages against the cumulative accumulator, but a warn-on-fire + // signal here lets us detect upstream surprises (e.g. Audible serving the same item + // in both a carousel and the main grid) without the noisy duplicate-key Postgres + // errors. Keep the first occurrence so Audible's editorial ordering is preserved. + const seenAsins = new Set(); + const dedupedBooks = books.filter((b) => { + if (!b?.asin || seenAsins.has(b.asin)) return false; + seenAsins.add(b.asin); + return true; + }); + const droppedCount = books.length - dedupedBooks.length; + if (droppedCount > 0) { + logger.warn( + `Dropped ${droppedCount} duplicate ASIN(s) from ${categoryId} input list before persist`, + ); + } + // Wipe previous entries for this section logger.info(`Clearing previous data for ${categoryId}...`); await prisma.audibleCacheCategory.deleteMany({ where: { categoryId }, }); - logger.info(`Cleared previous entries for ${categoryId}, saving ${books.length} books...`); + logger.info( + `Cleared previous entries for ${categoryId}, saving ${dedupedBooks.length} books...`, + ); let saved = 0; - for (let i = 0; i < books.length; i++) { - const book = books[i]; + for (let i = 0; i < dedupedBooks.length; i++) { + const book = dedupedBooks[i]; try { // Cache thumbnail if coverArtUrl exists let cachedCoverPath: string | null = null; diff --git a/src/lib/services/works.service.ts b/src/lib/services/works.service.ts index 45d989d..a8f8026 100644 --- a/src/lib/services/works.service.ts +++ b/src/lib/services/works.service.ts @@ -9,7 +9,8 @@ import { prisma } from '@/lib/db'; import { RMABLogger } from '@/lib/utils/logger'; -import type { DedupGroup } from '@/lib/utils/deduplicate-audiobooks'; +import { metadataScore, type DedupGroup } from '@/lib/utils/deduplicate-audiobooks'; +import type { AudibleAudiobook } from '@/lib/integrations/audible.service'; const logger = RMABLogger.create('WorksService'); @@ -182,6 +183,96 @@ export async function seedAsin( } } +// --------------------------------------------------------------------------- +// View-level collapse (consult the works table after local dedup) +// --------------------------------------------------------------------------- + +/** + * Collapse books that already share a Work record according to the works table. + * + * The local `deduplicateAndCollectGroups()` pass is title/narrator/duration-based + * and stateless — it can fail to merge ASINs whose source metadata diverges (e.g. + * a series-page scrape captures different "first narrators" for two ASINs of the + * same recording, or two paginated pages each contain one ASIN and never compare + * them). The works table is the durable source of truth for "same book" identity, + * populated by every prior dedup pass and by request-time seeding. This pass + * applies that knowledge to the current view. + * + * Behavior: + * - Books whose ASINs map to a shared workId collapse to a single representative + * chosen by `metadataScore()` (same ranking as local dedup). + * - Books not present in any work, or in single-ASIN works, pass through untouched. + * - Original ordering is preserved (the kept representative sits at the position + * of the first occurrence of its work in the input list). + * - DB failure is non-fatal: the input list is returned unchanged so the view + * still renders (degrades to local-dedup-only behavior). + */ +export async function collapseByExistingWorks( + books: AudibleAudiobook[], +): Promise { + if (books.length <= 1) return books; + + try { + const asins = books.map(b => b.asin); + const entries = await prisma.workAsin.findMany({ + where: { asin: { in: asins } }, + select: { asin: true, workId: true }, + }); + + if (entries.length === 0) return books; + + // Map ASIN → workId for fast lookup in the loop below + const asinToWorkId = new Map(); + for (const entry of entries) { + asinToWorkId.set(entry.asin, entry.workId); + } + + // Walk the input once, preserving position. For each work seen, keep a + // running "best" book; for books not in any work, emit immediately. + const result: AudibleAudiobook[] = []; + const workIdToResultIndex = new Map(); + + for (const book of books) { + const workId = asinToWorkId.get(book.asin); + if (!workId) { + result.push(book); + continue; + } + + const existingIndex = workIdToResultIndex.get(workId); + if (existingIndex === undefined) { + workIdToResultIndex.set(workId, result.length); + result.push(book); + continue; + } + + // A sibling from this work is already in the result. Keep whichever + // has the richer metadata; on tie, keep the earlier entry (already there). + const existing = result[existingIndex]; + if (metadataScore(book) > metadataScore(existing)) { + result[existingIndex] = book; + } + } + + const collapsed = books.length - result.length; + if (collapsed > 0) { + logger.debug('Collapsed books via works table', { + inputCount: books.length, + outputCount: result.length, + collapsed, + }); + } + + return result; + } catch (error) { + logger.error('collapseByExistingWorks failed; returning input unchanged', { + error: error instanceof Error ? error.message : String(error), + bookCount: books.length, + }); + return books; + } +} + // --------------------------------------------------------------------------- // Sibling ASIN lookup (for library matching expansion) // --------------------------------------------------------------------------- diff --git a/src/lib/utils/deduplicate-audiobooks.ts b/src/lib/utils/deduplicate-audiobooks.ts index fafa4cb..ebcd7c8 100644 --- a/src/lib/utils/deduplicate-audiobooks.ts +++ b/src/lib/utils/deduplicate-audiobooks.ts @@ -109,7 +109,12 @@ export function areDurationsCompatible(a?: number, b?: number): boolean { // Metadata scoring (for picking best representative) // --------------------------------------------------------------------------- -function metadataScore(book: AudibleAudiobook): number { +/** + * Score a book by how much metadata it carries. Used as the tie-breaker when + * collapsing duplicates — the entry with the richest metadata wins. Exported + * so the works-table collapse pass can apply the same ranking. + */ +export function metadataScore(book: AudibleAudiobook): number { let score = 0; if (book.coverArtUrl) score++; if (book.rating != null) score++; diff --git a/src/lib/utils/extract-narrator.ts b/src/lib/utils/extract-narrator.ts new file mode 100644 index 0000000..1a8427e --- /dev/null +++ b/src/lib/utils/extract-narrator.ts @@ -0,0 +1,37 @@ +/** + * Component: Narrator Extraction Utility + * Documentation: documentation/integrations/audible.md + * + * Shared helper for Audible HTML scrapers. Audible product listings render + * each narrator as a separate `` link; using + * `.first()` on that selector silently drops co-narrators and breaks dedup + * for multi-narrator productions (e.g. full-cast audiobooks). This helper + * captures every narrator link and joins them, falling back to the + * `.narratorLabel` span when no anchor links are present. + */ + +import type * as cheerio from 'cheerio'; +import type { AnyNode } from 'domhandler'; + +/** + * Extract a comma-joined narrator string from an Audible product list item. + * + * Order is not semantically significant — downstream `normalizeNarrator()` + * sorts before comparison — but document-order preserves a stable, legible + * value for caching and logging. + */ +export function extractAllNarrators( + $: cheerio.CheerioAPI, + $el: cheerio.Cheerio, +): string { + const links = $el.find('a[href*="searchNarrator="]'); + if (links.length > 0) { + const names: string[] = []; + links.each((_, link) => { + const name = $(link).text().trim(); + if (name) names.push(name); + }); + if (names.length > 0) return names.join(', '); + } + return $el.find('.narratorLabel').text().trim(); +} diff --git a/src/lib/utils/scrape-resilience.ts b/src/lib/utils/scrape-resilience.ts index fce4cbc..3572f1d 100644 --- a/src/lib/utils/scrape-resilience.ts +++ b/src/lib/utils/scrape-resilience.ts @@ -38,12 +38,18 @@ export function getBrowserHeaders(userAgent: string): Record { } /** - * Jittered exponential backoff: 2^attempt * baseMs * random(0.5, 1.5) + * Jittered exponential backoff: 2^attempt * baseMs * random(0.5, 1.5), + * optionally capped so high attempt counts don't produce absurd waits. * Avoids predictable retry timing that is trivially fingerprinted. */ -export function jitteredBackoff(attempt: number, baseMs: number = 1000): number { +export function jitteredBackoff( + attempt: number, + baseMs: number = 1000, + maxBackoffMs: number = Number.POSITIVE_INFINITY, +): number { const jitter = 0.5 + Math.random(); // 0.5 – 1.5 - return Math.round(Math.pow(2, attempt) * baseMs * jitter); + const raw = Math.pow(2, attempt) * baseMs * jitter; + return Math.round(Math.min(raw, maxBackoffMs)); } /** Random integer in [minMs, maxMs] */ diff --git a/tests/integrations/audible.service.test.ts b/tests/integrations/audible.service.test.ts index 9186a47..f006031 100644 --- a/tests/integrations/audible.service.test.ts +++ b/tests/integrations/audible.service.test.ts @@ -81,6 +81,122 @@ function apiResponse(envelope: object) { return { data: envelope }; } +// --------------------------------------------------------------------------- +// HTML fixture helpers (for getPopularAudiobooks / getNewReleases / getCategoryBooks, +// which scrape Audible's curated HTML pages) +// --------------------------------------------------------------------------- + +interface HtmlBookOverrides { + asin?: string; + title?: string; + author?: string; + authorAsin?: string; + /** Single-narrator shorthand; mutually exclusive with `narrators`. */ + narrator?: string; + /** Multi-narrator productions render each name as its own searchNarrator anchor. */ + narrators?: string[]; + coverArtUrl?: string; + rating?: number; +} + +/** Render one or more narrator anchor links suitable for embedding in .narratorLabel. */ +function renderNarratorLinks(names: string[]): string { + return names + .map( + (name) => + `${name}`, + ) + .join(', '); +} + +/** + * Produces a single .productListItem block matching the selectors parsed by + * parseProductListItems(). The parser looks for an `
  • ` descendant, + * with an `` fallback — using a real `
  • ` here both + * exercises the primary path and keeps the markup well-formed. + */ +function makeProductListItemHtml(overrides: HtmlBookOverrides = {}): string { + const { + asin = 'B000000001', + title = 'Test Book', + author = 'Test Author', + authorAsin = 'A000000001', + narrator = 'Test Narrator', + narrators, + coverArtUrl = 'https://images.example.com/cover._SL500_.jpg', + rating = 4.5, + } = overrides; + + // Real Audible storefront markup embeds each narrator as its own anchor inside + // .narratorLabel for multi-narrator productions. The single-narrator case keeps + // the original plain-text span for backward compatibility with existing tests. + const narratorMarkup = narrators && narrators.length > 0 + ? `Narrated by: ${renderNarratorLinks(narrators)}` + : `${narrator}`; + + return ` +
    + +
    + `; +} + +/** + * Produces a single .s-result-item block matching the selectors parsed by + * parseSearchResultItems(). Used for /search?node= category pages. + */ +function makeSearchResultItemHtml(overrides: HtmlBookOverrides = {}): string { + const { + asin = 'B000000001', + title = 'Test Book', + author = 'Test Author', + authorAsin = 'A000000001', + narrator = 'Test Narrator', + narrators, + coverArtUrl = 'https://images.example.com/cover._SL500_.jpg', + rating = 4.5, + } = overrides; + + const narratorLinks = narrators && narrators.length > 0 + ? renderNarratorLinks(narrators) + : `${narrator}`; + + return ` +
    + +
    + `; +} + +/** Wrap one or more item-HTML strings in a minimal page document. */ +function makeHtmlPage(items: string[]): string { + return `${items.join('')}`; +} + +/** + * Produces the value that client.get() should resolve to for HTML responses. + * cheerio.load() is called on response.data, so .data must be the raw HTML string. + */ +function htmlResponse(html: string) { + return { data: html }; +} + // --------------------------------------------------------------------------- // Test setup // --------------------------------------------------------------------------- @@ -683,61 +799,66 @@ describe('AudibleService', () => { }); // ------------------------------------------------------------------------- - // getPopularAudiobooks() + // getPopularAudiobooks() — HTML scraping of /adblbestsellers // ------------------------------------------------------------------------- describe('getPopularAudiobooks()', () => { - it('uses products_sort_by: BestSellers', async () => { - apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + it('hits /adblbestsellers on the htmlClient with pageSize=50', async () => { + htmlClientMock.get.mockResolvedValue(htmlResponse(makeHtmlPage([makeProductListItemHtml()]))); const service = new AudibleService(); await service.getPopularAudiobooks(1); - expect(apiClientMock.get.mock.calls[0][1].params.products_sort_by).toBe('BestSellers'); + expect(htmlClientMock.get).toHaveBeenCalledWith( + '/adblbestsellers', + expect.objectContaining({ + params: expect.objectContaining({ pageSize: 50 }), + }), + ); }); - it('subtracts 1 from public page=1 before calling the API', async () => { - apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + it('does not include a page param on the first request (only from page 2 onward)', async () => { + htmlClientMock.get.mockResolvedValue(htmlResponse(makeHtmlPage([makeProductListItemHtml()]))); const service = new AudibleService(); const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); await service.getPopularAudiobooks(1); - expect(apiClientMock.get.mock.calls[0][1].params.page).toBe(0); + expect(htmlClientMock.get.mock.calls[0][1].params.page).toBeUndefined(); delaySpy.mockRestore(); }); - it('makes a second call with page=1 when paginating to page 2', async () => { - const page1Products = Array.from({ length: 50 }, (_, i) => - makeProduct({ asin: `B${String(i).padStart(9, '0')}`, title: `Book ${i}` }), + it('includes page=2 on the second request when paginating', async () => { + const page1Items = Array.from({ length: 50 }, (_, i) => + makeProductListItemHtml({ asin: `B${String(i).padStart(9, '0')}`, title: `Book ${i}` }), ); - const page2Products = Array.from({ length: 25 }, (_, i) => - makeProduct({ asin: `B${String(i + 50).padStart(9, '0')}`, title: `Book ${i + 50}` }), + const page2Items = Array.from({ length: 25 }, (_, i) => + makeProductListItemHtml({ asin: `B${String(i + 50).padStart(9, '0')}`, title: `Book ${i + 50}` }), ); - apiClientMock.get - .mockResolvedValueOnce(apiResponse(makeProductsResponse(page1Products, 75))) - .mockResolvedValueOnce(apiResponse(makeProductsResponse(page2Products, 75))); + htmlClientMock.get + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page1Items))) + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page2Items))); const service = new AudibleService(); const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); await service.getPopularAudiobooks(75); - expect(apiClientMock.get.mock.calls[1][1].params.page).toBe(1); + expect(htmlClientMock.get.mock.calls[1][1].params.page).toBe(2); delaySpy.mockRestore(); }); - it('paginates and returns up to the requested limit', async () => { - const page1Products = Array.from({ length: 50 }, (_, i) => - makeProduct({ asin: `B${String(i).padStart(9, '0')}`, title: `Book ${i}` }), + it('paginates across pages and returns up to the requested limit', async () => { + const page1Items = Array.from({ length: 50 }, (_, i) => + makeProductListItemHtml({ asin: `B${String(i).padStart(9, '0')}`, title: `Book ${i}` }), ); - const page2Products = Array.from({ length: 25 }, (_, i) => - makeProduct({ asin: `B${String(i + 50).padStart(9, '0')}`, title: `Book ${i + 50}` }), + const page2Items = Array.from({ length: 25 }, (_, i) => + makeProductListItemHtml({ asin: `B${String(i + 50).padStart(9, '0')}`, title: `Book ${i + 50}` }), ); - apiClientMock.get - .mockResolvedValueOnce(apiResponse(makeProductsResponse(page1Products, 75))) - .mockResolvedValueOnce(apiResponse(makeProductsResponse(page2Products, 75))); + htmlClientMock.get + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page1Items))) + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page2Items))); const service = new AudibleService(); const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); @@ -747,176 +868,338 @@ describe('AudibleService', () => { delaySpy.mockRestore(); }); - it('stops early when a page returns fewer than the page size', async () => { - const products = [makeProduct()]; - apiClientMock.get.mockResolvedValueOnce(apiResponse(makeProductsResponse(products, 1))); + it('stops early when a page returns fewer than half the page size', async () => { + htmlClientMock.get.mockResolvedValueOnce( + htmlResponse(makeHtmlPage([makeProductListItemHtml()])), + ); const service = new AudibleService(); const results = await service.getPopularAudiobooks(50); expect(results).toHaveLength(1); - expect(apiClientMock.get).toHaveBeenCalledTimes(1); + expect(htmlClientMock.get).toHaveBeenCalledTimes(1); }); it('deduplicates by ASIN across pages', async () => { - const sharedProduct = makeProduct({ asin: 'BDUP000001', title: 'Duplicated Book' }); - const uniqueProduct = makeProduct({ asin: 'BUNIQ000001', title: 'Unique Book' }); + const sharedAsin = 'BDUP000001'; + const uniqueAsin = 'BUNIQ000001'; - apiClientMock.get - .mockResolvedValueOnce( - apiResponse(makeProductsResponse([sharedProduct], 51)), - ) - .mockResolvedValueOnce( - // page 2 returns the same ASIN plus a new one - apiResponse(makeProductsResponse([sharedProduct, uniqueProduct], 51)), - ); + // Build a "full" first page (50 items, all with the shared ASIN duplicated as filler) + // so the parser proceeds to page 2. + const page1Items = [ + makeProductListItemHtml({ asin: sharedAsin, title: 'Duplicated Book' }), + ...Array.from({ length: 49 }, (_, i) => + makeProductListItemHtml({ asin: `BFILL${String(i).padStart(5, '0')}`, title: `Filler ${i}` }), + ), + ]; + const page2Items = [ + makeProductListItemHtml({ asin: sharedAsin, title: 'Duplicated Book' }), + makeProductListItemHtml({ asin: uniqueAsin, title: 'Unique Book' }), + ...Array.from({ length: 48 }, (_, i) => + makeProductListItemHtml({ asin: `BFILL2${String(i).padStart(4, '0')}`, title: `Filler2 ${i}` }), + ), + ]; + + htmlClientMock.get + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page1Items))) + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page2Items))); const service = new AudibleService(); const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); - const results = await service.getPopularAudiobooks(100); + const results = await service.getPopularAudiobooks(150); const asins = results.map((r) => r.asin); - expect(asins.filter((a) => a === 'BDUP000001')).toHaveLength(1); + expect(asins.filter((a) => a === sharedAsin)).toHaveLength(1); + expect(asins).toContain(uniqueAsin); delaySpy.mockRestore(); }); it('returns empty array on error without throwing', async () => { const error: Error & { response?: { status: number } } = new Error('Not Found'); error.response = { status: 404 }; - apiClientMock.get.mockRejectedValue(error); + htmlClientMock.get.mockRejectedValue(error); const service = new AudibleService(); const results = await service.getPopularAudiobooks(5); expect(results).toEqual([]); }); + + it('uses htmlClient (not apiClient) for the request', async () => { + htmlClientMock.get.mockResolvedValue(htmlResponse(makeHtmlPage([makeProductListItemHtml()]))); + + const service = new AudibleService(); + await service.getPopularAudiobooks(1); + + expect(htmlClientMock.get).toHaveBeenCalled(); + expect(apiClientMock.get).not.toHaveBeenCalled(); + }); + + it('maps title, author, narrator, and rating from the parsed item', async () => { + htmlClientMock.get.mockResolvedValue( + htmlResponse( + makeHtmlPage([ + makeProductListItemHtml({ + asin: 'B0HTMLMAP1', + title: 'Mapped Title', + author: 'Mapped Author', + authorAsin: 'A00MAPAUTH', + narrator: 'Mapped Narrator', + rating: 4.7, + }), + ]), + ), + ); + + const service = new AudibleService(); + const [book] = await service.getPopularAudiobooks(1); + + expect(book.asin).toBe('B0HTMLMAP1'); + expect(book.title).toBe('Mapped Title'); + expect(book.author).toBe('Mapped Author'); + expect(book.authorAsin).toBe('A00MAPAUTH'); + expect(book.narrator).toBe('Mapped Narrator'); + expect(book.rating).toBeCloseTo(4.7); + }); + + it('captures every co-narrator on multi-narrator productions (regression: prior code took only the first link)', async () => { + htmlClientMock.get.mockResolvedValue( + htmlResponse( + makeHtmlPage([ + makeProductListItemHtml({ + asin: 'B0FULLCAST', + narrators: [ + 'Kristin Atherton', + 'Roy McMillan', + 'Clare Corbett', + 'Tom Bateman', + 'Patience Tomlinson', + 'Shaheen Khan', + ], + }), + ]), + ), + ); + + const service = new AudibleService(); + const [book] = await service.getPopularAudiobooks(1); + + // Every narrator must round-trip — order is not significant downstream, + // but document order should be preserved for stable cache values. + expect(book.narrator).toBe( + 'Kristin Atherton, Roy McMillan, Clare Corbett, Tom Bateman, Patience Tomlinson, Shaheen Khan', + ); + }); }); // ------------------------------------------------------------------------- - // getNewReleases() + // getNewReleases() — HTML scraping of /newreleases // ------------------------------------------------------------------------- describe('getNewReleases()', () => { - it('uses products_sort_by: -ReleaseDate', async () => { - apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + it('hits /newreleases on the htmlClient with pageSize=50', async () => { + htmlClientMock.get.mockResolvedValue(htmlResponse(makeHtmlPage([makeProductListItemHtml()]))); const service = new AudibleService(); await service.getNewReleases(1); - expect(apiClientMock.get.mock.calls[0][1].params.products_sort_by).toBe('-ReleaseDate'); + expect(htmlClientMock.get).toHaveBeenCalledWith( + '/newreleases', + expect.objectContaining({ + params: expect.objectContaining({ pageSize: 50 }), + }), + ); }); - it('subtracts 1 from public page=1 before calling the API', async () => { - apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + it('does not include a page param on the first request', async () => { + htmlClientMock.get.mockResolvedValue(htmlResponse(makeHtmlPage([makeProductListItemHtml()]))); const service = new AudibleService(); const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); await service.getNewReleases(1); - expect(apiClientMock.get.mock.calls[0][1].params.page).toBe(0); + expect(htmlClientMock.get.mock.calls[0][1].params.page).toBeUndefined(); delaySpy.mockRestore(); }); - it('subtracts 1 from public page=2 when paginating to the second page', async () => { - const page1Products = Array.from({ length: 50 }, (_, i) => - makeProduct({ asin: `B${String(i).padStart(9, '0')}` }), + it('includes page=2 on the second request when paginating', async () => { + const page1Items = Array.from({ length: 50 }, (_, i) => + makeProductListItemHtml({ asin: `B${String(i).padStart(9, '0')}` }), + ); + const page2Items = Array.from({ length: 50 }, (_, i) => + makeProductListItemHtml({ asin: `B${String(i + 50).padStart(9, '0')}` }), ); - const page2Products = [makeProduct({ asin: 'BNEW000099' })]; - apiClientMock.get - .mockResolvedValueOnce(apiResponse(makeProductsResponse(page1Products, 51))) - .mockResolvedValueOnce(apiResponse(makeProductsResponse(page2Products, 51))); + htmlClientMock.get + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page1Items))) + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page2Items))); const service = new AudibleService(); const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); - await service.getNewReleases(51); - expect(apiClientMock.get.mock.calls[1][1].params.page).toBe(1); + await service.getNewReleases(100); + expect(htmlClientMock.get.mock.calls[1][1].params.page).toBe(2); delaySpy.mockRestore(); }); it('deduplicates by ASIN across pages', async () => { - const sharedProduct = makeProduct({ asin: 'BDUP000002' }); - apiClientMock.get - .mockResolvedValueOnce(apiResponse(makeProductsResponse([sharedProduct], 51))) - .mockResolvedValueOnce(apiResponse(makeProductsResponse([sharedProduct], 51))); + const sharedAsin = 'BDUP000002'; + + const page1Items = [ + makeProductListItemHtml({ asin: sharedAsin }), + ...Array.from({ length: 49 }, (_, i) => + makeProductListItemHtml({ asin: `BNEW${String(i).padStart(6, '0')}` }), + ), + ]; + const page2Items = [ + makeProductListItemHtml({ asin: sharedAsin }), + ...Array.from({ length: 49 }, (_, i) => + makeProductListItemHtml({ asin: `BNEW2${String(i).padStart(5, '0')}` }), + ), + ]; + + htmlClientMock.get + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page1Items))) + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page2Items))); const service = new AudibleService(); const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); - const results = await service.getNewReleases(100); + const results = await service.getNewReleases(150); - expect(results.filter((r) => r.asin === 'BDUP000002')).toHaveLength(1); + expect(results.filter((r) => r.asin === sharedAsin)).toHaveLength(1); delaySpy.mockRestore(); }); it('returns empty array on error without throwing', async () => { const error: Error & { response?: { status: number } } = new Error('Not Found'); error.response = { status: 404 }; - apiClientMock.get.mockRejectedValue(error); + htmlClientMock.get.mockRejectedValue(error); const service = new AudibleService(); const results = await service.getNewReleases(5); expect(results).toEqual([]); }); + + it('uses htmlClient (not apiClient) for the request', async () => { + htmlClientMock.get.mockResolvedValue(htmlResponse(makeHtmlPage([makeProductListItemHtml()]))); + + const service = new AudibleService(); + await service.getNewReleases(1); + + expect(htmlClientMock.get).toHaveBeenCalled(); + expect(apiClientMock.get).not.toHaveBeenCalled(); + }); }); // ------------------------------------------------------------------------- - // getCategoryBooks() + // getCategoryBooks() — HTML scraping of /search?node= // ------------------------------------------------------------------------- describe('getCategoryBooks()', () => { - it('sends category_id and BestSellers sort param', async () => { - apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + it('hits /search on the htmlClient with node, pageSize, and popularity-rank sort', async () => { + htmlClientMock.get.mockResolvedValue( + htmlResponse(makeHtmlPage([makeSearchResultItemHtml()])), + ); const service = new AudibleService(); await service.getCategoryBooks('18685580011', 1); - const params = apiClientMock.get.mock.calls[0][1].params; - expect(params.category_id).toBe('18685580011'); - expect(params.products_sort_by).toBe('BestSellers'); + const params = htmlClientMock.get.mock.calls[0][1].params; + expect(htmlClientMock.get.mock.calls[0][0]).toBe('/search'); + expect(params.node).toBe('18685580011'); + expect(params.pageSize).toBe(50); + expect(params.sort).toBe('popularity-rank'); }); - it('subtracts 1 from public page=1 before calling the API', async () => { - apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse([]))); + it('does not include a page param on the first request', async () => { + htmlClientMock.get.mockResolvedValue( + htmlResponse(makeHtmlPage([makeSearchResultItemHtml()])), + ); const service = new AudibleService(); const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); await service.getCategoryBooks('CAT001', 1); - expect(apiClientMock.get.mock.calls[0][1].params.page).toBe(0); + expect(htmlClientMock.get.mock.calls[0][1].params.page).toBeUndefined(); delaySpy.mockRestore(); }); - it('subtracts 1 from public page=2 when paginating to the second page', async () => { - const page1Products = Array.from({ length: 50 }, (_, i) => - makeProduct({ asin: `B${String(i).padStart(9, '0')}` }), + it('includes page=2 on the second request when paginating', async () => { + const page1Items = Array.from({ length: 50 }, (_, i) => + makeSearchResultItemHtml({ asin: `B${String(i).padStart(9, '0')}` }), + ); + const page2Items = Array.from({ length: 50 }, (_, i) => + makeSearchResultItemHtml({ asin: `B${String(i + 50).padStart(9, '0')}` }), ); - const page2Products = [makeProduct({ asin: 'BCAT000099' })]; - apiClientMock.get - .mockResolvedValueOnce(apiResponse(makeProductsResponse(page1Products, 51))) - .mockResolvedValueOnce(apiResponse(makeProductsResponse(page2Products, 51))); + htmlClientMock.get + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page1Items))) + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page2Items))); const service = new AudibleService(); const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); - await service.getCategoryBooks('CAT001', 51); - expect(apiClientMock.get.mock.calls[1][1].params.page).toBe(1); + await service.getCategoryBooks('CAT001', 100); + expect(htmlClientMock.get.mock.calls[1][1].params.page).toBe(2); delaySpy.mockRestore(); }); it('deduplicates by ASIN across pages', async () => { - const sharedProduct = makeProduct({ asin: 'BDUP000003' }); - apiClientMock.get - .mockResolvedValueOnce(apiResponse(makeProductsResponse([sharedProduct], 51))) - .mockResolvedValueOnce(apiResponse(makeProductsResponse([sharedProduct], 51))); + const sharedAsin = 'BDUP000003'; + + const page1Items = [ + makeSearchResultItemHtml({ asin: sharedAsin }), + ...Array.from({ length: 49 }, (_, i) => + makeSearchResultItemHtml({ asin: `BCAT${String(i).padStart(6, '0')}` }), + ), + ]; + const page2Items = [ + makeSearchResultItemHtml({ asin: sharedAsin }), + ...Array.from({ length: 49 }, (_, i) => + makeSearchResultItemHtml({ asin: `BCAT2${String(i).padStart(5, '0')}` }), + ), + ]; + + htmlClientMock.get + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page1Items))) + .mockResolvedValueOnce(htmlResponse(makeHtmlPage(page2Items))); const service = new AudibleService(); const delaySpy = vi.spyOn(service as any, 'delay').mockResolvedValue(undefined); - const results = await service.getCategoryBooks('CAT001', 100); + const results = await service.getCategoryBooks('CAT001', 150); - expect(results.filter((r) => r.asin === 'BDUP000003')).toHaveLength(1); + expect(results.filter((r) => r.asin === sharedAsin)).toHaveLength(1); delaySpy.mockRestore(); }); + + it('uses htmlClient (not apiClient) for the request', async () => { + htmlClientMock.get.mockResolvedValue( + htmlResponse(makeHtmlPage([makeSearchResultItemHtml()])), + ); + + const service = new AudibleService(); + await service.getCategoryBooks('CAT001', 1); + + expect(htmlClientMock.get).toHaveBeenCalled(); + expect(apiClientMock.get).not.toHaveBeenCalled(); + }); + + it('captures every co-narrator on multi-narrator productions (regression: prior code took only the first link)', async () => { + htmlClientMock.get.mockResolvedValue( + htmlResponse( + makeHtmlPage([ + makeSearchResultItemHtml({ + asin: 'B0FULLCAST', + narrators: ['Alice', 'Bob', 'Carol', 'Dan'], + }), + ]), + ), + ); + + const service = new AudibleService(); + const [book] = await service.getCategoryBooks('CAT001', 1); + + expect(book.narrator).toBe('Alice, Bob, Carol, Dan'); + }); }); // ------------------------------------------------------------------------- diff --git a/tests/processors/audible-refresh.processor.test.ts b/tests/processors/audible-refresh.processor.test.ts index bdfd7b3..20fcf0c 100644 --- a/tests/processors/audible-refresh.processor.test.ts +++ b/tests/processors/audible-refresh.processor.test.ts @@ -198,4 +198,69 @@ describe('processAudibleRefresh', () => { const { processAudibleRefresh } = await import('@/lib/processors/audible-refresh.processor'); await expect(processAudibleRefresh({ jobId: 'job-2' })).rejects.toThrow('DB down'); }); + + it('deduplicates ASINs in the input list before persisting, preserving order', async () => { + // Two `A` entries should collapse to one. Final ranks must be contiguous + // (1, 2, 3) and follow Audible's editorial ordering (A, B, C). + const popular = [ + { asin: 'A', title: 'Book A', author: 'X', coverArtUrl: null }, + { asin: 'B', title: 'Book B', author: 'X', coverArtUrl: null }, + { asin: 'A', title: 'Book A (duplicate)', author: 'X', coverArtUrl: null }, + { asin: 'C', title: 'Book C', author: 'X', coverArtUrl: null }, + ]; + + audibleServiceMock.getPopularAudiobooks.mockResolvedValue(popular); + audibleServiceMock.getNewReleases.mockResolvedValue([]); + thumbnailCacheMock.cleanupUnusedThumbnails.mockResolvedValue(0); + prismaMock.audibleCache.upsert.mockResolvedValue({}); + prismaMock.audibleCacheCategory.deleteMany.mockResolvedValue({ count: 0 }); + prismaMock.audibleCacheCategory.create.mockResolvedValue({}); + prismaMock.userHomeSection.findMany.mockResolvedValue([]); + prismaMock.audibleCache.findMany.mockResolvedValue([]); + + const { processAudibleRefresh } = await import('@/lib/processors/audible-refresh.processor'); + const result = await processAudibleRefresh({ jobId: 'job-dedup' }); + + expect(result.popularSaved).toBe(3); + + // Only 3 category entries created — the duplicate `A` was dropped. + const popularCreates = (prismaMock.audibleCacheCategory.create.mock.calls as Array<[{ data: { asin: string; categoryId: string; rank: number } }]>) + .map((c) => c[0].data) + .filter((d) => d.categoryId === '__popular__'); + expect(popularCreates).toHaveLength(3); + expect(popularCreates.map((d) => d.asin)).toEqual(['A', 'B', 'C']); + expect(popularCreates.map((d) => d.rank)).toEqual([1, 2, 3]); + + // upsert called once per unique ASIN, not per input row. + expect(prismaMock.audibleCache.upsert).toHaveBeenCalledTimes(3); + }); + + it('drops entries with missing ASINs as part of dedup', async () => { + const popular = [ + { asin: 'A', title: 'Book A', author: 'X', coverArtUrl: null }, + { asin: '', title: 'Book with empty asin', author: 'X', coverArtUrl: null }, + { asin: null, title: 'Book with null asin', author: 'X', coverArtUrl: null }, + { asin: 'B', title: 'Book B', author: 'X', coverArtUrl: null }, + ]; + + audibleServiceMock.getPopularAudiobooks.mockResolvedValue(popular as any); + audibleServiceMock.getNewReleases.mockResolvedValue([]); + thumbnailCacheMock.cleanupUnusedThumbnails.mockResolvedValue(0); + prismaMock.audibleCache.upsert.mockResolvedValue({}); + prismaMock.audibleCacheCategory.deleteMany.mockResolvedValue({ count: 0 }); + prismaMock.audibleCacheCategory.create.mockResolvedValue({}); + prismaMock.userHomeSection.findMany.mockResolvedValue([]); + prismaMock.audibleCache.findMany.mockResolvedValue([]); + + const { processAudibleRefresh } = await import('@/lib/processors/audible-refresh.processor'); + const result = await processAudibleRefresh({ jobId: 'job-empty-asin' }); + + expect(result.popularSaved).toBe(2); + + const popularCreates = (prismaMock.audibleCacheCategory.create.mock.calls as Array<[{ data: { asin: string; categoryId: string; rank: number } }]>) + .map((c) => c[0].data) + .filter((d) => d.categoryId === '__popular__'); + expect(popularCreates.map((d) => d.asin)).toEqual(['A', 'B']); + expect(popularCreates.map((d) => d.rank)).toEqual([1, 2]); + }); }); diff --git a/tests/services/works.service.test.ts b/tests/services/works.service.test.ts index 5efca96..830a75d 100644 --- a/tests/services/works.service.test.ts +++ b/tests/services/works.service.test.ts @@ -6,6 +6,15 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { createPrismaMock } from '../helpers/prisma'; import type { DedupGroup } from '@/lib/utils/deduplicate-audiobooks'; +import type { AudibleAudiobook } from '@/lib/integrations/audible.service'; + +function makeBook(overrides: Partial & { asin: string }): AudibleAudiobook { + return { + title: 'Test Book', + author: 'Test Author', + ...overrides, + }; +} const prismaMock = createPrismaMock(); @@ -304,3 +313,183 @@ describe('getSiblingAsins', () => { expect(result.has('ASIN_LONELY')).toBe(false); }); }); + +describe('collapseByExistingWorks', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.resetModules(); + }); + + it('returns input unchanged when the list is empty or has one entry', async () => { + const { collapseByExistingWorks } = await import('@/lib/services/works.service'); + + expect(await collapseByExistingWorks([])).toEqual([]); + expect(prismaMock.workAsin.findMany).not.toHaveBeenCalled(); + + const single = [makeBook({ asin: 'A1' })]; + expect(await collapseByExistingWorks(single)).toEqual(single); + expect(prismaMock.workAsin.findMany).not.toHaveBeenCalled(); + }); + + it('returns input unchanged when none of the ASINs are in any work', async () => { + prismaMock.workAsin.findMany.mockResolvedValue([]); + + const { collapseByExistingWorks } = await import('@/lib/services/works.service'); + + const books = [ + makeBook({ asin: 'A1', title: 'Alpha' }), + makeBook({ asin: 'A2', title: 'Beta' }), + ]; + + const result = await collapseByExistingWorks(books); + expect(result).toEqual(books); + }); + + it('collapses two ASINs that share a work to a single representative', async () => { + prismaMock.workAsin.findMany.mockResolvedValue([ + { asin: 'A1', workId: 'work-1' }, + { asin: 'A2', workId: 'work-1' }, + ]); + + const { collapseByExistingWorks } = await import('@/lib/services/works.service'); + + const books = [ + makeBook({ asin: 'A1', title: 'The Passengers', coverArtUrl: 'cover.jpg' }), + makeBook({ asin: 'A2', title: 'The Passengers' }), + ]; + + const result = await collapseByExistingWorks(books); + expect(result).toHaveLength(1); + // A1 wins — it has the cover URL (higher metadata score) + expect(result[0].asin).toBe('A1'); + }); + + it('keeps the richest-metadata entry when collapsing, regardless of input order', async () => { + prismaMock.workAsin.findMany.mockResolvedValue([ + { asin: 'A1', workId: 'work-1' }, + { asin: 'A2', workId: 'work-1' }, + ]); + + const { collapseByExistingWorks } = await import('@/lib/services/works.service'); + + // A1 first (sparse), A2 second (rich) — A2 should win on score + const books = [ + makeBook({ asin: 'A1', title: 'Book' }), + makeBook({ + asin: 'A2', + title: 'Book', + coverArtUrl: 'cover.jpg', + rating: 4.5, + durationMinutes: 600, + narrator: 'Full Cast', + description: 'Rich book', + releaseDate: '2024-01-01', + genres: ['Fiction'], + }), + ]; + + const result = await collapseByExistingWorks(books); + expect(result).toHaveLength(1); + expect(result[0].asin).toBe('A2'); + }); + + it('preserves position of the work in the input order', async () => { + prismaMock.workAsin.findMany.mockResolvedValue([ + { asin: 'A2', workId: 'work-1' }, + { asin: 'A4', workId: 'work-1' }, + ]); + + const { collapseByExistingWorks } = await import('@/lib/services/works.service'); + + const books = [ + makeBook({ asin: 'A1', title: 'Alpha' }), + makeBook({ asin: 'A2', title: 'Beta' }), + makeBook({ asin: 'A3', title: 'Gamma' }), + makeBook({ asin: 'A4', title: 'Beta' }), + makeBook({ asin: 'A5', title: 'Delta' }), + ]; + + const result = await collapseByExistingWorks(books); + // A2 and A4 collapse to one entry at position 1 (the first occurrence) + expect(result.map(b => b.asin)).toEqual(['A1', 'A2', 'A3', 'A5']); + }); + + it('handles multiple independent works in the same batch', async () => { + prismaMock.workAsin.findMany.mockResolvedValue([ + { asin: 'A1', workId: 'work-1' }, + { asin: 'A2', workId: 'work-1' }, + { asin: 'B1', workId: 'work-2' }, + { asin: 'B2', workId: 'work-2' }, + { asin: 'B3', workId: 'work-2' }, + ]); + + const { collapseByExistingWorks } = await import('@/lib/services/works.service'); + + const books = [ + makeBook({ asin: 'A1' }), + makeBook({ asin: 'B1' }), + makeBook({ asin: 'A2' }), + makeBook({ asin: 'B2' }), + makeBook({ asin: 'B3' }), + makeBook({ asin: 'C1' }), + ]; + + const result = await collapseByExistingWorks(books); + expect(result.map(b => b.asin)).toEqual(['A1', 'B1', 'C1']); + }); + + it('passes through books that are not in any work alongside collapsed ones', async () => { + prismaMock.workAsin.findMany.mockResolvedValue([ + { asin: 'A1', workId: 'work-1' }, + { asin: 'A2', workId: 'work-1' }, + ]); + + const { collapseByExistingWorks } = await import('@/lib/services/works.service'); + + const books = [ + makeBook({ asin: 'STANDALONE_1', title: 'Standalone 1' }), + makeBook({ asin: 'A1', title: 'Same Book' }), + makeBook({ asin: 'STANDALONE_2', title: 'Standalone 2' }), + makeBook({ asin: 'A2', title: 'Same Book' }), + ]; + + const result = await collapseByExistingWorks(books); + expect(result).toHaveLength(3); + expect(result.map(b => b.asin)).toEqual(['STANDALONE_1', 'A1', 'STANDALONE_2']); + }); + + it('returns input unchanged on DB failure (does not throw)', async () => { + prismaMock.workAsin.findMany.mockRejectedValue(new Error('DB exploded')); + + const { collapseByExistingWorks } = await import('@/lib/services/works.service'); + + const books = [ + makeBook({ asin: 'A1' }), + makeBook({ asin: 'A2' }), + ]; + + const result = await collapseByExistingWorks(books); + expect(result).toEqual(books); + }); + + it('only queries the workAsin table once per call', async () => { + prismaMock.workAsin.findMany.mockResolvedValue([ + { asin: 'A1', workId: 'work-1' }, + { asin: 'A2', workId: 'work-1' }, + ]); + + const { collapseByExistingWorks } = await import('@/lib/services/works.service'); + + await collapseByExistingWorks([ + makeBook({ asin: 'A1' }), + makeBook({ asin: 'A2' }), + makeBook({ asin: 'A3' }), + ]); + + expect(prismaMock.workAsin.findMany).toHaveBeenCalledTimes(1); + expect(prismaMock.workAsin.findMany).toHaveBeenCalledWith({ + where: { asin: { in: ['A1', 'A2', 'A3'] } }, + select: { asin: true, workId: true }, + }); + }); +}); diff --git a/tests/utils/extract-narrator.test.ts b/tests/utils/extract-narrator.test.ts new file mode 100644 index 0000000..c9220d2 --- /dev/null +++ b/tests/utils/extract-narrator.test.ts @@ -0,0 +1,95 @@ +/** + * Component: Narrator Extraction Utility Tests + * Documentation: documentation/integrations/audible.md + */ + +import { describe, expect, it } from 'vitest'; +import * as cheerio from 'cheerio'; +import { extractAllNarrators } from '@/lib/utils/extract-narrator'; + +function load(html: string) { + const $ = cheerio.load(`
    ${html}
    `); + return { $, $el: $('#item') }; +} + +describe('extractAllNarrators', () => { + it('returns the single narrator name when only one searchNarrator link is present', () => { + const { $, $el } = load( + `Andy Serkis`, + ); + expect(extractAllNarrators($, $el)).toBe('Andy Serkis'); + }); + + it('joins multiple narrator names from separate searchNarrator links', () => { + const { $, $el } = load(` + Kristin Atherton, + Roy McMillan, + Clare Corbett, + Tom Bateman, + Patience Tomlinson, + Shaheen Khan + `); + expect(extractAllNarrators($, $el)).toBe( + 'Kristin Atherton, Roy McMillan, Clare Corbett, Tom Bateman, Patience Tomlinson, Shaheen Khan', + ); + }); + + it('preserves document order (downstream sorts before comparing, but order should be stable)', () => { + const { $, $el } = load(` + Zelda + Alice + Mallory + `); + expect(extractAllNarrators($, $el)).toBe('Zelda, Alice, Mallory'); + }); + + it('falls back to .narratorLabel text when no searchNarrator links exist', () => { + const { $, $el } = load( + `Narrated by: Single Narrator`, + ); + expect(extractAllNarrators($, $el)).toBe('Narrated by: Single Narrator'); + }); + + it('prefers searchNarrator links over .narratorLabel when both are present', () => { + const { $, $el } = load(` + Narrated by: ONLY ONE + First + Second + `); + expect(extractAllNarrators($, $el)).toBe('First, Second'); + }); + + it('returns empty string when neither links nor .narratorLabel exist', () => { + const { $, $el } = load(`some other content`); + expect(extractAllNarrators($, $el)).toBe(''); + }); + + it('skips empty link text and joins only non-empty names', () => { + const { $, $el } = load(` + + Bob + + Diana + `); + expect(extractAllNarrators($, $el)).toBe('Bob, Diana'); + }); + + it('trims whitespace from each captured name', () => { + const { $, $el } = load(` + Alice + + Bob + + `); + expect(extractAllNarrators($, $el)).toBe('Alice, Bob'); + }); + + it('falls back to .narratorLabel when all searchNarrator links are empty', () => { + const { $, $el } = load(` + + + Fallback Narrator + `); + expect(extractAllNarrators($, $el)).toBe('Fallback Narrator'); + }); +}); diff --git a/tests/utils/scrape-resilience.test.ts b/tests/utils/scrape-resilience.test.ts index 68b6ad1..64e1225 100644 --- a/tests/utils/scrape-resilience.test.ts +++ b/tests/utils/scrape-resilience.test.ts @@ -67,6 +67,24 @@ describe('jitteredBackoff', () => { expect(value).toBeGreaterThanOrEqual(250); expect(value).toBeLessThanOrEqual(750); }); + + it('caps the result at maxBackoffMs when the raw backoff would exceed it', () => { + // attempt=10 with base=1000 produces 2^10 * 1000 * [0.5..1.5] = 512_000..1_536_000, + // all of which exceed a 60_000ms cap. + for (let i = 0; i < 50; i++) { + const value = jitteredBackoff(10, 1000, 60_000); + expect(value).toBeLessThanOrEqual(60_000); + } + }); + + it('returns the un-capped jittered value when below the cap', () => { + // attempt=0 with base=1000 produces 500..1500, all below a 60_000ms cap. + for (let i = 0; i < 50; i++) { + const value = jitteredBackoff(0, 1000, 60_000); + expect(value).toBeGreaterThanOrEqual(500); + expect(value).toBeLessThanOrEqual(1500); + } + }); }); describe('randomDelay', () => { From 6c8ca9647d96029cee5437bf58895713a7dea207 Mon Sep 17 00:00:00 2001 From: kikootwo Date: Thu, 14 May 2026 15:33:30 -0400 Subject: [PATCH 09/12] Support language/format/publisher for Audible Expose language, formatType, and publisherName from the Audible catalog. Update audible.service to map format_type and publisher_name (and language) into the AudibleAudiobook model, update AudiobookDetailsModal to display language and format using the CSS "capitalize" class, and update documentation to list the new fields. Add unit tests to verify the mappings, details propagation, and behavior when fields are omitted. --- documentation/frontend/components.md | 2 +- documentation/integrations/audible.md | 3 ++ .../audiobooks/AudiobookDetailsModal.tsx | 4 +- src/lib/integrations/audible.service.ts | 5 ++ tests/integrations/audible.service.test.ts | 49 +++++++++++++++++++ 5 files changed, 60 insertions(+), 3 deletions(-) diff --git a/documentation/frontend/components.md b/documentation/frontend/components.md index 62f8e6f..fca5969 100644 --- a/documentation/frontend/components.md +++ b/documentation/frontend/components.md @@ -30,7 +30,7 @@ src/components/ **Audiobooks** - **AudiobookCard** ✅ - Cover, title, author, narrator, duration, request button, clickable to open details modal. Shows "Requested by [username]" when someone else has requested the book, "Requested" when current user has requested it - **AudiobookGrid** - Responsive grid (1/2/3/4 cols) -- **AudiobookDetailsModal** ✅ - Full-screen modal with comprehensive metadata (description, genres, rating, release date, narrator, request functionality). Shows requesting user's name when applicable +- **AudiobookDetailsModal** ✅ - Full-screen modal with comprehensive metadata (description, genres, rating, release date, narrator, language, format, publisher, request functionality). Shows requesting user's name when applicable **Requests** - **RequestCard** ✅ - Cover, title, author, status badge, progress bar, timestamps, action buttons (cancel, manual search, interactive search) diff --git a/documentation/integrations/audible.md b/documentation/integrations/audible.md index b7bac3b..b377b75 100644 --- a/documentation/integrations/audible.md +++ b/documentation/integrations/audible.md @@ -250,6 +250,9 @@ interface AudibleAudiobook { series?: string; seriesPart?: string; seriesAsin?: string; + language?: string; + formatType?: string; + publisherName?: string; } interface EnrichedAudibleAudiobook extends AudibleAudiobook { diff --git a/src/components/audiobooks/AudiobookDetailsModal.tsx b/src/components/audiobooks/AudiobookDetailsModal.tsx index 5bf23b8..9a7658c 100644 --- a/src/components/audiobooks/AudiobookDetailsModal.tsx +++ b/src/components/audiobooks/AudiobookDetailsModal.tsx @@ -552,7 +552,7 @@ export function AudiobookDetailsModal({ {audiobook.language && (

    Language

    -

    {audiobook.language.charAt(0).toUpperCase() + audiobook.language.slice(1)}

    +

    {audiobook.language}

    )} @@ -560,7 +560,7 @@ export function AudiobookDetailsModal({ {audiobook.formatType && (

    Format

    -

    {audiobook.formatType.charAt(0).toUpperCase() + audiobook.formatType.slice(1)}

    +

    {audiobook.formatType}

    )} diff --git a/src/lib/integrations/audible.service.ts b/src/lib/integrations/audible.service.ts index b19d721..9934171 100644 --- a/src/lib/integrations/audible.service.ts +++ b/src/lib/integrations/audible.service.ts @@ -108,6 +108,8 @@ interface CatalogProduct { runtime_length_min?: number; release_date?: string; language?: string; + format_type?: string; + publisher_name?: string; rating?: { overall_distribution?: { display_stars?: number; @@ -198,6 +200,9 @@ function mapCatalogProduct(product: CatalogProduct): AudibleAudiobook { series, seriesPart, seriesAsin, + language: product.language ?? undefined, + formatType: product.format_type ?? undefined, + publisherName: product.publisher_name ?? undefined, }; } diff --git a/tests/integrations/audible.service.test.ts b/tests/integrations/audible.service.test.ts index f006031..ea4d955 100644 --- a/tests/integrations/audible.service.test.ts +++ b/tests/integrations/audible.service.test.ts @@ -49,6 +49,8 @@ interface ProductOverrides { runtime_length_min?: number; release_date?: string; language?: string; + format_type?: string; + publisher_name?: string; rating?: { overall_distribution?: { display_stars?: number } }; category_ladders?: Array<{ ladder: Array<{ name: string }> }>; series?: Array<{ asin?: string; title?: string; sequence?: string }>; @@ -615,6 +617,47 @@ describe('AudibleService', () => { const genreSet = new Set(results[0].genres); expect(genreSet.size).toBe(5); }); + + it('maps language from catalog product', async () => { + const products = [makeProduct({ language: 'english' })]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].language).toBe('english'); + }); + + it('maps format_type to formatType from catalog product', async () => { + const products = [makeProduct({ format_type: 'unabridged' })]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].formatType).toBe('unabridged'); + }); + + it('maps publisher_name to publisherName from catalog product', async () => { + const products = [makeProduct({ publisher_name: 'Penguin Random House Audio' })]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].publisherName).toBe('Penguin Random House Audio'); + }); + + it('leaves formatType and publisherName undefined when catalog product omits them', async () => { + const products = [makeProduct()]; + apiClientMock.get.mockResolvedValue(apiResponse(makeProductsResponse(products))); + + const service = new AudibleService(); + const { results } = await service.search('test', 1); + + expect(results[0].formatType).toBeUndefined(); + expect(results[0].publisherName).toBeUndefined(); + }); }); // ------------------------------------------------------------------------- @@ -1262,6 +1305,9 @@ describe('AudibleService', () => { runtimeLengthMin: '300', genres: ['Fiction'], rating: '4.7', + language: 'english', + formatType: 'unabridged', + publisherName: 'Test Publisher', }, }); @@ -1271,6 +1317,9 @@ describe('AudibleService', () => { expect(details?.title).toBe('Audnexus Book'); expect(details?.author).toBe('Author A'); expect(details?.durationMinutes).toBe(300); + expect(details?.language).toBe('english'); + expect(details?.formatType).toBe('unabridged'); + expect(details?.publisherName).toBe('Test Publisher'); // Catalog API should NOT be called when Audnexus succeeds. expect(apiClientMock.get).not.toHaveBeenCalled(); }); From 247fe88b99a64b313ef58577c454cc4a9f69ffbe Mon Sep 17 00:00:00 2001 From: kikootwo Date: Thu, 14 May 2026 15:43:30 -0400 Subject: [PATCH 10/12] Refactor approval buttons into reusable component Extract LoadingSpinner and ApprovalActionButtons components and replace duplicated approve/search/deny button blocks with the new ApprovalActionButtons to reduce duplication and centralize behavior/styles. Remove the inline LoadingSpinner in PendingApprovalSection, add an aria-label to the details button, and update the details modal's adminActions to use ApprovalActionButtons with callbacks that handle approval/denial/search and close modals as needed. Improves DRY, maintainability, and consistency of loading state handling. --- src/app/admin/page.tsx | 176 ++++++++++++++++++++--------------------- 1 file changed, 84 insertions(+), 92 deletions(-) diff --git a/src/app/admin/page.tsx b/src/app/admin/page.tsx index de2b9cd..d54fd9e 100644 --- a/src/app/admin/page.tsx +++ b/src/app/admin/page.tsx @@ -58,6 +58,63 @@ function formatTorrentSize(bytes: number): string { return gb >= 1 ? `${gb.toFixed(1)} GB` : `${mb.toFixed(0)} MB`; } +function LoadingSpinner() { + return ( + + + + + ); +} + +interface ApprovalActionButtonsProps { + isLoading: boolean; + onApprove: () => void; + onSearch: () => void; + onDeny: () => void; +} + +function ApprovalActionButtons({ isLoading, onApprove, onSearch, onDeny }: ApprovalActionButtonsProps) { + return ( + <> + + + + + ); +} + function PendingApprovalSection({ requests }: { requests: PendingApprovalRequest[] }) { const toast = useToast(); const [loadingStates, setLoadingStates] = useState>({}); @@ -133,13 +190,6 @@ function PendingApprovalSection({ requests }: { requests: PendingApprovalRequest await mutate('/api/admin/metrics'); }; - const LoadingSpinner = () => ( - - - - - ); - return (
    {/* Section Header */} @@ -189,6 +239,7 @@ function PendingApprovalSection({ requests }: { requests: PendingApprovalRequest }} className="absolute top-2 right-2 z-10 p-1 text-gray-400 hover:text-blue-500 dark:hover:text-blue-400 transition-colors rounded-full hover:bg-gray-100 dark:hover:bg-gray-700" title="View book details" + aria-label="View book details" > @@ -336,42 +387,12 @@ function PendingApprovalSection({ requests }: { requests: PendingApprovalRequest {/* Action Buttons */}
    - - - - - + handleApproveRequest(request.id)} + onSearch={() => setSearchModalRequestId(request.id)} + onDeny={() => handleDenyRequest(request.id)} + />
    ); @@ -406,55 +427,26 @@ function PendingApprovalSection({ requests }: { requests: PendingApprovalRequest onClose={() => { setDetailsAsin(null); setDetailsRequestId(null); }} requestStatus="awaiting_approval" requestedByUsername={detailsRequest?.user.plexUsername ?? null} - adminActions={(() => { - const isLoading = loadingStates[detailsRequestId] || false; - return ( - <> - - - - - ); - })()} + adminActions={ + { + await handleApproveRequest(detailsRequestId); + setDetailsAsin(null); + setDetailsRequestId(null); + }} + onSearch={() => { + setSearchModalRequestId(detailsRequestId); + setDetailsAsin(null); + setDetailsRequestId(null); + }} + onDeny={async () => { + await handleDenyRequest(detailsRequestId); + setDetailsAsin(null); + setDetailsRequestId(null); + }} + /> + } /> )}
    • From 5e4a38a340d3798fba12a483c3cd1562ee04b4bc Mon Sep 17 00:00:00 2001 From: kikootwo Date: Thu, 14 May 2026 15:57:15 -0400 Subject: [PATCH 11/12] Normalize notification events and update grab flow Introduce a NotificationEventConfig interface and validate NOTIFICATION_EVENTS with `satisfies` for stronger typing and normalized metadata shape. Replace escaped emoji sequences with literal emoji, simplify helper functions (getEventMeta/getEventTitle) to use the typed registry, and clean up titleByRequestType typing. In download-torrent.processor: include the requesting user when setting status to downloading to avoid an extra DB query, and use that returned user to enqueue a non-blocking `request_grabbed` notification. Docs: note that `request_grabbed` notifications are opt-in for existing backends. Tests: add messageLabel rendering tests for Apprise and ntfy providers to validate emoji, label text, and type-specific titles. --- .../backend/services/notifications.md | 2 +- src/lib/constants/notification-events.ts | 50 ++++++++-------- .../processors/download-torrent.processor.ts | 41 ++++++------- tests/services/apprise.provider.test.ts | 58 +++++++++++++++++++ tests/services/ntfy.provider.test.ts | 58 +++++++++++++++++++ 5 files changed, 157 insertions(+), 52 deletions(-) diff --git a/documentation/backend/services/notifications.md b/documentation/backend/services/notifications.md index 9ad9be9..9bd8905 100644 --- a/documentation/backend/services/notifications.md +++ b/documentation/backend/services/notifications.md @@ -33,7 +33,7 @@ model NotificationBackend { |-------|---------|------------------------| | request_pending_approval | User creates request | Request needs admin approval | | request_approved | Admin approves OR auto-approval | Request approved (manual or auto) | -| request_grabbed | Torrent/NZB added to download client | Download handed off to configured download client (title resolves by type) | +| request_grabbed | Torrent/NZB added to download client | Download handed off to configured download client (title resolves by type) — **opt-in: existing backends do not auto-subscribe; enable in Settings** | | request_available | Plex/ABS scan or ebook download completes | Request available (title resolves by type) | | request_error | Download/import fails | Request failed at any stage | | issue_reported | User reports issue | User reports problem with available audiobook | diff --git a/src/lib/constants/notification-events.ts b/src/lib/constants/notification-events.ts index 37bfba4..d1e6542 100644 --- a/src/lib/constants/notification-events.ts +++ b/src/lib/constants/notification-events.ts @@ -1,4 +1,4 @@ -/** +/** * Component: Notification Event Constants * Documentation: documentation/backend/services/notifications.md * @@ -10,9 +10,9 @@ export type NotificationSeverity = 'info' | 'success' | 'error' | 'warning'; export type NotificationPriority = 'normal' | 'high'; /** - * Central registry of notification events. + * Normalized interface for event metadata. + * Each entry in NOTIFICATION_EVENTS is structurally validated against this via `satisfies`. * - * Each entry defines: * - `label`: Human-readable name shown in the UI * - `title`: Default title used in notification messages * - `titleByRequestType`: Optional map of request-type-specific titles (e.g. audiobook → "Audiobook Available") @@ -21,6 +21,17 @@ export type NotificationPriority = 'normal' | 'high'; * - `priority`: Drives notification urgency (Pushover/ntfy priority levels) * - `messageLabel`: Optional label for the `message` payload field (defaults to "Error" if omitted) */ +export interface NotificationEventConfig { + label: string; + title: string; + titleByRequestType?: Record; + emoji: string; + severity: NotificationSeverity; + priority: NotificationPriority; + messageLabel?: string; +} + +/** Central registry of notification events. */ export const NOTIFICATION_EVENTS = { request_pending_approval: { label: 'Request Pending Approval', @@ -32,7 +43,7 @@ export const NOTIFICATION_EVENTS = { request_approved: { label: 'Request Approved', title: 'Request Approved', - emoji: '\u2705', + emoji: '✅', severity: 'success' as const, priority: 'normal' as const, }, @@ -42,7 +53,7 @@ export const NOTIFICATION_EVENTS = { titleByRequestType: { audiobook: 'Audiobook Grabbed', ebook: 'Ebook Grabbed', - } as Record, + }, emoji: '\u{1F4E5}', severity: 'info' as const, priority: 'normal' as const, @@ -54,7 +65,7 @@ export const NOTIFICATION_EVENTS = { titleByRequestType: { audiobook: 'Audiobook Available', ebook: 'Ebook Available', - } as Record, + }, emoji: '\u{1F389}', severity: 'success' as const, priority: 'high' as const, @@ -62,7 +73,7 @@ export const NOTIFICATION_EVENTS = { request_error: { label: 'Request Error', title: 'Request Error', - emoji: '\u274C', + emoji: '❌', severity: 'error' as const, priority: 'high' as const, }, @@ -74,7 +85,7 @@ export const NOTIFICATION_EVENTS = { priority: 'high' as const, messageLabel: 'Reason', }, -} as const; +} satisfies Record; /** Union type of all valid notification event keys */ export type NotificationEvent = keyof typeof NOTIFICATION_EVENTS; @@ -85,24 +96,9 @@ export const NOTIFICATION_EVENT_KEYS = Object.keys(NOTIFICATION_EVENTS) as [Noti /** Metadata shape for a single notification event */ export type NotificationEventMeta = (typeof NOTIFICATION_EVENTS)[NotificationEvent]; -/** - * Normalized interface for event metadata consumed by providers. - * Broadens the `as const` literal union to make optional fields accessible. - */ -export interface NotificationEventConfig { - label: string; - title: string; - titleByRequestType?: Record; - emoji: string; - severity: NotificationSeverity; - priority: NotificationPriority; - /** Label for the `message` payload field. Defaults to "Error" in providers when absent. */ - messageLabel?: string; -} - /** Helper: get event metadata by key */ export function getEventMeta(event: NotificationEvent): NotificationEventConfig { - return NOTIFICATION_EVENTS[event] as NotificationEventConfig; + return NOTIFICATION_EVENTS[event]; } /** @@ -111,9 +107,9 @@ export function getEventMeta(event: NotificationEvent): NotificationEventConfig * returns the type-specific title. Otherwise falls back to the default `title`. */ export function getEventTitle(event: NotificationEvent, requestType?: string): string { - const meta = NOTIFICATION_EVENTS[event]; - if (requestType && 'titleByRequestType' in meta) { - const typeTitle = (meta as typeof meta & { titleByRequestType: Record }).titleByRequestType[requestType]; + const meta = getEventMeta(event); + if (requestType && meta.titleByRequestType) { + const typeTitle = meta.titleByRequestType[requestType]; if (typeTitle) return typeTitle; } return meta.title; diff --git a/src/lib/processors/download-torrent.processor.ts b/src/lib/processors/download-torrent.processor.ts index eabadfc..153da51 100644 --- a/src/lib/processors/download-torrent.processor.ts +++ b/src/lib/processors/download-torrent.processor.ts @@ -31,13 +31,16 @@ export async function processDownloadTorrent(payload: DownloadTorrentPayload): P try { // Update request status to downloading - await prisma.request.update({ + const request = await prisma.request.update({ where: { id: requestId }, data: { status: 'downloading', progress: 0, updatedAt: new Date(), }, + include: { + user: { select: { plexUsername: true } }, + }, }); // Detect protocol from result and get appropriate client @@ -103,30 +106,20 @@ export async function processDownloadTorrent(payload: DownloadTorrentPayload): P logger.info(`Created download history record: ${downloadHistory.id}`); - // Send grab notification - const requestWithUser = await prisma.request.findUnique({ - where: { id: requestId }, - include: { - user: { select: { plexUsername: true } }, - }, - }); - + // Send grab notification (non-blocking — failures here don't fail the download) const jobQueue = getJobQueueService(); - - if (requestWithUser) { - const grabMessage = `${torrent.title} via ${torrent.indexer} (${client.clientType})`; - await jobQueue.addNotificationJob( - 'request_grabbed', - requestId, - audiobook.title, - audiobook.author, - requestWithUser.user.plexUsername || 'Unknown User', - grabMessage, - requestWithUser.type - ).catch((error) => { - logger.error('Failed to queue grab notification', { error: error instanceof Error ? error.message : String(error) }); - }); - } + const grabMessage = `${torrent.title} via ${torrent.indexer} (${client.clientType})`; + await jobQueue.addNotificationJob( + 'request_grabbed', + requestId, + audiobook.title, + audiobook.author, + request.user.plexUsername || 'Unknown User', + grabMessage, + request.type + ).catch((error) => { + logger.error('Failed to queue grab notification', { error: error instanceof Error ? error.message : String(error) }); + }); // Trigger monitor download job with initial delay await jobQueue.addMonitorJob( diff --git a/tests/services/apprise.provider.test.ts b/tests/services/apprise.provider.test.ts index ac5ef8f..1d0bdea 100644 --- a/tests/services/apprise.provider.test.ts +++ b/tests/services/apprise.provider.test.ts @@ -458,6 +458,64 @@ describe('AppriseProvider', () => { }); }); + describe('messageLabel rendering by event', () => { + const basePayload = { + requestId: 'req-1', + title: 'Test Book', + author: 'Test Author', + userName: 'Test User', + timestamp: new Date('2024-01-01T00:00:00Z'), + }; + + it('renders "⚠️ Error:" with error emoji for request_error', async () => { + fetchMock.mockResolvedValue({ ok: true, text: async () => 'ok' }); + const { AppriseProvider } = await import('@/lib/services/notification'); + const provider = new AppriseProvider(); + + await provider.send( + { serverUrl: 'http://apprise:8000', urls: 'slack://token' }, + { ...basePayload, event: 'request_error', message: 'Boom' } + ); + + const body = JSON.parse(fetchMock.mock.calls[0][1].body); + expect(body.body).toContain('⚠️ Error: Boom'); + expect(body.body).not.toContain('📝'); + }); + + it('renders "📝 Reason:" with note emoji for issue_reported', async () => { + fetchMock.mockResolvedValue({ ok: true, text: async () => 'ok' }); + const { AppriseProvider } = await import('@/lib/services/notification'); + const provider = new AppriseProvider(); + + await provider.send( + { serverUrl: 'http://apprise:8000', urls: 'slack://token' }, + { ...basePayload, event: 'issue_reported', issueId: 'iss-1', message: 'Chapter 3 cuts off' } + ); + + const body = JSON.parse(fetchMock.mock.calls[0][1].body); + expect(body.body).toContain('📝 Reason: Chapter 3 cuts off'); + expect(body.body).not.toContain('⚠️'); + expect(body.body).not.toContain('Error:'); + }); + + it('renders "📝 Details:" with note emoji for request_grabbed', async () => { + fetchMock.mockResolvedValue({ ok: true, text: async () => 'ok' }); + const { AppriseProvider } = await import('@/lib/services/notification'); + const provider = new AppriseProvider(); + + await provider.send( + { serverUrl: 'http://apprise:8000', urls: 'slack://token' }, + { ...basePayload, event: 'request_grabbed', message: 'Test Book [M4B] via NZBGeek (SABnzbd)', requestType: 'audiobook' } + ); + + const body = JSON.parse(fetchMock.mock.calls[0][1].body); + expect(body.body).toContain('📝 Details: Test Book [M4B] via NZBGeek (SABnzbd)'); + expect(body.body).not.toContain('⚠️'); + expect(body.body).not.toContain('Error:'); + expect(body.title).toBe('Audiobook Grabbed'); + }); + }); + describe('integration with NotificationService.sendToBackend', () => { it('decrypts sensitive fields and sends to Apprise', async () => { fetchMock.mockResolvedValue({ diff --git a/tests/services/ntfy.provider.test.ts b/tests/services/ntfy.provider.test.ts index 366daf3..430e048 100644 --- a/tests/services/ntfy.provider.test.ts +++ b/tests/services/ntfy.provider.test.ts @@ -267,6 +267,64 @@ describe('NtfyProvider', () => { }); }); + describe('messageLabel rendering by event', () => { + const basePayload = { + requestId: 'req-1', + title: 'Test Book', + author: 'Test Author', + userName: 'Test User', + timestamp: new Date('2024-01-01T00:00:00Z'), + }; + + it('renders "⚠️ Error:" with error emoji for request_error', async () => { + fetchMock.mockResolvedValue({ ok: true, json: async () => ({ id: 'msg' }) }); + const { NtfyProvider } = await import('@/lib/services/notification'); + const provider = new NtfyProvider(); + + await provider.send( + { topic: 'audiobooks' }, + { ...basePayload, event: 'request_error', message: 'Boom' } + ); + + const body = JSON.parse(fetchMock.mock.calls[0][1].body); + expect(body.message).toContain('⚠️ Error: Boom'); + expect(body.message).not.toContain('📝'); + }); + + it('renders "📝 Reason:" with note emoji for issue_reported', async () => { + fetchMock.mockResolvedValue({ ok: true, json: async () => ({ id: 'msg' }) }); + const { NtfyProvider } = await import('@/lib/services/notification'); + const provider = new NtfyProvider(); + + await provider.send( + { topic: 'audiobooks' }, + { ...basePayload, event: 'issue_reported', issueId: 'iss-1', message: 'Chapter 3 cuts off' } + ); + + const body = JSON.parse(fetchMock.mock.calls[0][1].body); + expect(body.message).toContain('📝 Reason: Chapter 3 cuts off'); + expect(body.message).not.toContain('⚠️'); + expect(body.message).not.toContain('Error:'); + }); + + it('renders "📝 Details:" with note emoji for request_grabbed', async () => { + fetchMock.mockResolvedValue({ ok: true, json: async () => ({ id: 'msg' }) }); + const { NtfyProvider } = await import('@/lib/services/notification'); + const provider = new NtfyProvider(); + + await provider.send( + { topic: 'audiobooks' }, + { ...basePayload, event: 'request_grabbed', message: 'Test Book [M4B] via NZBGeek (SABnzbd)', requestType: 'audiobook' } + ); + + const body = JSON.parse(fetchMock.mock.calls[0][1].body); + expect(body.message).toContain('📝 Details: Test Book [M4B] via NZBGeek (SABnzbd)'); + expect(body.message).not.toContain('⚠️'); + expect(body.message).not.toContain('Error:'); + expect(body.title).toBe('Audiobook Grabbed'); + }); + }); + describe('integration with NotificationService.sendToBackend', () => { it('decrypts accessToken and sends to ntfy', async () => { fetchMock.mockResolvedValue({ From d1a980e2101cbfd8f46145970c1153b436e3f839 Mon Sep 17 00:00:00 2001 From: kikootwo Date: Thu, 14 May 2026 16:02:04 -0400 Subject: [PATCH 12/12] Enhance download-torrent test mocks Update tests/processors/download-torrent.processor.test.ts to better mock dependencies used by processDownloadTorrent. Add jobQueueMock.addNotificationJob.mockResolvedValue(undefined) to avoid unmocked job queue calls, and change prismaMock.request.update.mockResolvedValue from an empty object to include { type: 'audiobook', user: { plexUsername: 'testuser' } } in the affected test cases so the returned request shape matches code expectations. --- tests/processors/download-torrent.processor.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/processors/download-torrent.processor.test.ts b/tests/processors/download-torrent.processor.test.ts index 179b5f2..1d8d6fd 100644 --- a/tests/processors/download-torrent.processor.test.ts +++ b/tests/processors/download-torrent.processor.test.ts @@ -59,6 +59,7 @@ describe('processDownloadTorrent', () => { vi.clearAllMocks(); // Restore default implementations cleared by clearAllMocks configMock.getMany.mockResolvedValue({ prowlarr_api_key: null }); + jobQueueMock.addNotificationJob.mockResolvedValue(undefined); }); const torrentPayload = { @@ -110,7 +111,7 @@ describe('processDownloadTorrent', () => { enabled: true, category: 'readmeabook', }); - prismaMock.request.update.mockResolvedValue({}); + prismaMock.request.update.mockResolvedValue({ type: 'audiobook', user: { plexUsername: 'testuser' } }); prismaMock.downloadHistory.create.mockResolvedValue({ id: 'dh-1' }); const { processDownloadTorrent } = await import('@/lib/processors/download-torrent.processor'); @@ -141,7 +142,7 @@ describe('processDownloadTorrent', () => { enabled: true, category: 'readmeabook', }); - prismaMock.request.update.mockResolvedValue({}); + prismaMock.request.update.mockResolvedValue({ type: 'audiobook', user: { plexUsername: 'testuser' } }); prismaMock.downloadHistory.create.mockResolvedValue({ id: 'dh-2' }); const { processDownloadTorrent } = await import('@/lib/processors/download-torrent.processor'); @@ -186,7 +187,7 @@ describe('processDownloadTorrent', () => { enabled: true, category: 'readmeabook', }); - prismaMock.request.update.mockResolvedValue({}); + prismaMock.request.update.mockResolvedValue({ type: 'audiobook', user: { plexUsername: 'testuser' } }); prismaMock.downloadHistory.create.mockResolvedValue({ id: 'dh-1' }); const { processDownloadTorrent } = await import('@/lib/processors/download-torrent.processor');