From c8bfcdb6112ae6cad74f5c6b51b95839ccc6e478 Mon Sep 17 00:00:00 2001 From: kikootwo Date: Fri, 13 Mar 2026 12:03:21 -0400 Subject: [PATCH] Add admin Bulk Import feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a Bulk Import feature for admins to scan server folders, match discovered audiobook folders against Audible, review matches, and queue batch imports. What changed: - Added documentation: documentation/features/bulk-import.md and TABLEOFCONTENTS update. - Backend: SSE scan endpoint (POST /api/admin/bulk-import/scan) streams discovery and matching events; execute endpoint (POST /api/admin/bulk-import/execute) validates paths, creates/resolves audiobook & request records, and queues organize_files jobs. Both endpoints enforce admin-only access and validate allowed root directories (download_dir, media_dir, /bookdrop). - Frontend: Modal wizard and steps for folder selection, scan progress, and match review (BulkImportWizard + ScanFolderStep, ScanProgressStep, MatchReviewStep + shared types). - Utilities: bulk-import-scanner for folder discovery and ffprobe metadata extraction; shared types for scanned books/events. - UI: Added Bulk Import quick action to admin dashboard (src/app/admin/page.tsx). Key details: - Audible searches are rate-limited (≈1.5s) and matching results include library/request status checks. - Reuses existing organize_files job queue and manual-import pipeline; no new database tables introduced (state is ephemeral during the wizard). - Includes error handling, path normalization, and security checks for allowed directories. This commit wires frontend, backend, and docs together to provide an admin-only multi-step bulk import workflow. --- documentation/TABLEOFCONTENTS.md | 4 + documentation/features/bulk-import.md | 82 +++++ src/app/admin/page.tsx | 35 +- .../api/admin/bulk-import/execute/route.ts | 292 +++++++++++++++ src/app/api/admin/bulk-import/scan/route.ts | 271 ++++++++++++++ src/components/admin/BulkImportWizard.tsx | 348 ++++++++++++++++++ .../admin/bulk-import/MatchReviewStep.tsx | 344 +++++++++++++++++ .../admin/bulk-import/ScanFolderStep.tsx | 346 +++++++++++++++++ .../admin/bulk-import/ScanProgressStep.tsx | 179 +++++++++ src/components/admin/bulk-import/types.ts | 81 ++++ src/lib/utils/bulk-import-scanner.ts | 294 +++++++++++++++ 11 files changed, 2275 insertions(+), 1 deletion(-) create mode 100644 documentation/features/bulk-import.md create mode 100644 src/app/api/admin/bulk-import/execute/route.ts create mode 100644 src/app/api/admin/bulk-import/scan/route.ts create mode 100644 src/components/admin/BulkImportWizard.tsx create mode 100644 src/components/admin/bulk-import/MatchReviewStep.tsx create mode 100644 src/components/admin/bulk-import/ScanFolderStep.tsx create mode 100644 src/components/admin/bulk-import/ScanProgressStep.tsx create mode 100644 src/components/admin/bulk-import/types.ts create mode 100644 src/lib/utils/bulk-import-scanner.ts diff --git a/documentation/TABLEOFCONTENTS.md b/documentation/TABLEOFCONTENTS.md index 2375f31..ab1e724 100644 --- a/documentation/TABLEOFCONTENTS.md +++ b/documentation/TABLEOFCONTENTS.md @@ -99,6 +99,7 @@ ## Admin Features - **Dashboard (metrics, downloads, requests)** → [admin-dashboard.md](admin-dashboard.md) +- **Bulk import (scan folders, match Audible, batch import)** → [features/bulk-import.md](features/bulk-import.md) - **Jobs management UI** → [backend/services/scheduler.md](backend/services/scheduler.md) - **Request deletion (soft delete, seeding awareness)** → [admin-features/request-deletion.md](admin-features/request-deletion.md) - **Request approval system, auto-approve settings** → [admin-features/request-approval.md](admin-features/request-approval.md) @@ -167,3 +168,6 @@ **"How do Hardcover shelves work?"** → [backend/services/hardcover-sync.md](backend/services/hardcover-sync.md) **"How do I add a new shelf provider?"** → [backend/services/goodreads-sync.md](backend/services/goodreads-sync.md#adding-a-new-provider) **"How does the shelf sync core work?"** → [backend/services/goodreads-sync.md](backend/services/goodreads-sync.md#shared-sync-core) +**"How does bulk import work?"** → [features/bulk-import.md](features/bulk-import.md) +**"How do I import multiple audiobooks at once?"** → [features/bulk-import.md](features/bulk-import.md) +**"How does the bulk import scanner detect audiobooks?"** → [features/bulk-import.md](features/bulk-import.md) diff --git a/documentation/features/bulk-import.md b/documentation/features/bulk-import.md new file mode 100644 index 0000000..6b1c0f9 --- /dev/null +++ b/documentation/features/bulk-import.md @@ -0,0 +1,82 @@ +# Bulk Import Feature + +**Status:** ✅ Implemented | Admin-only | Multi-step wizard modal + +## Overview +Lets admins scan a server folder recursively, discover audiobook subfolders, match against Audible, review matches, and import selected books via the existing manual import pipeline. + +## Flow +1. **Select Folder** — Browse base folders (Downloads, Media Library, Book Drop), pick scan root +2. **Scan & Match** — Recursively discover audiobook folders (max 10 levels), read metadata via ffprobe, search Audible per book (1.5s rate limit) +3. **Review & Import** — Scrollable list with skip toggles, library status, confidence badges; Start Import queues organize_files jobs + +## Key Details +- **Access:** Admin-only, modal opened from admin dashboard Quick Actions +- **Audio detection:** Uses `AUDIO_EXTENSIONS` from `src/lib/constants/audio-formats.ts` +- **Audiobook boundary:** A folder containing audio files = one audiobook; subfolders not scanned further +- **Metadata extraction:** ffprobe reads `album` (title), `album_artist` (author), `composer` (narrator) from first audio file +- **Fallback:** If metadata tags are empty, folder name used as search term; "Low Confidence" badge shown +- **Author/narrator dedup:** Splits on `,;& ` delimiters, removes names appearing in both fields +- **Scan depth:** Max 10 levels recursion +- **Rate limiting:** 1.5s delay between Audible searches (same as existing scraping rate limit) +- **Library check:** Uses `findPlexMatch()` for ASIN-based availability detection +- **Import:** Reuses existing `organize_files` job queue (same as manual import) +- **No new database tables** — all state is ephemeral during wizard session + +## API Endpoints + +**POST /api/admin/bulk-import/scan** (SSE stream) +- Body: `{ rootPath: string }` +- Path validation: must be within download_dir, media_dir, or /bookdrop +- Streams events: `progress`, `discovery_complete`, `matching`, `book_matched`, `complete`, `error` +- Each `book_matched` event includes: folderPath, match (Audible data), inLibrary, hasActiveRequest, metadataSource + +**POST /api/admin/bulk-import/execute** +- Body: `{ imports: Array<{ folderPath: string, asin: string }> }` +- Creates audiobook records + requests, queues organize_files jobs +- Returns: `{ success, results[], summary: { total, succeeded, failed } }` + +## SSE Event Types + +| Event | Data | When | +|---|---|---| +| `progress` | `{ phase, foldersScanned, audiobooksFound, currentFolder }` | During folder discovery | +| `discovery_complete` | `{ totalFound, message }` | All folders scanned | +| `matching` | `{ current, total, folderName, searchTerm }` | Before each Audible search | +| `book_matched` | Full book result with match data | After each Audible search | +| `complete` | `{ audiobooks[], totalFound, matched, inLibrary }` | All matching done | +| `error` | `{ message }` | On failure | + +## UI States + +| State | Visual | +|---|---| +| Normal (will import) | Full opacity, blue toggle ON | +| Skipped by user | 40% opacity, gray toggle OFF | +| Already in library | 40% opacity, green "In Library" badge, toggle disabled | +| Active request exists | 40% opacity, purple "Requested" badge, toggle disabled | +| No Audible match | Red "No Match" badge, folder name shown, pre-skipped | +| Low confidence (folder name fallback) | Amber "Low Confidence" badge | + +## Files + +**Backend:** +- `src/lib/utils/bulk-import-scanner.ts` — Folder discovery + ffprobe metadata +- `src/app/api/admin/bulk-import/scan/route.ts` — SSE scan endpoint +- `src/app/api/admin/bulk-import/execute/route.ts` — Batch import endpoint + +**Frontend:** +- `src/components/admin/BulkImportWizard.tsx` — Modal orchestrator +- `src/components/admin/bulk-import/types.ts` — Shared types +- `src/components/admin/bulk-import/ScanFolderStep.tsx` — Folder browser +- `src/components/admin/bulk-import/ScanProgressStep.tsx` — Progress display +- `src/components/admin/bulk-import/MatchReviewStep.tsx` — Review list + import + +**Modified:** +- `src/app/admin/page.tsx` — Added Bulk Import quick action + modal + +## Related +- [Manual Import](manual-import.md) — Single-book import (reused pipeline) +- [File Organization](../phase3/file-organization.md) — organize_files job +- [Audible Integration](../integrations/audible.md) — Search/scraping +- [Background Jobs](../backend/services/jobs.md) — Job queue system diff --git a/src/app/admin/page.tsx b/src/app/admin/page.tsx index 845367c..a4837dc 100644 --- a/src/app/admin/page.tsx +++ b/src/app/admin/page.tsx @@ -14,6 +14,7 @@ import { RecentRequestsTable } from './components/RecentRequestsTable'; import { ToastProvider, useToast } from '@/components/ui/Toast'; import { ReportedIssuesSection } from './components/ReportedIssuesSection'; import { InteractiveTorrentSearchModal } from '@/components/requests/InteractiveTorrentSearchModal'; +import { BulkImportWizard } from '@/components/admin/BulkImportWizard'; import { TorrentResult } from '@/lib/utils/ranking-algorithm'; import { formatDistanceToNow } from 'date-fns'; import { useState } from 'react'; @@ -379,6 +380,8 @@ function PendingApprovalSection({ requests }: { requests: PendingApprovalRequest } function AdminDashboardContent() { + const [isBulkImportOpen, setIsBulkImportOpen] = useState(false); + // Fetch data with auto-refresh every 10 seconds const { data: metrics, error: metricsError } = useSWR( '/api/admin/metrics', @@ -572,7 +575,7 @@ function AdminDashboardContent() { {/* Quick Actions */} -
+
+ +
+ {/* Bulk Import Wizard Modal */} + setIsBulkImportOpen(false)} + /> + {/* Requests Awaiting Approval */} {pendingApprovalData?.requests && pendingApprovalData.requests.length > 0 && ( diff --git a/src/app/api/admin/bulk-import/execute/route.ts b/src/app/api/admin/bulk-import/execute/route.ts new file mode 100644 index 0000000..2abc2df --- /dev/null +++ b/src/app/api/admin/bulk-import/execute/route.ts @@ -0,0 +1,292 @@ +/** + * Component: Bulk Import Execute API + * Documentation: documentation/features/bulk-import.md + * + * Queues manual imports for multiple audiobooks at once. + * Reuses the same logic as the single manual import endpoint. + * Admin-only. + */ + +import { NextRequest, NextResponse } from 'next/server'; +import { requireAuth, requireAdmin, AuthenticatedRequest } from '@/lib/middleware/auth'; +import { prisma } from '@/lib/db'; +import { getJobQueueService } from '@/lib/services/job-queue.service'; +import { RMABLogger } from '@/lib/utils/logger'; +import { AUDIO_EXTENSIONS } from '@/lib/constants/audio-formats'; +import { getAudibleService } from '@/lib/integrations/audible.service'; + +const logger = RMABLogger.create('API.Admin.BulkImport.Execute'); + +const BOOKDROP_PATH = '/bookdrop'; + +/** Statuses that indicate the request is actively being worked on. */ +const ACTIVE_STATUSES = ['searching', 'downloading', 'processing', 'awaiting_import']; + +/** Statuses that can be recycled for a new manual import. */ +const RECYCLABLE_STATUSES = [ + 'failed', 'warn', 'cancelled', 'denied', 'pending', + 'awaiting_search', 'awaiting_approval', +]; + +interface ImportItem { + folderPath: string; + asin: string; +} + +interface ImportResult { + folderPath: string; + asin: string; + success: boolean; + requestId?: string; + error?: string; +} + +/** Check if a directory contains audio files. */ +async function hasAudioFiles(dirPath: string): Promise { + const fs = await import('fs/promises'); + const pathModule = await import('path'); + + try { + const children = await fs.readdir(dirPath, { withFileTypes: true }); + return children.some( + (child) => + child.isFile() && + (AUDIO_EXTENSIONS as readonly string[]).includes( + pathModule.extname(child.name).toLowerCase() + ) + ); + } catch { + return false; + } +} + +export async function POST(request: NextRequest) { + return requireAuth(request, async (req: AuthenticatedRequest) => { + return requireAdmin(req, async () => { + try { + const pathModule = await import('path'); + const fs = await import('fs/promises'); + + const body = await request.json(); + const { imports } = body as { imports: ImportItem[] }; + + if (!imports || !Array.isArray(imports) || imports.length === 0) { + return NextResponse.json( + { error: 'imports array is required and must not be empty' }, + { status: 400 } + ); + } + + // Load allowed roots + const [downloadDirConfig, mediaDirConfig] = await Promise.all([ + prisma.configuration.findUnique({ where: { key: 'download_dir' } }), + prisma.configuration.findUnique({ where: { key: 'media_dir' } }), + ]); + + const allowedRoots: string[] = []; + if (downloadDirConfig?.value) { + allowedRoots.push(pathModule.resolve(downloadDirConfig.value).replace(/\\/g, '/')); + } + if (mediaDirConfig?.value) { + allowedRoots.push(pathModule.resolve(mediaDirConfig.value).replace(/\\/g, '/')); + } + try { + const bookdropStat = await fs.stat(BOOKDROP_PATH); + if (bookdropStat.isDirectory()) { + allowedRoots.push(pathModule.resolve(BOOKDROP_PATH).replace(/\\/g, '/')); + } + } catch { + /* not mounted */ + } + + const userId = req.user!.id; + const audibleService = getAudibleService(); + const jobQueue = getJobQueueService(); + const results: ImportResult[] = []; + + for (const item of imports) { + const { folderPath, asin } = item; + + try { + // Validate path + const normalizedPath = pathModule.resolve(folderPath).replace(/\\/g, '/'); + const isAllowed = allowedRoots.some( + (root) => normalizedPath === root || normalizedPath.startsWith(root + '/') + ); + + if (!isAllowed) { + results.push({ folderPath, asin, success: false, error: 'Path outside allowed directories' }); + continue; + } + + // Verify directory exists and has audio files + try { + const stat = await fs.stat(normalizedPath); + if (!stat.isDirectory()) { + results.push({ folderPath, asin, success: false, error: 'Not a directory' }); + continue; + } + } catch { + results.push({ folderPath, asin, success: false, error: 'Directory not found' }); + continue; + } + + const hasAudio = await hasAudioFiles(normalizedPath); + if (!hasAudio) { + results.push({ folderPath, asin, success: false, error: 'No audio files' }); + continue; + } + + // Resolve or create audiobook record + let audiobookId: string; + let existingBook = await prisma.audiobook.findFirst({ + where: { audibleAsin: asin }, + }); + + if (existingBook) { + audiobookId = existingBook.id; + } else { + // Try Audible cache, then Audnexus + const cached = await prisma.audibleCache.findUnique({ where: { asin } }); + if (cached) { + const newBook = await prisma.audiobook.create({ + data: { + audibleAsin: asin, + title: cached.title, + author: cached.author, + coverArtUrl: cached.coverArtUrl, + narrator: cached.narrator, + status: 'pending', + }, + }); + audiobookId = newBook.id; + } else { + try { + const liveData = await audibleService.getAudiobookDetails(asin); + if (!liveData) { + results.push({ folderPath, asin, success: false, error: 'Audiobook not found' }); + continue; + } + const newBook = await prisma.audiobook.create({ + data: { + audibleAsin: asin, + title: liveData.title, + author: liveData.author, + coverArtUrl: liveData.coverArtUrl, + narrator: liveData.narrator, + series: liveData.series, + seriesPart: liveData.seriesPart, + seriesAsin: liveData.seriesAsin, + year: liveData.releaseDate + ? new Date(liveData.releaseDate).getFullYear() || undefined + : undefined, + status: 'pending', + }, + }); + audiobookId = newBook.id; + } catch { + results.push({ folderPath, asin, success: false, error: 'Failed to fetch audiobook details' }); + continue; + } + } + } + + // Check for existing request and recycle or create + const existingRequest = await prisma.request.findFirst({ + where: { + audiobookId, + type: 'audiobook', + deletedAt: null, + }, + orderBy: { createdAt: 'desc' }, + }); + + let requestId: string; + + if (existingRequest) { + if (ACTIVE_STATUSES.includes(existingRequest.status)) { + results.push({ folderPath, asin, success: false, error: 'Already being processed' }); + continue; + } + + if ( + RECYCLABLE_STATUSES.includes(existingRequest.status) || + existingRequest.status === 'downloaded' || + existingRequest.status === 'available' + ) { + await prisma.request.update({ + where: { id: existingRequest.id }, + data: { + status: 'processing', + progress: 100, + errorMessage: null, + importAttempts: 0, + updatedAt: new Date(), + }, + }); + requestId = existingRequest.id; + } else { + const newReq = await prisma.request.create({ + data: { + userId, + audiobookId, + type: 'audiobook', + status: 'processing', + progress: 100, + }, + }); + requestId = newReq.id; + } + } else { + const newReq = await prisma.request.create({ + data: { + userId, + audiobookId, + type: 'audiobook', + status: 'processing', + progress: 100, + }, + }); + requestId = newReq.id; + } + + // Queue organize_files job + await jobQueue.addOrganizeJob(requestId, audiobookId, normalizedPath); + + results.push({ folderPath, asin, success: true, requestId }); + logger.info(`Bulk import queued: asin=${asin}, path=${normalizedPath}, request=${requestId}`); + } catch (itemError) { + logger.error(`Bulk import item failed: asin=${asin}, path=${folderPath}`, { + error: itemError instanceof Error ? itemError.message : String(itemError), + }); + results.push({ + folderPath, + asin, + success: false, + error: itemError instanceof Error ? itemError.message : 'Import failed', + }); + } + } + + const succeeded = results.filter((r) => r.success).length; + const failed = results.filter((r) => !r.success).length; + + logger.info(`Bulk import execute complete: ${succeeded} queued, ${failed} failed`); + + return NextResponse.json({ + success: true, + results, + summary: { total: results.length, succeeded, failed }, + }); + } catch (error) { + logger.error('Bulk import execute failed', { + error: error instanceof Error ? error.message : String(error), + }); + return NextResponse.json( + { error: error instanceof Error ? error.message : 'Bulk import failed' }, + { status: 500 } + ); + } + }); + }); +} diff --git a/src/app/api/admin/bulk-import/scan/route.ts b/src/app/api/admin/bulk-import/scan/route.ts new file mode 100644 index 0000000..36bf408 --- /dev/null +++ b/src/app/api/admin/bulk-import/scan/route.ts @@ -0,0 +1,271 @@ +/** + * Component: Bulk Import Scan API (SSE) + * Documentation: documentation/features/bulk-import.md + * + * Streams audiobook discovery and Audible matching results via Server-Sent Events. + * Admin-only. Validates path is within allowed roots. + */ + +import { NextRequest, NextResponse } from 'next/server'; +import { requireAuth, requireAdmin, AuthenticatedRequest } from '@/lib/middleware/auth'; +import { prisma } from '@/lib/db'; +import { RMABLogger } from '@/lib/utils/logger'; +import { discoverAudiobooks } from '@/lib/utils/bulk-import-scanner'; +import { getAudibleService } from '@/lib/integrations/audible.service'; +import { findPlexMatch } from '@/lib/utils/audiobook-matcher'; + +const logger = RMABLogger.create('API.Admin.BulkImport.Scan'); + +const BOOKDROP_PATH = '/bookdrop'; +const AUDIBLE_SEARCH_DELAY_MS = 1500; + +/** Load allowed root directories from configuration. */ +async function getAllowedRoots(): Promise { + const pathModule = await import('path'); + const fs = await import('fs/promises'); + + const [downloadDirConfig, mediaDirConfig] = await Promise.all([ + prisma.configuration.findUnique({ where: { key: 'download_dir' } }), + prisma.configuration.findUnique({ where: { key: 'media_dir' } }), + ]); + + const roots: string[] = []; + if (downloadDirConfig?.value) { + roots.push(pathModule.resolve(downloadDirConfig.value).replace(/\\/g, '/')); + } + if (mediaDirConfig?.value) { + roots.push(pathModule.resolve(mediaDirConfig.value).replace(/\\/g, '/')); + } + try { + const stat = await fs.stat(BOOKDROP_PATH); + if (stat.isDirectory()) { + roots.push(pathModule.resolve(BOOKDROP_PATH).replace(/\\/g, '/')); + } + } catch { + /* not mounted */ + } + + return roots; +} + +/** Check if a path is within allowed roots. */ +function isPathAllowed(normalizedPath: string, roots: string[]): boolean { + return roots.some( + (root) => normalizedPath === root || normalizedPath.startsWith(root + '/') + ); +} + +/** Delay helper for rate limiting. */ +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function POST(request: NextRequest) { + return requireAuth(request, async (req: AuthenticatedRequest) => { + return requireAdmin(req, async () => { + const pathModule = await import('path'); + const fs = await import('fs/promises'); + + let body: any; + try { + body = await request.json(); + } catch { + return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 }); + } + + const { rootPath } = body; + if (!rootPath) { + return NextResponse.json({ error: 'rootPath is required' }, { status: 400 }); + } + + // Validate path + const allowedRoots = await getAllowedRoots(); + const normalizedPath = pathModule.resolve(rootPath).replace(/\\/g, '/'); + + if (!isPathAllowed(normalizedPath, allowedRoots)) { + return NextResponse.json( + { error: 'Access denied: path outside allowed directories' }, + { status: 403 } + ); + } + + // Verify directory exists + try { + const stat = await fs.stat(normalizedPath); + if (!stat.isDirectory()) { + return NextResponse.json({ error: 'Path is not a directory' }, { status: 400 }); + } + } catch { + return NextResponse.json({ error: 'Directory not found' }, { status: 404 }); + } + + logger.info(`Bulk import scan started: ${normalizedPath}`); + + // Create SSE stream + const encoder = new TextEncoder(); + const abortController = new AbortController(); + + const stream = new ReadableStream({ + async start(controller) { + const send = (event: string, data: any) => { + try { + controller.enqueue( + encoder.encode(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`) + ); + } catch { + /* stream closed */ + } + }; + + try { + // Phase 1: Discover audiobook folders + const audiobooks = await discoverAudiobooks( + normalizedPath, + (progress) => { + send('progress', progress); + }, + abortController.signal + ); + + if (audiobooks.length === 0) { + send('complete', { audiobooks: [], message: 'No audiobooks found' }); + controller.close(); + return; + } + + send('discovery_complete', { + totalFound: audiobooks.length, + message: `Found ${audiobooks.length} audiobook folders`, + }); + + // Phase 2: Match each audiobook against Audible + const audibleService = getAudibleService(); + const results: any[] = []; + + for (let i = 0; i < audiobooks.length; i++) { + if (abortController.signal.aborted) break; + + const book = audiobooks[i]; + + send('matching', { + current: i + 1, + total: audiobooks.length, + folderName: book.folderName, + searchTerm: book.searchTerm, + }); + + let match: any = null; + let inLibrary = false; + let hasActiveRequest = false; + + try { + const searchResult = await audibleService.search(book.searchTerm); + + if (searchResult.results.length > 0) { + match = searchResult.results[0]; + + // Check library availability + const plexMatch = await findPlexMatch({ + asin: match.asin, + title: match.title, + author: match.author, + narrator: match.narrator, + }); + inLibrary = plexMatch !== null; + + // Check for active requests + if (!inLibrary) { + const activeRequest = await prisma.request.findFirst({ + where: { + audiobook: { audibleAsin: match.asin }, + type: 'audiobook', + status: { + in: [ + 'pending', 'searching', 'downloading', 'processing', + 'awaiting_search', 'awaiting_import', 'awaiting_approval', + 'downloaded', 'available', + ], + }, + deletedAt: null, + }, + }); + hasActiveRequest = activeRequest !== null; + } + } + } catch (searchError) { + logger.warn( + `Audible search failed for "${book.searchTerm}": ${ + searchError instanceof Error ? searchError.message : String(searchError) + }` + ); + } + + const result = { + index: i, + folderPath: book.folderPath, + folderName: book.folderName, + relativePath: book.relativePath, + audioFileCount: book.audioFileCount, + totalSizeBytes: book.totalSizeBytes, + metadataSource: book.metadataSource, + searchTerm: book.searchTerm, + match: match + ? { + asin: match.asin, + title: match.title, + author: match.author, + narrator: match.narrator, + coverArtUrl: match.coverArtUrl, + durationMinutes: match.durationMinutes, + } + : null, + inLibrary, + hasActiveRequest, + }; + + results.push(result); + send('book_matched', result); + + // Rate limit: wait between Audible searches (except after last) + if (i < audiobooks.length - 1) { + await delay(AUDIBLE_SEARCH_DELAY_MS); + } + } + + send('complete', { + totalFound: results.length, + matched: results.filter((r) => r.match !== null).length, + inLibrary: results.filter((r) => r.inLibrary).length, + }); + } catch (error) { + logger.error('Bulk import scan failed', { + error: error instanceof Error ? error.message : String(error), + }); + send('error', { + message: error instanceof Error ? error.message : 'Scan failed', + }); + } finally { + try { + controller.close(); + } catch { + /* already closed */ + } + } + }, + cancel() { + abortController.abort(); + }, + }); + + // Cast to NextResponse: SSE streams require raw Response constructor, + // but requireAdmin types expect NextResponse. The Response is valid at runtime. + return new Response(stream, { + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }, + }) as unknown as NextResponse; + }); + }); +} diff --git a/src/components/admin/BulkImportWizard.tsx b/src/components/admin/BulkImportWizard.tsx new file mode 100644 index 0000000..bb6bc2b --- /dev/null +++ b/src/components/admin/BulkImportWizard.tsx @@ -0,0 +1,348 @@ +/** + * Component: Bulk Import Wizard + * Documentation: documentation/features/bulk-import.md + * + * Multi-step modal wizard for bulk importing audiobooks from server folders. + * Step 1: Select root folder to scan. + * Step 2: Scanning/matching progress. + * Step 3: Review matches and start import. + */ + +'use client'; + +import React, { useState, useCallback, useRef } from 'react'; +import { createPortal } from 'react-dom'; +import { XMarkIcon, FolderArrowDownIcon } from '@heroicons/react/24/outline'; +import { ScanFolderStep } from './bulk-import/ScanFolderStep'; +import { ScanProgressStep } from './bulk-import/ScanProgressStep'; +import { MatchReviewStep } from './bulk-import/MatchReviewStep'; +import { WizardStep, ScannedBook, ScanProgressEvent, MatchingProgressEvent } from './bulk-import/types'; +import { fetchWithAuth } from '@/lib/utils/api'; + +interface BulkImportWizardProps { + isOpen: boolean; + onClose: () => void; +} + +const STEP_LABELS: Record = { + select_folder: 'Select Folder', + scanning: 'Scanning', + review: 'Review & Import', +}; + +const STEP_ORDER: WizardStep[] = ['select_folder', 'scanning', 'review']; + +export function BulkImportWizard({ isOpen, onClose }: BulkImportWizardProps) { + const [step, setStep] = useState('select_folder'); + const [selectedRootPath, setSelectedRootPath] = useState(null); + + // Scanning state + const [scanProgress, setScanProgress] = useState(null); + const [matchingProgress, setMatchingProgress] = useState(null); + const [scanPhase, setScanPhase] = useState<'discovering' | 'matching' | 'idle'>('idle'); + const abortRef = useRef(null); + + // Results state + const [scannedBooks, setScannedBooks] = useState([]); + const [scanError, setScanError] = useState(null); + + // Import state + const [isImporting, setIsImporting] = useState(false); + const [importResults, setImportResults] = useState(null); + + const resetWizard = useCallback(() => { + setStep('select_folder'); + setSelectedRootPath(null); + setScanProgress(null); + setMatchingProgress(null); + setScanPhase('idle'); + setScannedBooks([]); + setScanError(null); + setIsImporting(false); + setImportResults(null); + if (abortRef.current) { + abortRef.current.abort(); + abortRef.current = null; + } + }, []); + + const handleClose = useCallback(() => { + if (abortRef.current) { + abortRef.current.abort(); + abortRef.current = null; + } + resetWizard(); + onClose(); + }, [onClose, resetWizard]); + + const handleFolderSelected = useCallback(async (rootPath: string) => { + setSelectedRootPath(rootPath); + setStep('scanning'); + setScanPhase('discovering'); + setScanError(null); + setScannedBooks([]); + + const controller = new AbortController(); + abortRef.current = controller; + + try { + const response = await fetchWithAuth('/api/admin/bulk-import/scan', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ rootPath }), + signal: controller.signal, + }); + + if (!response.ok) { + const errData = await response.json().catch(() => ({ error: 'Scan failed' })); + throw new Error(errData.error || 'Scan failed'); + } + + const reader = response.body?.getReader(); + if (!reader) throw new Error('No response stream'); + + const decoder = new TextDecoder(); + let buffer = ''; + let eventType = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + // Parse SSE events from buffer + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + + for (const line of lines) { + if (line.startsWith('event: ')) { + eventType = line.slice(7).trim(); + } else if (line.startsWith('data: ') && eventType) { + try { + const data = JSON.parse(line.slice(6)); + handleSSEEvent(eventType, data); + } catch { + /* ignore parse errors */ + } + eventType = ''; + } + } + } + } catch (error) { + if (controller.signal.aborted) return; + setScanError(error instanceof Error ? error.message : 'Scan failed'); + setScanPhase('idle'); + } + }, []); + + const handleSSEEvent = useCallback((event: string, data: any) => { + switch (event) { + case 'progress': + setScanProgress(data); + break; + + case 'discovery_complete': + setScanPhase('matching'); + break; + + case 'matching': + setMatchingProgress(data); + break; + + case 'book_matched': { + const book: ScannedBook = { + ...data, + skipped: data.inLibrary || data.hasActiveRequest || data.match === null, + }; + setScannedBooks((prev) => [...prev, book]); + break; + } + + case 'complete': + setScanPhase('idle'); + setStep('review'); + break; + + case 'error': + setScanError(data.message || 'Scan failed'); + setScanPhase('idle'); + break; + } + }, []); + + const handleCancelScan = useCallback(() => { + if (abortRef.current) { + abortRef.current.abort(); + abortRef.current = null; + } + setScanPhase('idle'); + setStep('select_folder'); + }, []); + + const handleToggleSkip = useCallback((index: number) => { + setScannedBooks((prev) => + prev.map((book) => + book.index === index ? { ...book, skipped: !book.skipped } : book + ) + ); + }, []); + + const handleStartImport = useCallback(async () => { + const booksToImport = scannedBooks.filter( + (b) => !b.skipped && b.match !== null + ); + + if (booksToImport.length === 0) return; + + setIsImporting(true); + + try { + const response = await fetchWithAuth('/api/admin/bulk-import/execute', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + imports: booksToImport.map((b) => ({ + folderPath: b.folderPath, + asin: b.match!.asin, + })), + }), + }); + + const data = await response.json(); + + if (!response.ok) { + throw new Error(data.error || 'Import failed'); + } + + setImportResults(data); + } catch (error) { + setImportResults({ + success: false, + error: error instanceof Error ? error.message : 'Import failed', + }); + } finally { + setIsImporting(false); + } + }, [scannedBooks]); + + const handleBackToFolderSelect = useCallback(() => { + setStep('select_folder'); + setScanError(null); + setScannedBooks([]); + setScanPhase('idle'); + }, []); + + if (!isOpen) return null; + + const currentStepIndex = STEP_ORDER.indexOf(step); + + const modalContent = ( +
+
e.stopPropagation()} + > + {/* Header */} +
+
+ +

+ Bulk Import +

+
+ +
+ + {/* Step Indicator */} +
+ {STEP_ORDER.map((s, i) => ( + + {i > 0 && ( +
+ )} +
+
+ {i < currentStepIndex ? ( + + + + ) : ( + i + 1 + )} +
+ +
+ + ))} +
+ + {/* Content */} +
+ {step === 'select_folder' && ( + + )} + + {step === 'scanning' && ( + selectedRootPath && handleFolderSelected(selectedRootPath)} + onBack={handleBackToFolderSelect} + /> + )} + + {step === 'review' && ( + + )} +
+
+
+ ); + + return createPortal(modalContent, document.body); +} diff --git a/src/components/admin/bulk-import/MatchReviewStep.tsx b/src/components/admin/bulk-import/MatchReviewStep.tsx new file mode 100644 index 0000000..5bce97c --- /dev/null +++ b/src/components/admin/bulk-import/MatchReviewStep.tsx @@ -0,0 +1,344 @@ +/** + * Component: Bulk Import - Match Review Step + * Documentation: documentation/features/bulk-import.md + * + * Scrollable list of discovered audiobooks with Audible matches, + * skip toggles, library status badges, and import controls. + */ + +'use client'; + +import React from 'react'; +import { + ArrowLeftIcon, + CheckCircleIcon, + ExclamationTriangleIcon, + MusicalNoteIcon, + XCircleIcon, +} from '@heroicons/react/24/outline'; +import { CheckCircleIcon as CheckCircleSolid } from '@heroicons/react/24/solid'; +import { ScannedBook, formatBytes } from './types'; + +interface MatchReviewStepProps { + books: ScannedBook[]; + onToggleSkip: (index: number) => void; + onStartImport: () => void; + isImporting: boolean; + importResults: any; + onClose: () => void; + onBack: () => void; +} + +function BookRow({ + book, + onToggleSkip, +}: { + book: ScannedBook; + onToggleSkip: () => void; +}) { + const isDisabled = book.inLibrary || book.hasActiveRequest; + const isSkipped = book.skipped; + const hasMatch = book.match !== null; + const isLowConfidence = book.metadataSource === 'file_name'; + + return ( +
+ {/* Cover Art */} +
+ {hasMatch && book.match!.coverArtUrl ? ( + /* eslint-disable-next-line @next/next/no-img-element */ + {book.match!.title} { + (e.target as HTMLImageElement).src = '/placeholder_cover.svg'; + }} + /> + ) : ( +
+ +
+ )} +
+ + {/* Book Info */} +
+ {hasMatch ? ( + <> +
+

+ {book.match!.title} +

+ {isLowConfidence && ( + + Low Confidence + + )} +
+

+ {book.match!.author} + {book.match!.narrator && ( + + {' '}· {book.match!.narrator} + + )} +

+ + ) : ( + <> +
+

+ {book.folderName} +

+ + No Match + +
+

+ Could not find this title on Audible +

+ + )} +

+ {book.relativePath} +

+
+ + {/* Badges */} +
+ {/* Audio file count */} + + + {book.audioFileCount} + + + {/* Status badges */} + {book.inLibrary && ( + + + In Library + + )} + {book.hasActiveRequest && !book.inLibrary && ( + + Requested + + )} +
+ + {/* Skip Toggle */} + +
+ ); +} + +export function MatchReviewStep({ + books, + onToggleSkip, + onStartImport, + isImporting, + importResults, + onClose, + onBack, +}: MatchReviewStepProps) { + const toImport = books.filter((b) => !b.skipped && b.match !== null); + const skippedCount = books.filter((b) => b.skipped).length; + const inLibraryCount = books.filter((b) => b.inLibrary).length; + const noMatchCount = books.filter((b) => b.match === null).length; + const matchedCount = books.filter((b) => b.match !== null).length; + + // Import completed state + if (importResults) { + const succeeded = importResults.summary?.succeeded || 0; + const failed = importResults.summary?.failed || 0; + + return ( +
+ {importResults.success !== false ? ( + <> + +

+ Import Started +

+

+ {succeeded} audiobook{succeeded !== 1 ? 's' : ''} queued for import. +

+ {failed > 0 && ( +

+ {failed} book{failed !== 1 ? 's' : ''} could not be queued. +

+ )} +

+ Files will be organized, tagged, and imported into your library. Check the admin + dashboard for progress. +

+ + + ) : ( + <> + +

+ Import Failed +

+

+ {importResults.error || 'An unexpected error occurred'} +

+ + + )} +
+ ); + } + + // Empty state (no audiobooks found) + if (books.length === 0) { + return ( +
+ +

+ No Audiobooks Found +

+

+ The selected folder does not contain any folders with audio files. Try selecting a + different folder. +

+ +
+ ); + } + + return ( +
+ {/* Summary header */} +
+
+ + {books.length} discovered + + · + + {matchedCount} matched + + {noMatchCount > 0 && ( + <> + · + + {noMatchCount} unmatched + + + )} + {inLibraryCount > 0 && ( + <> + · + + {inLibraryCount} in library + + + )} +
+
+ + {/* Scrollable book list */} +
+ {books.map((book) => ( + onToggleSkip(book.index)} + /> + ))} +
+ + {/* Import footer */} +
+ + +
+ + + {toImport.length} + {' '} + book{toImport.length !== 1 ? 's' : ''} to import + {skippedCount > 0 && ( + + {' '}({skippedCount} skipped) + + )} + + + +
+
+
+ ); +} diff --git a/src/components/admin/bulk-import/ScanFolderStep.tsx b/src/components/admin/bulk-import/ScanFolderStep.tsx new file mode 100644 index 0000000..fb63645 --- /dev/null +++ b/src/components/admin/bulk-import/ScanFolderStep.tsx @@ -0,0 +1,346 @@ +/** + * Component: Bulk Import - Folder Selection Step + * Documentation: documentation/features/bulk-import.md + * + * Filesystem browser for selecting a root folder to scan for audiobooks. + * Adapted from the manual import BrowsePhase patterns. + * Any folder is selectable (not just audio-containing folders). + */ + +'use client'; + +import React, { useState, useEffect, useCallback } from 'react'; +import { + FolderIcon, + FolderOpenIcon, + FolderArrowDownIcon, + InboxArrowDownIcon, + HomeIcon, + ChevronRightIcon, + ArrowLeftIcon, + MusicalNoteIcon, + ExclamationTriangleIcon, + ArrowPathIcon, + MagnifyingGlassIcon, +} from '@heroicons/react/24/outline'; +import { fetchWithAuth } from '@/lib/utils/api'; +import { RootEntry, DirectoryEntry, formatBytes } from './types'; + +function SkeletonRow() { + return ( +
+
+
+
+
+
+
+ ); +} + +interface ScanFolderStepProps { + onFolderSelected: (rootPath: string) => void; +} + +export function ScanFolderStep({ onFolderSelected }: ScanFolderStepProps) { + const [roots, setRoots] = useState([]); + const [currentPath, setCurrentPath] = useState(null); + const [entries, setEntries] = useState([]); + const [pathHistory, setPathHistory] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(null); + const [hoveredFolder, setHoveredFolder] = useState(null); + + useEffect(() => { + fetchRoots(); + }, []); + + const fetchRoots = async () => { + setIsLoading(true); + setError(null); + try { + const res = await fetchWithAuth('/api/admin/filesystem/browse'); + if (!res.ok) { + const data = await res.json().catch(() => ({ error: 'Failed to load' })); + throw new Error(data.error || 'Failed to load directories'); + } + const data = await res.json(); + setRoots(data.roots || []); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to load directories'); + } finally { + setIsLoading(false); + } + }; + + const fetchDirectory = useCallback(async (dirPath: string) => { + setIsLoading(true); + setError(null); + try { + const res = await fetchWithAuth( + `/api/admin/filesystem/browse?path=${encodeURIComponent(dirPath)}` + ); + if (!res.ok) { + const data = await res.json().catch(() => ({ error: 'Failed to load' })); + throw new Error(data.error || 'Failed to browse directory'); + } + const data = await res.json(); + setEntries(data.entries || []); + setCurrentPath(data.path || dirPath); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to browse directory'); + } finally { + setIsLoading(false); + } + }, []); + + const navigateInto = (dirPath: string) => { + if (currentPath) { + setPathHistory((prev) => [...prev, currentPath]); + } + fetchDirectory(dirPath); + }; + + const navigateBack = () => { + if (pathHistory.length > 0) { + const prevPath = pathHistory[pathHistory.length - 1]; + setPathHistory((prev) => prev.slice(0, -1)); + fetchDirectory(prevPath); + } else { + setCurrentPath(null); + setEntries([]); + } + }; + + const navigateToRoot = () => { + setCurrentPath(null); + setEntries([]); + setPathHistory([]); + }; + + const navigateToBreadcrumb = (index: number) => { + if (!currentPath) return; + const allPaths = [...pathHistory, currentPath]; + const targetPath = allPaths[index]; + if (targetPath) { + setPathHistory(allPaths.slice(0, index)); + fetchDirectory(targetPath); + } else { + navigateToRoot(); + } + }; + + // Build breadcrumb segments + const breadcrumbs = (() => { + if (!currentPath) return []; + const allPaths = [...pathHistory, currentPath]; + return allPaths.map((p) => { + const parts = p.replace(/\\/g, '/').split('/'); + return parts[parts.length - 1] || p; + }); + })(); + + const visibleBreadcrumbs = (() => { + if (breadcrumbs.length <= 3) return breadcrumbs.map((b, i) => ({ label: b, index: i })); + return [ + { label: breadcrumbs[0], index: 0 }, + { label: '...', index: -1 }, + { label: breadcrumbs[breadcrumbs.length - 1], index: breadcrumbs.length - 1 }, + ]; + })(); + + // Count total audio files and subfolders in current listing + const totalSubfolders = entries.reduce((sum, e) => sum + e.subfolderCount, 0); + const totalAudioInChildren = entries.reduce((sum, e) => sum + e.audioFileCount, 0); + + return ( +
+ {/* Breadcrumb bar */} + {currentPath && ( +
+ + {visibleBreadcrumbs.map((crumb, i) => ( + + + {crumb.index === -1 ? ( + ... + ) : i === visibleBreadcrumbs.length - 1 ? ( + + {crumb.label} + + ) : ( + + )} + + ))} +
+ )} + + {/* Listing */} +
+ {/* Loading */} + {isLoading && ( +
+ {[...Array(5)].map((_, i) => ( + + ))} +
+ )} + + {/* Error */} + {error && !isLoading && ( +
+ +

{error}

+ +
+ )} + + {/* Root view */} + {!currentPath && !isLoading && !error && ( +
+

+ Select a folder to scan for audiobooks. All subfolders will be searched recursively. +

+
+ {roots.map((root) => ( + + ))} +
+
+ )} + + {/* Directory listing */} + {currentPath && !isLoading && !error && entries.length > 0 && ( +
+ {entries.map((entry) => { + const hasAudio = entry.audioFileCount > 0; + const isHovered = hoveredFolder === entry.name; + + return ( + + ); + })} +
+ )} + + {/* Empty state */} + {currentPath && !isLoading && !error && entries.length === 0 && ( +
+ +

This folder is empty

+ +
+ )} +
+ + {/* Footer: Scan this folder */} + {currentPath && !isLoading && ( +
+
+

{currentPath}

+ {entries.length > 0 && ( +

+ {entries.length} subfolder{entries.length !== 1 ? 's' : ''} + {totalAudioInChildren > 0 && ( + · {totalAudioInChildren} audio files visible + )} +

+ )} +
+ +
+ )} +
+ ); +} diff --git a/src/components/admin/bulk-import/ScanProgressStep.tsx b/src/components/admin/bulk-import/ScanProgressStep.tsx new file mode 100644 index 0000000..169f44b --- /dev/null +++ b/src/components/admin/bulk-import/ScanProgressStep.tsx @@ -0,0 +1,179 @@ +/** + * Component: Bulk Import - Scan Progress Step + * Documentation: documentation/features/bulk-import.md + * + * Displays progress during folder discovery and Audible matching phases. + * Shows animated indicators, counts, and cancel/retry controls. + */ + +'use client'; + +import React from 'react'; +import { + FolderIcon, + ExclamationTriangleIcon, + ArrowPathIcon, + ArrowLeftIcon, + XMarkIcon, +} from '@heroicons/react/24/outline'; +import { ScanProgressEvent, MatchingProgressEvent } from './types'; + +interface ScanProgressStepProps { + scanProgress: ScanProgressEvent | null; + matchingProgress: MatchingProgressEvent | null; + scanPhase: 'discovering' | 'matching' | 'idle'; + error: string | null; + booksFound: number; + onCancel: () => void; + onRetry: () => void; + onBack: () => void; +} + +export function ScanProgressStep({ + scanProgress, + matchingProgress, + scanPhase, + error, + booksFound, + onCancel, + onRetry, + onBack, +}: ScanProgressStepProps) { + // Error state + if (error) { + return ( +
+ +

+ Scan Failed +

+

+ {error} +

+
+ + +
+
+ ); + } + + const matchPercent = matchingProgress + ? Math.round((matchingProgress.current / matchingProgress.total) * 100) + : 0; + + return ( +
+ {/* Animated icon */} +
+
+ +
+
+
+ + {/* Phase-specific content */} + {scanPhase === 'discovering' && ( + <> +

+ Scanning Folders +

+

+ Searching for folders containing audiobook files... +

+ + {scanProgress && ( +
+
+
+ {scanProgress.foldersScanned} +
+
+ Folders Scanned +
+
+
+
+
+ {scanProgress.audiobooksFound} +
+
+ Audiobooks Found +
+
+
+ )} + + {scanProgress?.currentFolder && ( +

+ {scanProgress.currentFolder} +

+ )} + + )} + + {scanPhase === 'matching' && ( + <> +

+ Matching Against Audible +

+

+ Searching Audible for each discovered audiobook... +

+ + {matchingProgress && ( + <> + {/* Progress bar */} +
+
+
+
+
+ +
+ {matchingProgress.current} / {matchingProgress.total} +
+ + {matchingProgress.folderName && ( +

+ {matchingProgress.folderName} +

+ )} + + {/* Books matched so far count */} + {booksFound > 0 && ( +

+ {booksFound} book{booksFound !== 1 ? 's' : ''} matched so far +

+ )} + + )} + + )} + + {/* Cancel button */} + +
+ ); +} diff --git a/src/components/admin/bulk-import/types.ts b/src/components/admin/bulk-import/types.ts new file mode 100644 index 0000000..f441b6b --- /dev/null +++ b/src/components/admin/bulk-import/types.ts @@ -0,0 +1,81 @@ +/** + * Component: Bulk Import Shared Types + * Documentation: documentation/features/bulk-import.md + */ + +/** Root directory entry from the filesystem browse API. */ +export interface RootEntry { + name: string; + path: string; + icon: string; +} + +/** Directory entry from the filesystem browse API. */ +export interface DirectoryEntry { + name: string; + type: 'directory'; + audioFileCount: number; + subfolderCount: number; + totalSize: number; +} + +/** Audible match data for a discovered audiobook. */ +export interface AudibleMatch { + asin: string; + title: string; + author: string; + narrator?: string; + coverArtUrl?: string; + durationMinutes?: number; +} + +/** A scanned audiobook result with its Audible match status. */ +export interface ScannedBook { + index: number; + folderPath: string; + folderName: string; + relativePath: string; + audioFileCount: number; + totalSizeBytes: number; + metadataSource: 'tags' | 'file_name'; + searchTerm: string; + match: AudibleMatch | null; + inLibrary: boolean; + hasActiveRequest: boolean; + /** User toggle: true = skip this book during import. */ + skipped: boolean; +} + +/** Progress event from the SSE scan stream. */ +export interface ScanProgressEvent { + phase: 'discovering' | 'reading_metadata'; + foldersScanned: number; + audiobooksFound: number; + currentFolder?: string; +} + +/** Matching progress event from the SSE scan stream. */ +export interface MatchingProgressEvent { + current: number; + total: number; + folderName: string; + searchTerm: string; +} + +/** Discovery complete event from the SSE scan stream. */ +export interface DiscoveryCompleteEvent { + totalFound: number; + message: string; +} + +/** Wizard step identifiers. */ +export type WizardStep = 'select_folder' | 'scanning' | 'review'; + +/** Format bytes into a human-readable string. */ +export function formatBytes(bytes: number): string { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`; +} diff --git a/src/lib/utils/bulk-import-scanner.ts b/src/lib/utils/bulk-import-scanner.ts new file mode 100644 index 0000000..00ff058 --- /dev/null +++ b/src/lib/utils/bulk-import-scanner.ts @@ -0,0 +1,294 @@ +/** + * Component: Bulk Import Scanner Utility + * Documentation: documentation/features/bulk-import.md + * + * Recursively discovers audiobook folders, reads embedded metadata via ffprobe, + * and prepares search terms for Audible matching. Used by the bulk import API. + */ + +import { exec } from 'child_process'; +import { promisify } from 'util'; +import path from 'path'; +import fs from 'fs/promises'; +import { AUDIO_EXTENSIONS } from '../constants/audio-formats'; + +const execPromise = promisify(exec); + +/** Maximum recursion depth for folder scanning. */ +export const MAX_SCAN_DEPTH = 10; + +/** Metadata extracted from an audio file via ffprobe. */ +export interface AudioFileMetadata { + title?: string; // From 'album' tag (book title) + author?: string; // From 'album_artist' tag + narrator?: string; // From 'composer' tag + contributingArtists?: string; // From 'artist' tag (contributing artists) + trackTitle?: string; // From 'title' tag (chapter/track name) +} + +/** A discovered audiobook folder with its metadata and file info. */ +export interface DiscoveredAudiobook { + folderPath: string; + folderName: string; + relativePath: string; // Relative to scan root + audioFileCount: number; + totalSizeBytes: number; + metadata: AudioFileMetadata; + searchTerm: string; // Constructed search query for Audible + metadataSource: 'tags' | 'file_name'; // Where the search term came from +} + +/** Progress callback for streaming updates to the caller. */ +export interface ScanProgress { + phase: 'discovering' | 'reading_metadata'; + foldersScanned: number; + audiobooksFound: number; + currentFolder?: string; +} + +/** + * Check if a file has a supported audio extension. + */ +function isAudioFile(filename: string): boolean { + const ext = path.extname(filename).toLowerCase(); + return (AUDIO_EXTENSIONS as readonly string[]).includes(ext); +} + +/** + * Read audio metadata from a file using ffprobe. + * Extracts album, album_artist, composer, and title tags. + * Returns empty metadata on any failure (non-blocking). + */ +export async function readAudioMetadata(filePath: string): Promise { + try { + const command = `ffprobe -v quiet -print_format json -show_format "${filePath}"`; + const { stdout } = await execPromise(command, { timeout: 15000 }); + const data = JSON.parse(stdout); + + const tags = data?.format?.tags || {}; + + // ffprobe tag names can be case-insensitive; check common variants + const album = tags.album || tags.ALBUM || tags.Album || undefined; + const albumArtist = tags.album_artist || tags.ALBUM_ARTIST || tags['Album Artist'] + || tags.albumartist || tags.ALBUMARTIST || undefined; + const composer = tags.composer || tags.COMPOSER || tags.Composer || undefined; + const artist = tags.artist || tags.ARTIST || tags.Artist + || tags['Contributing artists'] || tags['CONTRIBUTING ARTISTS'] || undefined; + const title = tags.title || tags.TITLE || tags.Title || undefined; + + return { + title: album || undefined, + author: albumArtist || undefined, + narrator: composer || undefined, + contributingArtists: artist || undefined, + trackTitle: title || undefined, + }; + } catch { + return {}; + } +} + +/** + * Deduplicate names across author, narrator, and contributing artists fields. + * Sometimes Album Artist contains "Author, Narrator" and Composer also has "Narrator", + * and Contributing Artists may overlap with both. + * We split on common delimiters and cross-reference to remove duplicates. + */ +export function deduplicateNames( + rawAuthor?: string, + rawNarrator?: string, + rawContributingArtists?: string +): { author?: string; narrator?: string; contributingArtists?: string } { + const splitNames = (str: string): string[] => + str.split(/[,;&]/).map((s) => s.trim()).filter(Boolean); + + const normalize = (s: string) => s.toLowerCase().replace(/\s+/g, ' ').trim(); + + const authorNames = rawAuthor ? splitNames(rawAuthor) : []; + const narratorNames = rawNarrator ? splitNames(rawNarrator) : []; + const contributingNames = rawContributingArtists ? splitNames(rawContributingArtists) : []; + + // Build sets for cross-referencing + const authorNormalized = new Set(authorNames.map(normalize)); + const narratorNormalized = new Set(narratorNames.map(normalize)); + + // Remove from author list any name that appears in narrator list + const dedupedAuthors = authorNames.filter( + (name) => !narratorNormalized.has(normalize(name)) + ); + + // Remove from contributing artists any name already in author or narrator + const allKnown = new Set([...authorNormalized, ...narratorNormalized]); + const dedupedContributing = contributingNames.filter( + (name) => !allKnown.has(normalize(name)) + ); + + return { + author: dedupedAuthors.length > 0 ? dedupedAuthors.join(', ') + : rawAuthor || undefined, + narrator: rawNarrator || undefined, + contributingArtists: dedupedContributing.length > 0 + ? dedupedContributing.join(', ') + : undefined, + }; +} + +/** + * Build a search term from metadata or file name. + * Returns the search term and the source it was derived from. + * When metadata tags are present, constructs "Title Author Narrator ContributingArtists". + * When tags are empty, falls back to the first audio file's name (cleaned). + */ +export function buildSearchTerm( + metadata: AudioFileMetadata, + firstFileName: string +): { searchTerm: string; source: 'tags' | 'file_name' } { + const { author, narrator, contributingArtists } = deduplicateNames( + metadata.author, + metadata.narrator, + metadata.contributingArtists + ); + const title = metadata.title; + + // If we have at least a title from metadata, use tags + if (title) { + const parts = [title]; + if (author) parts.push(author); + if (narrator) parts.push(narrator); + if (contributingArtists) parts.push(contributingArtists); + return { searchTerm: parts.join(' '), source: 'tags' }; + } + + // Fallback: clean up the first audio file name and use it as search term + const cleaned = firstFileName + .replace(/\.[^.]+$/, '') // Remove file extension + .replace(/[\[\(][A-Z0-9]{10}[\]\)]/g, '') // Remove ASIN in brackets + .replace(/[\[\(]\d{4}[\]\)]/g, '') // Remove year in brackets + .replace(/^\d+[\s._-]+/, '') // Remove leading track numbers + .replace(/[_]/g, ' ') // Underscores to spaces + .replace(/\s+/g, ' ') // Collapse whitespace + .trim(); + + return { searchTerm: cleaned || firstFileName, source: 'file_name' }; +} + +/** + * Scan a single directory for audio files. + * Returns audio file names and total size, or null if no audio files found. + */ +async function scanDirectoryForAudio( + dirPath: string +): Promise<{ audioFiles: string[]; totalSize: number } | null> { + try { + const children = await fs.readdir(dirPath, { withFileTypes: true }); + const audioFiles: string[] = []; + let totalSize = 0; + + for (const child of children) { + if (child.isFile() && isAudioFile(child.name)) { + audioFiles.push(child.name); + try { + const stat = await fs.stat(path.join(dirPath, child.name)); + totalSize += stat.size; + } catch { + /* skip unreadable files */ + } + } + } + + if (audioFiles.length === 0) return null; + + audioFiles.sort((a, b) => a.localeCompare(b)); + return { audioFiles, totalSize }; + } catch { + return null; + } +} + +/** + * Recursively discover audiobook folders starting from a root path. + * + * A folder is classified as an "audiobook folder" if it contains audio files. + * Once a folder is classified as an audiobook, its subfolders are NOT scanned + * further (the audio-containing folder is the audiobook boundary). + * + * @param rootPath - The root directory to scan + * @param onProgress - Optional callback for progress updates + * @param abortSignal - Optional AbortSignal to cancel the scan + * @returns Array of discovered audiobook folders with metadata + */ +export async function discoverAudiobooks( + rootPath: string, + onProgress?: (progress: ScanProgress) => void, + abortSignal?: AbortSignal +): Promise { + const results: DiscoveredAudiobook[] = []; + let foldersScanned = 0; + + async function walk(currentPath: string, depth: number): Promise { + if (depth > MAX_SCAN_DEPTH) return; + if (abortSignal?.aborted) return; + + foldersScanned++; + + onProgress?.({ + phase: 'discovering', + foldersScanned, + audiobooksFound: results.length, + currentFolder: path.basename(currentPath), + }); + + // Check if this folder contains audio files + const audioResult = await scanDirectoryForAudio(currentPath); + + if (audioResult) { + // This is an audiobook folder — read metadata and add to results + const firstFile = path.join(currentPath, audioResult.audioFiles[0]); + const metadata = await readAudioMetadata(firstFile); + + onProgress?.({ + phase: 'reading_metadata', + foldersScanned, + audiobooksFound: results.length + 1, + currentFolder: path.basename(currentPath), + }); + + const folderName = path.basename(currentPath); + const relativePath = path.relative(rootPath, currentPath).replace(/\\/g, '/'); + const firstFileName = audioResult.audioFiles[0]; + const { searchTerm, source } = buildSearchTerm(metadata, firstFileName); + + results.push({ + folderPath: currentPath.replace(/\\/g, '/'), + folderName, + relativePath: relativePath || folderName, + audioFileCount: audioResult.audioFiles.length, + totalSizeBytes: audioResult.totalSize, + metadata, + searchTerm, + metadataSource: source, + }); + + // Do NOT recurse into subfolders of audiobook folders + return; + } + + // No audio files here — recurse into subfolders + try { + const children = await fs.readdir(currentPath, { withFileTypes: true }); + const subdirs = children + .filter((c) => c.isDirectory() && !c.name.startsWith('.')) + .sort((a, b) => a.name.localeCompare(b.name)); + + for (const subdir of subdirs) { + if (abortSignal?.aborted) return; + await walk(path.join(currentPath, subdir.name), depth + 1); + } + } catch { + /* directory not readable — skip */ + } + } + + await walk(rootPath, 0); + return results; +}