Add admin Bulk Import feature

Introduce a Bulk Import feature for admins to scan server folders, match discovered audiobook folders against Audible, review matches, and queue batch imports.

What changed:
- Added documentation: documentation/features/bulk-import.md and TABLEOFCONTENTS update.
- Backend: SSE scan endpoint (POST /api/admin/bulk-import/scan) streams discovery and matching events; execute endpoint (POST /api/admin/bulk-import/execute) validates paths, creates/resolves audiobook & request records, and queues organize_files jobs. Both endpoints enforce admin-only access and validate allowed root directories (download_dir, media_dir, /bookdrop).
- Frontend: Modal wizard and steps for folder selection, scan progress, and match review (BulkImportWizard + ScanFolderStep, ScanProgressStep, MatchReviewStep + shared types).
- Utilities: bulk-import-scanner for folder discovery and ffprobe metadata extraction; shared types for scanned books/events.
- UI: Added Bulk Import quick action to admin dashboard (src/app/admin/page.tsx).

Key details:
- Audible searches are rate-limited (≈1.5s) and matching results include library/request status checks.
- Reuses existing organize_files job queue and manual-import pipeline; no new database tables introduced (state is ephemeral during the wizard).
- Includes error handling, path normalization, and security checks for allowed directories.

This commit wires frontend, backend, and docs together to provide an admin-only multi-step bulk import workflow.
This commit is contained in:
kikootwo
2026-03-13 12:03:21 -04:00
parent 6fc622c4e7
commit c8bfcdb611
11 changed files with 2275 additions and 1 deletions
+34 -1
View File
@@ -14,6 +14,7 @@ import { RecentRequestsTable } from './components/RecentRequestsTable';
import { ToastProvider, useToast } from '@/components/ui/Toast';
import { ReportedIssuesSection } from './components/ReportedIssuesSection';
import { InteractiveTorrentSearchModal } from '@/components/requests/InteractiveTorrentSearchModal';
import { BulkImportWizard } from '@/components/admin/BulkImportWizard';
import { TorrentResult } from '@/lib/utils/ranking-algorithm';
import { formatDistanceToNow } from 'date-fns';
import { useState } from 'react';
@@ -379,6 +380,8 @@ function PendingApprovalSection({ requests }: { requests: PendingApprovalRequest
}
function AdminDashboardContent() {
const [isBulkImportOpen, setIsBulkImportOpen] = useState(false);
// Fetch data with auto-refresh every 10 seconds
const { data: metrics, error: metricsError } = useSWR(
'/api/admin/metrics',
@@ -572,7 +575,7 @@ function AdminDashboardContent() {
</div>
{/* Quick Actions */}
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4 mb-8">
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-5 gap-4 mb-8">
<Link
href="/admin/settings"
className="block p-6 bg-white dark:bg-gray-800 border border-gray-200 dark:border-gray-700 rounded-lg hover:shadow-md transition-all"
@@ -657,8 +660,38 @@ function AdminDashboardContent() {
</span>
</div>
</Link>
<button
onClick={() => setIsBulkImportOpen(true)}
className="block p-6 bg-white dark:bg-gray-800 border border-gray-200 dark:border-gray-700 rounded-lg hover:shadow-md transition-all text-left"
>
<div className="flex items-center gap-3">
<svg
className="w-6 h-6 text-gray-600 dark:text-gray-400"
fill="none"
stroke="currentColor"
viewBox="0 0 24 24"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
strokeWidth={2}
d="M12 10v6m0 0l-3-3m3 3l3-3m2 8H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z"
/>
</svg>
<span className="font-medium text-gray-900 dark:text-gray-100">
Bulk Import
</span>
</div>
</button>
</div>
{/* Bulk Import Wizard Modal */}
<BulkImportWizard
isOpen={isBulkImportOpen}
onClose={() => setIsBulkImportOpen(false)}
/>
{/* Requests Awaiting Approval */}
{pendingApprovalData?.requests && pendingApprovalData.requests.length > 0 && (
<PendingApprovalSection requests={pendingApprovalData.requests} />
@@ -0,0 +1,292 @@
/**
* Component: Bulk Import Execute API
* Documentation: documentation/features/bulk-import.md
*
* Queues manual imports for multiple audiobooks at once.
* Reuses the same logic as the single manual import endpoint.
* Admin-only.
*/
import { NextRequest, NextResponse } from 'next/server';
import { requireAuth, requireAdmin, AuthenticatedRequest } from '@/lib/middleware/auth';
import { prisma } from '@/lib/db';
import { getJobQueueService } from '@/lib/services/job-queue.service';
import { RMABLogger } from '@/lib/utils/logger';
import { AUDIO_EXTENSIONS } from '@/lib/constants/audio-formats';
import { getAudibleService } from '@/lib/integrations/audible.service';
const logger = RMABLogger.create('API.Admin.BulkImport.Execute');
const BOOKDROP_PATH = '/bookdrop';
/** Statuses that indicate the request is actively being worked on. */
const ACTIVE_STATUSES = ['searching', 'downloading', 'processing', 'awaiting_import'];
/** Statuses that can be recycled for a new manual import. */
const RECYCLABLE_STATUSES = [
'failed', 'warn', 'cancelled', 'denied', 'pending',
'awaiting_search', 'awaiting_approval',
];
interface ImportItem {
folderPath: string;
asin: string;
}
interface ImportResult {
folderPath: string;
asin: string;
success: boolean;
requestId?: string;
error?: string;
}
/** Check if a directory contains audio files. */
async function hasAudioFiles(dirPath: string): Promise<boolean> {
const fs = await import('fs/promises');
const pathModule = await import('path');
try {
const children = await fs.readdir(dirPath, { withFileTypes: true });
return children.some(
(child) =>
child.isFile() &&
(AUDIO_EXTENSIONS as readonly string[]).includes(
pathModule.extname(child.name).toLowerCase()
)
);
} catch {
return false;
}
}
export async function POST(request: NextRequest) {
return requireAuth(request, async (req: AuthenticatedRequest) => {
return requireAdmin(req, async () => {
try {
const pathModule = await import('path');
const fs = await import('fs/promises');
const body = await request.json();
const { imports } = body as { imports: ImportItem[] };
if (!imports || !Array.isArray(imports) || imports.length === 0) {
return NextResponse.json(
{ error: 'imports array is required and must not be empty' },
{ status: 400 }
);
}
// Load allowed roots
const [downloadDirConfig, mediaDirConfig] = await Promise.all([
prisma.configuration.findUnique({ where: { key: 'download_dir' } }),
prisma.configuration.findUnique({ where: { key: 'media_dir' } }),
]);
const allowedRoots: string[] = [];
if (downloadDirConfig?.value) {
allowedRoots.push(pathModule.resolve(downloadDirConfig.value).replace(/\\/g, '/'));
}
if (mediaDirConfig?.value) {
allowedRoots.push(pathModule.resolve(mediaDirConfig.value).replace(/\\/g, '/'));
}
try {
const bookdropStat = await fs.stat(BOOKDROP_PATH);
if (bookdropStat.isDirectory()) {
allowedRoots.push(pathModule.resolve(BOOKDROP_PATH).replace(/\\/g, '/'));
}
} catch {
/* not mounted */
}
const userId = req.user!.id;
const audibleService = getAudibleService();
const jobQueue = getJobQueueService();
const results: ImportResult[] = [];
for (const item of imports) {
const { folderPath, asin } = item;
try {
// Validate path
const normalizedPath = pathModule.resolve(folderPath).replace(/\\/g, '/');
const isAllowed = allowedRoots.some(
(root) => normalizedPath === root || normalizedPath.startsWith(root + '/')
);
if (!isAllowed) {
results.push({ folderPath, asin, success: false, error: 'Path outside allowed directories' });
continue;
}
// Verify directory exists and has audio files
try {
const stat = await fs.stat(normalizedPath);
if (!stat.isDirectory()) {
results.push({ folderPath, asin, success: false, error: 'Not a directory' });
continue;
}
} catch {
results.push({ folderPath, asin, success: false, error: 'Directory not found' });
continue;
}
const hasAudio = await hasAudioFiles(normalizedPath);
if (!hasAudio) {
results.push({ folderPath, asin, success: false, error: 'No audio files' });
continue;
}
// Resolve or create audiobook record
let audiobookId: string;
let existingBook = await prisma.audiobook.findFirst({
where: { audibleAsin: asin },
});
if (existingBook) {
audiobookId = existingBook.id;
} else {
// Try Audible cache, then Audnexus
const cached = await prisma.audibleCache.findUnique({ where: { asin } });
if (cached) {
const newBook = await prisma.audiobook.create({
data: {
audibleAsin: asin,
title: cached.title,
author: cached.author,
coverArtUrl: cached.coverArtUrl,
narrator: cached.narrator,
status: 'pending',
},
});
audiobookId = newBook.id;
} else {
try {
const liveData = await audibleService.getAudiobookDetails(asin);
if (!liveData) {
results.push({ folderPath, asin, success: false, error: 'Audiobook not found' });
continue;
}
const newBook = await prisma.audiobook.create({
data: {
audibleAsin: asin,
title: liveData.title,
author: liveData.author,
coverArtUrl: liveData.coverArtUrl,
narrator: liveData.narrator,
series: liveData.series,
seriesPart: liveData.seriesPart,
seriesAsin: liveData.seriesAsin,
year: liveData.releaseDate
? new Date(liveData.releaseDate).getFullYear() || undefined
: undefined,
status: 'pending',
},
});
audiobookId = newBook.id;
} catch {
results.push({ folderPath, asin, success: false, error: 'Failed to fetch audiobook details' });
continue;
}
}
}
// Check for existing request and recycle or create
const existingRequest = await prisma.request.findFirst({
where: {
audiobookId,
type: 'audiobook',
deletedAt: null,
},
orderBy: { createdAt: 'desc' },
});
let requestId: string;
if (existingRequest) {
if (ACTIVE_STATUSES.includes(existingRequest.status)) {
results.push({ folderPath, asin, success: false, error: 'Already being processed' });
continue;
}
if (
RECYCLABLE_STATUSES.includes(existingRequest.status) ||
existingRequest.status === 'downloaded' ||
existingRequest.status === 'available'
) {
await prisma.request.update({
where: { id: existingRequest.id },
data: {
status: 'processing',
progress: 100,
errorMessage: null,
importAttempts: 0,
updatedAt: new Date(),
},
});
requestId = existingRequest.id;
} else {
const newReq = await prisma.request.create({
data: {
userId,
audiobookId,
type: 'audiobook',
status: 'processing',
progress: 100,
},
});
requestId = newReq.id;
}
} else {
const newReq = await prisma.request.create({
data: {
userId,
audiobookId,
type: 'audiobook',
status: 'processing',
progress: 100,
},
});
requestId = newReq.id;
}
// Queue organize_files job
await jobQueue.addOrganizeJob(requestId, audiobookId, normalizedPath);
results.push({ folderPath, asin, success: true, requestId });
logger.info(`Bulk import queued: asin=${asin}, path=${normalizedPath}, request=${requestId}`);
} catch (itemError) {
logger.error(`Bulk import item failed: asin=${asin}, path=${folderPath}`, {
error: itemError instanceof Error ? itemError.message : String(itemError),
});
results.push({
folderPath,
asin,
success: false,
error: itemError instanceof Error ? itemError.message : 'Import failed',
});
}
}
const succeeded = results.filter((r) => r.success).length;
const failed = results.filter((r) => !r.success).length;
logger.info(`Bulk import execute complete: ${succeeded} queued, ${failed} failed`);
return NextResponse.json({
success: true,
results,
summary: { total: results.length, succeeded, failed },
});
} catch (error) {
logger.error('Bulk import execute failed', {
error: error instanceof Error ? error.message : String(error),
});
return NextResponse.json(
{ error: error instanceof Error ? error.message : 'Bulk import failed' },
{ status: 500 }
);
}
});
});
}
+271
View File
@@ -0,0 +1,271 @@
/**
* Component: Bulk Import Scan API (SSE)
* Documentation: documentation/features/bulk-import.md
*
* Streams audiobook discovery and Audible matching results via Server-Sent Events.
* Admin-only. Validates path is within allowed roots.
*/
import { NextRequest, NextResponse } from 'next/server';
import { requireAuth, requireAdmin, AuthenticatedRequest } from '@/lib/middleware/auth';
import { prisma } from '@/lib/db';
import { RMABLogger } from '@/lib/utils/logger';
import { discoverAudiobooks } from '@/lib/utils/bulk-import-scanner';
import { getAudibleService } from '@/lib/integrations/audible.service';
import { findPlexMatch } from '@/lib/utils/audiobook-matcher';
const logger = RMABLogger.create('API.Admin.BulkImport.Scan');
const BOOKDROP_PATH = '/bookdrop';
const AUDIBLE_SEARCH_DELAY_MS = 1500;
/** Load allowed root directories from configuration. */
async function getAllowedRoots(): Promise<string[]> {
const pathModule = await import('path');
const fs = await import('fs/promises');
const [downloadDirConfig, mediaDirConfig] = await Promise.all([
prisma.configuration.findUnique({ where: { key: 'download_dir' } }),
prisma.configuration.findUnique({ where: { key: 'media_dir' } }),
]);
const roots: string[] = [];
if (downloadDirConfig?.value) {
roots.push(pathModule.resolve(downloadDirConfig.value).replace(/\\/g, '/'));
}
if (mediaDirConfig?.value) {
roots.push(pathModule.resolve(mediaDirConfig.value).replace(/\\/g, '/'));
}
try {
const stat = await fs.stat(BOOKDROP_PATH);
if (stat.isDirectory()) {
roots.push(pathModule.resolve(BOOKDROP_PATH).replace(/\\/g, '/'));
}
} catch {
/* not mounted */
}
return roots;
}
/** Check if a path is within allowed roots. */
function isPathAllowed(normalizedPath: string, roots: string[]): boolean {
return roots.some(
(root) => normalizedPath === root || normalizedPath.startsWith(root + '/')
);
}
/** Delay helper for rate limiting. */
function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export async function POST(request: NextRequest) {
return requireAuth(request, async (req: AuthenticatedRequest) => {
return requireAdmin(req, async () => {
const pathModule = await import('path');
const fs = await import('fs/promises');
let body: any;
try {
body = await request.json();
} catch {
return NextResponse.json({ error: 'Invalid JSON body' }, { status: 400 });
}
const { rootPath } = body;
if (!rootPath) {
return NextResponse.json({ error: 'rootPath is required' }, { status: 400 });
}
// Validate path
const allowedRoots = await getAllowedRoots();
const normalizedPath = pathModule.resolve(rootPath).replace(/\\/g, '/');
if (!isPathAllowed(normalizedPath, allowedRoots)) {
return NextResponse.json(
{ error: 'Access denied: path outside allowed directories' },
{ status: 403 }
);
}
// Verify directory exists
try {
const stat = await fs.stat(normalizedPath);
if (!stat.isDirectory()) {
return NextResponse.json({ error: 'Path is not a directory' }, { status: 400 });
}
} catch {
return NextResponse.json({ error: 'Directory not found' }, { status: 404 });
}
logger.info(`Bulk import scan started: ${normalizedPath}`);
// Create SSE stream
const encoder = new TextEncoder();
const abortController = new AbortController();
const stream = new ReadableStream({
async start(controller) {
const send = (event: string, data: any) => {
try {
controller.enqueue(
encoder.encode(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`)
);
} catch {
/* stream closed */
}
};
try {
// Phase 1: Discover audiobook folders
const audiobooks = await discoverAudiobooks(
normalizedPath,
(progress) => {
send('progress', progress);
},
abortController.signal
);
if (audiobooks.length === 0) {
send('complete', { audiobooks: [], message: 'No audiobooks found' });
controller.close();
return;
}
send('discovery_complete', {
totalFound: audiobooks.length,
message: `Found ${audiobooks.length} audiobook folders`,
});
// Phase 2: Match each audiobook against Audible
const audibleService = getAudibleService();
const results: any[] = [];
for (let i = 0; i < audiobooks.length; i++) {
if (abortController.signal.aborted) break;
const book = audiobooks[i];
send('matching', {
current: i + 1,
total: audiobooks.length,
folderName: book.folderName,
searchTerm: book.searchTerm,
});
let match: any = null;
let inLibrary = false;
let hasActiveRequest = false;
try {
const searchResult = await audibleService.search(book.searchTerm);
if (searchResult.results.length > 0) {
match = searchResult.results[0];
// Check library availability
const plexMatch = await findPlexMatch({
asin: match.asin,
title: match.title,
author: match.author,
narrator: match.narrator,
});
inLibrary = plexMatch !== null;
// Check for active requests
if (!inLibrary) {
const activeRequest = await prisma.request.findFirst({
where: {
audiobook: { audibleAsin: match.asin },
type: 'audiobook',
status: {
in: [
'pending', 'searching', 'downloading', 'processing',
'awaiting_search', 'awaiting_import', 'awaiting_approval',
'downloaded', 'available',
],
},
deletedAt: null,
},
});
hasActiveRequest = activeRequest !== null;
}
}
} catch (searchError) {
logger.warn(
`Audible search failed for "${book.searchTerm}": ${
searchError instanceof Error ? searchError.message : String(searchError)
}`
);
}
const result = {
index: i,
folderPath: book.folderPath,
folderName: book.folderName,
relativePath: book.relativePath,
audioFileCount: book.audioFileCount,
totalSizeBytes: book.totalSizeBytes,
metadataSource: book.metadataSource,
searchTerm: book.searchTerm,
match: match
? {
asin: match.asin,
title: match.title,
author: match.author,
narrator: match.narrator,
coverArtUrl: match.coverArtUrl,
durationMinutes: match.durationMinutes,
}
: null,
inLibrary,
hasActiveRequest,
};
results.push(result);
send('book_matched', result);
// Rate limit: wait between Audible searches (except after last)
if (i < audiobooks.length - 1) {
await delay(AUDIBLE_SEARCH_DELAY_MS);
}
}
send('complete', {
totalFound: results.length,
matched: results.filter((r) => r.match !== null).length,
inLibrary: results.filter((r) => r.inLibrary).length,
});
} catch (error) {
logger.error('Bulk import scan failed', {
error: error instanceof Error ? error.message : String(error),
});
send('error', {
message: error instanceof Error ? error.message : 'Scan failed',
});
} finally {
try {
controller.close();
} catch {
/* already closed */
}
}
},
cancel() {
abortController.abort();
},
});
// Cast to NextResponse: SSE streams require raw Response constructor,
// but requireAdmin types expect NextResponse. The Response is valid at runtime.
return new Response(stream, {
headers: {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
Connection: 'keep-alive',
},
}) as unknown as NextResponse;
});
});
}