Add per-user home sections & unified Audible cache

Introduce per-user configurable home page sections and a unified Audible cache/category model. Adds Prisma models (UserHomeSection, AudibleCacheCategory) and migrations to create tables and remove legacy popular/new_release flags; updates schema.prisma accordingly. Add API routes for user home sections, live Audible categories, and category-based audiobook listing, and refactor popular/new-releases/covers routes to read from AudibleCacheCategory. Frontend: new HomeSection component, HomeSectionConfigModal, useHomeSections hook, and homepage changes to render dynamic sections plus image fallback to a placeholder SVG. Also add placeholder_cover.svg and tests for home sections and the audible refresh processor.
This commit is contained in:
kikootwo
2026-03-05 11:30:39 -05:00
parent 248bd5359c
commit cc8e106a2b
40 changed files with 2582 additions and 655 deletions
+119
View File
@@ -0,0 +1,119 @@
/**
* Component: Home Sections Hook
* Documentation: documentation/features/home-sections.md
*
* Manages user home section configuration (CRUD) and category fetching.
*/
'use client';
import useSWR, { mutate as globalMutate } from 'swr';
import { authenticatedFetcher } from '@/lib/utils/api';
import { useCallback, useRef } from 'react';
export interface HomeSection {
id: string;
sectionType: 'popular' | 'new_releases' | 'category';
categoryId: string | null;
categoryName: string | null;
sortOrder: number;
}
export interface HomeSectionsResponse {
success: boolean;
sections: HomeSection[];
nextRefresh: string | null;
}
export interface AudibleCategory {
id: string;
name: string;
}
const HOME_SECTIONS_KEY = '/api/user/home-sections';
/**
* Hook to fetch and manage user home sections.
*/
export function useHomeSections() {
const { data, error, isLoading, mutate } = useSWR<HomeSectionsResponse>(
HOME_SECTIONS_KEY,
authenticatedFetcher,
{
revalidateOnFocus: false,
dedupingInterval: 30000,
}
);
const saveSections = useCallback(
async (sections: Omit<HomeSection, 'id'>[]) => {
const { fetchJSON } = await import('@/lib/utils/api');
const result = await fetchJSON<HomeSectionsResponse>(HOME_SECTIONS_KEY, {
method: 'PUT',
body: JSON.stringify({ sections }),
});
// Update local cache
mutate(result, false);
return result;
},
[mutate]
);
return {
sections: data?.sections || [],
nextRefresh: data?.nextRefresh || null,
isLoading,
error,
saveSections,
mutate,
};
}
/**
* Hook to fetch Audible categories (live scrape, for config modal).
*/
export function useAudibleCategories() {
const { data, error, isLoading } = useSWR<{ success: boolean; categories: AudibleCategory[] }>(
null, // Don't fetch automatically — use fetchCategories
authenticatedFetcher,
{ revalidateOnFocus: false }
);
return {
categories: data?.categories || [],
isLoading,
error,
};
}
/**
* Hook to fetch category audiobooks (same pattern as useAudiobooks).
*/
export function useCategoryAudiobooks(
categoryId: string | null,
limit: number = 20,
page: number = 1,
hideAvailable: boolean = false
) {
const hideParam = hideAvailable ? '&hideAvailable=true' : '';
const endpoint = categoryId
? `/api/audiobooks/category/${categoryId}?page=${page}&limit=${limit}${hideParam}`
: null;
const { data, error, isLoading } = useSWR(endpoint, authenticatedFetcher, {
revalidateOnFocus: false,
revalidateOnReconnect: false,
dedupingInterval: 60000,
});
return {
audiobooks: data?.audiobooks || [],
totalCount: data?.totalCount || 0,
totalPages: data?.totalPages || 0,
currentPage: data?.page || page,
hasMore: data?.hasMore || false,
message: data?.message || null,
isLoading,
error,
};
}
+158
View File
@@ -256,6 +256,15 @@ export class AudibleService {
throw error;
}
// Don't retry on deterministic 500 errors (e.g. "Release date is in the future")
if (status === 500) {
const message = error.response?.data?.message || '';
if (message.includes('Release date is in the future')) {
logger.info(` External API returned non-retryable error: ${message}`);
throw error;
}
}
// Don't retry on last attempt
if (attempt === maxRetries) {
break;
@@ -1172,6 +1181,155 @@ export class AudibleService {
}
}
/**
* Get top-level categories from Audible's categories page.
* Scrapes {baseUrl}/categories and returns {id, name}[] for top-level nodes.
*/
async getCategories(): Promise<{ id: string; name: string }[]> {
await this.initialize();
logger.info('Fetching Audible categories...');
try {
const { data: response } = await this.fetchWithRetry('/categories', {
params: { ipRedirectOverride: 'true' },
});
const $ = cheerio.load(response.data);
const categories: { id: string; name: string }[] = [];
// Top-level category links are in the main categories grid
// They follow the pattern /cat/{name}/{nodeId}
$('a[href*="/cat/"]').each((_index, element) => {
const $el = $(element);
const href = $el.attr('href') || '';
const match = href.match(/\/cat\/[^\/]+\/(\d+)/);
if (!match) return;
const id = match[1];
const name = $el.text().trim();
if (name && !categories.some((c) => c.id === id)) {
categories.push({ id, name });
}
});
logger.info(`Found ${categories.length} top-level categories`);
return categories;
} catch (error) {
logger.error('Failed to fetch categories', {
error: error instanceof Error ? error.message : String(error),
});
return [];
}
}
/**
* Get audiobooks for a specific category using Audible search with node parameter.
* Scrapes {baseUrl}/search?node={categoryId}&pageSize=50, up to `limit` results.
*/
async getCategoryBooks(categoryId: string, limit: number = 200): Promise<AudibleAudiobook[]> {
await this.initialize();
logger.info(`Fetching category books for node ${categoryId} (limit: ${limit})...`);
const audiobooks: AudibleAudiobook[] = [];
let page = 1;
const maxPages = Math.ceil(limit / AUDIBLE_PAGE_SIZE);
this.pacer.reset();
while (audiobooks.length < limit && page <= maxPages) {
try {
const { data: response, meta } = await this.fetchWithRetry('/search', {
params: {
ipRedirectOverride: 'true',
node: categoryId,
pageSize: AUDIBLE_PAGE_SIZE,
sort: 'popularity-rank',
...(page > 1 ? { page } : {}),
},
});
const $ = cheerio.load(response.data);
let foundOnPage = 0;
// Parse search results — same selectors as search()
$('.s-result-item, .productListItem').each((_index, element) => {
if (audiobooks.length >= limit) return false;
const $el = $(element);
const asin =
$el.find('li').attr('data-asin') ||
$el.find('a').attr('href')?.match(/\/(?:pd|ac)\/[^\/]+\/([A-Z0-9]{10})/)?.[1] ||
'';
if (!asin || audiobooks.some((b) => b.asin === asin)) return;
const title =
$el.find('h2').first().text().trim() ||
$el.find('h3 a').text().trim() ||
$el.find('.bc-heading a').text().trim();
const authorLink = $el.find('a[href*="/author/"]').first();
const authorText =
authorLink.text().trim() ||
$el.find('.authorLabel').text().trim();
const authorHref = authorLink.attr('href') || '';
const authorAsinMatch = authorHref.match(/\/author\/[^\/]+\/([A-Z0-9]{10})/);
const narratorText =
$el.find('a[href*="searchNarrator="]').first().text().trim() ||
$el.find('.narratorLabel').text().trim();
const coverArtUrl = $el.find('img').attr('src') || '';
const langConfig = this.getLangConfig();
const runtimeText =
$el.find('.runtimeLabel').text().trim() ||
$el.find(buildContainsSelector('span', langConfig.scraping.lengthLabels)).text().trim();
const durationMinutes = this.parseRuntime(runtimeText);
const ratingText =
$el.find('.ratingsLabel').text().trim() ||
$el.find('.a-icon-star span').first().text().trim();
const rating = ratingText ? parseFloat(ratingText.split(' ')[0]) : undefined;
audiobooks.push({
asin,
title,
author: stripPrefixes(authorText, langConfig.scraping.authorPrefixes),
authorAsin: authorAsinMatch?.[1] || undefined,
narrator: stripPrefixes(narratorText, langConfig.scraping.narratorPrefixes),
coverArtUrl: coverArtUrl.replace(/\._.*_\./, '._SL500_.'),
durationMinutes,
rating,
});
foundOnPage++;
});
logger.info(`Category ${categoryId}: found ${foundOnPage} books on page ${page}`);
if (foundOnPage < AUDIBLE_PAGE_SIZE / 2) break;
page++;
if (page <= maxPages && audiobooks.length < limit) {
await this.delay(this.pacer.reportPageResult(meta));
}
} catch (error) {
logger.error(`Failed to fetch category ${categoryId} page ${page}`, {
error: error instanceof Error ? error.message : String(error),
collectedSoFar: audiobooks.length,
});
break;
}
}
logger.info(`Category ${categoryId}: collected ${audiobooks.length} books across ${page - 1} pages`);
return audiobooks;
}
/**
* Add delay between requests to respect rate limits
*/
+144 -118
View File
@@ -2,12 +2,18 @@
* Component: Audible Refresh Processor
* Documentation: documentation/backend/services/scheduler.md
*
* Fetches popular and new release audiobooks from Audible and caches them
* Fetches popular, new release, and category audiobooks from Audible and caches them.
* All section data is stored uniformly in AudibleCacheCategory with reserved IDs
* '__popular__' and '__new_releases__' for built-in sections.
*/
import { prisma } from '../db';
import { RMABLogger } from '../utils/logger';
/** Reserved category IDs for built-in home sections */
export const POPULAR_CATEGORY_ID = '__popular__';
export const NEW_RELEASES_CATEGORY_ID = '__new_releases__';
export interface AudibleRefreshPayload {
jobId?: string;
scheduledJobId?: string;
@@ -25,22 +31,7 @@ export async function processAudibleRefresh(payload: AudibleRefreshPayload): Pro
const thumbnailCache = getThumbnailCacheService();
try {
// Clear previous popular/new-release flags for fresh data
await prisma.audibleCache.updateMany({
where: {
OR: [
{ isPopular: true },
{ isNewRelease: true },
],
},
data: {
isPopular: false,
isNewRelease: false,
popularRank: null,
newReleaseRank: null,
},
});
logger.info('Cleared previous popular/new-release flags in audible_cache');
const syncTime = new Date();
// Fetch popular and new releases - 200 items each
const popular = await audibleService.getPopularAudiobooks(200);
@@ -54,113 +45,63 @@ export async function processAudibleRefresh(payload: AudibleRefreshPayload): Pro
logger.info(`Fetched ${popular.length} popular, ${newReleases.length} new releases from Audible`);
// Persist to audible_cache
let popularSaved = 0;
let newReleasesSaved = 0;
const syncTime = new Date();
// Persist popular audiobooks via AudibleCacheCategory
const popularSaved = await persistSectionBooks(
popular, POPULAR_CATEGORY_ID, syncTime, thumbnailCache, logger, 'popular audiobook'
);
for (let i = 0; i < popular.length; i++) {
const audiobook = popular[i];
try {
// Cache thumbnail if coverArtUrl exists
let cachedCoverPath: string | null = null;
if (audiobook.coverArtUrl) {
cachedCoverPath = await thumbnailCache.cacheThumbnail(audiobook.asin, audiobook.coverArtUrl);
// Persist new releases via AudibleCacheCategory
const newReleasesSaved = await persistSectionBooks(
newReleases, NEW_RELEASES_CATEGORY_ID, syncTime, thumbnailCache, logger, 'new release'
);
logger.info(`Saved ${popularSaved} popular and ${newReleasesSaved} new releases`);
// --- Category scraping ---
// Query distinct categoryIds from all users' home sections
let categoriesSynced = 0;
try {
const categorySections = await prisma.userHomeSection.findMany({
where: { sectionType: 'category', categoryId: { not: null } },
select: { categoryId: true },
distinct: ['categoryId'],
});
const categoryIds = categorySections
.map((s) => s.categoryId)
.filter((id): id is string => id !== null);
if (categoryIds.length > 0) {
logger.info(`Refreshing ${categoryIds.length} user-configured categories...`);
for (const catId of categoryIds) {
try {
// Batch cooldown between categories
const catCooldownMs = 10000 + Math.floor(Math.random() * 10000);
logger.info(`Category cooldown: waiting ${Math.round(catCooldownMs / 1000)}s before category ${catId}...`);
await new Promise(resolve => setTimeout(resolve, catCooldownMs));
// Scrape category books
const books = await audibleService.getCategoryBooks(catId, 200);
logger.info(`Category ${catId}: fetched ${books.length} books`);
const saved = await persistSectionBooks(
books, catId, syncTime, thumbnailCache, logger, 'category book'
);
categoriesSynced++;
logger.info(`Category ${catId}: saved ${saved} entries`);
} catch (error) {
logger.error(`Failed to refresh category ${catId}: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
}
await prisma.audibleCache.upsert({
where: { asin: audiobook.asin },
create: {
asin: audiobook.asin,
title: audiobook.title,
author: audiobook.author,
narrator: audiobook.narrator,
description: audiobook.description,
coverArtUrl: audiobook.coverArtUrl,
cachedCoverPath: cachedCoverPath,
durationMinutes: audiobook.durationMinutes,
releaseDate: audiobook.releaseDate ? new Date(audiobook.releaseDate) : null,
rating: audiobook.rating ? audiobook.rating : null,
genres: audiobook.genres || [],
isPopular: true,
popularRank: i + 1,
lastSyncedAt: syncTime,
},
update: {
title: audiobook.title,
author: audiobook.author,
narrator: audiobook.narrator,
description: audiobook.description,
coverArtUrl: audiobook.coverArtUrl,
cachedCoverPath: cachedCoverPath,
durationMinutes: audiobook.durationMinutes,
releaseDate: audiobook.releaseDate ? new Date(audiobook.releaseDate) : null,
rating: audiobook.rating ? audiobook.rating : null,
genres: audiobook.genres || [],
isPopular: true,
popularRank: i + 1,
lastSyncedAt: syncTime,
},
});
popularSaved++;
} catch (error) {
logger.error(`Failed to save popular audiobook ${audiobook.title}: ${error instanceof Error ? error.message : 'Unknown error'}`);
logger.info(`Category refresh complete: ${categoriesSynced}/${categoryIds.length} categories synced`);
}
} catch (error) {
logger.error(`Category refresh failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
for (let i = 0; i < newReleases.length; i++) {
const audiobook = newReleases[i];
try {
// Cache thumbnail if coverArtUrl exists
let cachedCoverPath: string | null = null;
if (audiobook.coverArtUrl) {
cachedCoverPath = await thumbnailCache.cacheThumbnail(audiobook.asin, audiobook.coverArtUrl);
}
await prisma.audibleCache.upsert({
where: { asin: audiobook.asin },
create: {
asin: audiobook.asin,
title: audiobook.title,
author: audiobook.author,
narrator: audiobook.narrator,
description: audiobook.description,
coverArtUrl: audiobook.coverArtUrl,
cachedCoverPath: cachedCoverPath,
durationMinutes: audiobook.durationMinutes,
releaseDate: audiobook.releaseDate ? new Date(audiobook.releaseDate) : null,
rating: audiobook.rating ? audiobook.rating : null,
genres: audiobook.genres || [],
isNewRelease: true,
newReleaseRank: i + 1,
lastSyncedAt: syncTime,
},
update: {
title: audiobook.title,
author: audiobook.author,
narrator: audiobook.narrator,
description: audiobook.description,
coverArtUrl: audiobook.coverArtUrl,
cachedCoverPath: cachedCoverPath,
durationMinutes: audiobook.durationMinutes,
releaseDate: audiobook.releaseDate ? new Date(audiobook.releaseDate) : null,
rating: audiobook.rating ? audiobook.rating : null,
genres: audiobook.genres || [],
isNewRelease: true,
newReleaseRank: i + 1,
lastSyncedAt: syncTime,
},
});
newReleasesSaved++;
} catch (error) {
logger.error(`Failed to save new release ${audiobook.title}: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
}
logger.info(`Saved ${popularSaved} popular and ${newReleasesSaved} new releases to audible_cache`);
// Cleanup unused thumbnails
logger.info('Cleaning up unused thumbnails...');
const allActiveAsins = await prisma.audibleCache.findMany({
@@ -175,6 +116,7 @@ export async function processAudibleRefresh(payload: AudibleRefreshPayload): Pro
message: 'Audible refresh completed',
popularSaved,
newReleasesSaved,
categoriesSynced,
thumbnailsDeleted: deletedCount,
};
} catch (error) {
@@ -182,3 +124,87 @@ export async function processAudibleRefresh(payload: AudibleRefreshPayload): Pro
throw error;
}
}
/**
* Wipe previous entries for a category, upsert book metadata into AudibleCache,
* and insert ranked entries into AudibleCacheCategory.
* Returns the number of books successfully saved.
*/
async function persistSectionBooks(
books: any[],
categoryId: string,
syncTime: Date,
thumbnailCache: { cacheThumbnail: (asin: string, url: string) => Promise<string | null> },
logger: ReturnType<typeof RMABLogger.forJob>,
labelForErrors: string,
): Promise<number> {
// Wipe previous entries for this section
logger.info(`Clearing previous data for ${categoryId}...`);
await prisma.audibleCacheCategory.deleteMany({
where: { categoryId },
});
logger.info(`Cleared previous entries for ${categoryId}, saving ${books.length} books...`);
let saved = 0;
for (let i = 0; i < books.length; i++) {
const book = books[i];
try {
// Cache thumbnail if coverArtUrl exists
let cachedCoverPath: string | null = null;
if (book.coverArtUrl) {
cachedCoverPath = await thumbnailCache.cacheThumbnail(book.asin, book.coverArtUrl);
if (!cachedCoverPath) {
logger.warn(`Cover cache failed for "${book.title}" (${book.asin}) - falling back to remote URL`);
}
}
// Upsert book metadata into AudibleCache
await prisma.audibleCache.upsert({
where: { asin: book.asin },
create: {
asin: book.asin,
title: book.title,
author: book.author,
narrator: book.narrator,
description: book.description,
coverArtUrl: book.coverArtUrl,
cachedCoverPath,
durationMinutes: book.durationMinutes,
releaseDate: book.releaseDate ? new Date(book.releaseDate) : null,
rating: book.rating ? book.rating : null,
genres: book.genres || [],
lastSyncedAt: syncTime,
},
update: {
title: book.title,
author: book.author,
narrator: book.narrator,
description: book.description,
coverArtUrl: book.coverArtUrl,
cachedCoverPath,
durationMinutes: book.durationMinutes,
releaseDate: book.releaseDate ? new Date(book.releaseDate) : null,
rating: book.rating ? book.rating : null,
genres: book.genres || [],
lastSyncedAt: syncTime,
},
});
// Insert ranked entry into AudibleCacheCategory
await prisma.audibleCacheCategory.create({
data: {
asin: book.asin,
categoryId,
rank: i + 1,
lastSyncedAt: syncTime,
},
});
saved++;
} catch (error) {
logger.error(`Failed to save ${labelForErrors} ${book.title}: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
}
return saved;
}
+9 -13
View File
@@ -24,7 +24,7 @@ export class ThumbnailCacheService {
try {
await fs.mkdir(CACHE_DIR, { recursive: true });
} catch (error) {
logger.error('Failed to create cache directory', { error: error instanceof Error ? error.message : String(error) });
logger.error(`Failed to create cache directory: ${error instanceof Error ? error.message : String(error)}`);
throw error;
}
}
@@ -36,7 +36,7 @@ export class ThumbnailCacheService {
try {
await fs.mkdir(LIBRARY_CACHE_DIR, { recursive: true });
} catch (error) {
logger.error('Failed to create library cache directory', { error: error instanceof Error ? error.message : String(error) });
logger.error(`Failed to create library cache directory: ${error instanceof Error ? error.message : String(error)}`);
throw error;
}
}
@@ -127,8 +127,8 @@ export class ThumbnailCacheService {
logger.info(`Cached thumbnail for ${asin}: ${filePath}`);
return filePath;
} catch (error) {
// Log error but don't throw - we'll fall back to the original URL
logger.error(`Failed to cache thumbnail for ${asin}`, { error: error instanceof Error ? error.message : String(error) });
// Log warning but don't throw - we'll fall back to the original URL
logger.warn(`Failed to cache thumbnail for ${asin}: ${error instanceof Error ? error.message : String(error)} - will use remote URL`);
return null;
}
}
@@ -203,10 +203,8 @@ export class ThumbnailCacheService {
logger.info(`Cached library thumbnail for ${plexGuid}: ${filePath}`);
return filePath;
} catch (error) {
// Log error but don't throw - graceful degradation
logger.warn(`Failed to cache library thumbnail for ${plexGuid}`, {
error: error instanceof Error ? error.message : String(error),
});
// Log warning but don't throw - graceful degradation
logger.warn(`Failed to cache library thumbnail for ${plexGuid}: ${error instanceof Error ? error.message : String(error)}`);
return null;
}
}
@@ -227,7 +225,7 @@ export class ThumbnailCacheService {
logger.info(`Deleted thumbnail: ${filePath}`);
}
} catch (error) {
logger.error(`Failed to delete thumbnail for ${asin}`, { error: error instanceof Error ? error.message : String(error) });
logger.error(`Failed to delete thumbnail for ${asin}: ${error instanceof Error ? error.message : String(error)}`);
}
}
@@ -258,7 +256,7 @@ export class ThumbnailCacheService {
logger.info(`Cleanup complete: ${deletedCount} thumbnails deleted`);
return deletedCount;
} catch (error) {
logger.error('Failed to cleanup thumbnails', { error: error instanceof Error ? error.message : String(error) });
logger.error(`Failed to cleanup thumbnails: ${error instanceof Error ? error.message : String(error)}`);
return 0;
}
}
@@ -299,9 +297,7 @@ export class ThumbnailCacheService {
logger.info(`Library cleanup complete: ${deletedCount} thumbnails deleted`);
return deletedCount;
} catch (error) {
logger.error('Failed to cleanup library thumbnails', {
error: error instanceof Error ? error.message : String(error),
});
logger.error(`Failed to cleanup library thumbnails: ${error instanceof Error ? error.message : String(error)}`);
return 0;
}
}