Add Kindle EPUB compatibility fixer

Introduce an optional Kindle EPUB compatibility fixer and integrate it into the ebook organization flow. Adds a new config key (ebook_kindle_fix_enabled, default false), a settings API update, and a UI toggle (visible when preferred format is EPUB). Implements src/lib/utils/epub-fixer.ts (uses adm-zip and cheerio) to apply fixes: add UTF-8 XML declarations, remove body/#bodymatter fragments from links, validate/normalize dc:language, and remove stray <img> tags without src. organize-files.processor now detects EPUB downloads, runs the fixer (produces a temp fixed EPUB), uses the fixed file for organization, logs fixes, and cleans up temporary files; fix failures are non-blocking and the original download is preserved. Adds dependencies adm-zip and @types/adm-zip and updates documentation and types/UI to expose the new setting. Also includes helper functions to detect EPUB paths in downloads.
This commit is contained in:
kikootwo
2026-02-03 16:34:57 -05:00
parent 863f8466ea
commit 2ef9ac7be1
11 changed files with 677 additions and 233 deletions
+2
View File
@@ -115,6 +115,8 @@ export interface EbookSettings {
// General settings (shared across sources)
preferredFormat: string;
autoGrabEnabled: boolean;
// Kindle compatibility
kindleFixEnabled: boolean;
}
/**
@@ -254,6 +254,32 @@ export function EbookTab({ ebook, onChange, onSuccess, onError, markAsSaved }: E
</p>
</div>
</div>
{/* Kindle Fix Toggle - Only shown when EPUB is selected */}
{(ebook.preferredFormat === 'epub' || !ebook.preferredFormat) && (
<div className="flex items-start gap-4 pt-2 border-t border-gray-200 dark:border-gray-700 mt-4">
<input
type="checkbox"
id="kindle-fix-enabled"
checked={ebook.kindleFixEnabled ?? false}
onChange={(e) => updateEbook('kindleFixEnabled', e.target.checked)}
className="mt-1 h-5 w-5 rounded border-gray-300 text-blue-600 focus:ring-blue-500"
/>
<div className="flex-1">
<label
htmlFor="kindle-fix-enabled"
className="block text-sm font-medium text-gray-900 dark:text-gray-100 cursor-pointer"
>
Fix EPUB for Kindle import
</label>
<p className="text-sm text-gray-600 dark:text-gray-400 mt-1">
Apply compatibility fixes before organizing EPUB files. Fixes encoding declarations,
broken hyperlinks, invalid language tags, and orphaned image elements that can
cause Kindle import failures.
</p>
</div>
</div>
)}
</div>
</div>
)}
@@ -83,6 +83,7 @@ export function useEbookSettings({ ebook, onChange, onSuccess, onError, markAsSa
baseUrl: ebook.baseUrl || 'https://annas-archive.li',
flaresolverrUrl: ebook.flaresolverrUrl || '',
autoGrabEnabled: ebook.autoGrabEnabled ?? true,
kindleFixEnabled: ebook.kindleFixEnabled ?? false,
}),
});
+8 -1
View File
@@ -14,7 +14,7 @@ export async function PUT(request: NextRequest) {
return requireAdmin(req, async () => {
try {
// Parse request body - new structure with separate source toggles
const { annasArchiveEnabled, indexerSearchEnabled, format, baseUrl, flaresolverrUrl, autoGrabEnabled } = await request.json();
const { annasArchiveEnabled, indexerSearchEnabled, format, baseUrl, flaresolverrUrl, autoGrabEnabled, kindleFixEnabled } = await request.json();
// Enforce: auto-grab must be false if no sources are enabled
const effectiveAutoGrabEnabled = (annasArchiveEnabled || indexerSearchEnabled) ? (autoGrabEnabled ?? true) : false;
@@ -88,6 +88,13 @@ export async function PUT(request: NextRequest) {
category: 'ebook',
description: 'FlareSolverr URL for bypassing Cloudflare protection',
},
// Kindle compatibility
{
key: 'ebook_kindle_fix_enabled',
value: kindleFixEnabled ? 'true' : 'false',
category: 'ebook',
description: 'Apply compatibility fixes to EPUB files for Kindle import',
},
];
await configService.setMany(configs);
+2
View File
@@ -141,6 +141,8 @@ export async function GET(request: NextRequest) {
preferredFormat: configMap.get('ebook_sidecar_preferred_format') || 'epub',
// Auto-grab: default true to preserve existing behavior
autoGrabEnabled: configMap.get('ebook_auto_grab_enabled') !== 'false',
// Kindle compatibility fixes: default false
kindleFixEnabled: configMap.get('ebook_kindle_fix_enabled') === 'true',
},
general: {
appName: configMap.get('app_name') || 'ReadMeABook',
+123 -1
View File
@@ -10,6 +10,7 @@ import { RMABLogger } from '../utils/logger';
import { getLibraryService } from '../services/library';
import { getConfigService } from '../services/config.service';
import { generateFilesHash } from '../utils/files-hash';
import { fixEpubForKindle, cleanupFixedEpub } from '../utils/epub-fixer';
/**
* Process organize files job
@@ -585,10 +586,52 @@ async function processEbookOrganization(
});
const template = templateConfig?.value || '{author}/{title} {asin}';
// Check if Kindle EPUB fix is needed
let effectiveDownloadPath = downloadPath;
let fixedEpubPath: string | null = null;
// Detect the actual EPUB file path (handles both single file and directory downloads)
const epubFilePath = await detectEpubFilePath(downloadPath);
// Only apply Kindle fix for EPUB files when enabled
if (epubFilePath) {
const configService = getConfigService();
const kindleFixEnabled = await configService.get('ebook_kindle_fix_enabled');
if (kindleFixEnabled === 'true') {
logger.info('Kindle EPUB fix enabled - applying compatibility fixes');
const tempDir = process.env.TEMP_DIR || '/tmp/readmeabook';
const fixResult = await fixEpubForKindle(
epubFilePath,
tempDir,
jobId ? { jobId, context: 'EpubFixer' } : undefined
);
if (fixResult.success && fixResult.outputPath) {
fixedEpubPath = fixResult.outputPath;
effectiveDownloadPath = fixResult.outputPath;
logger.info(`Using fixed EPUB: ${fixResult.outputPath}`);
// Log fixes applied
const { encodingFixes, bodyIdLinkFixes, languageFix, strayImgFixes } = fixResult.fixesApplied;
const totalFixes = encodingFixes + bodyIdLinkFixes + (languageFix ? 1 : 0) + strayImgFixes;
if (totalFixes > 0) {
logger.info(`Kindle fixes applied: encoding=${encodingFixes}, bodyIdLinks=${bodyIdLinkFixes}, language=${languageFix}, strayImages=${strayImgFixes}`);
}
} else {
// Fix failed - continue with original file
logger.warn(`Kindle EPUB fix failed: ${fixResult.error}. Continuing with original file.`);
}
} else {
logger.info('Kindle EPUB fix disabled - organizing original file');
}
}
// Organize ebook files (organizer will detect ebook type and skip audio-specific processing)
// Pass all metadata that could be used in path templates (same as audiobooks)
const result = await organizer.organizeEbook(
downloadPath,
effectiveDownloadPath,
{
title: book.title,
author: book.author,
@@ -603,6 +646,12 @@ async function processEbookOrganization(
isIndexerDownload
);
// Clean up fixed EPUB temp file after organization (regardless of success)
if (fixedEpubPath) {
await cleanupFixedEpub(fixedEpubPath);
logger.info('Cleaned up temporary fixed EPUB');
}
if (!result.success) {
throw new Error(`Ebook organization failed: ${result.errors.join(', ')}`);
}
@@ -857,3 +906,76 @@ async function createEbookRequestIfEnabled(
logger.error(`Failed to create ebook request: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
}
// =========================================================================
// HELPER FUNCTIONS
// =========================================================================
/**
* Detect the path to an EPUB file from download path
* Handles both single file downloads (direct path) and directory downloads (indexer)
*
* @param downloadPath - Path to the download (file or directory)
* @returns Full path to EPUB file, or null if no EPUB found
*/
async function detectEpubFilePath(downloadPath: string): Promise<string | null> {
const fs = await import('fs/promises');
const path = await import('path');
try {
const stats = await fs.stat(downloadPath);
if (stats.isFile()) {
// Single file - check if it's an EPUB
if (path.extname(downloadPath).toLowerCase() === '.epub') {
return downloadPath;
}
return null;
}
// Directory - search for EPUB file
const files = await walkDirectory(downloadPath);
const epubFile = files.find(file =>
path.extname(file).toLowerCase() === '.epub'
);
if (epubFile) {
return path.join(downloadPath, epubFile);
}
return null;
} catch {
return null;
}
}
/**
* Recursively walk directory to find all files
* Returns relative paths from the base directory
*/
async function walkDirectory(dir: string, baseDir: string = ''): Promise<string[]> {
const fs = await import('fs/promises');
const path = await import('path');
const files: string[] = [];
try {
const entries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
const relativePath = baseDir ? path.join(baseDir, entry.name) : entry.name;
if (entry.isDirectory()) {
const subFiles = await walkDirectory(fullPath, relativePath);
files.push(...subFiles);
} else {
files.push(relativePath);
}
}
} catch {
// Directory not accessible
}
return files;
}
+465
View File
@@ -0,0 +1,465 @@
/**
* Component: EPUB Kindle Compatibility Fixer
* Documentation: documentation/integrations/ebook-sidecar.md
*
* Applies compatibility fixes to EPUB files for Kindle import.
* Based on: https://github.com/innocenat/kindle-epub-fix
*
* Fixes applied:
* 1. Encoding declaration - Adds UTF-8 XML declaration to files missing it
* 2. Body ID link fix - Removes #bodyid fragments from hyperlinks
* 3. Language validation - Ensures dc:language uses Amazon KDP-approved codes
* 4. Stray IMG removal - Removes <img> tags without src attributes
*/
import AdmZip from 'adm-zip';
import * as cheerio from 'cheerio';
import path from 'path';
import fs from 'fs/promises';
import { RMABLogger } from './logger';
const moduleLogger = RMABLogger.create('EpubFixer');
/**
* Amazon KDP approved language codes
* Source: https://kdp.amazon.com/en_US/help/topic/G200673300
*/
const AMAZON_APPROVED_LANGUAGES: Set<string> = new Set([
// ISO 639-1 codes (2-letter)
'af', 'sq', 'ar', 'hy', 'az', 'eu', 'be', 'bn', 'bs', 'br', 'bg', 'ca',
'zh', 'hr', 'cs', 'da', 'nl', 'en', 'eo', 'et', 'fo', 'fi', 'fr', 'fy',
'gl', 'ka', 'de', 'el', 'gu', 'he', 'hi', 'hu', 'is', 'id', 'ga', 'it',
'ja', 'kn', 'kk', 'ko', 'ku', 'ky', 'la', 'lv', 'lt', 'lb', 'mk', 'ms',
'ml', 'mt', 'mr', 'mn', 'ne', 'no', 'nb', 'nn', 'oc', 'or', 'ps', 'fa',
'pl', 'pt', 'pa', 'rm', 'ro', 'ru', 'gd', 'sr', 'sk', 'sl', 'es', 'sw',
'sv', 'tl', 'ta', 'te', 'th', 'tr', 'uk', 'ur', 'uz', 'vi', 'cy', 'yi',
// ISO 639-2 codes (3-letter) commonly used
'eng', 'fra', 'deu', 'spa', 'ita', 'por', 'rus', 'jpn', 'zho', 'kor',
'ara', 'hin', 'nld', 'pol', 'tur', 'swe', 'dan', 'nor', 'fin', 'ces',
// Regional variants
'en-us', 'en-gb', 'en-au', 'en-ca', 'en-nz', 'en-ie', 'en-za',
'pt-br', 'pt-pt', 'zh-cn', 'zh-tw', 'zh-hk', 'es-es', 'es-mx', 'es-ar',
'fr-fr', 'fr-ca', 'de-de', 'de-at', 'de-ch', 'it-it', 'nl-nl', 'nl-be',
]);
/**
* Content file extensions that should be processed
*/
const CONTENT_EXTENSIONS = ['.html', '.xhtml', '.htm', '.xml'];
/**
* Result of the EPUB fixing process
*/
export interface EpubFixResult {
success: boolean;
outputPath: string | null;
fixesApplied: {
encodingFixes: number;
bodyIdLinkFixes: number;
languageFix: boolean;
strayImgFixes: number;
};
error?: string;
}
/**
* Logger interface for job-aware logging
*/
interface LoggerConfig {
jobId: string;
context: string;
}
/**
* Fix EPUB file for Kindle compatibility
*
* @param sourcePath - Path to the source EPUB file
* @param tempDir - Directory to write the fixed EPUB to
* @param loggerConfig - Optional logger configuration for job-aware logging
* @returns Result with path to fixed EPUB or error
*/
export async function fixEpubForKindle(
sourcePath: string,
tempDir: string,
loggerConfig?: LoggerConfig
): Promise<EpubFixResult> {
const logger = loggerConfig
? RMABLogger.forJob(loggerConfig.jobId, loggerConfig.context)
: null;
const result: EpubFixResult = {
success: false,
outputPath: null,
fixesApplied: {
encodingFixes: 0,
bodyIdLinkFixes: 0,
languageFix: false,
strayImgFixes: 0,
},
};
try {
await logger?.info(`Starting Kindle EPUB fix for: ${path.basename(sourcePath)}`);
// Verify source file exists
try {
await fs.access(sourcePath, fs.constants.R_OK);
} catch {
throw new Error(`Source EPUB not found or not readable: ${sourcePath}`);
}
// Load the EPUB (ZIP file)
const zip = new AdmZip(sourcePath);
const zipEntries = zip.getEntries();
await logger?.info(`Loaded EPUB with ${zipEntries.length} entries`);
// Track OPF file for language fix
let opfEntry: AdmZip.IZipEntry | null = null;
let opfPath = '';
// Find content files and OPF
for (const entry of zipEntries) {
const entryPath = entry.entryName.toLowerCase();
// Find OPF file (metadata)
if (entryPath.endsWith('.opf')) {
opfEntry = entry;
opfPath = entry.entryName;
}
}
// Process content files (HTML/XHTML)
for (const entry of zipEntries) {
if (entry.isDirectory) continue;
const ext = path.extname(entry.entryName).toLowerCase();
if (!CONTENT_EXTENSIONS.includes(ext)) continue;
// Read file content
let content = entry.getData().toString('utf8');
let modified = false;
// Fix 1: Encoding declaration
const encodingResult = fixEncoding(content);
if (encodingResult.modified) {
content = encodingResult.content;
modified = true;
result.fixesApplied.encodingFixes++;
}
// Fix 2: Body ID links
const bodyIdResult = fixBodyIdLinks(content);
if (bodyIdResult.modified) {
content = bodyIdResult.content;
modified = true;
result.fixesApplied.bodyIdLinkFixes += bodyIdResult.count;
}
// Fix 4: Stray IMG tags (applied to HTML content)
const strayImgResult = fixStrayImages(content);
if (strayImgResult.modified) {
content = strayImgResult.content;
modified = true;
result.fixesApplied.strayImgFixes += strayImgResult.count;
}
// Update entry if modified
if (modified) {
zip.updateFile(entry.entryName, Buffer.from(content, 'utf8'));
}
}
// Fix 3: Language validation (in OPF file)
if (opfEntry) {
const opfContent = opfEntry.getData().toString('utf8');
const languageResult = fixLanguage(opfContent);
if (languageResult.modified) {
zip.updateFile(opfPath, Buffer.from(languageResult.content, 'utf8'));
result.fixesApplied.languageFix = true;
await logger?.info(`Fixed language tag: "${languageResult.originalLang}" -> "${languageResult.newLang}"`);
}
}
// Log fixes applied
const totalFixes =
result.fixesApplied.encodingFixes +
result.fixesApplied.bodyIdLinkFixes +
(result.fixesApplied.languageFix ? 1 : 0) +
result.fixesApplied.strayImgFixes;
if (totalFixes > 0) {
await logger?.info(
`Applied ${totalFixes} fixes: ` +
`encoding=${result.fixesApplied.encodingFixes}, ` +
`bodyIdLinks=${result.fixesApplied.bodyIdLinkFixes}, ` +
`language=${result.fixesApplied.languageFix}, ` +
`strayImages=${result.fixesApplied.strayImgFixes}`
);
} else {
await logger?.info('No fixes needed - EPUB is already Kindle-compatible');
}
// Create unique temp subdirectory to avoid filename conflicts
// This preserves the original filename for the final organized file
const uniqueDir = path.join(tempDir, `kindle-fix-${Date.now()}`);
await fs.mkdir(uniqueDir, { recursive: true });
// Keep original filename
const sourceFilename = path.basename(sourcePath);
const outputPath = path.join(uniqueDir, sourceFilename);
// Write fixed EPUB
zip.writeZip(outputPath);
await logger?.info(`Fixed EPUB written to temp directory, preserving filename: ${sourceFilename}`);
result.success = true;
result.outputPath = outputPath;
return result;
} catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
await logger?.error(`EPUB fix failed: ${errorMessage}`);
result.error = errorMessage;
return result;
}
}
/**
* Fix 1: Add UTF-8 XML encoding declaration if missing
*
* Many EPUBs lack the XML declaration, which can cause Kindle import issues.
* Adds: <?xml version="1.0" encoding="utf-8"?>
*/
function fixEncoding(content: string): { content: string; modified: boolean } {
// Check if already has XML declaration
const xmlDeclRegex = /^\s*<\?xml[^?]*\?>/i;
if (xmlDeclRegex.test(content)) {
// Already has declaration, check if it has encoding
const hasEncoding = /encoding\s*=\s*["'][^"']+["']/i.test(content);
if (hasEncoding) {
return { content, modified: false };
}
// Has declaration but no encoding - add encoding attribute
const updatedContent = content.replace(
/(<\?xml[^?]*?)(\?>)/i,
'$1 encoding="utf-8"$2'
);
return { content: updatedContent, modified: true };
}
// No declaration - add one at the beginning
const declaration = '<?xml version="1.0" encoding="utf-8"?>\n';
return { content: declaration + content.trimStart(), modified: true };
}
/**
* Fix 2: Remove body ID fragments from hyperlinks
*
* Links like "file.html#body" or "file.xhtml#bodymatter" can break on Kindle.
* This removes the fragment when it targets body-related IDs.
*/
function fixBodyIdLinks(content: string): { content: string; modified: boolean; count: number } {
// Pattern to match href attributes with body-related ID fragments
// Matches: href="file.html#body", href="page.xhtml#bodymatter", etc.
const bodyIdPattern = /href\s*=\s*["']([^"'#]+)#(body[^"']*|bodymatter)["']/gi;
let count = 0;
const updatedContent = content.replace(bodyIdPattern, (match, file) => {
count++;
return `href="${file}"`;
});
return {
content: updatedContent,
modified: count > 0,
count,
};
}
/**
* Fix 3: Validate and fix dc:language in OPF metadata
*
* Ensures the language tag is one approved by Amazon KDP.
* If invalid or missing, defaults to "en" (English).
*/
function fixLanguage(opfContent: string): {
content: string;
modified: boolean;
originalLang: string;
newLang: string;
} {
const result = {
content: opfContent,
modified: false,
originalLang: '',
newLang: '',
};
// Parse with cheerio (XML mode)
const $ = cheerio.load(opfContent, { xmlMode: true });
// Find dc:language element (handle namespace variations)
let langElement = $('dc\\:language, language');
if (langElement.length === 0) {
// No language tag found - add one
// Find the metadata element to insert into
const metadata = $('metadata');
if (metadata.length > 0) {
// Add language element
metadata.append('\n <dc:language>en</dc:language>');
result.content = $.xml();
result.modified = true;
result.originalLang = '(missing)';
result.newLang = 'en';
}
return result;
}
// Get current language value
const currentLang = langElement.first().text().trim().toLowerCase();
result.originalLang = currentLang;
// Check if it's a valid Amazon language
if (isValidAmazonLanguage(currentLang)) {
return result; // No fix needed
}
// Try to normalize the language
const normalizedLang = normalizeLanguage(currentLang);
if (normalizedLang !== currentLang) {
langElement.first().text(normalizedLang);
result.content = $.xml();
result.modified = true;
result.newLang = normalizedLang;
}
return result;
}
/**
* Check if a language code is approved by Amazon KDP
*/
function isValidAmazonLanguage(lang: string): boolean {
const normalized = lang.toLowerCase().trim();
// Direct match
if (AMAZON_APPROVED_LANGUAGES.has(normalized)) {
return true;
}
// Check base language (e.g., "en-us" -> "en")
const baseLang = normalized.split('-')[0];
return AMAZON_APPROVED_LANGUAGES.has(baseLang);
}
/**
* Normalize a language code to an Amazon-approved format
*/
function normalizeLanguage(lang: string): string {
const normalized = lang.toLowerCase().trim();
// If already valid, return as-is
if (isValidAmazonLanguage(normalized)) {
return normalized;
}
// Try base language
const baseLang = normalized.split('-')[0];
if (AMAZON_APPROVED_LANGUAGES.has(baseLang)) {
return baseLang;
}
// Common mappings for non-standard codes
const mappings: Record<string, string> = {
'english': 'en',
'french': 'fr',
'german': 'de',
'spanish': 'es',
'italian': 'it',
'portuguese': 'pt',
'russian': 'ru',
'japanese': 'ja',
'chinese': 'zh',
'korean': 'ko',
'dutch': 'nl',
'polish': 'pl',
'swedish': 'sv',
'danish': 'da',
'norwegian': 'no',
'finnish': 'fi',
'und': 'en', // "undetermined" -> default to English
'mul': 'en', // "multiple" -> default to English
'': 'en', // empty -> default to English
};
return mappings[normalized] || 'en';
}
/**
* Fix 4: Remove stray IMG tags without src attributes
*
* IMG tags without src attributes can cause Kindle import failures.
*/
function fixStrayImages(content: string): { content: string; modified: boolean; count: number } {
// Parse with cheerio
const $ = cheerio.load(content, { xmlMode: true });
let count = 0;
// Find all img tags
$('img').each((_, element) => {
const $img = $(element);
const src = $img.attr('src');
// Remove if src is missing or empty
if (!src || src.trim() === '') {
$img.remove();
count++;
}
});
if (count > 0) {
return {
content: $.xml(),
modified: true,
count,
};
}
return { content, modified: false, count: 0 };
}
/**
* Check if a file is an EPUB based on extension
*/
export function isEpubFile(filePath: string): boolean {
return path.extname(filePath).toLowerCase() === '.epub';
}
/**
* Clean up a temporary fixed EPUB file and its parent directory
* The parent directory is a unique temp dir created during the fix process
*/
export async function cleanupFixedEpub(fixedPath: string): Promise<void> {
try {
// Remove the file first
await fs.unlink(fixedPath);
// Remove the parent temp directory (e.g., kindle-fix-1234567890)
const parentDir = path.dirname(fixedPath);
if (parentDir.includes('kindle-fix-')) {
await fs.rmdir(parentDir);
}
moduleLogger.debug(`Cleaned up fixed EPUB and temp directory: ${path.basename(fixedPath)}`);
} catch {
// Ignore cleanup errors
}
}