From 2ef9ac7be18c9915fb24ba3f3efd08b8bc4a2bcc Mon Sep 17 00:00:00 2001 From: kikootwo Date: Tue, 3 Feb 2026 16:34:57 -0500 Subject: [PATCH] Add Kindle EPUB compatibility fixer Introduce an optional Kindle EPUB compatibility fixer and integrate it into the ebook organization flow. Adds a new config key (ebook_kindle_fix_enabled, default false), a settings API update, and a UI toggle (visible when preferred format is EPUB). Implements src/lib/utils/epub-fixer.ts (uses adm-zip and cheerio) to apply fixes: add UTF-8 XML declarations, remove body/#bodymatter fragments from links, validate/normalize dc:language, and remove stray tags without src. organize-files.processor now detects EPUB downloads, runs the fixer (produces a temp fixed EPUB), uses the fixed file for organization, logs fixes, and cleans up temporary files; fix failures are non-blocking and the original download is preserved. Adds dependencies adm-zip and @types/adm-zip and updates documentation and types/UI to expose the new setting. Also includes helper functions to detect EPUB paths in downloads. --- documentation/integrations/ebook-sidecar.md | 25 +- documentation/settings-pages.md | 3 + package-lock.json | 251 +--------- package.json | 2 + src/app/admin/settings/lib/types.ts | 2 + .../admin/settings/tabs/EbookTab/EbookTab.tsx | 26 + .../tabs/EbookTab/useEbookSettings.ts | 1 + src/app/api/admin/settings/ebook/route.ts | 9 +- src/app/api/admin/settings/route.ts | 2 + .../processors/organize-files.processor.ts | 124 ++++- src/lib/utils/epub-fixer.ts | 465 ++++++++++++++++++ 11 files changed, 677 insertions(+), 233 deletions(-) create mode 100644 src/lib/utils/epub-fixer.ts diff --git a/documentation/integrations/ebook-sidecar.md b/documentation/integrations/ebook-sidecar.md index 3116e30..6ef526e 100644 --- a/documentation/integrations/ebook-sidecar.md +++ b/documentation/integrations/ebook-sidecar.md @@ -66,8 +66,30 @@ Ebooks are first-class citizens in RMAB, with their own request type, tracking, |-----|---------|---------|-------------| | `ebook_sidecar_preferred_format` | `epub` | `epub, pdf, mobi, azw3, any` | Preferred format | | `ebook_auto_grab_enabled` | `true` | `true, false` | Auto-create ebook requests after audiobook downloads | +| `ebook_kindle_fix_enabled` | `false` | `true, false` | Apply Kindle compatibility fixes to EPUB files | -*Note: Auto-grab is automatically disabled if no ebook sources are enabled. Manual fetch via admin buttons still works.* +*Notes:* +- *Auto-grab is automatically disabled if no ebook sources are enabled. Manual fetch via admin buttons still works.* +- *Kindle fix toggle only visible when preferred format is EPUB.* + +### Kindle EPUB Fix + +**Purpose:** Apply compatibility fixes to EPUB files before organizing, ensuring successful Kindle import. + +**Fixes Applied:** +1. **Encoding declaration** - Adds UTF-8 XML declaration to files missing it +2. **Body ID link fix** - Removes `#body`/`#bodymatter` fragments from hyperlinks that break on Kindle +3. **Language validation** - Ensures `dc:language` uses Amazon KDP-approved codes (defaults to `en` if invalid) +4. **Stray IMG removal** - Removes `` tags without `src` attributes + +**How It Works:** +- Enabled via toggle in E-book Sidecar settings (only visible when EPUB format selected) +- Applied during `organize_files` job, before copying to final location +- Creates temp fixed file → organizes temp file → cleans up temp file +- Original download file stays intact (important for seeding torrents) +- Non-blocking: if fix fails, continues with original file + +**Source:** Based on [kindle-epub-fix](https://github.com/innocenat/kindle-epub-fix) ## Database Schema @@ -204,6 +226,7 @@ Search: https://annas-archive.li/search?q=Title+Author&ext=epub&lang=en - `src/lib/utils/file-organizer.ts` (`organizeEbook` method) - `src/lib/utils/ranking-algorithm.ts` (`rankEbookTorrents` function) - `src/lib/utils/indexer-grouping.ts` (supports `'ebook'` type) +- `src/lib/utils/epub-fixer.ts` (Kindle EPUB compatibility fixes) **UI:** - `src/components/requests/RequestCard.tsx` (ebook badge) diff --git a/documentation/settings-pages.md b/documentation/settings-pages.md index 935dfe9..1c788f8 100644 --- a/documentation/settings-pages.md +++ b/documentation/settings-pages.md @@ -91,6 +91,7 @@ src/app/admin/settings/ 3. **General Settings Section** (visible when any source enabled) - Preferred format: EPUB (recommended), PDF, MOBI, AZW3, Any - Auto-grab toggle: Automatically create ebook requests after audiobook downloads + - Kindle fix toggle: Apply compatibility fixes to EPUB files (only visible when EPUB format selected) **Configuration Keys:** | Key | Default | Description | @@ -99,6 +100,7 @@ src/app/admin/settings/ | `ebook_indexer_search_enabled` | `false` | Enable Indexer Search via Prowlarr | | `ebook_sidecar_preferred_format` | `epub` | Preferred format | | `ebook_auto_grab_enabled` | `true` | Auto-create ebook requests after audiobook downloads | +| `ebook_kindle_fix_enabled` | `false` | Apply Kindle compatibility fixes to EPUB files | | `ebook_sidecar_base_url` | `https://annas-archive.li` | Anna's Archive mirror | | `ebook_sidecar_flaresolverr_url` | `` | FlareSolverr URL | @@ -107,6 +109,7 @@ src/app/admin/settings/ - If Indexer Search enabled → Falls back to indexer search if Anna's Archive fails/disabled - If both disabled → Ebook downloads completely off - If auto-grab disabled → Manual "Fetch Ebook" button only (admin buttons still work) +- If Kindle fix enabled (and EPUB format) → Applies compatibility fixes during organization ## Indexer Categories (Tabbed) diff --git a/package-lock.json b/package-lock.json index 8f07910..9cc1c26 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,9 +8,9 @@ "name": "readmeabook", "version": "0.1.0", "dependencies": { - "@headlessui/react": "^2.2.9", "@heroicons/react": "^2.2.0", "@prisma/client": "^6.19.0", + "adm-zip": "^0.5.16", "axios": "^1.7.2", "bcrypt": "^5.1.1", "bull": "^4.12.0", @@ -26,7 +26,6 @@ "parse-torrent": "^11.0.19", "react": "19.2.1", "react-dom": "19.2.1", - "react-hook-form": "^7.66.0", "react-swipeable": "^7.0.1", "string-similarity": "^4.0.4", "swr": "^2.3.6", @@ -38,6 +37,7 @@ "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.1", "@testing-library/user-event": "^14.6.1", + "@types/adm-zip": "^0.5.6", "@types/bcrypt": "^5.0.2", "@types/bull": "^4.10.0", "@types/jsonwebtoken": "^9.0.6", @@ -1217,79 +1217,6 @@ } } }, - "node_modules/@floating-ui/core": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.3.tgz", - "integrity": "sha512-sGnvb5dmrJaKEZ+LDIpguvdX3bDlEllmv4/ClQ9awcmCZrlx5jQyyMWFM5kBI+EyNOCDDiKk8il0zeuX3Zlg/w==", - "license": "MIT", - "dependencies": { - "@floating-ui/utils": "^0.2.10" - } - }, - "node_modules/@floating-ui/dom": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.4.tgz", - "integrity": "sha512-OOchDgh4F2CchOX94cRVqhvy7b3AFb+/rQXyswmzmGakRfkMgoWVjfnLWkRirfLEfuD4ysVW16eXzwt3jHIzKA==", - "license": "MIT", - "dependencies": { - "@floating-ui/core": "^1.7.3", - "@floating-ui/utils": "^0.2.10" - } - }, - "node_modules/@floating-ui/react": { - "version": "0.26.28", - "resolved": "https://registry.npmjs.org/@floating-ui/react/-/react-0.26.28.tgz", - "integrity": "sha512-yORQuuAtVpiRjpMhdc0wJj06b9JFjrYF4qp96j++v2NBpbi6SEGF7donUJ3TMieerQ6qVkAv1tgr7L4r5roTqw==", - "license": "MIT", - "dependencies": { - "@floating-ui/react-dom": "^2.1.2", - "@floating-ui/utils": "^0.2.8", - "tabbable": "^6.0.0" - }, - "peerDependencies": { - "react": ">=16.8.0", - "react-dom": ">=16.8.0" - } - }, - "node_modules/@floating-ui/react-dom": { - "version": "2.1.6", - "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.6.tgz", - "integrity": "sha512-4JX6rEatQEvlmgU80wZyq9RT96HZJa88q8hp0pBd+LrczeDI4o6uA2M+uvxngVHo4Ihr8uibXxH6+70zhAFrVw==", - "license": "MIT", - "dependencies": { - "@floating-ui/dom": "^1.7.4" - }, - "peerDependencies": { - "react": ">=16.8.0", - "react-dom": ">=16.8.0" - } - }, - "node_modules/@floating-ui/utils": { - "version": "0.2.10", - "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.10.tgz", - "integrity": "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ==", - "license": "MIT" - }, - "node_modules/@headlessui/react": { - "version": "2.2.9", - "resolved": "https://registry.npmjs.org/@headlessui/react/-/react-2.2.9.tgz", - "integrity": "sha512-Mb+Un58gwBn0/yWZfyrCh0TJyurtT+dETj7YHleylHk5od3dv2XqETPGWMyQ5/7sYN7oWdyM1u9MvC0OC8UmzQ==", - "license": "MIT", - "dependencies": { - "@floating-ui/react": "^0.26.16", - "@react-aria/focus": "^3.20.2", - "@react-aria/interactions": "^3.25.0", - "@tanstack/react-virtual": "^3.13.9", - "use-sync-external-store": "^1.5.0" - }, - "engines": { - "node": ">=10" - }, - "peerDependencies": { - "react": "^18 || ^19 || ^19.0.0-rc", - "react-dom": "^18 || ^19 || ^19.0.0-rc" - } - }, "node_modules/@heroicons/react": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@heroicons/react/-/react-2.2.0.tgz", @@ -2270,103 +2197,6 @@ "@prisma/debug": "6.19.0" } }, - "node_modules/@react-aria/focus": { - "version": "3.21.2", - "resolved": "https://registry.npmjs.org/@react-aria/focus/-/focus-3.21.2.tgz", - "integrity": "sha512-JWaCR7wJVggj+ldmM/cb/DXFg47CXR55lznJhZBh4XVqJjMKwaOOqpT5vNN7kpC1wUpXicGNuDnJDN1S/+6dhQ==", - "license": "Apache-2.0", - "dependencies": { - "@react-aria/interactions": "^3.25.6", - "@react-aria/utils": "^3.31.0", - "@react-types/shared": "^3.32.1", - "@swc/helpers": "^0.5.0", - "clsx": "^2.0.0" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1", - "react-dom": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, - "node_modules/@react-aria/interactions": { - "version": "3.25.6", - "resolved": "https://registry.npmjs.org/@react-aria/interactions/-/interactions-3.25.6.tgz", - "integrity": "sha512-5UgwZmohpixwNMVkMvn9K1ceJe6TzlRlAfuYoQDUuOkk62/JVJNDLAPKIf5YMRc7d2B0rmfgaZLMtbREb0Zvkw==", - "license": "Apache-2.0", - "dependencies": { - "@react-aria/ssr": "^3.9.10", - "@react-aria/utils": "^3.31.0", - "@react-stately/flags": "^3.1.2", - "@react-types/shared": "^3.32.1", - "@swc/helpers": "^0.5.0" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1", - "react-dom": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, - "node_modules/@react-aria/ssr": { - "version": "3.9.10", - "resolved": "https://registry.npmjs.org/@react-aria/ssr/-/ssr-3.9.10.tgz", - "integrity": "sha512-hvTm77Pf+pMBhuBm760Li0BVIO38jv1IBws1xFm1NoL26PU+fe+FMW5+VZWyANR6nYL65joaJKZqOdTQMkO9IQ==", - "license": "Apache-2.0", - "dependencies": { - "@swc/helpers": "^0.5.0" - }, - "engines": { - "node": ">= 12" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, - "node_modules/@react-aria/utils": { - "version": "3.31.0", - "resolved": "https://registry.npmjs.org/@react-aria/utils/-/utils-3.31.0.tgz", - "integrity": "sha512-ABOzCsZrWzf78ysswmguJbx3McQUja7yeGj6/vZo4JVsZNlxAN+E9rs381ExBRI0KzVo6iBTeX5De8eMZPJXig==", - "license": "Apache-2.0", - "dependencies": { - "@react-aria/ssr": "^3.9.10", - "@react-stately/flags": "^3.1.2", - "@react-stately/utils": "^3.10.8", - "@react-types/shared": "^3.32.1", - "@swc/helpers": "^0.5.0", - "clsx": "^2.0.0" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1", - "react-dom": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, - "node_modules/@react-stately/flags": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@react-stately/flags/-/flags-3.1.2.tgz", - "integrity": "sha512-2HjFcZx1MyQXoPqcBGALwWWmgFVUk2TuKVIQxCbRq7fPyWXIl6VHcakCLurdtYC2Iks7zizvz0Idv48MQ38DWg==", - "license": "Apache-2.0", - "dependencies": { - "@swc/helpers": "^0.5.0" - } - }, - "node_modules/@react-stately/utils": { - "version": "3.10.8", - "resolved": "https://registry.npmjs.org/@react-stately/utils/-/utils-3.10.8.tgz", - "integrity": "sha512-SN3/h7SzRsusVQjQ4v10LaVsDc81jyyR0DD5HnsQitm/I5WDpaSr2nRHtyloPFU48jlql1XX/S04T2DLQM7Y3g==", - "license": "Apache-2.0", - "dependencies": { - "@swc/helpers": "^0.5.0" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, - "node_modules/@react-types/shared": { - "version": "3.32.1", - "resolved": "https://registry.npmjs.org/@react-types/shared/-/shared-3.32.1.tgz", - "integrity": "sha512-famxyD5emrGGpFuUlgOP6fVW2h/ZaF405G5KDi3zPHzyjAWys/8W6NAVJtNbkCkhedmvL0xOhvt8feGXyXaw5w==", - "license": "Apache-2.0", - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.55.1", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.55.1.tgz", @@ -2743,15 +2573,6 @@ "devOptional": true, "license": "MIT" }, - "node_modules/@swc/helpers": { - "version": "0.5.17", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.17.tgz", - "integrity": "sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.8.0" - } - }, "node_modules/@szmarczak/http-timer": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.6.tgz", @@ -3035,33 +2856,6 @@ "tailwindcss": "4.1.17" } }, - "node_modules/@tanstack/react-virtual": { - "version": "3.13.12", - "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.13.12.tgz", - "integrity": "sha512-Gd13QdxPSukP8ZrkbgS2RwoZseTTbQPLnQEn7HY/rqtM+8Zt95f7xKC7N0EsKs7aoz0WzZ+fditZux+F8EzYxA==", - "license": "MIT", - "dependencies": { - "@tanstack/virtual-core": "3.13.12" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/tannerlinsley" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" - } - }, - "node_modules/@tanstack/virtual-core": { - "version": "3.13.12", - "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.13.12.tgz", - "integrity": "sha512-1YBOJfRHV4sXUmWsFSf5rQor4Ss82G8dQWLRbnk3GA4jeP8hQt1hxXh0tmflpC0dz3VgEv/1+qwPyLeWkQuPFA==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/tannerlinsley" - } - }, "node_modules/@testing-library/dom": { "version": "10.4.1", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz", @@ -3214,6 +3008,16 @@ "tslib": "^2.4.0" } }, + "node_modules/@types/adm-zip": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.5.7.tgz", + "integrity": "sha512-DNEs/QvmyRLurdQPChqq0Md4zGvPwHerAJYWk9l2jCbD1VPpnzRJorOdiq4zsw09NFbYnhfsoEhWtxIzXpn2yw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/aria-query": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", @@ -4179,6 +3983,15 @@ "node": ">=0.4.0" } }, + "node_modules/adm-zip": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, "node_modules/agent-base": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", @@ -9871,22 +9684,6 @@ "react": "^19.2.1" } }, - "node_modules/react-hook-form": { - "version": "7.66.0", - "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.66.0.tgz", - "integrity": "sha512-xXBqsWGKrY46ZqaHDo+ZUYiMUgi8suYu5kdrS20EG8KiL7VRQitEbNjm+UcrDYrNi1YLyfpmAeGjCZYXLT9YBw==", - "license": "MIT", - "engines": { - "node": ">=18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/react-hook-form" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17 || ^18 || ^19" - } - }, "node_modules/react-is": { "version": "16.13.1", "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", @@ -10833,12 +10630,6 @@ "dev": true, "license": "MIT" }, - "node_modules/tabbable": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.3.0.tgz", - "integrity": "sha512-EIHvdY5bPLuWForiR/AN2Bxngzpuwn1is4asboytXtpTgsArc+WmSJKVLlhdh71u7jFcryDqB2A8lQvj78MkyQ==", - "license": "MIT" - }, "node_modules/tailwindcss": { "version": "4.1.17", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.1.17.tgz", diff --git a/package.json b/package.json index fa58690..cf966c1 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "dependencies": { "@heroicons/react": "^2.2.0", "@prisma/client": "^6.19.0", + "adm-zip": "^0.5.16", "axios": "^1.7.2", "bcrypt": "^5.1.1", "bull": "^4.12.0", @@ -42,6 +43,7 @@ "devDependencies": { "@tailwindcss/postcss": "^4", "@testing-library/jest-dom": "^6.9.1", + "@types/adm-zip": "^0.5.6", "@testing-library/react": "^16.3.1", "@testing-library/user-event": "^14.6.1", "@types/bcrypt": "^5.0.2", diff --git a/src/app/admin/settings/lib/types.ts b/src/app/admin/settings/lib/types.ts index c4d1f3a..9c290bd 100644 --- a/src/app/admin/settings/lib/types.ts +++ b/src/app/admin/settings/lib/types.ts @@ -115,6 +115,8 @@ export interface EbookSettings { // General settings (shared across sources) preferredFormat: string; autoGrabEnabled: boolean; + // Kindle compatibility + kindleFixEnabled: boolean; } /** diff --git a/src/app/admin/settings/tabs/EbookTab/EbookTab.tsx b/src/app/admin/settings/tabs/EbookTab/EbookTab.tsx index 90c35be..6719e93 100644 --- a/src/app/admin/settings/tabs/EbookTab/EbookTab.tsx +++ b/src/app/admin/settings/tabs/EbookTab/EbookTab.tsx @@ -254,6 +254,32 @@ export function EbookTab({ ebook, onChange, onSuccess, onError, markAsSaved }: E

+ + {/* Kindle Fix Toggle - Only shown when EPUB is selected */} + {(ebook.preferredFormat === 'epub' || !ebook.preferredFormat) && ( +
+ updateEbook('kindleFixEnabled', e.target.checked)} + className="mt-1 h-5 w-5 rounded border-gray-300 text-blue-600 focus:ring-blue-500" + /> +
+ +

+ Apply compatibility fixes before organizing EPUB files. Fixes encoding declarations, + broken hyperlinks, invalid language tags, and orphaned image elements that can + cause Kindle import failures. +

+
+
+ )} )} diff --git a/src/app/admin/settings/tabs/EbookTab/useEbookSettings.ts b/src/app/admin/settings/tabs/EbookTab/useEbookSettings.ts index 381fe94..8d0b359 100644 --- a/src/app/admin/settings/tabs/EbookTab/useEbookSettings.ts +++ b/src/app/admin/settings/tabs/EbookTab/useEbookSettings.ts @@ -83,6 +83,7 @@ export function useEbookSettings({ ebook, onChange, onSuccess, onError, markAsSa baseUrl: ebook.baseUrl || 'https://annas-archive.li', flaresolverrUrl: ebook.flaresolverrUrl || '', autoGrabEnabled: ebook.autoGrabEnabled ?? true, + kindleFixEnabled: ebook.kindleFixEnabled ?? false, }), }); diff --git a/src/app/api/admin/settings/ebook/route.ts b/src/app/api/admin/settings/ebook/route.ts index 0f9a0c8..766429c 100644 --- a/src/app/api/admin/settings/ebook/route.ts +++ b/src/app/api/admin/settings/ebook/route.ts @@ -14,7 +14,7 @@ export async function PUT(request: NextRequest) { return requireAdmin(req, async () => { try { // Parse request body - new structure with separate source toggles - const { annasArchiveEnabled, indexerSearchEnabled, format, baseUrl, flaresolverrUrl, autoGrabEnabled } = await request.json(); + const { annasArchiveEnabled, indexerSearchEnabled, format, baseUrl, flaresolverrUrl, autoGrabEnabled, kindleFixEnabled } = await request.json(); // Enforce: auto-grab must be false if no sources are enabled const effectiveAutoGrabEnabled = (annasArchiveEnabled || indexerSearchEnabled) ? (autoGrabEnabled ?? true) : false; @@ -88,6 +88,13 @@ export async function PUT(request: NextRequest) { category: 'ebook', description: 'FlareSolverr URL for bypassing Cloudflare protection', }, + // Kindle compatibility + { + key: 'ebook_kindle_fix_enabled', + value: kindleFixEnabled ? 'true' : 'false', + category: 'ebook', + description: 'Apply compatibility fixes to EPUB files for Kindle import', + }, ]; await configService.setMany(configs); diff --git a/src/app/api/admin/settings/route.ts b/src/app/api/admin/settings/route.ts index 1b020be..7fd98f3 100644 --- a/src/app/api/admin/settings/route.ts +++ b/src/app/api/admin/settings/route.ts @@ -141,6 +141,8 @@ export async function GET(request: NextRequest) { preferredFormat: configMap.get('ebook_sidecar_preferred_format') || 'epub', // Auto-grab: default true to preserve existing behavior autoGrabEnabled: configMap.get('ebook_auto_grab_enabled') !== 'false', + // Kindle compatibility fixes: default false + kindleFixEnabled: configMap.get('ebook_kindle_fix_enabled') === 'true', }, general: { appName: configMap.get('app_name') || 'ReadMeABook', diff --git a/src/lib/processors/organize-files.processor.ts b/src/lib/processors/organize-files.processor.ts index b1c716c..af0e3db 100644 --- a/src/lib/processors/organize-files.processor.ts +++ b/src/lib/processors/organize-files.processor.ts @@ -10,6 +10,7 @@ import { RMABLogger } from '../utils/logger'; import { getLibraryService } from '../services/library'; import { getConfigService } from '../services/config.service'; import { generateFilesHash } from '../utils/files-hash'; +import { fixEpubForKindle, cleanupFixedEpub } from '../utils/epub-fixer'; /** * Process organize files job @@ -585,10 +586,52 @@ async function processEbookOrganization( }); const template = templateConfig?.value || '{author}/{title} {asin}'; + // Check if Kindle EPUB fix is needed + let effectiveDownloadPath = downloadPath; + let fixedEpubPath: string | null = null; + + // Detect the actual EPUB file path (handles both single file and directory downloads) + const epubFilePath = await detectEpubFilePath(downloadPath); + + // Only apply Kindle fix for EPUB files when enabled + if (epubFilePath) { + const configService = getConfigService(); + const kindleFixEnabled = await configService.get('ebook_kindle_fix_enabled'); + + if (kindleFixEnabled === 'true') { + logger.info('Kindle EPUB fix enabled - applying compatibility fixes'); + + const tempDir = process.env.TEMP_DIR || '/tmp/readmeabook'; + const fixResult = await fixEpubForKindle( + epubFilePath, + tempDir, + jobId ? { jobId, context: 'EpubFixer' } : undefined + ); + + if (fixResult.success && fixResult.outputPath) { + fixedEpubPath = fixResult.outputPath; + effectiveDownloadPath = fixResult.outputPath; + logger.info(`Using fixed EPUB: ${fixResult.outputPath}`); + + // Log fixes applied + const { encodingFixes, bodyIdLinkFixes, languageFix, strayImgFixes } = fixResult.fixesApplied; + const totalFixes = encodingFixes + bodyIdLinkFixes + (languageFix ? 1 : 0) + strayImgFixes; + if (totalFixes > 0) { + logger.info(`Kindle fixes applied: encoding=${encodingFixes}, bodyIdLinks=${bodyIdLinkFixes}, language=${languageFix}, strayImages=${strayImgFixes}`); + } + } else { + // Fix failed - continue with original file + logger.warn(`Kindle EPUB fix failed: ${fixResult.error}. Continuing with original file.`); + } + } else { + logger.info('Kindle EPUB fix disabled - organizing original file'); + } + } + // Organize ebook files (organizer will detect ebook type and skip audio-specific processing) // Pass all metadata that could be used in path templates (same as audiobooks) const result = await organizer.organizeEbook( - downloadPath, + effectiveDownloadPath, { title: book.title, author: book.author, @@ -603,6 +646,12 @@ async function processEbookOrganization( isIndexerDownload ); + // Clean up fixed EPUB temp file after organization (regardless of success) + if (fixedEpubPath) { + await cleanupFixedEpub(fixedEpubPath); + logger.info('Cleaned up temporary fixed EPUB'); + } + if (!result.success) { throw new Error(`Ebook organization failed: ${result.errors.join(', ')}`); } @@ -857,3 +906,76 @@ async function createEbookRequestIfEnabled( logger.error(`Failed to create ebook request: ${error instanceof Error ? error.message : 'Unknown error'}`); } } + +// ========================================================================= +// HELPER FUNCTIONS +// ========================================================================= + +/** + * Detect the path to an EPUB file from download path + * Handles both single file downloads (direct path) and directory downloads (indexer) + * + * @param downloadPath - Path to the download (file or directory) + * @returns Full path to EPUB file, or null if no EPUB found + */ +async function detectEpubFilePath(downloadPath: string): Promise { + const fs = await import('fs/promises'); + const path = await import('path'); + + try { + const stats = await fs.stat(downloadPath); + + if (stats.isFile()) { + // Single file - check if it's an EPUB + if (path.extname(downloadPath).toLowerCase() === '.epub') { + return downloadPath; + } + return null; + } + + // Directory - search for EPUB file + const files = await walkDirectory(downloadPath); + const epubFile = files.find(file => + path.extname(file).toLowerCase() === '.epub' + ); + + if (epubFile) { + return path.join(downloadPath, epubFile); + } + + return null; + } catch { + return null; + } +} + +/** + * Recursively walk directory to find all files + * Returns relative paths from the base directory + */ +async function walkDirectory(dir: string, baseDir: string = ''): Promise { + const fs = await import('fs/promises'); + const path = await import('path'); + + const files: string[] = []; + + try { + const entries = await fs.readdir(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + const relativePath = baseDir ? path.join(baseDir, entry.name) : entry.name; + + if (entry.isDirectory()) { + const subFiles = await walkDirectory(fullPath, relativePath); + files.push(...subFiles); + } else { + files.push(relativePath); + } + } + } catch { + // Directory not accessible + } + + return files; +} diff --git a/src/lib/utils/epub-fixer.ts b/src/lib/utils/epub-fixer.ts new file mode 100644 index 0000000..fafe2cd --- /dev/null +++ b/src/lib/utils/epub-fixer.ts @@ -0,0 +1,465 @@ +/** + * Component: EPUB Kindle Compatibility Fixer + * Documentation: documentation/integrations/ebook-sidecar.md + * + * Applies compatibility fixes to EPUB files for Kindle import. + * Based on: https://github.com/innocenat/kindle-epub-fix + * + * Fixes applied: + * 1. Encoding declaration - Adds UTF-8 XML declaration to files missing it + * 2. Body ID link fix - Removes #bodyid fragments from hyperlinks + * 3. Language validation - Ensures dc:language uses Amazon KDP-approved codes + * 4. Stray IMG removal - Removes tags without src attributes + */ + +import AdmZip from 'adm-zip'; +import * as cheerio from 'cheerio'; +import path from 'path'; +import fs from 'fs/promises'; +import { RMABLogger } from './logger'; + +const moduleLogger = RMABLogger.create('EpubFixer'); + +/** + * Amazon KDP approved language codes + * Source: https://kdp.amazon.com/en_US/help/topic/G200673300 + */ +const AMAZON_APPROVED_LANGUAGES: Set = new Set([ + // ISO 639-1 codes (2-letter) + 'af', 'sq', 'ar', 'hy', 'az', 'eu', 'be', 'bn', 'bs', 'br', 'bg', 'ca', + 'zh', 'hr', 'cs', 'da', 'nl', 'en', 'eo', 'et', 'fo', 'fi', 'fr', 'fy', + 'gl', 'ka', 'de', 'el', 'gu', 'he', 'hi', 'hu', 'is', 'id', 'ga', 'it', + 'ja', 'kn', 'kk', 'ko', 'ku', 'ky', 'la', 'lv', 'lt', 'lb', 'mk', 'ms', + 'ml', 'mt', 'mr', 'mn', 'ne', 'no', 'nb', 'nn', 'oc', 'or', 'ps', 'fa', + 'pl', 'pt', 'pa', 'rm', 'ro', 'ru', 'gd', 'sr', 'sk', 'sl', 'es', 'sw', + 'sv', 'tl', 'ta', 'te', 'th', 'tr', 'uk', 'ur', 'uz', 'vi', 'cy', 'yi', + // ISO 639-2 codes (3-letter) commonly used + 'eng', 'fra', 'deu', 'spa', 'ita', 'por', 'rus', 'jpn', 'zho', 'kor', + 'ara', 'hin', 'nld', 'pol', 'tur', 'swe', 'dan', 'nor', 'fin', 'ces', + // Regional variants + 'en-us', 'en-gb', 'en-au', 'en-ca', 'en-nz', 'en-ie', 'en-za', + 'pt-br', 'pt-pt', 'zh-cn', 'zh-tw', 'zh-hk', 'es-es', 'es-mx', 'es-ar', + 'fr-fr', 'fr-ca', 'de-de', 'de-at', 'de-ch', 'it-it', 'nl-nl', 'nl-be', +]); + +/** + * Content file extensions that should be processed + */ +const CONTENT_EXTENSIONS = ['.html', '.xhtml', '.htm', '.xml']; + +/** + * Result of the EPUB fixing process + */ +export interface EpubFixResult { + success: boolean; + outputPath: string | null; + fixesApplied: { + encodingFixes: number; + bodyIdLinkFixes: number; + languageFix: boolean; + strayImgFixes: number; + }; + error?: string; +} + +/** + * Logger interface for job-aware logging + */ +interface LoggerConfig { + jobId: string; + context: string; +} + +/** + * Fix EPUB file for Kindle compatibility + * + * @param sourcePath - Path to the source EPUB file + * @param tempDir - Directory to write the fixed EPUB to + * @param loggerConfig - Optional logger configuration for job-aware logging + * @returns Result with path to fixed EPUB or error + */ +export async function fixEpubForKindle( + sourcePath: string, + tempDir: string, + loggerConfig?: LoggerConfig +): Promise { + const logger = loggerConfig + ? RMABLogger.forJob(loggerConfig.jobId, loggerConfig.context) + : null; + + const result: EpubFixResult = { + success: false, + outputPath: null, + fixesApplied: { + encodingFixes: 0, + bodyIdLinkFixes: 0, + languageFix: false, + strayImgFixes: 0, + }, + }; + + try { + await logger?.info(`Starting Kindle EPUB fix for: ${path.basename(sourcePath)}`); + + // Verify source file exists + try { + await fs.access(sourcePath, fs.constants.R_OK); + } catch { + throw new Error(`Source EPUB not found or not readable: ${sourcePath}`); + } + + // Load the EPUB (ZIP file) + const zip = new AdmZip(sourcePath); + const zipEntries = zip.getEntries(); + + await logger?.info(`Loaded EPUB with ${zipEntries.length} entries`); + + // Track OPF file for language fix + let opfEntry: AdmZip.IZipEntry | null = null; + let opfPath = ''; + + // Find content files and OPF + for (const entry of zipEntries) { + const entryPath = entry.entryName.toLowerCase(); + + // Find OPF file (metadata) + if (entryPath.endsWith('.opf')) { + opfEntry = entry; + opfPath = entry.entryName; + } + } + + // Process content files (HTML/XHTML) + for (const entry of zipEntries) { + if (entry.isDirectory) continue; + + const ext = path.extname(entry.entryName).toLowerCase(); + if (!CONTENT_EXTENSIONS.includes(ext)) continue; + + // Read file content + let content = entry.getData().toString('utf8'); + let modified = false; + + // Fix 1: Encoding declaration + const encodingResult = fixEncoding(content); + if (encodingResult.modified) { + content = encodingResult.content; + modified = true; + result.fixesApplied.encodingFixes++; + } + + // Fix 2: Body ID links + const bodyIdResult = fixBodyIdLinks(content); + if (bodyIdResult.modified) { + content = bodyIdResult.content; + modified = true; + result.fixesApplied.bodyIdLinkFixes += bodyIdResult.count; + } + + // Fix 4: Stray IMG tags (applied to HTML content) + const strayImgResult = fixStrayImages(content); + if (strayImgResult.modified) { + content = strayImgResult.content; + modified = true; + result.fixesApplied.strayImgFixes += strayImgResult.count; + } + + // Update entry if modified + if (modified) { + zip.updateFile(entry.entryName, Buffer.from(content, 'utf8')); + } + } + + // Fix 3: Language validation (in OPF file) + if (opfEntry) { + const opfContent = opfEntry.getData().toString('utf8'); + const languageResult = fixLanguage(opfContent); + + if (languageResult.modified) { + zip.updateFile(opfPath, Buffer.from(languageResult.content, 'utf8')); + result.fixesApplied.languageFix = true; + await logger?.info(`Fixed language tag: "${languageResult.originalLang}" -> "${languageResult.newLang}"`); + } + } + + // Log fixes applied + const totalFixes = + result.fixesApplied.encodingFixes + + result.fixesApplied.bodyIdLinkFixes + + (result.fixesApplied.languageFix ? 1 : 0) + + result.fixesApplied.strayImgFixes; + + if (totalFixes > 0) { + await logger?.info( + `Applied ${totalFixes} fixes: ` + + `encoding=${result.fixesApplied.encodingFixes}, ` + + `bodyIdLinks=${result.fixesApplied.bodyIdLinkFixes}, ` + + `language=${result.fixesApplied.languageFix}, ` + + `strayImages=${result.fixesApplied.strayImgFixes}` + ); + } else { + await logger?.info('No fixes needed - EPUB is already Kindle-compatible'); + } + + // Create unique temp subdirectory to avoid filename conflicts + // This preserves the original filename for the final organized file + const uniqueDir = path.join(tempDir, `kindle-fix-${Date.now()}`); + await fs.mkdir(uniqueDir, { recursive: true }); + + // Keep original filename + const sourceFilename = path.basename(sourcePath); + const outputPath = path.join(uniqueDir, sourceFilename); + + // Write fixed EPUB + zip.writeZip(outputPath); + + await logger?.info(`Fixed EPUB written to temp directory, preserving filename: ${sourceFilename}`); + + result.success = true; + result.outputPath = outputPath; + + return result; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + await logger?.error(`EPUB fix failed: ${errorMessage}`); + result.error = errorMessage; + return result; + } +} + +/** + * Fix 1: Add UTF-8 XML encoding declaration if missing + * + * Many EPUBs lack the XML declaration, which can cause Kindle import issues. + * Adds: + */ +function fixEncoding(content: string): { content: string; modified: boolean } { + // Check if already has XML declaration + const xmlDeclRegex = /^\s*<\?xml[^?]*\?>/i; + + if (xmlDeclRegex.test(content)) { + // Already has declaration, check if it has encoding + const hasEncoding = /encoding\s*=\s*["'][^"']+["']/i.test(content); + if (hasEncoding) { + return { content, modified: false }; + } + + // Has declaration but no encoding - add encoding attribute + const updatedContent = content.replace( + /(<\?xml[^?]*?)(\?>)/i, + '$1 encoding="utf-8"$2' + ); + return { content: updatedContent, modified: true }; + } + + // No declaration - add one at the beginning + const declaration = '\n'; + return { content: declaration + content.trimStart(), modified: true }; +} + +/** + * Fix 2: Remove body ID fragments from hyperlinks + * + * Links like "file.html#body" or "file.xhtml#bodymatter" can break on Kindle. + * This removes the fragment when it targets body-related IDs. + */ +function fixBodyIdLinks(content: string): { content: string; modified: boolean; count: number } { + // Pattern to match href attributes with body-related ID fragments + // Matches: href="file.html#body", href="page.xhtml#bodymatter", etc. + const bodyIdPattern = /href\s*=\s*["']([^"'#]+)#(body[^"']*|bodymatter)["']/gi; + + let count = 0; + const updatedContent = content.replace(bodyIdPattern, (match, file) => { + count++; + return `href="${file}"`; + }); + + return { + content: updatedContent, + modified: count > 0, + count, + }; +} + +/** + * Fix 3: Validate and fix dc:language in OPF metadata + * + * Ensures the language tag is one approved by Amazon KDP. + * If invalid or missing, defaults to "en" (English). + */ +function fixLanguage(opfContent: string): { + content: string; + modified: boolean; + originalLang: string; + newLang: string; +} { + const result = { + content: opfContent, + modified: false, + originalLang: '', + newLang: '', + }; + + // Parse with cheerio (XML mode) + const $ = cheerio.load(opfContent, { xmlMode: true }); + + // Find dc:language element (handle namespace variations) + let langElement = $('dc\\:language, language'); + + if (langElement.length === 0) { + // No language tag found - add one + // Find the metadata element to insert into + const metadata = $('metadata'); + if (metadata.length > 0) { + // Add language element + metadata.append('\n en'); + result.content = $.xml(); + result.modified = true; + result.originalLang = '(missing)'; + result.newLang = 'en'; + } + return result; + } + + // Get current language value + const currentLang = langElement.first().text().trim().toLowerCase(); + result.originalLang = currentLang; + + // Check if it's a valid Amazon language + if (isValidAmazonLanguage(currentLang)) { + return result; // No fix needed + } + + // Try to normalize the language + const normalizedLang = normalizeLanguage(currentLang); + + if (normalizedLang !== currentLang) { + langElement.first().text(normalizedLang); + result.content = $.xml(); + result.modified = true; + result.newLang = normalizedLang; + } + + return result; +} + +/** + * Check if a language code is approved by Amazon KDP + */ +function isValidAmazonLanguage(lang: string): boolean { + const normalized = lang.toLowerCase().trim(); + + // Direct match + if (AMAZON_APPROVED_LANGUAGES.has(normalized)) { + return true; + } + + // Check base language (e.g., "en-us" -> "en") + const baseLang = normalized.split('-')[0]; + return AMAZON_APPROVED_LANGUAGES.has(baseLang); +} + +/** + * Normalize a language code to an Amazon-approved format + */ +function normalizeLanguage(lang: string): string { + const normalized = lang.toLowerCase().trim(); + + // If already valid, return as-is + if (isValidAmazonLanguage(normalized)) { + return normalized; + } + + // Try base language + const baseLang = normalized.split('-')[0]; + if (AMAZON_APPROVED_LANGUAGES.has(baseLang)) { + return baseLang; + } + + // Common mappings for non-standard codes + const mappings: Record = { + 'english': 'en', + 'french': 'fr', + 'german': 'de', + 'spanish': 'es', + 'italian': 'it', + 'portuguese': 'pt', + 'russian': 'ru', + 'japanese': 'ja', + 'chinese': 'zh', + 'korean': 'ko', + 'dutch': 'nl', + 'polish': 'pl', + 'swedish': 'sv', + 'danish': 'da', + 'norwegian': 'no', + 'finnish': 'fi', + 'und': 'en', // "undetermined" -> default to English + 'mul': 'en', // "multiple" -> default to English + '': 'en', // empty -> default to English + }; + + return mappings[normalized] || 'en'; +} + +/** + * Fix 4: Remove stray IMG tags without src attributes + * + * IMG tags without src attributes can cause Kindle import failures. + */ +function fixStrayImages(content: string): { content: string; modified: boolean; count: number } { + // Parse with cheerio + const $ = cheerio.load(content, { xmlMode: true }); + + let count = 0; + + // Find all img tags + $('img').each((_, element) => { + const $img = $(element); + const src = $img.attr('src'); + + // Remove if src is missing or empty + if (!src || src.trim() === '') { + $img.remove(); + count++; + } + }); + + if (count > 0) { + return { + content: $.xml(), + modified: true, + count, + }; + } + + return { content, modified: false, count: 0 }; +} + +/** + * Check if a file is an EPUB based on extension + */ +export function isEpubFile(filePath: string): boolean { + return path.extname(filePath).toLowerCase() === '.epub'; +} + +/** + * Clean up a temporary fixed EPUB file and its parent directory + * The parent directory is a unique temp dir created during the fix process + */ +export async function cleanupFixedEpub(fixedPath: string): Promise { + try { + // Remove the file first + await fs.unlink(fixedPath); + + // Remove the parent temp directory (e.g., kindle-fix-1234567890) + const parentDir = path.dirname(fixedPath); + if (parentDir.includes('kindle-fix-')) { + await fs.rmdir(parentDir); + } + + moduleLogger.debug(`Cleaned up fixed EPUB and temp directory: ${path.basename(fixedPath)}`); + } catch { + // Ignore cleanup errors + } +}