diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..8886bdd2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,57 @@ +# Changelog + +## [Unreleased] + +### New Features + +Bulk Annotation Sync: The "Synchronize data" button now syncs all annotations from all books in your reading history at once (previously only the currently open book). Sync on suspend still syncs statistics and annotations for the currently open book only (keeping suspend snappy). + +### Bug Fixes + +Fixed Book Duplicates: Some users saw the same book twice in KoInsight, one with statistics, one with annotations. We now match books using their unique MD5 checksum instead of title. + +If you have duplicates, either: + +1. Delete your database and re-sync (recommended - clean start) +2. Manually delete duplicate books in the web interface (the faulty one) + +New syncs won't create duplicates. If you still see duplicates, you most likely have duplicates in your KoReader statistics database and KoInsight makes those visible. + +### Breaking Changes + +Plugin version 0.3.0 required. Update it before syncing. + +--- + +## [0.2.2] - 2026-01-11 + +### Added + +- Annotation sync support for currently open book +- Mark deleted annotations in the database + +### Fixed + +- Annotations now properly marked as deleted when removed in KoReader +- Docker build issues + +## [0.2.0] - 2026-01-11 + +### Added + +- Plugin versioning system +- Server validates plugin version before accepting data + +### Changed + +- **BREAKING:** Server now requires specific plugin version + +--- + +## Earlier Versions + +See git history for changes prior to v0.2.0. + +[Unreleased]: https://github.com/GeorgeSG/koinsight/compare/v0.2.2...HEAD +[0.2.2]: https://github.com/GeorgeSG/koinsight/compare/v0.2.0...v0.2.2 +[0.2.0]: https://github.com/GeorgeSG/koinsight/releases/tag/v0.2.0 diff --git a/apps/server/src/koplugin/koplugin-router.test.ts b/apps/server/src/koplugin/koplugin-router.test.ts index f8ebbf63..03ec6fc5 100644 --- a/apps/server/src/koplugin/koplugin-router.test.ts +++ b/apps/server/src/koplugin/koplugin-router.test.ts @@ -3,20 +3,18 @@ import request from 'supertest'; import { createDevice } from '../db/factories/device-factory'; import { fakeKoReaderAnnotation } from '../db/factories/koreader-annotation-factory'; import { db } from '../knex'; -import { kopluginRouter } from './koplugin-router'; +import { kopluginRouter, REQUIRED_PLUGIN_VERSION } from './koplugin-router'; describe('koplugin-router', () => { const app = express(); app.use(express.json()); app.use('/koplugin', kopluginRouter); - const PLUGIN_VERSION = '0.2.0'; - describe('POST /koplugin/device', () => { it('registers a device', async () => { const response = await request(app) .post('/koplugin/device') - .send({ id: 'device-123', model: 'Kindle Paperwhite', version: PLUGIN_VERSION }); + .send({ id: 'device-123', model: 'Kindle Paperwhite', version: REQUIRED_PLUGIN_VERSION }); expect(response.status).toBe(200); expect(response.body).toEqual({ message: 'Device registered successfully' }); @@ -33,7 +31,7 @@ describe('koplugin-router', () => { it('returns 400 when device ID is missing', async () => { const response = await request(app) .post('/koplugin/device') - .send({ model: 'Kindle', version: PLUGIN_VERSION }); + .send({ model: 'Kindle', version: REQUIRED_PLUGIN_VERSION }); expect(response.status).toBe(400); expect(response.body).toEqual({ error: 'Missing device ID or model' }); @@ -42,7 +40,7 @@ describe('koplugin-router', () => { it('returns 400 when model is missing', async () => { const response = await request(app) .post('/koplugin/device') - .send({ id: 'device-123', version: PLUGIN_VERSION }); + .send({ id: 'device-123', version: REQUIRED_PLUGIN_VERSION }); expect(response.status).toBe(400); expect(response.body).toEqual({ error: 'Missing device ID or model' }); @@ -75,7 +73,7 @@ describe('koplugin-router', () => { const response = await request(app) .post('/koplugin/import') .send({ - version: PLUGIN_VERSION, + version: REQUIRED_PLUGIN_VERSION, books: [ { md5: bookMd5, @@ -130,7 +128,7 @@ describe('koplugin-router', () => { const response = await request(app) .post('/koplugin/import') .send({ - version: PLUGIN_VERSION, + version: REQUIRED_PLUGIN_VERSION, books: [ { md5: bookMd5, @@ -222,7 +220,7 @@ describe('koplugin-router', () => { it('returns health status', async () => { const response = await request(app) .get('/koplugin/health') - .send({ version: PLUGIN_VERSION }); + .send({ version: REQUIRED_PLUGIN_VERSION }); expect(response.status).toBe(200); expect(response.body).toEqual({ message: 'Plugin is healthy' }); diff --git a/apps/server/src/koplugin/koplugin-router.ts b/apps/server/src/koplugin/koplugin-router.ts index f7d517f0..10bef0b0 100644 --- a/apps/server/src/koplugin/koplugin-router.ts +++ b/apps/server/src/koplugin/koplugin-router.ts @@ -11,7 +11,7 @@ import { UploadService } from '../upload/upload-service'; // Router for KoInsight koreader plugin const router = Router(); -const REQUIRED_PLUGIN_VERSION = '0.2.0'; +export const REQUIRED_PLUGIN_VERSION = '0.3.0'; const rejectOldPluginVersion = (req: Request, res: Response, next: NextFunction) => { const { version } = req.body; @@ -53,6 +53,7 @@ router.post('/import', rejectOldPluginVersion, async (req, res) => { const koreaderBooks: KoReaderBook[] = req.body.books; const newPageStats: PageStat[] = req.body.stats; const annotations: Record = req.body.annotations || {}; + const deviceId: string | undefined = req.body.device_id; // For annotation sync path try { console.debug('Importing books:', koreaderBooks); @@ -63,7 +64,7 @@ router.post('/import', rejectOldPluginVersion, async (req, res) => { 'books with annotations' ); - await UploadService.uploadStatisticData(koreaderBooks, newPageStats, annotations); + await UploadService.uploadStatisticData(koreaderBooks, newPageStats, annotations, deviceId); res.status(200).json({ message: 'Upload successful' }); } catch (err) { console.error(err); diff --git a/apps/server/src/upload/upload-service.ts b/apps/server/src/upload/upload-service.ts index ec9c1688..febf9375 100644 --- a/apps/server/src/upload/upload-service.ts +++ b/apps/server/src/upload/upload-service.ts @@ -41,7 +41,8 @@ export class UploadService { static uploadStatisticData( booksToImport: KoReaderBook[], newPageStats: PageStat[], - annotationsByBook?: Record + annotationsByBook?: Record, + deviceIdOverride?: string // For annotation sync path without stats ) { return db.transaction(async (trx) => { // Insert books @@ -58,8 +59,13 @@ export class UploadService { newBooks.map(({ id, ...book }) => trx('book').insert(book).onConflict('md5').ignore()) ); - const hasUnknownDevices = - newPageStats.length > 0 && newPageStats[0].device_id === this.UNKNOWN_DEVICE_ID; + // Determine device ID: from stats, override, or fall back to unknown device + const deviceId = + newPageStats.length > 0 + ? newPageStats[0].device_id + : deviceIdOverride || this.UNKNOWN_DEVICE_ID; + + const hasUnknownDevices = deviceId === this.UNKNOWN_DEVICE_ID; if (hasUnknownDevices) { let unknownDevice = await trx('device') @@ -76,47 +82,54 @@ export class UploadService { } const newBookDevices: Omit[] = booksToImport.map((book) => ({ - device_id: newPageStats[0].device_id, + device_id: deviceId, book_md5: book.md5, last_open: book.last_open, pages: book.pages, notes: book.notes, highlights: book.highlights, - total_read_pages: book.total_read_pages, - total_read_time: book.total_read_time, + total_read_pages: book.total_read_pages ?? 0, + total_read_time: book.total_read_time ?? 0, })); await Promise.all( - newBookDevices.map((bookDevice) => - trx('book_device') + newBookDevices.map((bookDevice) => { + const { book_md5, device_id, total_read_time, total_read_pages, ...otherFields } = + bookDevice; + + // Always merge these fields + const fieldsToMerge: (keyof BookDevice)[] = ['last_open', 'pages', 'notes', 'highlights']; + + // Only merge statistics fields if they have actual values (if on statistics.db sync path) + // This prevents annotation-only syncs from overwriting with zeros + if (total_read_time !== undefined && total_read_time > 0) { + fieldsToMerge.push('total_read_time'); + } + if (total_read_pages !== undefined && total_read_pages > 0) { + fieldsToMerge.push('total_read_pages'); + } + + return trx('book_device') .insert(bookDevice) .onConflict(['book_md5', 'device_id']) - .merge([ - 'last_open', - 'pages', - 'notes', - 'highlights', - 'total_read_time', - 'total_read_pages', - ]) - ) + .merge(fieldsToMerge); + }) ); - // Insert page stats - await Promise.all( - newPageStats.map((pageStat) => - trx('page_stat') - .insert(pageStat) - .onConflict(['device_id', 'book_md5', 'page', 'start_time']) - .merge(['duration', 'total_pages']) - ) - ); + // Insert page stats (only on stats sync path! there are non for annotation sync path) + if (newPageStats.length > 0) { + await Promise.all( + newPageStats.map((pageStat) => + trx('page_stat') + .insert(pageStat) + .onConflict(['device_id', 'book_md5', 'page', 'start_time']) + .merge(['duration', 'total_pages']) + ) + ); + } // Insert annotations if provided if (annotationsByBook) { - const deviceId = - newPageStats.length > 0 ? newPageStats[0].device_id : this.UNKNOWN_DEVICE_ID; - await Promise.all( Object.entries(annotationsByBook).map(([bookMd5, annotations]) => AnnotationsRepository.bulkInsert(bookMd5, deviceId, annotations, trx) diff --git a/packages/common/types/book.ts b/packages/common/types/book.ts index 4a4f2e25..690c8d67 100644 --- a/packages/common/types/book.ts +++ b/packages/common/types/book.ts @@ -9,8 +9,9 @@ export type KoReaderBook = { pages: number; series: string; language: string; - total_read_time: number; - total_read_pages: number; + // These fields only come from statistics.db sync, not annotation sync + total_read_time?: number; + total_read_pages?: number; }; export type DbBook = { diff --git a/plugins/koinsight.koplugin/annotation_reader.lua b/plugins/koinsight.koplugin/annotation_reader.lua index 321f87d0..7a32f882 100644 --- a/plugins/koinsight.koplugin/annotation_reader.lua +++ b/plugins/koinsight.koplugin/annotation_reader.lua @@ -3,10 +3,34 @@ local logger = require("logger") local KoInsightAnnotationReader = {} +-- NOTE: +-- This module is used both inside the reader (normal annotation sync) +-- and outside (bulk sync). +-- There is a chance that after rebooting KoReader, the ReaderUI is not +-- yet available, so requiring it can blow up. +-- Therefore we lazy-load ReaderUI behind pcall() the first time we actually need it. +-- This keeps the module safe to require in any context while still allowing us to +-- use the live reader UI when it exists. +local ReaderUI_ok, ReaderUI = nil, nil +local function get_live_ui() + if ReaderUI_ok == nil then + ReaderUI_ok, ReaderUI = pcall(require, "apps/reader/readerui") + end + return (ReaderUI_ok and ReaderUI and ReaderUI.instance) or nil +end + +-- KoReader has this API, ideal for bulk operations +local function open_sidecar_readonly(doc_path) + local sidecar = DocSettings:findSidecarFile(doc_path) + if not sidecar then + return nil + end + return DocSettings.openSettingsFile(sidecar) +end + -- Get the currently opened document function KoInsightAnnotationReader.getCurrentDocument() - local ReaderUI = require("apps/reader/readerui") - local ui = ReaderUI.instance + local ui = get_live_ui() if ui and ui.document and ui.document.file then return ui.document.file @@ -17,49 +41,21 @@ end -- Get the MD5 hash for the currently open document function KoInsightAnnotationReader.getCurrentBookMd5() - local current_doc = KoInsightAnnotationReader.getCurrentDocument() - if not current_doc then - return nil - end - - -- Get document info from ReaderUI - local ReaderUI = require("apps/reader/readerui") - local ui = ReaderUI.instance - - if ui and ui.document and ui.document.info then - local doc_props = ui.document:getProps() - if doc_props and doc_props.title then - -- Try to find book by title in statistics database - local SQ3 = require("lua-ljsqlite3/init") - local DataStorage = require("datastorage") - local db_location = DataStorage:getSettingsDir() .. "/statistics.sqlite3" - - local conn = SQ3.open(db_location) - - -- Escape single quotes in title for SQL - local safe_title = doc_props.title:gsub("'", "''") - local query = string.format("SELECT md5 FROM book WHERE title = '%s'", safe_title) - - logger.dbg("[KoInsight] Looking for book with title:", doc_props.title) - - local result, rows = conn:exec(query) - conn:close() - - if rows > 0 and result[1] and result[1][1] then - local md5 = result[1][1] - logger.info("[KoInsight] Found MD5 for current book:", md5) - return md5 - else - logger.warn("[KoInsight] Book not found in statistics database:", doc_props.title) - end - end + -- when inside reader + local ui = get_live_ui() + if ui and ui.doc_settings then + return ui.doc_settings:readSetting("partial_md5_checksum") end - return nil + -- fallback (if called outside reader, e.g. in bulk operation) + local current_doc = KoInsightAnnotationReader.getCurrentDocument() + local ds = current_doc and open_sidecar_readonly(current_doc) + return ds and ds:readSetting("partial_md5_checksum") or nil end -- Get annotations for the currently opened book function KoInsightAnnotationReader.getCurrentBookAnnotations() + local ui = get_live_ui() local current_doc = KoInsightAnnotationReader.getCurrentDocument() if not current_doc then @@ -71,14 +67,18 @@ function KoInsightAnnotationReader.getCurrentBookAnnotations() -- Force flush any in-memory changes to disk before reading -- Otherwise changes are not reflected - local ReaderUI = require("apps/reader/readerui") - local ui = ReaderUI.instance + -- + -- IMPORTANT: + -- If we are inside the reader, ui.doc_settings is the freshest source (in-memory). + -- We flush to ensure sidecar on disk is up-to-date for other codepaths. if ui and ui.doc_settings then logger.dbg("[KoInsight] Flushing doc settings to disk") ui.doc_settings:flush() end - local doc_settings = DocSettings:open(current_doc) + -- Prefer live doc_settings when inside reader (fresh, no extra sidecar open) + -- Fall back to read-only sidecar open (outside reader withouth live settings) + local doc_settings = (ui and ui.doc_settings) or open_sidecar_readonly(current_doc) if not doc_settings then logger.dbg("[KoInsight] No doc settings found for:", current_doc) return nil @@ -99,6 +99,9 @@ function KoInsightAnnotationReader.getCurrentBookAnnotations() if ui and ui.document then total_pages = ui.document:getPageCount() logger.dbg("[KoInsight] Document has", total_pages, "total pages") + else + -- Fallback for outside of reader, where we have no live ui.document + total_pages = doc_settings:readSetting("doc_pages") end logger.info("[KoInsight] Found", #annotations, "annotations for current book") @@ -109,8 +112,8 @@ end function KoInsightAnnotationReader.getAnnotationsByBook() local annotations_by_book = {} - -- For now, only get annotations from currently opened book - -- TODO: check bulk-syncing possibilities + -- Get annotations from currently opened book + -- Bulk syncing is another code path since we need to open sidecar files for bulk syncing local current_annotations, total_pages = KoInsightAnnotationReader.getCurrentBookAnnotations() if not current_annotations or #current_annotations == 0 then @@ -127,10 +130,21 @@ function KoInsightAnnotationReader.getAnnotationsByBook() end -- Clean up annotations for JSON serialization - local cleaned_annotations = {} - for _, annotation in ipairs(current_annotations) do - -- Only include fields that are needed - local cleaned = { + local cleaned_annotations = + KoInsightAnnotationReader.cleanAnnotations(current_annotations, total_pages) + + annotations_by_book[book_md5] = cleaned_annotations + logger.info("[KoInsight] Prepared", #cleaned_annotations, "annotations for book", book_md5) + + return annotations_by_book +end + +-- Clean annotations for JSON serialization +-- Removes unnecessary fields and formats data for server +function KoInsightAnnotationReader.cleanAnnotations(annotations, total_pages) + local cleaned = {} + for _, annotation in ipairs(annotations) do + local cleaned_annotation = { datetime = annotation.datetime, drawer = annotation.drawer, color = annotation.color, @@ -139,29 +153,206 @@ function KoInsightAnnotationReader.getAnnotationsByBook() chapter = annotation.chapter, pageno = annotation.pageno, page = annotation.page, - total_pages = total_pages, -- Current document total pages (captured at sync time) + total_pages = total_pages, } - -- Include datetime_updated if it exists + -- Include optional fields if present if annotation.datetime_updated then - cleaned.datetime_updated = annotation.datetime_updated + cleaned_annotation.datetime_updated = annotation.datetime_updated end - - -- Include position data for highlights (not bookmarks) if annotation.pos0 then - cleaned.pos0 = annotation.pos0 + cleaned_annotation.pos0 = annotation.pos0 end if annotation.pos1 then - cleaned.pos1 = annotation.pos1 + cleaned_annotation.pos1 = annotation.pos1 end - table.insert(cleaned_annotations, cleaned) + table.insert(cleaned, cleaned_annotation) end + return cleaned +end - annotations_by_book[book_md5] = cleaned_annotations - logger.info("[KoInsight] Prepared", #cleaned_annotations, "annotations for book", book_md5) +-- Extract all necessary data from a book's sidecar file in one read +-- Returns: md5, annotations, total_pages, book_metadata (or nil if no annotations/md5) +function KoInsightAnnotationReader.getBookDataFromSidecar(file_path) + if not file_path then + return nil + end - return annotations_by_book + -- Read-only sidecar open: ideal for bulk operations + local doc_settings = open_sidecar_readonly(file_path) + if not doc_settings then + return nil + end + + -- Check if book has annotations first + local annotations = doc_settings:readSetting("annotations") + if not annotations or #annotations == 0 then + return nil + end + + -- Get MD5 + local md5 = doc_settings:readSetting("partial_md5_checksum") + if not md5 then + logger.warn("[KoInsight] No MD5 found in sidecar for:", file_path) + return nil + end + + -- Get total pages + local total_pages = doc_settings:readSetting("doc_pages") + + -- Extract book metadata from sidecar + local doc_props = doc_settings:readSetting("doc_props") + local stats = doc_settings:readSetting("stats") + local summary = doc_settings:readSetting("summary") + local percent_finished = doc_settings:readSetting("percent_finished") + + local book_metadata = { + md5 = md5, + title = (doc_props and doc_props.title) or "Unknown", + authors = (doc_props and doc_props.authors) or "Unknown", + series = doc_props and doc_props.series, + language = doc_props and doc_props.language, + pages = total_pages or 0, + highlights = (stats and stats.highlights) or 0, + notes = (stats and stats.notes) or 0, + last_open = (summary and summary.modified) or os.time(), + } + + return md5, annotations, total_pages, book_metadata +end + +-- Get annotations for a specific book file path +function KoInsightAnnotationReader.getAnnotationsForBook(file_path) + if not file_path then + logger.warn("[KoInsight] No file path provided") + return nil, nil + end + + logger.dbg("[KoInsight] Reading annotations for:", file_path) + + -- Read-only sidecar open: avoids unintended writes during bulk reads + local doc_settings = open_sidecar_readonly(file_path) + if not doc_settings then + logger.dbg("[KoInsight] No doc settings found for:", file_path) + return nil, nil + end + + local annotations = doc_settings:readSetting("annotations") + if not annotations or #annotations == 0 then + logger.dbg("[KoInsight] No annotations found in doc settings") + return nil, nil + end + + -- Try to get total pages from doc settings (stored per-book) + local total_pages = doc_settings:readSetting("doc_pages") + + logger.info("[KoInsight] Found", #annotations, "annotations for:", file_path) + return annotations, total_pages +end + +-- Get MD5 hash for a book directly from its sidecar file +function KoInsightAnnotationReader.getMd5ForPath(file_path) + if not file_path then + return nil + end + + -- Read-only sidecar open: avoids unintended writes during bulk reads + local doc_settings = open_sidecar_readonly(file_path) + if not doc_settings then + return nil + end + + -- Read MD5 directly from sidecar file + local md5 = doc_settings:readSetting("partial_md5_checksum") + + if md5 then + logger.dbg("[KoInsight] Found MD5 in sidecar:", md5) + else + logger.warn("[KoInsight] No MD5 checksum found in sidecar for:", file_path) + end + + return md5 +end + +-- Get all books with annotations from reading history +function KoInsightAnnotationReader.getAllBooksWithAnnotations() + local ReadHistory = require("readhistory") + + logger.info("[KoInsight] Starting bulk annotation collection from reading history") + + -- Force flush currently open book settings to disk first + -- Only needed if the user is currently in a book, other books should already + -- have been flushed settings + local ui = get_live_ui() + if ui and ui.doc_settings then + logger.dbg("[KoInsight] Flushing currently open book's doc settings to disk") + ui.doc_settings:flush() + end + + if not ReadHistory.hist or #ReadHistory.hist == 0 then + logger.info("[KoInsight] No books found in reading history") + return {} + end + + logger.info("[KoInsight] Found", #ReadHistory.hist, "books in reading history") + + local books_with_annotations = {} + local processed_count = 0 + local skipped_count = 0 + local error_count = 0 + + -- Iterate through all books in history + for _, history_entry in ipairs(ReadHistory.hist) do + local file_path = history_entry.file + processed_count = processed_count + 1 + + -- Skip deleted files + if history_entry.dim then + skipped_count = skipped_count + 1 + goto continue + end + + -- Get all book data in one sidecar read + local success, md5, annotations, total_pages, book_metadata = + pcall(KoInsightAnnotationReader.getBookDataFromSidecar, file_path) + + if not success then + logger.warn("[KoInsight] Error reading sidecar for:", file_path) + error_count = error_count + 1 + goto continue + end + + -- Skip books without annotations or MD5 + if not md5 or not annotations then + skipped_count = skipped_count + 1 + goto continue + end + + table.insert(books_with_annotations, { + md5 = md5, + file_path = file_path, + annotations = annotations, + total_pages = total_pages, + annotation_count = #annotations, + book_metadata = book_metadata, + }) + logger.info("[KoInsight] Collected", #annotations, "annotations for:", book_metadata.title) + + ::continue:: + end + + logger.info( + string.format( + "[KoInsight] Bulk collection complete: %d books processed, %d with annotations, %d skipped, %d errors", + processed_count, + #books_with_annotations, + skipped_count, + error_count + ) + ) + + return books_with_annotations end return KoInsightAnnotationReader diff --git a/plugins/koinsight.koplugin/const.lua b/plugins/koinsight.koplugin/const.lua index 86c57fb1..6c83f8a8 100644 --- a/plugins/koinsight.koplugin/const.lua +++ b/plugins/koinsight.koplugin/const.lua @@ -1,5 +1,5 @@ local const = {} -const.VERSION = "0.2.0" +const.VERSION = "0.3.0" return const diff --git a/plugins/koinsight.koplugin/main.lua b/plugins/koinsight.koplugin/main.lua index 8a50e51d..1f3c33c9 100644 --- a/plugins/koinsight.koplugin/main.lua +++ b/plugins/koinsight.koplugin/main.lua @@ -2,7 +2,7 @@ local _ = require("gettext") local Dispatcher = require("dispatcher") -- luacheck:ignore local InfoMessage = require("ui/widget/infomessage") local logger = require("logger") -local onUpload = require("upload") +local KoInsightUpload = require("upload") local UIManager = require("ui/uimanager") local WidgetContainer = require("ui/widget/container/widgetcontainer") local KoInsightSettings = require("settings") @@ -26,37 +26,13 @@ function koinsight:addToMainMenu(menu_items) text = _("KoInsight"), sorting_hint = "tools", sub_item_table = { - -- 1) Synchronize data + -- 1) Synchronize data (all books) { text = _("Synchronize data"), - separator = true, -- draws a separator line *after* this item callback = function() - -- Prefer the robust helper if present… - if self.syncNow then - self:syncNow(false) - return - end - -- …otherwise do a safe inline sync: bring Wi-Fi online, then upload. - local NetworkMgr = require("ui/network/manager") - local url = self.koinsight_settings:getServerURL() - if not url or url == "" then - UIManager:show( - InfoMessage:new({ text = _("KoInsight server URL is not configured."), timeout = 3 }) - ) - return - end - NetworkMgr:runWhenOnline(function() - local ok, err = pcall(function() - onUpload(url, false) -- not silent for manual trigger - end) - if ok then - UIManager:show(InfoMessage:new({ text = _("KoInsight sync finished."), timeout = 2 })) - else - logger.err("[KoInsight] Manual sync failed: " .. tostring(err)) - UIManager:show(InfoMessage:new({ text = _("KoInsight sync failed."), timeout = 3 })) - end - end) + self:performFullSync() end, + separator = true, -- separator line }, -- 2) Sync on suspend @@ -123,18 +99,81 @@ function koinsight:addToMainMenu(menu_items) } end --- Register sync action to make it available in gestures +-- Register sync actions to make them available in gestures function koinsight:onDispatcherRegisterActions() Dispatcher:registerAction("koinsight_sync", { category = "none", event = "KoInsightSync", - title = _("KoInsight: Sync stats"), + title = _("KoInsight: Sync all books"), general = true, }) end function koinsight:onKoInsightSync() - onUpload(self.koinsight_settings:getServerURL(), false) + self:performFullSync() +end + +-- Perform full sync of all books with progress UI +function koinsight:performFullSync() + local url = self.koinsight_settings:getServerURL() + if not url or url == "" then + UIManager:show( + InfoMessage:new({ text = _("KoInsight server URL is not configured."), timeout = 3 }) + ) + return + end + + -- Show initial message + local progress_info = InfoMessage:new({ + text = _("Starting sync..\nScanning reading history for books with annotations."), + }) + UIManager:show(progress_info) + + -- Run sync in background with progress updates + local NetworkMgr = require("ui/network/manager") + NetworkMgr:runWhenOnline(function() + local ok, err = pcall(function() + KoInsightUpload.syncAllBooks(url, function(progress) + -- Update progress UI + if progress.phase == "syncing" then + UIManager:close(progress_info) + progress_info = InfoMessage:new({ + text = string.format( + _("Syncing: %d/%d books\n%d annotations for current book"), + progress.current, + progress.total, + progress.annotation_count + ), + }) + UIManager:show(progress_info) + elseif progress.phase == "complete" then + UIManager:close(progress_info) + if progress.total == 0 then + UIManager:show(InfoMessage:new({ + text = _("No books with annotations found in reading history."), + timeout = 3, + })) + else + UIManager:show(InfoMessage:new({ + text = string.format( + _("Sync complete!\n%d/%d books synced successfully\n%d failed"), + progress.success, + progress.total, + progress.failed + ), + timeout = 5, + })) + end + end + end) + end) + + if not ok then + UIManager:close(progress_info) + logger.err("[KoInsight] Full sync failed: " .. tostring(err)) + UIManager:show(InfoMessage:new({ text = _("Sync failed: " .. tostring(err)), timeout = 5 })) + end + end) end -- Sync when device suspends @@ -201,7 +240,7 @@ function koinsight:performSyncOnSuspend() -- Perform sync in a protected call to avoid crashing on suspend local success, error_msg = pcall(function() - onUpload(server_url, true) -- true = silent mode + KoInsightUpload.syncCurrentBook(server_url, true) -- true = silent mode end) if not success then @@ -260,7 +299,7 @@ function koinsight:performAggressiveSyncOnSuspend() -- Perform the actual sync logger.info("[KoInsight] Performing sync") - onUpload(server_url, true) -- true = silent mode + KoInsightUpload.syncCurrentBook(server_url, true) -- true = silent mode -- Turn off WiFi if we turned it on if not was_wifi_on then @@ -306,7 +345,6 @@ function koinsight:isWiFiConnected() logger.dbg("[KoInsight] WiFi status - On:", result and "true" or "false") return result end - function koinsight:initMenuOrder() local menu_order_modules = { "ui/elements/filemanager_menu_order", diff --git a/plugins/koinsight.koplugin/upload.lua b/plugins/koinsight.koplugin/upload.lua index d033fa6c..912bc953 100644 --- a/plugins/koinsight.koplugin/upload.lua +++ b/plugins/koinsight.koplugin/upload.lua @@ -12,6 +12,8 @@ local Device = require("device") local API_UPLOAD_LOCATION = "/api/plugin/import" local API_DEVICE_LOCATION = "/api/plugin/device" +local KoInsightUpload = {} + function get_headers(body) local headers = { ["Content-Type"] = "application/json", @@ -83,7 +85,151 @@ function send_statistics_data(server_url, silent) end end -return function(server_url, silent) +-- Send annotations for a specific book +function send_book_annotations(server_url, book_md5, annotations, total_pages, book_metadata) + local url = server_url .. API_UPLOAD_LOCATION + local device_id = G_reader_settings:readSetting("device_id") + + -- Clean up annotations for JSON serialization + local cleaned_annotations = KoInsightAnnotationReader.cleanAnnotations(annotations, total_pages) + + -- Use provided book metadata instead of querying database + -- This allows bulk sync to work even if book isn't in statistics DB yet + local book_to_send = book_metadata + + -- Fallback: try to get from statistics database if metadata not provided + if not book_to_send then + local all_books = KoInsightDbReader.bookData() + for _, book in ipairs(all_books) do + if book.md5 == book_md5 then + book_to_send = book + break + end + end + end + + -- WARN: We MUST have book metadata to send annotations + -- The server has a foreign key constraint: annotations.book_md5 -> book.md5 + -- If we don't send book data, annotation insert will fail + if not book_to_send then + logger.err( + "[KoInsight] Cannot sync annotations for book " .. book_md5 .. ": no book metadata available" + ) + return false, { error = "No book metadata available" } + end + + -- Create minimal payload + local annotations_by_book = {} + annotations_by_book[book_md5] = cleaned_annotations + + local body = { + stats = {}, -- empty stats on annotations sync path, handled server side + books = { book_to_send }, -- Always send book metadata for FK constraint + annotations = annotations_by_book, + device_id = device_id, + version = const.VERSION, + } + + body = JSON.encode(body) + return callApi("POST", url, get_headers(body), body) +end + +-- Bulk sync all books with annotations +function bulk_sync_all_books(server_url, progress_callback) + logger.info("[KoInsight] Starting bulk sync of all books") + + -- Get all books with annotations from reading history + local books_with_annotations = KoInsightAnnotationReader.getAllBooksWithAnnotations() + + if #books_with_annotations == 0 then + logger.info("[KoInsight] No books with annotations found") + if progress_callback then + progress_callback({ + phase = "complete", + total = 0, + success = 0, + failed = 0, + message = "No books with annotations found", + }) + end + return + end + + logger.info("[KoInsight] Found", #books_with_annotations, "books to sync") + + local total_books = #books_with_annotations + local success_count = 0 + local failed_count = 0 + + -- Sync each book one by one + for i, book_info in ipairs(books_with_annotations) do + logger.info( + string.format( + "[KoInsight] Syncing book %d/%d (MD5: %s, %d annotations)", + i, + total_books, + book_info.md5, + book_info.annotation_count + ) + ) + + -- Report progress + if progress_callback then + progress_callback({ + phase = "syncing", + current = i, + total = total_books, + book_md5 = book_info.md5, + annotation_count = book_info.annotation_count, + }) + end + + -- Send annotations for this book + local ok, response = send_book_annotations( + server_url, + book_info.md5, + book_info.annotations, + book_info.total_pages, + book_info.book_metadata -- Pass metadata from sidecar + ) + + if ok then + success_count = success_count + 1 + logger.info("[KoInsight] Successfully synced book:", book_info.md5) + else + failed_count = failed_count + 1 + logger.err("[KoInsight] Failed to sync book:", book_info.md5) + end + + -- Small delay between requests to avoid overwhelming the server + -- and to allow UI to update + if i < total_books then + UIManager:nextTick(function() end) + end + end + + logger.info( + string.format( + "[KoInsight] Bulk sync complete: %d/%d books synced successfully, %d failed", + success_count, + total_books, + failed_count + ) + ) + + -- Report completion + if progress_callback then + progress_callback({ + phase = "complete", + total = total_books, + success = success_count, + failed = failed_count, + }) + end +end + +-- Sync current book only (stats + current book annotations) +function KoInsightUpload.syncCurrentBook(server_url, silent) if silent == nil then silent = false end @@ -97,3 +243,24 @@ return function(server_url, silent) send_device_data(server_url, silent) send_statistics_data(server_url, silent) end + +-- Sync all books (stats + all book annotations) +function KoInsightUpload.syncAllBooks(server_url, progress_callback) + if server_url == nil or server_url == "" then + UIManager:show(InfoMessage:new({ + text = _("Please configure the server URL first."), + })) + return + end + + send_device_data(server_url, true) -- silent + + -- First, sync all statistics data from the database + -- This includes all reading progress (page_stat_data) and book metadata + send_statistics_data(server_url, true) -- silent + + -- Then, sync all annotations for all books + bulk_sync_all_books(server_url, progress_callback) +end + +return KoInsightUpload