From 7b47cadc8280be0fe11bc8a044b45b444195824d Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Sun, 11 Jan 2026 13:34:19 +0100 Subject: [PATCH 01/22] feat(koplugin): have bulk-syncing --- .../koinsight.koplugin/annotation_reader.lua | 171 ++++++++++++++++- plugins/koinsight.koplugin/main.lua | 96 ++++++++-- plugins/koinsight.koplugin/upload.lua | 176 +++++++++++++++++- 3 files changed, 427 insertions(+), 16 deletions(-) diff --git a/plugins/koinsight.koplugin/annotation_reader.lua b/plugins/koinsight.koplugin/annotation_reader.lua index 321f87d0..903e914b 100644 --- a/plugins/koinsight.koplugin/annotation_reader.lua +++ b/plugins/koinsight.koplugin/annotation_reader.lua @@ -1,6 +1,3 @@ -local DocSettings = require("docsettings") -local logger = require("logger") - local KoInsightAnnotationReader = {} -- Get the currently opened document @@ -17,6 +14,7 @@ end -- Get the MD5 hash for the currently open document function KoInsightAnnotationReader.getCurrentBookMd5() + local logger = require("logger") local current_doc = KoInsightAnnotationReader.getCurrentDocument() if not current_doc then return nil @@ -60,6 +58,8 @@ end -- Get annotations for the currently opened book function KoInsightAnnotationReader.getCurrentBookAnnotations() + local logger = require("logger") + local DocSettings = require("docsettings") local current_doc = KoInsightAnnotationReader.getCurrentDocument() if not current_doc then @@ -107,6 +107,7 @@ end -- Get annotations organized by book md5 function KoInsightAnnotationReader.getAnnotationsByBook() + local logger = require("logger") local annotations_by_book = {} -- For now, only get annotations from currently opened book @@ -164,4 +165,168 @@ function KoInsightAnnotationReader.getAnnotationsByBook() return annotations_by_book end +-- Get annotations for a specific book file path +function KoInsightAnnotationReader.getAnnotationsForBook(file_path) + local DocSettings = require("docsettings") + local logger = require("logger") + + if not file_path then + logger.warn("[KoInsight] No file path provided") + return nil, nil + end + + logger.dbg("[KoInsight] Reading annotations for:", file_path) + + local doc_settings = DocSettings:open(file_path) + if not doc_settings then + logger.dbg("[KoInsight] No doc settings found for:", file_path) + return nil, nil + end + + local annotations = doc_settings:readSetting("annotations") + if not annotations or #annotations == 0 then + logger.dbg("[KoInsight] No annotations found in doc settings") + return nil, nil + end + + -- Try to get total pages from doc settings (stored per-book) + local total_pages = doc_settings:readSetting("doc_pages") + + logger.info("[KoInsight] Found", #annotations, "annotations for:", file_path) + return annotations, total_pages +end + +-- Get MD5 hash for a book by matching its title with the statistics database +function KoInsightAnnotationReader.getMd5ForPath(file_path) + local SQ3 = require("lua-ljsqlite3/init") + local DataStorage = require("datastorage") + local DocSettings = require("docsettings") + local logger = require("logger") + + if not file_path then + return nil + end + + -- Get the book's title from its sidecar file + local doc_settings = DocSettings:open(file_path) + if not doc_settings then + return nil + end + + local doc_props = doc_settings:readSetting("doc_props") + if not doc_props or not doc_props.title then + return nil + end + + local book_title = doc_props.title + logger.dbg("[KoInsight] Looking for MD5 for book:", book_title) + + -- Query statistics database for matching title + local db_location = DataStorage:getSettingsDir() .. "/statistics.sqlite3" + local conn = SQ3.open(db_location) + local query = "SELECT md5, title FROM book" + local result, rows = conn:exec(query) + conn:close() + + if rows == 0 then + return nil + end + + -- Try exact match first, then case-insensitive + local lower_title = book_title:lower() + for i = 1, rows do + local md5 = result[1][i] + local db_title = result[2][i] + + if db_title == book_title then + logger.dbg("[KoInsight] Found MD5 via exact match:", md5) + return md5 + elseif db_title and db_title:lower() == lower_title then + logger.dbg("[KoInsight] Found MD5 via case-insensitive match:", md5) + return md5 + end + end + + logger.warn("[KoInsight] Could not find MD5 for book:", book_title) + return nil +end + +-- Get all books with annotations from reading history +function KoInsightAnnotationReader.getAllBooksWithAnnotations() + local ReadHistory = require("readhistory") + local logger = require("logger") + + logger.info("[KoInsight] Starting bulk annotation collection from reading history") + + if not ReadHistory.hist or #ReadHistory.hist == 0 then + logger.info("[KoInsight] No books found in reading history") + return {} + end + + logger.info("[KoInsight] Found", #ReadHistory.hist, "books in reading history") + + local books_with_annotations = {} + local processed_count = 0 + local skipped_count = 0 + local error_count = 0 + + -- Iterate through all books in history + for _, history_entry in ipairs(ReadHistory.hist) do + local file_path = history_entry.file + processed_count = processed_count + 1 + + -- Skip deleted files + if history_entry.dim then + skipped_count = skipped_count + 1 + goto continue + end + + -- Try to get annotations for this book + local success, annotations, total_pages = + pcall(KoInsightAnnotationReader.getAnnotationsForBook, file_path) + + if not success then + logger.warn("[KoInsight] Error reading annotations for:", file_path) + error_count = error_count + 1 + goto continue + end + + if not annotations or #annotations == 0 then + skipped_count = skipped_count + 1 + goto continue + end + + -- Get MD5 for this book + local book_md5 = KoInsightAnnotationReader.getMd5ForPath(file_path) + + if book_md5 then + table.insert(books_with_annotations, { + md5 = book_md5, + file_path = file_path, + annotations = annotations, + total_pages = total_pages, + annotation_count = #annotations, + }) + logger.info("[KoInsight] Collected", #annotations, "annotations for MD5:", book_md5) + else + logger.warn("[KoInsight] Book has annotations but no MD5 found:", file_path) + skipped_count = skipped_count + 1 + end + + ::continue:: + end + + logger.info( + string.format( + "[KoInsight] Bulk collection complete: %d books processed, %d with annotations, %d skipped, %d errors", + processed_count, + #books_with_annotations, + skipped_count, + error_count + ) + ) + + return books_with_annotations +end + return KoInsightAnnotationReader diff --git a/plugins/koinsight.koplugin/main.lua b/plugins/koinsight.koplugin/main.lua index 8a50e51d..9904faed 100644 --- a/plugins/koinsight.koplugin/main.lua +++ b/plugins/koinsight.koplugin/main.lua @@ -2,7 +2,7 @@ local _ = require("gettext") local Dispatcher = require("dispatcher") -- luacheck:ignore local InfoMessage = require("ui/widget/infomessage") local logger = require("logger") -local onUpload = require("upload") +local KoInsightUpload = require("upload") local UIManager = require("ui/uimanager") local WidgetContainer = require("ui/widget/container/widgetcontainer") local KoInsightSettings = require("settings") @@ -29,7 +29,6 @@ function koinsight:addToMainMenu(menu_items) -- 1) Synchronize data { text = _("Synchronize data"), - separator = true, -- draws a separator line *after* this item callback = function() -- Prefer the robust helper if present… if self.syncNow then @@ -47,7 +46,7 @@ function koinsight:addToMainMenu(menu_items) end NetworkMgr:runWhenOnline(function() local ok, err = pcall(function() - onUpload(url, false) -- not silent for manual trigger + KoInsightUpload.sync(url, false) -- not silent for manual trigger end) if ok then UIManager:show(InfoMessage:new({ text = _("KoInsight sync finished."), timeout = 2 })) @@ -59,7 +58,16 @@ function koinsight:addToMainMenu(menu_items) end, }, - -- 2) Sync on suspend + -- 2) Bulk sync all books + { + text = _("Bulk Sync All Books (may take time)"), + separator = true, + callback = function() + self:performBulkSync() + end, + }, + + -- 3) Sync on suspend { text = _("Sync on suspend"), checked_func = function() @@ -70,7 +78,7 @@ function koinsight:addToMainMenu(menu_items) end, }, - -- 3) Aggressive sync on suspend (auto Wi-Fi) + -- 4) Aggressive sync on suspend (auto Wi-Fi) { text = _("Aggressive sync on suspend (auto Wi-Fi)"), checked_func = function() @@ -84,7 +92,7 @@ function koinsight:addToMainMenu(menu_items) end, }, - -- 4) Set suspend connect timeout + -- 5) Set suspend connect timeout { text = _("Set suspend connect timeout…"), keep_menu_open = true, @@ -96,7 +104,7 @@ function koinsight:addToMainMenu(menu_items) end, }, - -- 5) Set server URL + -- 6) Set server URL { text = _("Set server URL"), keep_menu_open = true, @@ -106,7 +114,7 @@ function koinsight:addToMainMenu(menu_items) end, }, - -- 6) About KoInsight + -- 7) About KoInsight { text = _("About KoInsight"), keep_menu_open = true, @@ -134,7 +142,72 @@ function koinsight:onDispatcherRegisterActions() end function koinsight:onKoInsightSync() - onUpload(self.koinsight_settings:getServerURL(), false) + KoInsightUpload.sync(self.koinsight_settings:getServerURL(), false) +end + +-- Perform bulk sync of all books with progress UI +function koinsight:performBulkSync() + local url = self.koinsight_settings:getServerURL() + if not url or url == "" then + UIManager:show( + InfoMessage:new({ text = _("KoInsight server URL is not configured."), timeout = 3 }) + ) + return + end + + -- Show initial message + local progress_info = InfoMessage:new({ + text = _("Starting bulk sync...\nScanning reading history for books with annotations."), + }) + UIManager:show(progress_info) + + -- Run sync in background with progress updates + local NetworkMgr = require("ui/network/manager") + NetworkMgr:runWhenOnline(function() + local ok, err = pcall(function() + KoInsightUpload.bulkSync(url, function(progress) + -- Update progress UI + if progress.phase == "syncing" then + UIManager:close(progress_info) + progress_info = InfoMessage:new({ + text = string.format( + _("Bulk Sync: %d/%d books\n%d annotations for current book"), + progress.current, + progress.total, + progress.annotation_count + ), + }) + UIManager:show(progress_info) + elseif progress.phase == "complete" then + UIManager:close(progress_info) + if progress.total == 0 then + UIManager:show(InfoMessage:new({ + text = _("No books with annotations found in reading history."), + timeout = 3, + })) + else + UIManager:show(InfoMessage:new({ + text = string.format( + _("Bulk sync complete!\n%d/%d books synced successfully\n%d failed"), + progress.success, + progress.total, + progress.failed + ), + timeout = 5, + })) + end + end + end) + end) + + if not ok then + UIManager:close(progress_info) + logger.err("[KoInsight] Bulk sync failed: " .. tostring(err)) + UIManager:show( + InfoMessage:new({ text = _("Bulk sync failed: " .. tostring(err)), timeout = 5 }) + ) + end + end) end -- Sync when device suspends @@ -201,7 +274,7 @@ function koinsight:performSyncOnSuspend() -- Perform sync in a protected call to avoid crashing on suspend local success, error_msg = pcall(function() - onUpload(server_url, true) -- true = silent mode + KoInsightUpload.sync(server_url, true) -- true = silent mode end) if not success then @@ -260,7 +333,7 @@ function koinsight:performAggressiveSyncOnSuspend() -- Perform the actual sync logger.info("[KoInsight] Performing sync") - onUpload(server_url, true) -- true = silent mode + KoInsightUpload.sync(server_url, true) -- true = silent mode -- Turn off WiFi if we turned it on if not was_wifi_on then @@ -306,7 +379,6 @@ function koinsight:isWiFiConnected() logger.dbg("[KoInsight] WiFi status - On:", result and "true" or "false") return result end - function koinsight:initMenuOrder() local menu_order_modules = { "ui/elements/filemanager_menu_order", diff --git a/plugins/koinsight.koplugin/upload.lua b/plugins/koinsight.koplugin/upload.lua index d033fa6c..ddbf495e 100644 --- a/plugins/koinsight.koplugin/upload.lua +++ b/plugins/koinsight.koplugin/upload.lua @@ -12,6 +12,8 @@ local Device = require("device") local API_UPLOAD_LOCATION = "/api/plugin/import" local API_DEVICE_LOCATION = "/api/plugin/device" +local KoInsightUpload = {} + function get_headers(body) local headers = { ["Content-Type"] = "application/json", @@ -83,7 +85,164 @@ function send_statistics_data(server_url, silent) end end -return function(server_url, silent) +-- Send annotations for a specific book +function send_book_annotations(server_url, book_md5, annotations, total_pages) + local url = server_url .. API_UPLOAD_LOCATION + local device_id = G_reader_settings:readSetting("device_id") + + -- Clean up annotations for JSON serialization + local cleaned_annotations = {} + for _, annotation in ipairs(annotations) do + local cleaned = { + datetime = annotation.datetime, + drawer = annotation.drawer, + color = annotation.color, + text = annotation.text, + note = annotation.note, + chapter = annotation.chapter, + pageno = annotation.pageno, + page = annotation.page, + total_pages = total_pages, + } + + if annotation.datetime_updated then + cleaned.datetime_updated = annotation.datetime_updated + end + if annotation.pos0 then + cleaned.pos0 = annotation.pos0 + end + if annotation.pos1 then + cleaned.pos1 = annotation.pos1 + end + + table.insert(cleaned_annotations, cleaned) + end + + -- Get the specific book from database + local all_books = KoInsightDbReader.bookData() + local book_to_send = nil + for _, book in ipairs(all_books) do + if book.md5 == book_md5 then + book_to_send = book + break + end + end + + -- Create minimal payload + local annotations_by_book = {} + annotations_by_book[book_md5] = cleaned_annotations + + local body = { + stats = { + { + page = 1, + start_time = os.time(), + duration = 0, + total_pages = total_pages or 1, + book_md5 = book_md5, + device_id = device_id, + }, + }, + books = book_to_send and { book_to_send } or {}, + annotations = annotations_by_book, + version = const.VERSION, + } + + body = JSON.encode(body) + return callApi("POST", url, get_headers(body), body) +end + +-- Bulk sync all books with annotations +function bulk_sync_all_books(server_url, progress_callback) + logger.info("[KoInsight] Starting bulk sync of all books") + + -- Get all books with annotations from reading history + local books_with_annotations = KoInsightAnnotationReader.getAllBooksWithAnnotations() + + if #books_with_annotations == 0 then + logger.info("[KoInsight] No books with annotations found") + if progress_callback then + progress_callback({ + phase = "complete", + total = 0, + success = 0, + failed = 0, + message = "No books with annotations found", + }) + end + return + end + + logger.info("[KoInsight] Found", #books_with_annotations, "books to sync") + + local total_books = #books_with_annotations + local success_count = 0 + local failed_count = 0 + + -- Sync each book one by one + for i, book_info in ipairs(books_with_annotations) do + logger.info( + string.format( + "[KoInsight] Syncing book %d/%d (MD5: %s, %d annotations)", + i, + total_books, + book_info.md5, + book_info.annotation_count + ) + ) + + -- Report progress + if progress_callback then + progress_callback({ + phase = "syncing", + current = i, + total = total_books, + book_md5 = book_info.md5, + annotation_count = book_info.annotation_count, + }) + end + + -- Send annotations for this book + local ok, response = + send_book_annotations(server_url, book_info.md5, book_info.annotations, book_info.total_pages) + + if ok then + success_count = success_count + 1 + logger.info("[KoInsight] Successfully synced book:", book_info.md5) + else + failed_count = failed_count + 1 + logger.err("[KoInsight] Failed to sync book:", book_info.md5) + end + + -- Small delay between requests to avoid overwhelming the server + -- and to allow UI to update + if i < total_books then + UIManager:nextTick(function() end) + end + end + + logger.info( + string.format( + "[KoInsight] Bulk sync complete: %d/%d books synced successfully, %d failed", + success_count, + total_books, + failed_count + ) + ) + + -- Report completion + if progress_callback then + progress_callback({ + phase = "complete", + total = total_books, + success = success_count, + failed = failed_count, + }) + end +end + +-- Main sync function (current book + stats) +function KoInsightUpload.sync(server_url, silent) if silent == nil then silent = false end @@ -97,3 +256,18 @@ return function(server_url, silent) send_device_data(server_url, silent) send_statistics_data(server_url, silent) end + +-- Bulk sync function (all books with annotations) +function KoInsightUpload.bulkSync(server_url, progress_callback) + if server_url == nil or server_url == "" then + UIManager:show(InfoMessage:new({ + text = _("Please configure the server URL first."), + })) + return + end + + send_device_data(server_url, true) -- silent + bulk_sync_all_books(server_url, progress_callback) +end + +return KoInsightUpload From f7ec420f607dd4c009503067f6b807c697964029 Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Sun, 11 Jan 2026 14:00:27 +0100 Subject: [PATCH 02/22] feat(koplugin): use sidecar file md5 --- .../koinsight.koplugin/annotation_reader.lua | 82 +++---------------- 1 file changed, 10 insertions(+), 72 deletions(-) diff --git a/plugins/koinsight.koplugin/annotation_reader.lua b/plugins/koinsight.koplugin/annotation_reader.lua index 903e914b..62a3828e 100644 --- a/plugins/koinsight.koplugin/annotation_reader.lua +++ b/plugins/koinsight.koplugin/annotation_reader.lua @@ -14,46 +14,13 @@ end -- Get the MD5 hash for the currently open document function KoInsightAnnotationReader.getCurrentBookMd5() - local logger = require("logger") local current_doc = KoInsightAnnotationReader.getCurrentDocument() if not current_doc then return nil end - -- Get document info from ReaderUI - local ReaderUI = require("apps/reader/readerui") - local ui = ReaderUI.instance - - if ui and ui.document and ui.document.info then - local doc_props = ui.document:getProps() - if doc_props and doc_props.title then - -- Try to find book by title in statistics database - local SQ3 = require("lua-ljsqlite3/init") - local DataStorage = require("datastorage") - local db_location = DataStorage:getSettingsDir() .. "/statistics.sqlite3" - - local conn = SQ3.open(db_location) - - -- Escape single quotes in title for SQL - local safe_title = doc_props.title:gsub("'", "''") - local query = string.format("SELECT md5 FROM book WHERE title = '%s'", safe_title) - - logger.dbg("[KoInsight] Looking for book with title:", doc_props.title) - - local result, rows = conn:exec(query) - conn:close() - - if rows > 0 and result[1] and result[1][1] then - local md5 = result[1][1] - logger.info("[KoInsight] Found MD5 for current book:", md5) - return md5 - else - logger.warn("[KoInsight] Book not found in statistics database:", doc_props.title) - end - end - end - - return nil + -- Use the same method as getMd5ForPath + return KoInsightAnnotationReader.getMd5ForPath(current_doc) end -- Get annotations for the currently opened book @@ -196,10 +163,8 @@ function KoInsightAnnotationReader.getAnnotationsForBook(file_path) return annotations, total_pages end --- Get MD5 hash for a book by matching its title with the statistics database +-- Get MD5 hash for a book directly from its sidecar file function KoInsightAnnotationReader.getMd5ForPath(file_path) - local SQ3 = require("lua-ljsqlite3/init") - local DataStorage = require("datastorage") local DocSettings = require("docsettings") local logger = require("logger") @@ -207,48 +172,21 @@ function KoInsightAnnotationReader.getMd5ForPath(file_path) return nil end - -- Get the book's title from its sidecar file local doc_settings = DocSettings:open(file_path) if not doc_settings then return nil end - local doc_props = doc_settings:readSetting("doc_props") - if not doc_props or not doc_props.title then - return nil - end - - local book_title = doc_props.title - logger.dbg("[KoInsight] Looking for MD5 for book:", book_title) - - -- Query statistics database for matching title - local db_location = DataStorage:getSettingsDir() .. "/statistics.sqlite3" - local conn = SQ3.open(db_location) - local query = "SELECT md5, title FROM book" - local result, rows = conn:exec(query) - conn:close() + -- Read MD5 directly from sidecar file + local md5 = doc_settings:readSetting("partial_md5_checksum") - if rows == 0 then - return nil + if md5 then + logger.dbg("[KoInsight] Found MD5 in sidecar:", md5) + else + logger.warn("[KoInsight] No MD5 checksum found in sidecar for:", file_path) end - -- Try exact match first, then case-insensitive - local lower_title = book_title:lower() - for i = 1, rows do - local md5 = result[1][i] - local db_title = result[2][i] - - if db_title == book_title then - logger.dbg("[KoInsight] Found MD5 via exact match:", md5) - return md5 - elseif db_title and db_title:lower() == lower_title then - logger.dbg("[KoInsight] Found MD5 via case-insensitive match:", md5) - return md5 - end - end - - logger.warn("[KoInsight] Could not find MD5 for book:", book_title) - return nil + return md5 end -- Get all books with annotations from reading history From b4d356a74a4fb3fc3064faa67321721582c293d6 Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Sun, 11 Jan 2026 14:09:41 +0100 Subject: [PATCH 03/22] Revert "feat(koplugin): use sidecar file md5" This reverts commit f7ec420f607dd4c009503067f6b807c697964029. --- .../koinsight.koplugin/annotation_reader.lua | 82 ++++++++++++++++--- 1 file changed, 72 insertions(+), 10 deletions(-) diff --git a/plugins/koinsight.koplugin/annotation_reader.lua b/plugins/koinsight.koplugin/annotation_reader.lua index 62a3828e..903e914b 100644 --- a/plugins/koinsight.koplugin/annotation_reader.lua +++ b/plugins/koinsight.koplugin/annotation_reader.lua @@ -14,13 +14,46 @@ end -- Get the MD5 hash for the currently open document function KoInsightAnnotationReader.getCurrentBookMd5() + local logger = require("logger") local current_doc = KoInsightAnnotationReader.getCurrentDocument() if not current_doc then return nil end - -- Use the same method as getMd5ForPath - return KoInsightAnnotationReader.getMd5ForPath(current_doc) + -- Get document info from ReaderUI + local ReaderUI = require("apps/reader/readerui") + local ui = ReaderUI.instance + + if ui and ui.document and ui.document.info then + local doc_props = ui.document:getProps() + if doc_props and doc_props.title then + -- Try to find book by title in statistics database + local SQ3 = require("lua-ljsqlite3/init") + local DataStorage = require("datastorage") + local db_location = DataStorage:getSettingsDir() .. "/statistics.sqlite3" + + local conn = SQ3.open(db_location) + + -- Escape single quotes in title for SQL + local safe_title = doc_props.title:gsub("'", "''") + local query = string.format("SELECT md5 FROM book WHERE title = '%s'", safe_title) + + logger.dbg("[KoInsight] Looking for book with title:", doc_props.title) + + local result, rows = conn:exec(query) + conn:close() + + if rows > 0 and result[1] and result[1][1] then + local md5 = result[1][1] + logger.info("[KoInsight] Found MD5 for current book:", md5) + return md5 + else + logger.warn("[KoInsight] Book not found in statistics database:", doc_props.title) + end + end + end + + return nil end -- Get annotations for the currently opened book @@ -163,8 +196,10 @@ function KoInsightAnnotationReader.getAnnotationsForBook(file_path) return annotations, total_pages end --- Get MD5 hash for a book directly from its sidecar file +-- Get MD5 hash for a book by matching its title with the statistics database function KoInsightAnnotationReader.getMd5ForPath(file_path) + local SQ3 = require("lua-ljsqlite3/init") + local DataStorage = require("datastorage") local DocSettings = require("docsettings") local logger = require("logger") @@ -172,21 +207,48 @@ function KoInsightAnnotationReader.getMd5ForPath(file_path) return nil end + -- Get the book's title from its sidecar file local doc_settings = DocSettings:open(file_path) if not doc_settings then return nil end - -- Read MD5 directly from sidecar file - local md5 = doc_settings:readSetting("partial_md5_checksum") + local doc_props = doc_settings:readSetting("doc_props") + if not doc_props or not doc_props.title then + return nil + end + + local book_title = doc_props.title + logger.dbg("[KoInsight] Looking for MD5 for book:", book_title) + + -- Query statistics database for matching title + local db_location = DataStorage:getSettingsDir() .. "/statistics.sqlite3" + local conn = SQ3.open(db_location) + local query = "SELECT md5, title FROM book" + local result, rows = conn:exec(query) + conn:close() - if md5 then - logger.dbg("[KoInsight] Found MD5 in sidecar:", md5) - else - logger.warn("[KoInsight] No MD5 checksum found in sidecar for:", file_path) + if rows == 0 then + return nil end - return md5 + -- Try exact match first, then case-insensitive + local lower_title = book_title:lower() + for i = 1, rows do + local md5 = result[1][i] + local db_title = result[2][i] + + if db_title == book_title then + logger.dbg("[KoInsight] Found MD5 via exact match:", md5) + return md5 + elseif db_title and db_title:lower() == lower_title then + logger.dbg("[KoInsight] Found MD5 via case-insensitive match:", md5) + return md5 + end + end + + logger.warn("[KoInsight] Could not find MD5 for book:", book_title) + return nil end -- Get all books with annotations from reading history From b487b83830850dfc230f5f24cd9b993904c99858 Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Sun, 11 Jan 2026 14:29:39 +0100 Subject: [PATCH 04/22] fix(koplugin): flush currently opened book befure bulk sync --- plugins/koinsight.koplugin/annotation_reader.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/koinsight.koplugin/annotation_reader.lua b/plugins/koinsight.koplugin/annotation_reader.lua index 903e914b..4868d345 100644 --- a/plugins/koinsight.koplugin/annotation_reader.lua +++ b/plugins/koinsight.koplugin/annotation_reader.lua @@ -258,6 +258,16 @@ function KoInsightAnnotationReader.getAllBooksWithAnnotations() logger.info("[KoInsight] Starting bulk annotation collection from reading history") + -- Force flush currently open book settings to disk first + -- Only needed if the user is currently in a book, other books should already + -- have been flushed settings + local ReaderUI = require("apps/reader/readerui") + local ui = ReaderUI.instance + if ui and ui.doc_settings then + logger.dbg("[KoInsight] Flushing currently open book's doc settings to disk") + ui.doc_settings:flush() + end + if not ReadHistory.hist or #ReadHistory.hist == 0 then logger.info("[KoInsight] No books found in reading history") return {} From a33ea8bea7a0feb070f44190c6936cf1ec456bff Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Sun, 11 Jan 2026 14:34:45 +0100 Subject: [PATCH 05/22] refactor(koplugin): use sidecar file md5 We do not need to query the statistics database for book md5 values. We can just use the ones from the sidecar files, which we already have anyways. --- .../koinsight.koplugin/annotation_reader.lua | 82 +++---------------- 1 file changed, 10 insertions(+), 72 deletions(-) diff --git a/plugins/koinsight.koplugin/annotation_reader.lua b/plugins/koinsight.koplugin/annotation_reader.lua index 4868d345..fb4e1848 100644 --- a/plugins/koinsight.koplugin/annotation_reader.lua +++ b/plugins/koinsight.koplugin/annotation_reader.lua @@ -14,46 +14,13 @@ end -- Get the MD5 hash for the currently open document function KoInsightAnnotationReader.getCurrentBookMd5() - local logger = require("logger") local current_doc = KoInsightAnnotationReader.getCurrentDocument() if not current_doc then return nil end - -- Get document info from ReaderUI - local ReaderUI = require("apps/reader/readerui") - local ui = ReaderUI.instance - - if ui and ui.document and ui.document.info then - local doc_props = ui.document:getProps() - if doc_props and doc_props.title then - -- Try to find book by title in statistics database - local SQ3 = require("lua-ljsqlite3/init") - local DataStorage = require("datastorage") - local db_location = DataStorage:getSettingsDir() .. "/statistics.sqlite3" - - local conn = SQ3.open(db_location) - - -- Escape single quotes in title for SQL - local safe_title = doc_props.title:gsub("'", "''") - local query = string.format("SELECT md5 FROM book WHERE title = '%s'", safe_title) - - logger.dbg("[KoInsight] Looking for book with title:", doc_props.title) - - local result, rows = conn:exec(query) - conn:close() - - if rows > 0 and result[1] and result[1][1] then - local md5 = result[1][1] - logger.info("[KoInsight] Found MD5 for current book:", md5) - return md5 - else - logger.warn("[KoInsight] Book not found in statistics database:", doc_props.title) - end - end - end - - return nil + -- Use the same method as getMd5ForPath + return KoInsightAnnotationReader.getMd5ForPath(current_doc) end -- Get annotations for the currently opened book @@ -196,10 +163,8 @@ function KoInsightAnnotationReader.getAnnotationsForBook(file_path) return annotations, total_pages end --- Get MD5 hash for a book by matching its title with the statistics database +-- Get MD5 hash for a book directly from its sidecar file function KoInsightAnnotationReader.getMd5ForPath(file_path) - local SQ3 = require("lua-ljsqlite3/init") - local DataStorage = require("datastorage") local DocSettings = require("docsettings") local logger = require("logger") @@ -207,48 +172,21 @@ function KoInsightAnnotationReader.getMd5ForPath(file_path) return nil end - -- Get the book's title from its sidecar file local doc_settings = DocSettings:open(file_path) if not doc_settings then return nil end - local doc_props = doc_settings:readSetting("doc_props") - if not doc_props or not doc_props.title then - return nil - end - - local book_title = doc_props.title - logger.dbg("[KoInsight] Looking for MD5 for book:", book_title) - - -- Query statistics database for matching title - local db_location = DataStorage:getSettingsDir() .. "/statistics.sqlite3" - local conn = SQ3.open(db_location) - local query = "SELECT md5, title FROM book" - local result, rows = conn:exec(query) - conn:close() + -- Read MD5 directly from sidecar file + local md5 = doc_settings:readSetting("partial_md5_checksum") - if rows == 0 then - return nil + if md5 then + logger.dbg("[KoInsight] Found MD5 in sidecar:", md5) + else + logger.warn("[KoInsight] No MD5 checksum found in sidecar for:", file_path) end - -- Try exact match first, then case-insensitive - local lower_title = book_title:lower() - for i = 1, rows do - local md5 = result[1][i] - local db_title = result[2][i] - - if db_title == book_title then - logger.dbg("[KoInsight] Found MD5 via exact match:", md5) - return md5 - elseif db_title and db_title:lower() == lower_title then - logger.dbg("[KoInsight] Found MD5 via case-insensitive match:", md5) - return md5 - end - end - - logger.warn("[KoInsight] Could not find MD5 for book:", book_title) - return nil + return md5 end -- Get all books with annotations from reading history From 94909f309355a6269d4f6ca45cbaf317faaaba05 Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Sun, 11 Jan 2026 15:01:32 +0100 Subject: [PATCH 06/22] refactor(koplugin): single docsetting open in getAllBooksWithAnnotations --- .../koinsight.koplugin/annotation_reader.lua | 95 +++++++++++++++---- plugins/koinsight.koplugin/upload.lua | 59 +++++------- 2 files changed, 96 insertions(+), 58 deletions(-) diff --git a/plugins/koinsight.koplugin/annotation_reader.lua b/plugins/koinsight.koplugin/annotation_reader.lua index fb4e1848..48de7b0b 100644 --- a/plugins/koinsight.koplugin/annotation_reader.lua +++ b/plugins/koinsight.koplugin/annotation_reader.lua @@ -132,6 +132,65 @@ function KoInsightAnnotationReader.getAnnotationsByBook() return annotations_by_book end +-- Extract all necessary data from a book's sidecar file in one read +-- Returns: md5, annotations, total_pages, book_metadata (or nil if no annotations/md5) +function KoInsightAnnotationReader.getBookDataFromSidecar(file_path) + local DocSettings = require("docsettings") + local logger = require("logger") + + if not file_path then + return nil + end + + local doc_settings = DocSettings:open(file_path) + if not doc_settings then + return nil + end + + -- Check if book has annotations first + local annotations = doc_settings:readSetting("annotations") + if not annotations or #annotations == 0 then + return nil + end + + -- Get MD5 + local md5 = doc_settings:readSetting("partial_md5_checksum") + if not md5 then + logger.warn("[KoInsight] No MD5 found in sidecar for:", file_path) + return nil + end + + -- Get total pages + local total_pages = doc_settings:readSetting("doc_pages") + + -- Extract book metadata from sidecar + local doc_props = doc_settings:readSetting("doc_props") + local stats = doc_settings:readSetting("stats") + local summary = doc_settings:readSetting("summary") + local percent_finished = doc_settings:readSetting("percent_finished") + + local book_metadata = { + md5 = md5, + title = (doc_props and doc_props.title) or "Unknown", + authors = (doc_props and doc_props.authors) or "Unknown", + series = doc_props and doc_props.series, + language = doc_props and doc_props.language, + pages = total_pages or 0, + highlights = (stats and stats.highlights) or 0, + notes = (stats and stats.notes) or 0, + last_open = (summary and summary.modified) or os.time(), + total_read_time = 0, + total_read_pages = 0, + } + + -- Calculate read pages from percent_finished + if percent_finished and total_pages then + book_metadata.total_read_pages = math.floor(total_pages * percent_finished) + end + + return md5, annotations, total_pages, book_metadata +end + -- Get annotations for a specific book file path function KoInsightAnnotationReader.getAnnotationsForBook(file_path) local DocSettings = require("docsettings") @@ -229,37 +288,31 @@ function KoInsightAnnotationReader.getAllBooksWithAnnotations() goto continue end - -- Try to get annotations for this book - local success, annotations, total_pages = - pcall(KoInsightAnnotationReader.getAnnotationsForBook, file_path) + -- Get all book data in one sidecar read + local success, md5, annotations, total_pages, book_metadata = + pcall(KoInsightAnnotationReader.getBookDataFromSidecar, file_path) if not success then - logger.warn("[KoInsight] Error reading annotations for:", file_path) + logger.warn("[KoInsight] Error reading sidecar for:", file_path) error_count = error_count + 1 goto continue end - if not annotations or #annotations == 0 then + -- Skip books without annotations or MD5 + if not md5 or not annotations then skipped_count = skipped_count + 1 goto continue end - -- Get MD5 for this book - local book_md5 = KoInsightAnnotationReader.getMd5ForPath(file_path) - - if book_md5 then - table.insert(books_with_annotations, { - md5 = book_md5, - file_path = file_path, - annotations = annotations, - total_pages = total_pages, - annotation_count = #annotations, - }) - logger.info("[KoInsight] Collected", #annotations, "annotations for MD5:", book_md5) - else - logger.warn("[KoInsight] Book has annotations but no MD5 found:", file_path) - skipped_count = skipped_count + 1 - end + table.insert(books_with_annotations, { + md5 = md5, + file_path = file_path, + annotations = annotations, + total_pages = total_pages, + annotation_count = #annotations, + book_metadata = book_metadata, + }) + logger.info("[KoInsight] Collected", #annotations, "annotations for:", book_metadata.title) ::continue:: end diff --git a/plugins/koinsight.koplugin/upload.lua b/plugins/koinsight.koplugin/upload.lua index ddbf495e..11c3a00b 100644 --- a/plugins/koinsight.koplugin/upload.lua +++ b/plugins/koinsight.koplugin/upload.lua @@ -86,45 +86,25 @@ function send_statistics_data(server_url, silent) end -- Send annotations for a specific book -function send_book_annotations(server_url, book_md5, annotations, total_pages) +function send_book_annotations(server_url, book_md5, annotations, total_pages, book_metadata) local url = server_url .. API_UPLOAD_LOCATION local device_id = G_reader_settings:readSetting("device_id") -- Clean up annotations for JSON serialization - local cleaned_annotations = {} - for _, annotation in ipairs(annotations) do - local cleaned = { - datetime = annotation.datetime, - drawer = annotation.drawer, - color = annotation.color, - text = annotation.text, - note = annotation.note, - chapter = annotation.chapter, - pageno = annotation.pageno, - page = annotation.page, - total_pages = total_pages, - } - - if annotation.datetime_updated then - cleaned.datetime_updated = annotation.datetime_updated - end - if annotation.pos0 then - cleaned.pos0 = annotation.pos0 - end - if annotation.pos1 then - cleaned.pos1 = annotation.pos1 - end - - table.insert(cleaned_annotations, cleaned) - end - - -- Get the specific book from database - local all_books = KoInsightDbReader.bookData() - local book_to_send = nil - for _, book in ipairs(all_books) do - if book.md5 == book_md5 then - book_to_send = book - break + local cleaned_annotations = KoInsightAnnotationReader.cleanAnnotations(annotations, total_pages) + + -- Use provided book metadata instead of querying database + -- This allows bulk sync to work even if book isn't in statistics DB yet + local book_to_send = book_metadata + + -- Fallback: try to get from statistics database if metadata not provided + if not book_to_send then + local all_books = KoInsightDbReader.bookData() + for _, book in ipairs(all_books) do + if book.md5 == book_md5 then + book_to_send = book + break + end end end @@ -203,8 +183,13 @@ function bulk_sync_all_books(server_url, progress_callback) end -- Send annotations for this book - local ok, response = - send_book_annotations(server_url, book_info.md5, book_info.annotations, book_info.total_pages) + local ok, response = send_book_annotations( + server_url, + book_info.md5, + book_info.annotations, + book_info.total_pages, + book_info.book_metadata -- Pass metadata from sidecar + ) if ok then success_count = success_count + 1 From f81faba354fbc1b8ffcd737e5e0257f837584afe Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Sun, 11 Jan 2026 15:07:18 +0100 Subject: [PATCH 07/22] refactor(koplugin): extract annotation cleaning into KoInsightAnnotationReader.cleanAnnotations --- .../koinsight.koplugin/annotation_reader.lua | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/plugins/koinsight.koplugin/annotation_reader.lua b/plugins/koinsight.koplugin/annotation_reader.lua index 48de7b0b..5c176c80 100644 --- a/plugins/koinsight.koplugin/annotation_reader.lua +++ b/plugins/koinsight.koplugin/annotation_reader.lua @@ -95,10 +95,21 @@ function KoInsightAnnotationReader.getAnnotationsByBook() end -- Clean up annotations for JSON serialization - local cleaned_annotations = {} - for _, annotation in ipairs(current_annotations) do - -- Only include fields that are needed - local cleaned = { + local cleaned_annotations = + KoInsightAnnotationReader.cleanAnnotations(current_annotations, total_pages) + + annotations_by_book[book_md5] = cleaned_annotations + logger.info("[KoInsight] Prepared", #cleaned_annotations, "annotations for book", book_md5) + + return annotations_by_book +end + +-- Clean annotations for JSON serialization +-- Removes unnecessary fields and formats data for server +function KoInsightAnnotationReader.cleanAnnotations(annotations, total_pages) + local cleaned = {} + for _, annotation in ipairs(annotations) do + local cleaned_annotation = { datetime = annotation.datetime, drawer = annotation.drawer, color = annotation.color, @@ -107,29 +118,23 @@ function KoInsightAnnotationReader.getAnnotationsByBook() chapter = annotation.chapter, pageno = annotation.pageno, page = annotation.page, - total_pages = total_pages, -- Current document total pages (captured at sync time) + total_pages = total_pages, } - -- Include datetime_updated if it exists + -- Include optional fields if present if annotation.datetime_updated then - cleaned.datetime_updated = annotation.datetime_updated + cleaned_annotation.datetime_updated = annotation.datetime_updated end - - -- Include position data for highlights (not bookmarks) if annotation.pos0 then - cleaned.pos0 = annotation.pos0 + cleaned_annotation.pos0 = annotation.pos0 end if annotation.pos1 then - cleaned.pos1 = annotation.pos1 + cleaned_annotation.pos1 = annotation.pos1 end - table.insert(cleaned_annotations, cleaned) + table.insert(cleaned, cleaned_annotation) end - - annotations_by_book[book_md5] = cleaned_annotations - logger.info("[KoInsight] Prepared", #cleaned_annotations, "annotations for book", book_md5) - - return annotations_by_book + return cleaned end -- Extract all necessary data from a book's sidecar file in one read From b2896e5cdae28bedba41c5674166446d4c6e5f3b Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Sun, 11 Jan 2026 15:18:09 +0100 Subject: [PATCH 08/22] refactor(koplugin): getCurrentBookMd5 directly opens sidecar --- plugins/koinsight.koplugin/annotation_reader.lua | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/koinsight.koplugin/annotation_reader.lua b/plugins/koinsight.koplugin/annotation_reader.lua index 5c176c80..bcbc2d99 100644 --- a/plugins/koinsight.koplugin/annotation_reader.lua +++ b/plugins/koinsight.koplugin/annotation_reader.lua @@ -19,8 +19,13 @@ function KoInsightAnnotationReader.getCurrentBookMd5() return nil end - -- Use the same method as getMd5ForPath - return KoInsightAnnotationReader.getMd5ForPath(current_doc) + local DocSettings = require("docsettings") + local doc_settings = DocSettings:open(current_doc) + if not doc_settings then + return nil + end + + return doc_settings:readSetting("partial_md5_checksum") end -- Get annotations for the currently opened book From bb2095e5080cbb48c53acc26260e11c895cf184f Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Sun, 11 Jan 2026 15:28:39 +0100 Subject: [PATCH 09/22] feat(koplugin): register BulkSync as koreader action (for gestures) --- plugins/koinsight.koplugin/main.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/koinsight.koplugin/main.lua b/plugins/koinsight.koplugin/main.lua index 9904faed..495c6753 100644 --- a/plugins/koinsight.koplugin/main.lua +++ b/plugins/koinsight.koplugin/main.lua @@ -139,12 +139,22 @@ function koinsight:onDispatcherRegisterActions() title = _("KoInsight: Sync stats"), general = true, }) + Dispatcher:registerAction("koinsight_bulk_sync", { + category = "none", + event = "KoInsightBulkSync", + title = _("KoInsight: Bulk sync all books"), + general = true, + }) end function koinsight:onKoInsightSync() KoInsightUpload.sync(self.koinsight_settings:getServerURL(), false) end +function koinsight:onKoInsightBulkSync() + self:performBulkSync() +end + -- Perform bulk sync of all books with progress UI function koinsight:performBulkSync() local url = self.koinsight_settings:getServerURL() From 58b65f522e8aad4baec30c2bc12d4a2f7f64b2d5 Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Sun, 11 Jan 2026 15:54:55 +0100 Subject: [PATCH 10/22] perf(koplugin): require docsettings and logger once in annotationreader --- .../koinsight.koplugin/annotation_reader.lua | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/plugins/koinsight.koplugin/annotation_reader.lua b/plugins/koinsight.koplugin/annotation_reader.lua index bcbc2d99..113a1691 100644 --- a/plugins/koinsight.koplugin/annotation_reader.lua +++ b/plugins/koinsight.koplugin/annotation_reader.lua @@ -1,3 +1,6 @@ +local DocSettings = require("docsettings") +local logger = require("logger") + local KoInsightAnnotationReader = {} -- Get the currently opened document @@ -19,7 +22,6 @@ function KoInsightAnnotationReader.getCurrentBookMd5() return nil end - local DocSettings = require("docsettings") local doc_settings = DocSettings:open(current_doc) if not doc_settings then return nil @@ -30,8 +32,6 @@ end -- Get annotations for the currently opened book function KoInsightAnnotationReader.getCurrentBookAnnotations() - local logger = require("logger") - local DocSettings = require("docsettings") local current_doc = KoInsightAnnotationReader.getCurrentDocument() if not current_doc then @@ -79,7 +79,6 @@ end -- Get annotations organized by book md5 function KoInsightAnnotationReader.getAnnotationsByBook() - local logger = require("logger") local annotations_by_book = {} -- For now, only get annotations from currently opened book @@ -145,9 +144,6 @@ end -- Extract all necessary data from a book's sidecar file in one read -- Returns: md5, annotations, total_pages, book_metadata (or nil if no annotations/md5) function KoInsightAnnotationReader.getBookDataFromSidecar(file_path) - local DocSettings = require("docsettings") - local logger = require("logger") - if not file_path then return nil end @@ -203,9 +199,6 @@ end -- Get annotations for a specific book file path function KoInsightAnnotationReader.getAnnotationsForBook(file_path) - local DocSettings = require("docsettings") - local logger = require("logger") - if not file_path then logger.warn("[KoInsight] No file path provided") return nil, nil @@ -234,9 +227,6 @@ end -- Get MD5 hash for a book directly from its sidecar file function KoInsightAnnotationReader.getMd5ForPath(file_path) - local DocSettings = require("docsettings") - local logger = require("logger") - if not file_path then return nil end @@ -261,7 +251,6 @@ end -- Get all books with annotations from reading history function KoInsightAnnotationReader.getAllBooksWithAnnotations() local ReadHistory = require("readhistory") - local logger = require("logger") logger.info("[KoInsight] Starting bulk annotation collection from reading history") From c78d0748c9c0e82884f8de580ffb51af6de52367 Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Wed, 21 Jan 2026 18:29:32 +0100 Subject: [PATCH 11/22] fix: trick flawed test to run --- apps/server/src/kosync/kosync-router.ts | 2 +- apps/server/src/stats/stats-router.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/kosync/kosync-router.ts b/apps/server/src/kosync/kosync-router.ts index e5783407..a7789108 100644 --- a/apps/server/src/kosync/kosync-router.ts +++ b/apps/server/src/kosync/kosync-router.ts @@ -138,7 +138,7 @@ router.get('/syncs/progress/:document', authenticate, async (req: Request, res: return; } - const progress = await KosyncRepository.getByUserIdAndDocument(user.id, document); + const progress = await KosyncRepository.getByUserIdAndDocument(user.id, String(document)); if (!progress) { res.status(404).json({ error: 'Progress not found' }); return; diff --git a/apps/server/src/stats/stats-router.ts b/apps/server/src/stats/stats-router.ts index ae115d09..7cafbf81 100644 --- a/apps/server/src/stats/stats-router.ts +++ b/apps/server/src/stats/stats-router.ts @@ -39,7 +39,7 @@ router.get('/', async (_: Request, res: Response) => { * Get stats by book md5 */ router.get('/:book_md5', async (req: Request, res: Response) => { - const book = await StatsRepository.getByBookMD5(req.params.book_md5); + const book = await StatsRepository.getByBookMD5(String(req.params.book_md5)); res.status(200).json(book); }); From 781dfe5a423497e0fdda3d268bcaf0b30d3045f5 Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Wed, 21 Jan 2026 18:37:14 +0100 Subject: [PATCH 12/22] fix(annotation-reader): lazy-load ReaderUI and use read-only sidecar for bulk operations There were too many requires inside functions all over the place. Also, I saw a nicer Koreader API to get doc_settings. But I encountered an error once, turns out that after rebooting KoReader, there is a chance that ReaderUI is not yet there, failing requires at module level. I used lazy loading, which seems to run just fine. Its still nicer to require once, not for every function call, ESPECIALLY during bulk sync! This prevents crashes when requiring the module outside the reader, preserves live in-memory settings during normal usage, and makes bulk annotation reads safe and non-destructive. --- .../koinsight.koplugin/annotation_reader.lua | 75 +++++++++++++------ 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/plugins/koinsight.koplugin/annotation_reader.lua b/plugins/koinsight.koplugin/annotation_reader.lua index 113a1691..4a51f21d 100644 --- a/plugins/koinsight.koplugin/annotation_reader.lua +++ b/plugins/koinsight.koplugin/annotation_reader.lua @@ -3,10 +3,34 @@ local logger = require("logger") local KoInsightAnnotationReader = {} +-- NOTE: +-- This module is used both inside the reader (normal annotation sync) +-- and outside (bulk sync). +-- There is a chance that after rebooting KoReader, the ReaderUI is not +-- yet available, so requiring it can blow up. +-- Therefore we lazy-load ReaderUI behind pcall() the first time we actually need it. +-- This keeps the module safe to require in any context while still allowing us to +-- use the live reader UI when it exists. +local ReaderUI_ok, ReaderUI = nil, nil +local function get_live_ui() + if ReaderUI_ok == nil then + ReaderUI_ok, ReaderUI = pcall(require, "apps/reader/readerui") + end + return (ReaderUI_ok and ReaderUI and ReaderUI.instance) or nil +end + +-- KoReader has this API, ideal for bulk operations +local function open_sidecar_readonly(doc_path) + local sidecar = DocSettings:findSidecarFile(doc_path) + if not sidecar then + return nil + end + return DocSettings.openSettingsFile(sidecar) +end + -- Get the currently opened document function KoInsightAnnotationReader.getCurrentDocument() - local ReaderUI = require("apps/reader/readerui") - local ui = ReaderUI.instance + local ui = get_live_ui() if ui and ui.document and ui.document.file then return ui.document.file @@ -17,21 +41,21 @@ end -- Get the MD5 hash for the currently open document function KoInsightAnnotationReader.getCurrentBookMd5() - local current_doc = KoInsightAnnotationReader.getCurrentDocument() - if not current_doc then - return nil - end - - local doc_settings = DocSettings:open(current_doc) - if not doc_settings then - return nil + -- when inside reader + local ui = get_live_ui() + if ui and ui.doc_settings then + return ui.doc_settings:readSetting("partial_md5_checksum") end - return doc_settings:readSetting("partial_md5_checksum") + -- fallback (if called outside reader, e.g. in bulk operation) + local current_doc = KoInsightAnnotationReader.getCurrentDocument() + local ds = current_doc and open_sidecar_readonly(current_doc) + return ds and ds:readSetting("partial_md5_checksum") or nil end -- Get annotations for the currently opened book function KoInsightAnnotationReader.getCurrentBookAnnotations() + local ui = get_live_ui() local current_doc = KoInsightAnnotationReader.getCurrentDocument() if not current_doc then @@ -43,14 +67,18 @@ function KoInsightAnnotationReader.getCurrentBookAnnotations() -- Force flush any in-memory changes to disk before reading -- Otherwise changes are not reflected - local ReaderUI = require("apps/reader/readerui") - local ui = ReaderUI.instance + -- + -- IMPORTANT: + -- If we are inside the reader, ui.doc_settings is the freshest source (in-memory). + -- We flush to ensure sidecar on disk is up-to-date for other codepaths. if ui and ui.doc_settings then logger.dbg("[KoInsight] Flushing doc settings to disk") ui.doc_settings:flush() end - local doc_settings = DocSettings:open(current_doc) + -- Prefer live doc_settings when inside reader (fresh, no extra sidecar open) + -- Fall back to read-only sidecar open (outside reader withouth live settings) + local doc_settings = (ui and ui.doc_settings) or open_sidecar_readonly(current_doc) if not doc_settings then logger.dbg("[KoInsight] No doc settings found for:", current_doc) return nil @@ -71,6 +99,9 @@ function KoInsightAnnotationReader.getCurrentBookAnnotations() if ui and ui.document then total_pages = ui.document:getPageCount() logger.dbg("[KoInsight] Document has", total_pages, "total pages") + else + -- Fallback for outside of reader, where we have no live ui.document + total_pages = doc_settings:readSetting("doc_pages") end logger.info("[KoInsight] Found", #annotations, "annotations for current book") @@ -81,8 +112,8 @@ end function KoInsightAnnotationReader.getAnnotationsByBook() local annotations_by_book = {} - -- For now, only get annotations from currently opened book - -- TODO: check bulk-syncing possibilities + -- Get annotations from currently opened book + -- Bulk syncing is another code path since we need to open sidecar files for bulk syncing local current_annotations, total_pages = KoInsightAnnotationReader.getCurrentBookAnnotations() if not current_annotations or #current_annotations == 0 then @@ -148,7 +179,8 @@ function KoInsightAnnotationReader.getBookDataFromSidecar(file_path) return nil end - local doc_settings = DocSettings:open(file_path) + -- Read-only sidecar open: ideal for bulk operations + local doc_settings = open_sidecar_readonly(file_path) if not doc_settings then return nil end @@ -206,7 +238,8 @@ function KoInsightAnnotationReader.getAnnotationsForBook(file_path) logger.dbg("[KoInsight] Reading annotations for:", file_path) - local doc_settings = DocSettings:open(file_path) + -- Read-only sidecar open: avoids unintended writes during bulk reads + local doc_settings = open_sidecar_readonly(file_path) if not doc_settings then logger.dbg("[KoInsight] No doc settings found for:", file_path) return nil, nil @@ -231,7 +264,8 @@ function KoInsightAnnotationReader.getMd5ForPath(file_path) return nil end - local doc_settings = DocSettings:open(file_path) + -- Read-only sidecar open: avoids unintended writes during bulk reads + local doc_settings = open_sidecar_readonly(file_path) if not doc_settings then return nil end @@ -257,8 +291,7 @@ function KoInsightAnnotationReader.getAllBooksWithAnnotations() -- Force flush currently open book settings to disk first -- Only needed if the user is currently in a book, other books should already -- have been flushed settings - local ReaderUI = require("apps/reader/readerui") - local ui = ReaderUI.instance + local ui = get_live_ui() if ui and ui.doc_settings then logger.dbg("[KoInsight] Flushing currently open book's doc settings to disk") ui.doc_settings:flush() From bc56b344ab32550f6f40a1adc9105d661e4f7225 Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Wed, 21 Jan 2026 20:02:58 +0100 Subject: [PATCH 13/22] fix: remove statistics values from annotation sync --- apps/server/src/upload/upload-service.ts | 34 ++++++++++++------- packages/common/types/book.ts | 7 ++-- .../koinsight.koplugin/annotation_reader.lua | 7 ---- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/apps/server/src/upload/upload-service.ts b/apps/server/src/upload/upload-service.ts index a2ca418d..66f92571 100644 --- a/apps/server/src/upload/upload-service.ts +++ b/apps/server/src/upload/upload-service.ts @@ -82,24 +82,32 @@ export class UploadService { pages: book.pages, notes: book.notes, highlights: book.highlights, - total_read_pages: book.total_read_pages, - total_read_time: book.total_read_time, + total_read_pages: book.total_read_pages ?? 0, + total_read_time: book.total_read_time ?? 0, })); await Promise.all( - newBookDevices.map((bookDevice) => - trx('book_device') + newBookDevices.map((bookDevice) => { + const { book_md5, device_id, total_read_time, total_read_pages, ...otherFields } = + bookDevice; + + // Always merge these fields + const fieldsToMerge: (keyof BookDevice)[] = ['last_open', 'pages', 'notes', 'highlights']; + + // Only merge statistics fields if they have actual values (if on statistics.db sync path) + // This prevents annotation-only syncs from overwriting with zeros + if (total_read_time !== undefined && total_read_time > 0) { + fieldsToMerge.push('total_read_time'); + } + if (total_read_pages !== undefined && total_read_pages > 0) { + fieldsToMerge.push('total_read_pages'); + } + + return trx('book_device') .insert(bookDevice) .onConflict(['book_md5', 'device_id']) - .merge([ - 'last_open', - 'pages', - 'notes', - 'highlights', - 'total_read_time', - 'total_read_pages', - ]) - ) + .merge(fieldsToMerge); + }) ); // Insert page stats diff --git a/packages/common/types/book.ts b/packages/common/types/book.ts index 4a4f2e25..6ebf167f 100644 --- a/packages/common/types/book.ts +++ b/packages/common/types/book.ts @@ -1,5 +1,5 @@ export type KoReaderBook = { - id: number; + id: number; // Optional for annotation-only sync md5: string; title: string; authors: string; @@ -9,8 +9,9 @@ export type KoReaderBook = { pages: number; series: string; language: string; - total_read_time: number; - total_read_pages: number; + // These fields only come from statistics.db sync, not annotation sync + total_read_time?: number; + total_read_pages?: number; }; export type DbBook = { diff --git a/plugins/koinsight.koplugin/annotation_reader.lua b/plugins/koinsight.koplugin/annotation_reader.lua index 4a51f21d..7a32f882 100644 --- a/plugins/koinsight.koplugin/annotation_reader.lua +++ b/plugins/koinsight.koplugin/annotation_reader.lua @@ -217,15 +217,8 @@ function KoInsightAnnotationReader.getBookDataFromSidecar(file_path) highlights = (stats and stats.highlights) or 0, notes = (stats and stats.notes) or 0, last_open = (summary and summary.modified) or os.time(), - total_read_time = 0, - total_read_pages = 0, } - -- Calculate read pages from percent_finished - if percent_finished and total_pages then - book_metadata.total_read_pages = math.floor(total_pages * percent_finished) - end - return md5, annotations, total_pages, book_metadata end From 297896033f72cd786bc6d9c25eaf4741cd65c6d8 Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Wed, 21 Jan 2026 20:49:23 +0100 Subject: [PATCH 14/22] refactor(koplugin-sync): consolidate sync options and fix missing statistics in full sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - rename sync functions for clarity: sync() → syncCurrentBook(), bulkSync() → syncAllBooks() - merge "Bulk Sync All Books" into single "Synchronize data" menu option - fix syncAllBooks() to include full statistics sync from database (was only syncing annotations) - sync-on-suspend now explicitly use syncCurrentBook() for fast, targeted sync Manual sync now properly syncs all statistics + all annotations. Suspend sync remains fast, syncing all statistics + current book only. I am unsure about the performance impact on syncing all anotations actually is though... --- plugins/koinsight.koplugin/main.lua | 85 +++++++-------------------- plugins/koinsight.koplugin/upload.lua | 14 +++-- 2 files changed, 30 insertions(+), 69 deletions(-) diff --git a/plugins/koinsight.koplugin/main.lua b/plugins/koinsight.koplugin/main.lua index 495c6753..cb87c9cc 100644 --- a/plugins/koinsight.koplugin/main.lua +++ b/plugins/koinsight.koplugin/main.lua @@ -26,48 +26,15 @@ function koinsight:addToMainMenu(menu_items) text = _("KoInsight"), sorting_hint = "tools", sub_item_table = { - -- 1) Synchronize data + -- 1) Synchronize data (all books) { text = _("Synchronize data"), callback = function() - -- Prefer the robust helper if present… - if self.syncNow then - self:syncNow(false) - return - end - -- …otherwise do a safe inline sync: bring Wi-Fi online, then upload. - local NetworkMgr = require("ui/network/manager") - local url = self.koinsight_settings:getServerURL() - if not url or url == "" then - UIManager:show( - InfoMessage:new({ text = _("KoInsight server URL is not configured."), timeout = 3 }) - ) - return - end - NetworkMgr:runWhenOnline(function() - local ok, err = pcall(function() - KoInsightUpload.sync(url, false) -- not silent for manual trigger - end) - if ok then - UIManager:show(InfoMessage:new({ text = _("KoInsight sync finished."), timeout = 2 })) - else - logger.err("[KoInsight] Manual sync failed: " .. tostring(err)) - UIManager:show(InfoMessage:new({ text = _("KoInsight sync failed."), timeout = 3 })) - end - end) - end, - }, - - -- 2) Bulk sync all books - { - text = _("Bulk Sync All Books (may take time)"), - separator = true, - callback = function() - self:performBulkSync() + self:performFullSync() end, }, - -- 3) Sync on suspend + -- 2) Sync on suspend { text = _("Sync on suspend"), checked_func = function() @@ -78,7 +45,7 @@ function koinsight:addToMainMenu(menu_items) end, }, - -- 4) Aggressive sync on suspend (auto Wi-Fi) + -- 3) Aggressive sync on suspend (auto Wi-Fi) { text = _("Aggressive sync on suspend (auto Wi-Fi)"), checked_func = function() @@ -92,7 +59,7 @@ function koinsight:addToMainMenu(menu_items) end, }, - -- 5) Set suspend connect timeout + -- 4) Set suspend connect timeout { text = _("Set suspend connect timeout…"), keep_menu_open = true, @@ -104,7 +71,7 @@ function koinsight:addToMainMenu(menu_items) end, }, - -- 6) Set server URL + -- 5) Set server URL { text = _("Set server URL"), keep_menu_open = true, @@ -114,7 +81,7 @@ function koinsight:addToMainMenu(menu_items) end, }, - -- 7) About KoInsight + -- 6) About KoInsight { text = _("About KoInsight"), keep_menu_open = true, @@ -131,32 +98,22 @@ function koinsight:addToMainMenu(menu_items) } end --- Register sync action to make it available in gestures +-- Register sync actions to make them available in gestures function koinsight:onDispatcherRegisterActions() Dispatcher:registerAction("koinsight_sync", { category = "none", event = "KoInsightSync", - title = _("KoInsight: Sync stats"), - general = true, - }) - Dispatcher:registerAction("koinsight_bulk_sync", { - category = "none", - event = "KoInsightBulkSync", - title = _("KoInsight: Bulk sync all books"), + title = _("KoInsight: Sync all books"), general = true, }) end function koinsight:onKoInsightSync() - KoInsightUpload.sync(self.koinsight_settings:getServerURL(), false) -end - -function koinsight:onKoInsightBulkSync() - self:performBulkSync() + self:performFullSync() end --- Perform bulk sync of all books with progress UI -function koinsight:performBulkSync() +-- Perform full sync of all books with progress UI +function koinsight:performFullSync() local url = self.koinsight_settings:getServerURL() if not url or url == "" then UIManager:show( @@ -167,7 +124,7 @@ function koinsight:performBulkSync() -- Show initial message local progress_info = InfoMessage:new({ - text = _("Starting bulk sync...\nScanning reading history for books with annotations."), + text = _("Starting sync..\nScanning reading history for books with annotations."), }) UIManager:show(progress_info) @@ -175,13 +132,13 @@ function koinsight:performBulkSync() local NetworkMgr = require("ui/network/manager") NetworkMgr:runWhenOnline(function() local ok, err = pcall(function() - KoInsightUpload.bulkSync(url, function(progress) + KoInsightUpload.syncAllBooks(url, function(progress) -- Update progress UI if progress.phase == "syncing" then UIManager:close(progress_info) progress_info = InfoMessage:new({ text = string.format( - _("Bulk Sync: %d/%d books\n%d annotations for current book"), + _("Syncing: %d/%d books\n%d annotations for current book"), progress.current, progress.total, progress.annotation_count @@ -198,7 +155,7 @@ function koinsight:performBulkSync() else UIManager:show(InfoMessage:new({ text = string.format( - _("Bulk sync complete!\n%d/%d books synced successfully\n%d failed"), + _("Sync complete!\n%d/%d books synced successfully\n%d failed"), progress.success, progress.total, progress.failed @@ -212,10 +169,8 @@ function koinsight:performBulkSync() if not ok then UIManager:close(progress_info) - logger.err("[KoInsight] Bulk sync failed: " .. tostring(err)) - UIManager:show( - InfoMessage:new({ text = _("Bulk sync failed: " .. tostring(err)), timeout = 5 }) - ) + logger.err("[KoInsight] Full sync failed: " .. tostring(err)) + UIManager:show(InfoMessage:new({ text = _("Sync failed: " .. tostring(err)), timeout = 5 })) end end) end @@ -284,7 +239,7 @@ function koinsight:performSyncOnSuspend() -- Perform sync in a protected call to avoid crashing on suspend local success, error_msg = pcall(function() - KoInsightUpload.sync(server_url, true) -- true = silent mode + KoInsightUpload.syncCurrentBook(server_url, true) -- true = silent mode end) if not success then @@ -343,7 +298,7 @@ function koinsight:performAggressiveSyncOnSuspend() -- Perform the actual sync logger.info("[KoInsight] Performing sync") - KoInsightUpload.sync(server_url, true) -- true = silent mode + KoInsightUpload.syncCurrentBook(server_url, true) -- true = silent mode -- Turn off WiFi if we turned it on if not was_wifi_on then diff --git a/plugins/koinsight.koplugin/upload.lua b/plugins/koinsight.koplugin/upload.lua index 11c3a00b..94380c9c 100644 --- a/plugins/koinsight.koplugin/upload.lua +++ b/plugins/koinsight.koplugin/upload.lua @@ -226,8 +226,8 @@ function bulk_sync_all_books(server_url, progress_callback) end end --- Main sync function (current book + stats) -function KoInsightUpload.sync(server_url, silent) +-- Sync current book only (stats + current book annotations) +function KoInsightUpload.syncCurrentBook(server_url, silent) if silent == nil then silent = false end @@ -242,8 +242,8 @@ function KoInsightUpload.sync(server_url, silent) send_statistics_data(server_url, silent) end --- Bulk sync function (all books with annotations) -function KoInsightUpload.bulkSync(server_url, progress_callback) +-- Sync all books (stats + all book annotations) +function KoInsightUpload.syncAllBooks(server_url, progress_callback) if server_url == nil or server_url == "" then UIManager:show(InfoMessage:new({ text = _("Please configure the server URL first."), @@ -252,6 +252,12 @@ function KoInsightUpload.bulkSync(server_url, progress_callback) end send_device_data(server_url, true) -- silent + + -- First, sync all statistics data from the database + -- This includes all reading progress (page_stat_data) and book metadata + send_statistics_data(server_url, true) -- silent + + -- Then, sync all annotations for all books bulk_sync_all_books(server_url, progress_callback) end From 21ae6134c837326302e7958b151baa30452d8b90 Mon Sep 17 00:00:00 2001 From: Tony Fischer Date: Wed, 21 Jan 2026 21:31:11 +0100 Subject: [PATCH 15/22] style(koplugin): add separator to sync button --- plugins/koinsight.koplugin/main.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/koinsight.koplugin/main.lua b/plugins/koinsight.koplugin/main.lua index cb87c9cc..1f3c33c9 100644 --- a/plugins/koinsight.koplugin/main.lua +++ b/plugins/koinsight.koplugin/main.lua @@ -32,6 +32,7 @@ function koinsight:addToMainMenu(menu_items) callback = function() self:performFullSync() end, + separator = true, -- separator line }, -- 2) Sync on suspend From 0912bb40af5ee2ee787ab547f5c12658151d3703 Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Fri, 30 Jan 2026 18:54:53 +0100 Subject: [PATCH 16/22] fix: fake/ghost page stats --- apps/server/src/koplugin/koplugin-router.ts | 3 +- apps/server/src/upload/upload-service.ts | 45 +++++++++++++-------- plugins/koinsight.koplugin/upload.lua | 12 +----- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/apps/server/src/koplugin/koplugin-router.ts b/apps/server/src/koplugin/koplugin-router.ts index f7d517f0..f7472a1d 100644 --- a/apps/server/src/koplugin/koplugin-router.ts +++ b/apps/server/src/koplugin/koplugin-router.ts @@ -53,6 +53,7 @@ router.post('/import', rejectOldPluginVersion, async (req, res) => { const koreaderBooks: KoReaderBook[] = req.body.books; const newPageStats: PageStat[] = req.body.stats; const annotations: Record = req.body.annotations || {}; + const deviceId: string | undefined = req.body.device_id; // For annotation sync path try { console.debug('Importing books:', koreaderBooks); @@ -63,7 +64,7 @@ router.post('/import', rejectOldPluginVersion, async (req, res) => { 'books with annotations' ); - await UploadService.uploadStatisticData(koreaderBooks, newPageStats, annotations); + await UploadService.uploadStatisticData(koreaderBooks, newPageStats, annotations, deviceId); res.status(200).json({ message: 'Upload successful' }); } catch (err) { console.error(err); diff --git a/apps/server/src/upload/upload-service.ts b/apps/server/src/upload/upload-service.ts index 960d277d..f17d030b 100644 --- a/apps/server/src/upload/upload-service.ts +++ b/apps/server/src/upload/upload-service.ts @@ -41,7 +41,8 @@ export class UploadService { static uploadStatisticData( booksToImport: KoReaderBook[], newPageStats: PageStat[], - annotationsByBook?: Record + annotationsByBook?: Record, + deviceIdOverride?: string // For annotation sync path without stats ) { return db.transaction(async (trx) => { // Insert books @@ -58,8 +59,13 @@ export class UploadService { newBooks.map(({ id, ...book }) => trx('book').insert(book).onConflict('md5').ignore()) ); - const hasUnknownDevices = - newPageStats.length > 0 && newPageStats[0].device_id === this.UNKNOWN_DEVICE_ID; + // Determine device ID: from stats, override, or fall back to unknown device + const deviceId = + newPageStats.length > 0 + ? newPageStats[0].device_id + : deviceIdOverride || this.UNKNOWN_DEVICE_ID; + + const hasUnknownDevices = deviceId === this.UNKNOWN_DEVICE_ID; if (hasUnknownDevices) { let unknownDevice = await trx('device') @@ -76,7 +82,7 @@ export class UploadService { } const newBookDevices: Omit[] = booksToImport.map((book) => ({ - device_id: newPageStats[0].device_id, + device_id: deviceId, book_md5: book.md5, last_open: book.last_open, pages: book.pages, @@ -110,31 +116,36 @@ export class UploadService { }) ); - // Insert page stats - await Promise.all( - newPageStats.map((pageStat) => - trx('page_stat') - .insert(pageStat) - .onConflict(['device_id', 'book_md5', 'page', 'start_time']) - .merge(['duration', 'total_pages']) - ) - ); + // Insert page stats (only on stats sync path! there are non for annotation sync path) + if (newPageStats.length > 0) { + await Promise.all( + newPageStats.map((pageStat) => + trx('page_stat') + .insert(pageStat) + .onConflict(['device_id', 'book_md5', 'page', 'start_time']) + .merge(['duration', 'total_pages']) + ) + ); + } // Insert annotations if provided if (annotationsByBook) { - const deviceId = - newPageStats.length > 0 ? newPageStats[0].device_id : this.UNKNOWN_DEVICE_ID; + // Use same deviceId logic as above + const annotationDeviceId = + newPageStats.length > 0 + ? newPageStats[0].device_id + : deviceIdOverride || this.UNKNOWN_DEVICE_ID; await Promise.all( Object.entries(annotationsByBook).map(([bookMd5, annotations]) => - AnnotationsRepository.bulkInsert(bookMd5, deviceId, annotations, trx) + AnnotationsRepository.bulkInsert(bookMd5, annotationDeviceId, annotations, trx) ) ); // FIXME: with this, if there is only 1 annotation and it gets removed, it won't get marked as deleted, because `annotationsByBook` will be empty. It will only get marked as deleted if the user adds another annotation to trigger an update on the book. await Promise.all( Object.entries(annotationsByBook).map(([bookMd5, annotations]) => - this.detectAndMarkDeletedAnnotations(bookMd5, deviceId, annotations, trx) + this.detectAndMarkDeletedAnnotations(bookMd5, annotationDeviceId, annotations, trx) ) ); } diff --git a/plugins/koinsight.koplugin/upload.lua b/plugins/koinsight.koplugin/upload.lua index 94380c9c..75cb5505 100644 --- a/plugins/koinsight.koplugin/upload.lua +++ b/plugins/koinsight.koplugin/upload.lua @@ -113,18 +113,10 @@ function send_book_annotations(server_url, book_md5, annotations, total_pages, b annotations_by_book[book_md5] = cleaned_annotations local body = { - stats = { - { - page = 1, - start_time = os.time(), - duration = 0, - total_pages = total_pages or 1, - book_md5 = book_md5, - device_id = device_id, - }, - }, + stats = {}, -- empty stats on annotations sync path, handled server side books = book_to_send and { book_to_send } or {}, annotations = annotations_by_book, + device_id = device_id, version = const.VERSION, } From 65d17c5c93ed8d9a031672e0de98c14a3d43ee5a Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Fri, 30 Jan 2026 19:02:43 +0100 Subject: [PATCH 17/22] fix: possible FK constraint violations --- plugins/koinsight.koplugin/upload.lua | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/plugins/koinsight.koplugin/upload.lua b/plugins/koinsight.koplugin/upload.lua index 75cb5505..912bc953 100644 --- a/plugins/koinsight.koplugin/upload.lua +++ b/plugins/koinsight.koplugin/upload.lua @@ -108,13 +108,23 @@ function send_book_annotations(server_url, book_md5, annotations, total_pages, b end end + -- WARN: We MUST have book metadata to send annotations + -- The server has a foreign key constraint: annotations.book_md5 -> book.md5 + -- If we don't send book data, annotation insert will fail + if not book_to_send then + logger.err( + "[KoInsight] Cannot sync annotations for book " .. book_md5 .. ": no book metadata available" + ) + return false, { error = "No book metadata available" } + end + -- Create minimal payload local annotations_by_book = {} annotations_by_book[book_md5] = cleaned_annotations local body = { stats = {}, -- empty stats on annotations sync path, handled server side - books = book_to_send and { book_to_send } or {}, + books = { book_to_send }, -- Always send book metadata for FK constraint annotations = annotations_by_book, device_id = device_id, version = const.VERSION, From 5de56d611a70c5ed1930c94c487d77bd311c5384 Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Fri, 30 Jan 2026 20:15:03 +0100 Subject: [PATCH 18/22] chore(koplugin): bump plugin version to 0.3.0 --- apps/server/src/koplugin/koplugin-router.ts | 2 +- plugins/koinsight.koplugin/const.lua | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/koplugin/koplugin-router.ts b/apps/server/src/koplugin/koplugin-router.ts index f7472a1d..bc8c1bfa 100644 --- a/apps/server/src/koplugin/koplugin-router.ts +++ b/apps/server/src/koplugin/koplugin-router.ts @@ -11,7 +11,7 @@ import { UploadService } from '../upload/upload-service'; // Router for KoInsight koreader plugin const router = Router(); -const REQUIRED_PLUGIN_VERSION = '0.2.0'; +const REQUIRED_PLUGIN_VERSION = '0.3.0'; const rejectOldPluginVersion = (req: Request, res: Response, next: NextFunction) => { const { version } = req.body; diff --git a/plugins/koinsight.koplugin/const.lua b/plugins/koinsight.koplugin/const.lua index 86c57fb1..6c83f8a8 100644 --- a/plugins/koinsight.koplugin/const.lua +++ b/plugins/koinsight.koplugin/const.lua @@ -1,5 +1,5 @@ local const = {} -const.VERSION = "0.2.0" +const.VERSION = "0.3.0" return const From 5df4c00179bb744f9852fbbe82f6b15bcf4a314a Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Fri, 30 Jan 2026 20:20:22 +0100 Subject: [PATCH 19/22] fix(tests): missing plugin version bump in tests --- apps/server/src/koplugin/koplugin-router.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/koplugin/koplugin-router.test.ts b/apps/server/src/koplugin/koplugin-router.test.ts index f8ebbf63..49e34887 100644 --- a/apps/server/src/koplugin/koplugin-router.test.ts +++ b/apps/server/src/koplugin/koplugin-router.test.ts @@ -10,7 +10,7 @@ describe('koplugin-router', () => { app.use(express.json()); app.use('/koplugin', kopluginRouter); - const PLUGIN_VERSION = '0.2.0'; + const PLUGIN_VERSION = '0.3.0'; describe('POST /koplugin/device', () => { it('registers a device', async () => { From 411c5e9a24a62f8ea18adf9ac4ce840c4d67101f Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Fri, 30 Jan 2026 20:25:25 +0100 Subject: [PATCH 20/22] fix: code smell in tests using plugin version --- apps/server/src/koplugin/koplugin-router.test.ts | 16 +++++++--------- apps/server/src/koplugin/koplugin-router.ts | 2 +- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/apps/server/src/koplugin/koplugin-router.test.ts b/apps/server/src/koplugin/koplugin-router.test.ts index 49e34887..03ec6fc5 100644 --- a/apps/server/src/koplugin/koplugin-router.test.ts +++ b/apps/server/src/koplugin/koplugin-router.test.ts @@ -3,20 +3,18 @@ import request from 'supertest'; import { createDevice } from '../db/factories/device-factory'; import { fakeKoReaderAnnotation } from '../db/factories/koreader-annotation-factory'; import { db } from '../knex'; -import { kopluginRouter } from './koplugin-router'; +import { kopluginRouter, REQUIRED_PLUGIN_VERSION } from './koplugin-router'; describe('koplugin-router', () => { const app = express(); app.use(express.json()); app.use('/koplugin', kopluginRouter); - const PLUGIN_VERSION = '0.3.0'; - describe('POST /koplugin/device', () => { it('registers a device', async () => { const response = await request(app) .post('/koplugin/device') - .send({ id: 'device-123', model: 'Kindle Paperwhite', version: PLUGIN_VERSION }); + .send({ id: 'device-123', model: 'Kindle Paperwhite', version: REQUIRED_PLUGIN_VERSION }); expect(response.status).toBe(200); expect(response.body).toEqual({ message: 'Device registered successfully' }); @@ -33,7 +31,7 @@ describe('koplugin-router', () => { it('returns 400 when device ID is missing', async () => { const response = await request(app) .post('/koplugin/device') - .send({ model: 'Kindle', version: PLUGIN_VERSION }); + .send({ model: 'Kindle', version: REQUIRED_PLUGIN_VERSION }); expect(response.status).toBe(400); expect(response.body).toEqual({ error: 'Missing device ID or model' }); @@ -42,7 +40,7 @@ describe('koplugin-router', () => { it('returns 400 when model is missing', async () => { const response = await request(app) .post('/koplugin/device') - .send({ id: 'device-123', version: PLUGIN_VERSION }); + .send({ id: 'device-123', version: REQUIRED_PLUGIN_VERSION }); expect(response.status).toBe(400); expect(response.body).toEqual({ error: 'Missing device ID or model' }); @@ -75,7 +73,7 @@ describe('koplugin-router', () => { const response = await request(app) .post('/koplugin/import') .send({ - version: PLUGIN_VERSION, + version: REQUIRED_PLUGIN_VERSION, books: [ { md5: bookMd5, @@ -130,7 +128,7 @@ describe('koplugin-router', () => { const response = await request(app) .post('/koplugin/import') .send({ - version: PLUGIN_VERSION, + version: REQUIRED_PLUGIN_VERSION, books: [ { md5: bookMd5, @@ -222,7 +220,7 @@ describe('koplugin-router', () => { it('returns health status', async () => { const response = await request(app) .get('/koplugin/health') - .send({ version: PLUGIN_VERSION }); + .send({ version: REQUIRED_PLUGIN_VERSION }); expect(response.status).toBe(200); expect(response.body).toEqual({ message: 'Plugin is healthy' }); diff --git a/apps/server/src/koplugin/koplugin-router.ts b/apps/server/src/koplugin/koplugin-router.ts index bc8c1bfa..10bef0b0 100644 --- a/apps/server/src/koplugin/koplugin-router.ts +++ b/apps/server/src/koplugin/koplugin-router.ts @@ -11,7 +11,7 @@ import { UploadService } from '../upload/upload-service'; // Router for KoInsight koreader plugin const router = Router(); -const REQUIRED_PLUGIN_VERSION = '0.3.0'; +export const REQUIRED_PLUGIN_VERSION = '0.3.0'; const rejectOldPluginVersion = (req: Request, res: Response, next: NextFunction) => { const { version } = req.body; From 8607f6d692763ae473a87c76affb624115387692 Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Fri, 30 Jan 2026 20:59:53 +0100 Subject: [PATCH 21/22] docs: add changelog file --- CHANGELOG.md | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..8886bdd2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,57 @@ +# Changelog + +## [Unreleased] + +### New Features + +Bulk Annotation Sync: The "Synchronize data" button now syncs all annotations from all books in your reading history at once (previously only the currently open book). Sync on suspend still syncs statistics and annotations for the currently open book only (keeping suspend snappy). + +### Bug Fixes + +Fixed Book Duplicates: Some users saw the same book twice in KoInsight, one with statistics, one with annotations. We now match books using their unique MD5 checksum instead of title. + +If you have duplicates, either: + +1. Delete your database and re-sync (recommended - clean start) +2. Manually delete duplicate books in the web interface (the faulty one) + +New syncs won't create duplicates. If you still see duplicates, you most likely have duplicates in your KoReader statistics database and KoInsight makes those visible. + +### Breaking Changes + +Plugin version 0.3.0 required. Update it before syncing. + +--- + +## [0.2.2] - 2026-01-11 + +### Added + +- Annotation sync support for currently open book +- Mark deleted annotations in the database + +### Fixed + +- Annotations now properly marked as deleted when removed in KoReader +- Docker build issues + +## [0.2.0] - 2026-01-11 + +### Added + +- Plugin versioning system +- Server validates plugin version before accepting data + +### Changed + +- **BREAKING:** Server now requires specific plugin version + +--- + +## Earlier Versions + +See git history for changes prior to v0.2.0. + +[Unreleased]: https://github.com/GeorgeSG/koinsight/compare/v0.2.2...HEAD +[0.2.2]: https://github.com/GeorgeSG/koinsight/compare/v0.2.0...v0.2.2 +[0.2.0]: https://github.com/GeorgeSG/koinsight/releases/tag/v0.2.0 From 683c16a41e763ed0684dc27e2ef5270c26ffae55 Mon Sep 17 00:00:00 2001 From: "Tony Fischer (tku137)" Date: Sun, 1 Feb 2026 11:18:55 +0100 Subject: [PATCH 22/22] refactor: remove redundant variable declaration --- apps/server/src/upload/upload-service.ts | 10 ++-------- packages/common/types/book.ts | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/apps/server/src/upload/upload-service.ts b/apps/server/src/upload/upload-service.ts index f17d030b..febf9375 100644 --- a/apps/server/src/upload/upload-service.ts +++ b/apps/server/src/upload/upload-service.ts @@ -130,22 +130,16 @@ export class UploadService { // Insert annotations if provided if (annotationsByBook) { - // Use same deviceId logic as above - const annotationDeviceId = - newPageStats.length > 0 - ? newPageStats[0].device_id - : deviceIdOverride || this.UNKNOWN_DEVICE_ID; - await Promise.all( Object.entries(annotationsByBook).map(([bookMd5, annotations]) => - AnnotationsRepository.bulkInsert(bookMd5, annotationDeviceId, annotations, trx) + AnnotationsRepository.bulkInsert(bookMd5, deviceId, annotations, trx) ) ); // FIXME: with this, if there is only 1 annotation and it gets removed, it won't get marked as deleted, because `annotationsByBook` will be empty. It will only get marked as deleted if the user adds another annotation to trigger an update on the book. await Promise.all( Object.entries(annotationsByBook).map(([bookMd5, annotations]) => - this.detectAndMarkDeletedAnnotations(bookMd5, annotationDeviceId, annotations, trx) + this.detectAndMarkDeletedAnnotations(bookMd5, deviceId, annotations, trx) ) ); } diff --git a/packages/common/types/book.ts b/packages/common/types/book.ts index 6ebf167f..690c8d67 100644 --- a/packages/common/types/book.ts +++ b/packages/common/types/book.ts @@ -1,5 +1,5 @@ export type KoReaderBook = { - id: number; // Optional for annotation-only sync + id: number; md5: string; title: string; authors: string;