feat(koplugin): bulk sync annotations

CHANGELOG.md

@@ -0,0 +1,57 @@
+# Changelog
+
+## [Unreleased]
+
+### New Features
+
+Bulk Annotation Sync: The "Synchronize data" button now syncs all annotations from all books in your reading history at once (previously only the currently open book). Sync on suspend still syncs statistics and annotations for the currently open book only (keeping suspend snappy).
+
+### Bug Fixes
+
+Fixed Book Duplicates: Some users saw the same book twice in KoInsight, one with statistics, one with annotations. We now match books using their unique MD5 checksum instead of title.
+
+If you have duplicates, either:
+
+1. Delete your database and re-sync (recommended - clean start)
+2. Manually delete duplicate books in the web interface (the faulty one)
+
+New syncs won't create duplicates. If you still see duplicates, you most likely have duplicates in your KoReader statistics database and KoInsight makes those visible.
+
+### Breaking Changes
+
+Plugin version 0.3.0 required. Update it before syncing.
+
+---
+
+## [0.2.2] - 2026-01-11
+
+### Added
+
+- Annotation sync support for currently open book
+- Mark deleted annotations in the database
+
+### Fixed
+
+- Annotations now properly marked as deleted when removed in KoReader
+- Docker build issues
+
+## [0.2.0] - 2026-01-11
+
+### Added
+
+- Plugin versioning system
+- Server validates plugin version before accepting data
+
+### Changed
+
+- **BREAKING:** Server now requires specific plugin version
+
+---
+
+## Earlier Versions
+
+See git history for changes prior to v0.2.0.
+
+[Unreleased]: https://github.com/GeorgeSG/koinsight/compare/v0.2.2...HEAD
+[0.2.2]: https://github.com/GeorgeSG/koinsight/compare/v0.2.0...v0.2.2
+[0.2.0]: https://github.com/GeorgeSG/koinsight/releases/tag/v0.2.0

apps/server/src/koplugin/koplugin-router.test.ts

@@ -3,20 +3,18 @@ import request from 'supertest';
 import { createDevice } from '../db/factories/device-factory';
 import { fakeKoReaderAnnotation } from '../db/factories/koreader-annotation-factory';
 import { db } from '../knex';
-import { kopluginRouter } from './koplugin-router';
+import { kopluginRouter, REQUIRED_PLUGIN_VERSION } from './koplugin-router';
 
 describe('koplugin-router', () => {
   const app = express();
   app.use(express.json());
   app.use('/koplugin', kopluginRouter);
 
-  const PLUGIN_VERSION = '0.2.0';
-
   describe('POST /koplugin/device', () => {
     it('registers a device', async () => {
       const response = await request(app)
         .post('/koplugin/device')
-        .send({ id: 'device-123', model: 'Kindle Paperwhite', version: PLUGIN_VERSION });
+        .send({ id: 'device-123', model: 'Kindle Paperwhite', version: REQUIRED_PLUGIN_VERSION });
 
       expect(response.status).toBe(200);
       expect(response.body).toEqual({ message: 'Device registered successfully' });
@@ -33,7 +31,7 @@ describe('koplugin-router', () => {
     it('returns 400 when device ID is missing', async () => {
       const response = await request(app)
         .post('/koplugin/device')
-        .send({ model: 'Kindle', version: PLUGIN_VERSION });
+        .send({ model: 'Kindle', version: REQUIRED_PLUGIN_VERSION });
 
       expect(response.status).toBe(400);
       expect(response.body).toEqual({ error: 'Missing device ID or model' });
@@ -42,7 +40,7 @@ describe('koplugin-router', () => {
     it('returns 400 when model is missing', async () => {
       const response = await request(app)
         .post('/koplugin/device')
-        .send({ id: 'device-123', version: PLUGIN_VERSION });
+        .send({ id: 'device-123', version: REQUIRED_PLUGIN_VERSION });
 
       expect(response.status).toBe(400);
       expect(response.body).toEqual({ error: 'Missing device ID or model' });
@@ -75,7 +73,7 @@ describe('koplugin-router', () => {
       const response = await request(app)
         .post('/koplugin/import')
         .send({
-          version: PLUGIN_VERSION,
+          version: REQUIRED_PLUGIN_VERSION,
           books: [
             {
               md5: bookMd5,
@@ -130,7 +128,7 @@ describe('koplugin-router', () => {
       const response = await request(app)
         .post('/koplugin/import')
         .send({
-          version: PLUGIN_VERSION,
+          version: REQUIRED_PLUGIN_VERSION,
           books: [
             {
               md5: bookMd5,
@@ -222,7 +220,7 @@ describe('koplugin-router', () => {
     it('returns health status', async () => {
       const response = await request(app)
         .get('/koplugin/health')
-        .send({ version: PLUGIN_VERSION });
+        .send({ version: REQUIRED_PLUGIN_VERSION });
 
       expect(response.status).toBe(200);
       expect(response.body).toEqual({ message: 'Plugin is healthy' });

apps/server/src/koplugin/koplugin-router.ts

@@ -11,7 +11,7 @@ import { UploadService } from '../upload/upload-service';
 // Router for KoInsight koreader plugin
 const router = Router();
 
-const REQUIRED_PLUGIN_VERSION = '0.2.0';
+export const REQUIRED_PLUGIN_VERSION = '0.3.0';
 
 const rejectOldPluginVersion = (req: Request, res: Response, next: NextFunction) => {
   const { version } = req.body;
@@ -53,6 +53,7 @@ router.post('/import', rejectOldPluginVersion, async (req, res) => {
   const koreaderBooks: KoReaderBook[] = req.body.books;
   const newPageStats: PageStat[] = req.body.stats;
   const annotations: Record<string, KoReaderAnnotation[]> = req.body.annotations || {};
+  const deviceId: string | undefined = req.body.device_id; // For annotation sync path
 
   try {
     console.debug('Importing books:', koreaderBooks);
@@ -63,7 +64,7 @@ router.post('/import', rejectOldPluginVersion, async (req, res) => {
       'books with annotations'
     );
 
-    await UploadService.uploadStatisticData(koreaderBooks, newPageStats, annotations);
+    await UploadService.uploadStatisticData(koreaderBooks, newPageStats, annotations, deviceId);
     res.status(200).json({ message: 'Upload successful' });
   } catch (err) {
     console.error(err);

apps/server/src/upload/upload-service.ts

@@ -41,7 +41,8 @@ export class UploadService {
   static uploadStatisticData(
     booksToImport: KoReaderBook[],
     newPageStats: PageStat[],
-    annotationsByBook?: Record<string, KoReaderAnnotation[]>
+    annotationsByBook?: Record<string, KoReaderAnnotation[]>,
+    deviceIdOverride?: string // For annotation sync path without stats
   ) {
     return db.transaction(async (trx) => {
       // Insert books
@@ -58,8 +59,13 @@ export class UploadService {
         newBooks.map(({ id, ...book }) => trx<Book>('book').insert(book).onConflict('md5').ignore())
       );
 
-      const hasUnknownDevices =
-        newPageStats.length > 0 && newPageStats[0].device_id === this.UNKNOWN_DEVICE_ID;
+      // Determine device ID: from stats, override, or fall back to unknown device
+      const deviceId =
+        newPageStats.length > 0
+          ? newPageStats[0].device_id
+          : deviceIdOverride || this.UNKNOWN_DEVICE_ID;
+
+      const hasUnknownDevices = deviceId === this.UNKNOWN_DEVICE_ID;
 
       if (hasUnknownDevices) {
         let unknownDevice = await trx<Device>('device')
@@ -76,47 +82,54 @@ export class UploadService {
       }
 
       const newBookDevices: Omit<BookDevice, 'id'>[] = booksToImport.map((book) => ({
-        device_id: newPageStats[0].device_id,
+        device_id: deviceId,
         book_md5: book.md5,
         last_open: book.last_open,
         pages: book.pages,
         notes: book.notes,
         highlights: book.highlights,
-        total_read_pages: book.total_read_pages,
-        total_read_time: book.total_read_time,
+        total_read_pages: book.total_read_pages ?? 0,
+        total_read_time: book.total_read_time ?? 0,
       }));
 
       await Promise.all(
-        newBookDevices.map((bookDevice) =>
-          trx<BookDevice>('book_device')
+        newBookDevices.map((bookDevice) => {
+          const { book_md5, device_id, total_read_time, total_read_pages, ...otherFields } =
+            bookDevice;
+
+          // Always merge these fields
+          const fieldsToMerge: (keyof BookDevice)[] = ['last_open', 'pages', 'notes', 'highlights'];
+
+          // Only merge statistics fields if they have actual values (if on statistics.db sync path)
+          // This prevents annotation-only syncs from overwriting with zeros
+          if (total_read_time !== undefined && total_read_time > 0) {
+            fieldsToMerge.push('total_read_time');
+          }
+          if (total_read_pages !== undefined && total_read_pages > 0) {
+            fieldsToMerge.push('total_read_pages');
+          }
+
+          return trx<BookDevice>('book_device')
             .insert(bookDevice)
             .onConflict(['book_md5', 'device_id'])
-            .merge([
-              'last_open',
-              'pages',
-              'notes',
-              'highlights',
-              'total_read_time',
-              'total_read_pages',
-            ])
-        )
+            .merge(fieldsToMerge);
+        })
       );
 
-      // Insert page stats
-      await Promise.all(
-        newPageStats.map((pageStat) =>
-          trx<PageStat>('page_stat')
-            .insert(pageStat)
-            .onConflict(['device_id', 'book_md5', 'page', 'start_time'])
-            .merge(['duration', 'total_pages'])
-        )
-      );
+      // Insert page stats (only on stats sync path! there are non for annotation sync path)
+      if (newPageStats.length > 0) {
+        await Promise.all(
+          newPageStats.map((pageStat) =>
+            trx<PageStat>('page_stat')
+              .insert(pageStat)
+              .onConflict(['device_id', 'book_md5', 'page', 'start_time'])
+              .merge(['duration', 'total_pages'])
+          )
+        );
+      }
 
       // Insert annotations if provided
       if (annotationsByBook) {
-        const deviceId =
-          newPageStats.length > 0 ? newPageStats[0].device_id : this.UNKNOWN_DEVICE_ID;
-
         await Promise.all(
           Object.entries(annotationsByBook).map(([bookMd5, annotations]) =>
             AnnotationsRepository.bulkInsert(bookMd5, deviceId, annotations, trx)

packages/common/types/book.ts

@@ -9,8 +9,9 @@ export type KoReaderBook = {
   pages: number;
   series: string;
   language: string;
-  total_read_time: number;
-  total_read_pages: number;
+  // These fields only come from statistics.db sync, not annotation sync
+  total_read_time?: number;
+  total_read_pages?: number;
 };
 
 export type DbBook = {