Hashtable is now prebuilt

nitotm · nitotm · commit d657c65c45c3 · 2026-06-16T11:55:02.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,5 +14,20 @@ All notable changes to this project will be documented in this file.
 ### Changed
 - 
 
+### Fixed
+- 
+
+## [0.2.0] - 2026-06-16
+
+> **Important**: This is a 0.x release. The API and functions are **unstable** and may change in future versions.
+
+### Added
+- 
+
+### Changed
+- Hashtable is now prebuilt
+- Hashtable is now 16MB instead of 32MB
+- set_faster function deleted
+
 ### Fixed
 - 
diff --git a/README.md b/README.md
@@ -115,7 +115,6 @@ $ffi = FFI::cdef('
     const char *eldc_set_languages(const char *codes);
     void        eldc_set_scheme(const char *scheme);
     void        eldc_set_scores(int n);
-    void        eldc_set_faster(int flag);
 ', './libeldc.so');  // Windows: 'eldc.dll', macOS: './libeldc.dylib'
 
 $ffi->eldc_init();
@@ -189,7 +188,6 @@ If we use `--scores` or `--reliable`, it will return JSON, if not, a simple unqu
 -h, --help              This message
     --list-languages    Print all supported codes and exit
 -v, --verbose           Loading info, timing, throughput
-    --faster            Uses 2x hashtable (32 MB → 64 MB), minimal speedup
 -l, --languages CODES   Restrict to a subset, e.g. -l "es,en,de,fr"
                         Accepts ISO 639-1 or ISO 639-2/T codes.
 -s, --scores [N]        Output compact JSON with top-N normalised [0,1] scores
@@ -294,10 +292,9 @@ It is the default ELD language scheme. `--scheme iso639-1`
 ***
 
 ### More info
-* ELD-C executable is 26MB, and loads a 32MB hashtable (64MB if --faster), using a total of ~55MB of memory. 
+* ELD-C executable is 24MB, memory use is arround ~30MB. 
 * ELD-C only reads first 1000 bytes of the input string (benchmarks are fair, with all lines under), but could be modded, if you feel an increased `--limit` flag/option is necessary, open a discussion.
 * Unlike other versions of ELD, ELD-C only comes with the 'large' database size, as that is the optimal one, but other sizes could be added.
-* ELD-C is optimized for multi-detection, for single detection a B-tree would offer faster loading and lower memory usage than the current hashtable. I haven't anticipated this being the most common use case, but it could be optimized for.
 * Next improvement could be a better training data set, my own "small" 60GB of data are not as clean as I wish, `fineweb-2` looks good.
 
 #### Donations and suggestions
diff --git a/examples/demo_eldc_lib.cs b/examples/demo_eldc_lib.cs
@@ -36,14 +36,10 @@ static class Eldc {
     [DllImport(LIB, EntryPoint="eldc_set_languages")] public static extern IntPtr SetLanguages(string codes);
     [DllImport(LIB, EntryPoint="eldc_set_scheme")]    public static extern void   SetScheme(string scheme);
     [DllImport(LIB, EntryPoint="eldc_set_scores")]    public static extern void   SetScores(int n);
-    [DllImport(LIB, EntryPoint="eldc_set_faster")]    public static extern void   SetFaster(int flag);
 }
 
 static string S(IntPtr p) => p != IntPtr.Zero ? Marshal.PtrToStringAnsi(p)! : "und";
 
-// ── 0. set_faster (Not worth it. Call before init()) ─────────────────────────
-// Eldc.SetFaster(1);  // 64 MB table (32MB default) Minimal speedup.
-
 // ── 1. init ──────────────────────────────────────────────────────────────────
 Eldc.Init();
 Console.WriteLine("=== eldc .NET demo ===\n");
diff --git a/examples/demo_eldc_lib.go b/examples/demo_eldc_lib.go
@@ -29,7 +29,6 @@ extern const char *eldc_detect_details(const char *text, EldcDetectResult *r);
 extern const char *eldc_set_languages(const char *codes);
 extern void        eldc_set_scheme(const char *scheme);
 extern void        eldc_set_scores(int n);
-extern void        eldc_set_faster(int flag);
 */
 import "C"
 import (
@@ -42,8 +41,6 @@ func gs(p *C.char) string { return C.GoString(p) }
 func free(p *C.char)      { C.free(unsafe.Pointer(p)) }
 
 func main() {
-   // ── 0. set_faster (Not worth it. Call before init()) ─────────────────────
-	// C.eldc_set_faster(1) // 64 MB table (32MB default) Minimal speedup.
 
 	// ── 1. init ──────────────────────────────────────────────────────────────
 	C.eldc_init()
diff --git a/examples/demo_eldc_lib.java b/examples/demo_eldc_lib.java
@@ -40,17 +40,13 @@ public interface EldcLib extends Library {
         String  eldc_set_languages(String codes);
         void    eldc_set_scheme(String scheme);
         void    eldc_set_scores(int n);
-        void    eldc_set_faster(int flag);
     }
 
     static String lang(Pointer p) { return p != null ? p.getString(0) : "und"; }
 
     public static void main(String[] args) {
         var lib = EldcLib.LIB;
 
-        // ── 0. set_faster (Not worth it. Call before init()) ─────────────────
-        // lib.eldc_set_faster(1); // 64 MB table (32MB default) Minimal speedup
-
         // ── 1. init ──────────────────────────────────────────────────────────
         lib.eldc_init();
         System.out.println("=== eldc Java demo ===\n");
diff --git a/examples/demo_eldc_lib.php b/examples/demo_eldc_lib.php
@@ -30,12 +30,8 @@
     const char *eldc_set_languages(const char *codes);
     void        eldc_set_scheme(const char *scheme);
     void        eldc_set_scores(int n);
-    void        eldc_set_faster(int flag);
 ', $lib_name);
 
-// ── 0. set_faster (Not worth it. Call before init()) ─────────────────────────
-// $ffi->eldc_set_faster(1);  // 64 MB table (32MB default) Minimal speedup.
-
 // ── 1. init ──────────────────────────────────────────────────────────────────
 $ffi->eldc_init();
 echo "=== eldc PHP demo ===\n\n";
diff --git a/examples/demo_eldc_lib.py b/examples/demo_eldc_lib.py
@@ -66,16 +66,10 @@ class EldcDetectResult(ctypes.Structure):
 lib.eldc_set_scores.restype      = None
 lib.eldc_set_scores.argtypes     = [ctypes.c_int]
 
-lib.eldc_set_faster.restype      = None
-lib.eldc_set_faster.argtypes     = [ctypes.c_int]
 
 # ── Helper: decode c_char_p safely ───────────────────────────────────────────
 def s(b): return b.decode("utf-8") if b else ""
 
-# ── 0. eldc_set_faster() — larger hash table, call before init() ─────────────
-# 0 → 32 MB (default).  1 → 64 MB. Not recommended, minimal speedup.
-# lib.eldc_set_faster(1)
-
 # ── 1. Initialise ────────────────────────────────────────────────────────────
 # eldc_init() is the only required call before detection.
 lib.eldc_init()
diff --git a/examples/demo_eldc_lib.rb b/examples/demo_eldc_lib.rb
@@ -37,14 +37,10 @@ class DetectResult < FFI::Struct
   attach_function :eldc_set_languages,  [:string],                    :string
   attach_function :eldc_set_scheme,     [:string],                    :void
   attach_function :eldc_set_scores,     [:int],                       :void
-  attach_function :eldc_set_faster,     [:int],                       :void
 
   def self.lang(ptr) = ptr.null? ? 'und' : ptr.read_string
 end
 
-# ── 0. set_faster (Not worth it. Call before init) ─────────────────────────
-# ELD.eldc_set_faster(1)  # 64 MB table
-
 # ── 1. init ──────────────────────────────────────────────────────────────────
 ELD.eldc_init
 puts "=== eldc Ruby demo ===\n\n"
diff --git a/examples/demo_eldc_lib.rs b/examples/demo_eldc_lib.rs
@@ -45,10 +45,6 @@ fn main() {
         let set_lang:  Symbol<unsafe extern "C" fn(*const c_char) -> *const c_char> = lib.get(b"eldc_set_languages\0").unwrap();
         let set_scheme:Symbol<unsafe extern "C" fn(*const c_char)>              = lib.get(b"eldc_set_scheme\0").unwrap();
         let set_scores:Symbol<unsafe extern "C" fn(c_int)>                      = lib.get(b"eldc_set_scores\0").unwrap();
-        let set_faster:Symbol<unsafe extern "C" fn(c_int)>                      = lib.get(b"eldc_set_faster\0").unwrap();
-
-        // ── 0. set_faster (Not worth it. Call before init()) ─────────────────
-        // set_faster(1);  // 64 MB table (32MB default) Minimal speedup.
 
         // ── 1. init ──────────────────────────────────────────────────────────
         init();
diff --git a/examples/demo_eldc_lib.ts b/examples/demo_eldc_lib.ts
@@ -43,10 +43,6 @@ const eldc_detect_details = lib.func('const char *eldc_detect_details(const char
 const eldc_set_languages  = lib.func('const char *eldc_set_languages(const char *codes)');
 const eldc_set_scheme     = lib.func('void eldc_set_scheme(const char *scheme)');
 const eldc_set_scores     = lib.func('void eldc_set_scores(int n)');
-const eldc_set_faster     = lib.func('void eldc_set_faster(int flag)');
-
-// ── 0. set_faster (Not worth it. Call before init()) ─────────────────────────
-// eldc_set_faster(1);  // 64 MB table (32MB default) Minimal speedup.
 
 // ── 1. init ──────────────────────────────────────────────────────────────────
 eldc_init();
diff --git a/examples/demo_eldc_package.py b/examples/demo_eldc_package.py
@@ -10,11 +10,6 @@
 import concurrent.futures
 import eldc   # the Python wrapper around _eldc
 
-# ── 0. Optional set_faster, not recomended ────────────────────────────────────
-# Must be called before init(). 
-#   eldc.set_faster(false)  → 32 MB hashtable (default, recommended)
-#   eldc.set_faster(true)   → 64 MB hashtable (marginal speedup)
-
 # ── 1. Initialise ────────────────────────────────────────────────────────────
 # Must be called before any detection.  Loads the n-gram DB into a hash table.
 eldc.init()
@@ -42,7 +37,7 @@
 # Minimum 1 (detect_details always returns scores).  Default: 3. Max 20.
 print("-- set_scores(10) --")
 eldc.set_scores(10)
-result = eldc.detect_details("Was ist das? A ship. Todo bien en la costa.")
+result = eldc.detect_details("Was ist das? Todo bien en la costa.")
 print(f"language : {result.language}")            # de
 print(f"scores   : {result.scores}")              # top-10 only
 eldc.set_scores(3)                                # reset to default
diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name            = "eldc"
-version         = "0.1.2"
+version         = "0.2.0"
 description     = "The fastest high-accuracy natural language detector. Compiled in C. Nito-ELDC, ELDC, ELD-C."
 readme          = "README.md"
 license         = {text = "Apache-2.0"}
diff --git a/src/eldc/__init__.py b/src/eldc/__init__.py
@@ -18,7 +18,6 @@
     set_scores,
     set_languages,
     set_scheme,
-    set_faster,
     LANGUAGES,
     LANGUAGES_ISO2T,
     MAX_LANGUAGES,
@@ -32,8 +31,7 @@
     "detect_details",
     "set_scores",
     "set_languages",
-    "set_scheme",
-    "set_faster",    
+    "set_scheme",  
     "LANGUAGES",
     "LANGUAGES_ISO2T",
     "MAX_LANGUAGES",
diff --git a/src/eldc/_eldc_module.c b/src/eldc/_eldc_module.c
@@ -32,14 +32,8 @@
 #endif
 #include <math.h>
 
-#if defined(__has_include) && __has_include("large_db.h")
-#  include "large_db.h"
-#endif
 #include "eld_core.c"
 
-#ifndef ELD_HT_BITS
-#  define ELD_HT_BITS 21
-#endif
 
 /* ═══════════════════════════════════════════════════════════════════════════
  * Module-level configuration
@@ -48,8 +42,7 @@ static uint64_t _cfg_lang_mask = (uint64_t)-1;
 static int      _cfg_subset    = 0;
 static int      _cfg_scheme    = 0;   /* 0=iso639-1, 1=iso639-2t */
 static int      _cfg_scores    = 3; /* min 1; N=top-N in detect_details */
-static int      _cfg_bits      = ELD_HT_BITS; /* hash-table size: ELD_HT_BITS, or +1 via set_faster(True) */
-static int      _cfg_inited    = 0; /* 1 once init() has been called; blocks further set_faster() calls */
+static int      _cfg_inited    = 0; /* 1 once init() has been called; */
 
 /* Pre-interned PyUnicode objects for all language codes in both schemes.
  *   _lang_str[0][i]  ISO 639-1 code  e.g. "fr"  (2-letter)
@@ -408,37 +401,13 @@ static PyObject *py_set_scheme(PyObject *module, PyObject *args)
     Py_RETURN_NONE;
 }
 
-/* Makes 2x size ngram hastable, double memory use,  minimal speedup */
-static PyObject *py_set_faster(PyObject *module, PyObject *args)
-{
-    PyObject *arg;
-
-    /* 1. Parse the arguments from Python */
-    if (!PyArg_ParseTuple(args, "O", &arg)) return NULL;
-
-    /* 2. Check initialization status first */
-    if (_cfg_inited == 1) {
-        PyErr_SetString(PyExc_RuntimeError, "Execute set_faster() before init()");
-        return NULL;
-    }
-
-    /* 3. Check type and apply */
-    if (PyBool_Check(arg)) {
-        _cfg_bits = (arg == Py_True) ? ELD_HT_BITS + 1 : ELD_HT_BITS;
-    } else {
-        PyErr_SetString(PyExc_TypeError, "set_faster expects bool");
-        return NULL;
-    }
-    
-    Py_RETURN_NONE;
-}
 
 /* init() / reinit(): same operation, init() is the primary name.
  * Calling init() again with a different ht_bits is always safe — it frees
  * the old table and allocates a fresh one.  Do this before any detection. */
 static PyObject *py_init(PyObject *module, PyObject *Py_UNUSED(ignored))
 {
-    init_detector(_cfg_bits);
+    init_detector();
 	 _cfg_inited = 1;
     Py_RETURN_NONE;
 }
@@ -484,13 +453,7 @@ static PyMethodDef eldc_methods[] = {
 
     {"set_scheme",    py_set_scheme,    METH_VARARGS,
      "set_scheme(scheme: str) -> None\n"
-     "Output scheme: 'iso639-1' (default, 'en') or 'iso639-2t' ('eng')."},
-	  
-    {"set_faster",    py_set_faster,    METH_VARARGS,
-     "set_faster(bool) -> None\n"
-     "Call before init()"
-	  "  False — 32 MB table (default, recommended)\n"
-	  "  True — 64 MB table (marginal speedup)"},	  
+     "Output scheme: 'iso639-1' (default, 'en') or 'iso639-2t' ('eng')."}, 
 
     {"init",          py_init,          METH_NOARGS,
      "init() -> None\n"
@@ -508,8 +471,8 @@ PyMODINIT_FUNC PyInit__eldc(void)
     if (PyType_Ready(&EldcResult_Type) < 0) return NULL;
 
     /* No init_detector() here: the hash table is NOT loaded on import.
-     * Call eldc.init() (optionally with ht_bits) before any detection.
-     * This keeps import cheap — no 32 MB allocation just for importing. */
+     * Call eldc.init()
+     * This keeps import cheap — no 16 MB allocation just for importing. */
 
     PyObject *m = PyModule_Create(&eldc_module_def);
     if (!m) return NULL;
diff --git a/src/eldc/eld.c b/src/eldc/eld.c
@@ -12,7 +12,7 @@
  * the wrapper, and the binary runs ~7% faster on real workloads.
  *
  * Custom DB or hash-table size:
- *   gcc ... -DELD_HT_BITS=22 -DELD_DB_INCLUDE='"my_db.h"' ...
+ *   gcc ... -DELD_DB_INCLUDE='"my_db.h"' ...
  *
  * Thread safety: see eld_core.c header comment.
  *
@@ -61,7 +61,6 @@ int main(int argc, char *argv[])
 {
     EldConfig cfg = { 0, 0, 0, (uint64_t)-1, 0 };
     int   verbose           = 0;
-    int   ht_bits           = 21;
     char *input_text        = NULL;
     int   force_interactive = 0;
 
@@ -73,8 +72,6 @@ int main(int argc, char *argv[])
             print_lang_list(); return 0;
         } else if (!strcmp(a,"-v")||!strcmp(a,"--verbose")) {
             verbose = 1;
-        } else if (!strcmp(a,"--faster")) {
-            ht_bits = 22;
         } else if (!strcmp(a,"--interactive")) {
             force_interactive = 1;
         } else if (!strcmp(a,"-r")||!strcmp(a,"--reliable")) {
@@ -117,9 +114,7 @@ int main(int argc, char *argv[])
     }
 
     if (verbose) {
-        fprintf(stderr,"Loading (HT_BITS=%d, %u slots, scheme=%s",
-                ht_bits, 1u<<ht_bits,
-                g_scheme==SCHEME_ISO639_2T?"iso639-2t":"iso639-1");
+        fprintf(stderr,"Loading (scheme=%s",g_scheme==SCHEME_ISO639_2T?"iso639-2t":"iso639-1");
         if (cfg.subset) {
             int cnt=0; for(int i=0;i<MAX_LANGUAGES;i++) if((cfg.lang_mask>>i)&1) cnt++;
             fprintf(stderr,", filter=%d lang%s", cnt, cnt==1?"":"s");
@@ -128,7 +123,7 @@ int main(int argc, char *argv[])
     }
 
     clock_t t0 = clock();
-    init_detector(ht_bits);
+    init_detector();
 
     if (verbose) {
         fprintf(stderr,"Done! (%.1f ms)\n----------------------------------------\n",
diff --git a/src/eldc/eld_cli_utils.c b/src/eldc/eld_cli_utils.c
@@ -134,7 +134,6 @@ static void eld_print_help(const char *prog, int show_threads)
         "  -h, --help              This message\n"
         "      --list-languages    Print all supported codes and exit\n"
         "  -v, --verbose           Loading info, timing, throughput\n"
-        "      --faster            Uses 2x hashtable (32 MB → 64 MB), minimal speedup\n"
         "  -l, --languages CODES   Restrict to a subset, e.g. -l \"es,en,de,fr\"\n"
         "                          Accepts ISO 639-1 or ISO 639-2/T codes.\n"
         "  -s, --scores [N]        Output compact JSON with top-N normalised [0,1] scores\n"
diff --git a/src/eldc/eld_core.c b/src/eldc/eld_core.c
diff --git a/src/eldc/eld_mt.c b/src/eldc/eld_mt.c
diff --git a/src/eldc/eldc_lib.c b/src/eldc/eldc_lib.c
diff --git a/src/eldc/eldc_lib.h b/src/eldc/eldc_lib.h
diff --git a/src/eldc/large_db.h b/src/eldc/large_db.h
