pad token decoding should use literal decode and not decode which is symantic and may include extra formatting (#54)

Author: Qubitium

pyproject.toml

@@ -21,7 +21,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "TokeNicer"
-version = "0.0.10"
+version = "0.0.11"
 description = "A (nicer) tokenizer you want to use for model `inference` and `training`: with all known peventable `gotchas` normalized or auto-fixed."
 readme = "README.md"
 requires-python = ">=3"

tests/test_pad_token.py

@@ -30,6 +30,7 @@ class TestPadToken(unittest.TestCase):
             ('/monster/data/model/Llama-3.2-1B-Instruct', '<|finetune_right_pad_id|>'),
             ('/monster/data/model/Qwen2.5-0.5B-Instruct', '<|fim_pad|>'),
             ('/monster/data/model/Qwen2-VL-2B-Instruct', '<|endoftext|>'),
+            ('/monster/data/model/ERNIE-4.5-0.3B-PT/', '<unk>', None, True),
             ('/monster/data/model/gemma-2-9b', '<pad>'),
             ('/monster/data/model/Hymba-1.5B-Instruct', '<unk>', None, True),
             ('/monster/data/model/Mistral-7B-Instruct-v0.2', '<unk>'),

tokenicer/tokenicer.py

@@ -160,7 +160,12 @@ def auto_fix_pad_token(
                 pad_token_id = getattr(self.tokenizer, "pad_token_id", None)
                 if pad_token_id is not None:
                     if getattr(self.tokenizer, "pad_token", None) is None:
-                        self.tokenizer.pad_token = self.tokenizer.decode([pad_token_id])
+                        # Do not round-trip special token ids through decode().
+                        # decode() returns rendered text, not the canonical token literal,
+                        # and some custom tokenizers intentionally inject formatting.
+                        # ERNIE 4.5 is the concrete regression here: decode([0]) returns
+                        # " <unk>" while the real special token is "<unk>".
+                        self.tokenizer.pad_token = self._token_literal_for_id(pad_token_id)
                     return
 
                 if not strict and getattr(self.tokenizer, "eos_token_id", None) is not None:
@@ -208,7 +213,10 @@ def auto_fix_pad_token(
                 )
 
         self.tokenizer.pad_token_id = pad_token_id
-        self.tokenizer.pad_token = self.tokenizer.decode([pad_token_id])
+        # Preserve the canonical special-token spelling instead of using
+        # decode([pad_token_id]). For tokenizers like ERNIE 4.5, decode() is a
+        # detokenization API and may prepend whitespace to special tokens.
+        self.tokenizer.pad_token = self._token_literal_for_id(pad_token_id)
 
         if getattr(model_config, "pad_token_id", None) is None:
             # Keep the resolved text config aligned so downstream callers do
@@ -217,6 +225,39 @@ def auto_fix_pad_token(
 
         logger.info(f"Tokenicer: Auto fixed pad_token_id={pad_token_id} (token='{self.tokenizer.pad_token}').")
 
+    def _token_literal_for_id(self, token_id: int) -> str:
+        # Prefer declared special-token metadata over decode(). The metadata is
+        # the canonical token literal used by save/load, while decode() is a
+        # text-rendering path and may be lossy for standalone special tokens.
+        special_token_attrs = (
+            "pad_token",
+            "unk_token",
+            "eos_token",
+            "bos_token",
+            "cls_token",
+            "sep_token",
+            "mask_token",
+        )
+
+        for attr in special_token_attrs:
+            attr_id = getattr(self.tokenizer, f"{attr}_id", None)
+            attr_value = getattr(self.tokenizer, attr, None)
+            if attr_id == token_id and attr_value is not None:
+                return attr_value
+
+        special_tokens_map = getattr(self.tokenizer, "special_tokens_map", {}) or {}
+        for attr_value in special_tokens_map.values():
+            if isinstance(attr_value, list):
+                for item in attr_value:
+                    if isinstance(item, str) and self.tokenizer.convert_tokens_to_ids(item) == token_id:
+                        return item
+            elif isinstance(attr_value, str) and self.tokenizer.convert_tokens_to_ids(attr_value) == token_id:
+                return attr_value
+
+        # Final fallback for tokenizers that do not expose enough special-token
+        # metadata to recover the literal directly.
+        return self.tokenizer.decode([token_id])
+
     @staticmethod
     def _resolve_text_model_config(model_config, seen=None):
         if model_config is None: