Upload char_encoder.py with huggingface_hub

char_encoder.py

@@ -191,21 +191,34 @@ class CharEncoder(nn.Module):
             from huggingface_hub import snapshot_download
             local_dir = snapshot_download(
                 pretrained_dir_or_repo,
-                allow_patterns=["config.json", "prefix_encoder.pt"],
+                allow_patterns=["config.json", "prefix_encoder.pt", "char_vocab.json"],
             )
         except (ImportError, ValueError):
             local_dir = pretrained_dir_or_repo
 
+        # Vocab size comes from char_vocab.json (the ground truth), not config.json.
+        # config.json can drift if it was hand-edited.
+        vocab_size = None
+        embedding_dim = EMBEDDING_DIM
+        vocab_path = os.path.join(local_dir, "char_vocab.json")
+        if os.path.isfile(vocab_path):
+            with open(vocab_path, encoding="utf-8") as f:
+                data = json.load(f)
+            if isinstance(data, dict) and "itos" in data:
+                vocab_size = len(data["itos"])
+
         config_path = os.path.join(local_dir, "config.json")
-        if not os.path.isfile(config_path):
-            raise FileNotFoundError(f"config.json not found in {local_dir}")
-        with open(config_path, encoding="utf-8") as f:
-            cfg = json.load(f)
-
-        model = cls(
-            vocab_size=cfg["vocab_size"],
-            embedding_dim=cfg["embedding_dim"],
-        )
+        if os.path.isfile(config_path):
+            with open(config_path, encoding="utf-8") as f:
+                cfg = json.load(f)
+            if vocab_size is None:
+                vocab_size = cfg.get("vocab_size", 45)
+            embedding_dim = cfg.get("embedding_dim", EMBEDDING_DIM)
+
+        if vocab_size is None:
+            raise FileNotFoundError(f"Cannot determine vocab_size from {local_dir}")
+
+        model = cls(vocab_size=vocab_size, embedding_dim=embedding_dim)
         weights_path = os.path.join(local_dir, "prefix_encoder.pt")
         if not os.path.isfile(weights_path):
             raise FileNotFoundError(f"prefix_encoder.pt not found in {local_dir}")