Update tokenizergen.py

tokenizergen.py

@@ -1,6 +1,9 @@
-from transformers import AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer
 
-# Download and save the tokenizer files
-tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2")
-tokenizer.save_pretrained("BuddAi")  # Creates a folder with all tokenizer files
-print("Tokenizer files saved to 'BuddAi/' directory!")
+model_id = "teknium/OpenHermes-2.5-Mistral-7B"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(model_id)
+
+# Save to BuddAi/
+model.save_pretrained("BuddAi", safe_serialization=True)  # Creates model.safetensors
+tokenizer.save_pretrained("BuddAi")  # Creates tokenizer files