Stage 1 HF Integration. Added upload_embeddings.py

scripts/upload_embeddings.py

@@ -0,0 +1,36 @@
+"""Upload embeddings to HuggingFace"""
+import sys
+from pathlib import Path
+from huggingface_hub import HfApi
+from dotenv import load_dotenv
+
+# Add parent directory to path
+sys.path.append(str(Path(__file__).parent.parent))
+from config.settings import Settings
+
+
+def main():
+    """Upload embeddings directory to HuggingFace dataset"""
+    print("Starting upload process...")
+    
+    # Print size info
+    total_size = sum(
+        f.stat().st_size for f in Settings.EMBEDDINGS_DIR.glob('**/*') if f.is_file()
+    )
+    print(f"Found embeddings: {total_size / 1024 / 1024:.2f} MB")
+    
+    api = HfApi(token=Settings.HF_TOKEN)
+    
+    print(f"Uploading to {Settings.HF_DATASET}...")
+    api.upload_folder(
+        folder_path=str(Settings.EMBEDDINGS_DIR),
+        repo_id=Settings.HF_DATASET,
+        repo_type="dataset"
+    )
+    
+    print("Upload complete!")
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    main()