from contextlib import AsyncExitStack, asynccontextmanager from fastapi.staticfiles import StaticFiles import requests, warnings from dotenv import load_dotenv from fastapi import FastAPI, HTTPException from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, StreamingResponse from bs4 import BeautifulSoup import requests as req_lib from huggingface_hub.utils import configure_http_backend from schemas import * from indexers import ( get_folder_name, get_spec_3gpp_indexer, get_spec_etsi_indexer, get_tdoc_indexer, indexing_locks, meetings_mapping, ) from mcp_indexing import indexing_mcp def hf_client_factory() -> req_lib.Session: session = req_lib.Session() session.verify = False return session configure_http_backend(backend_factory=hf_client_factory) warnings.filterwarnings("ignore") load_dotenv() @asynccontextmanager async def lifespan(app: FastAPI): # The MCP server's session manager owns its own background task group; it must be # entered as part of this app's lifespan since mounting its Starlette sub-app does not # run its lifespan on its own. async with AsyncExitStack() as stack: await stack.enter_async_context(indexing_mcp.session_manager.run()) yield app = FastAPI(lifespan=lifespan) app.add_middleware(CORSMiddleware, allow_credentials=True, allow_headers=["*"], allow_origins=["*"]) app.mount("/static", StaticFiles(directory="static"), name="static") # Claude Code (or any MCP client) connects over Streamable HTTP at /index/mcp app.mount("/index", indexing_mcp.streamable_http_app()) @app.get('/') def main(): return FileResponse("index.html") @app.get("/get_meetings/{working_group}") def get_meetings(working_group: str): category, wg_number = get_folder_name(working_group) folder = meetings_mapping[category][wg_number] url = f"https://www.3gpp.org/ftp/{meetings_mapping[category][0]}/{folder}" response = requests.get(url, verify=False, timeout=(10, 30)) responseHTML = response.text soup = BeautifulSoup(responseHTML, "html.parser") return {"url": url, "meetings": [item.get_text() for item in soup.select("tr td a") if item.get_text().startswith("TSG") or item.get_text().startswith("CT")]} @app.post("/index_tdocs/working_group") def index_tdocs_wg_progress(req: IndexTDoc): if not req.wg: raise HTTPException(status_code=400, detail="Working Group not defined !") if not indexing_locks["tdoc"].acquire(blocking=False): raise HTTPException(status_code=409, detail="TDoc indexing already in progress") category, wg_number = get_folder_name(req.wg) folder = meetings_mapping[category][wg_number] url = f"https://www.3gpp.org/ftp/{meetings_mapping[category][0]}" indexer = get_tdoc_indexer() def generate_events(): try: yield f"event: info\ndata: {req.wg}\n\n" for content in indexer.process_workgroup(folder, url): yield content indexer.save_indexer() yield "event: end\ndata: Indexation ended successfully !\n\n" finally: indexing_locks["tdoc"].release() return StreamingResponse(generate_events(), media_type="text/event-stream") @app.post("/index_tdocs/meeting") def index_tdocs_meeting_progress(req: IndexTDoc): if not req.wg: raise HTTPException(status_code=400, detail="Working Group not defined !") if not req.meetings: raise HTTPException(status_code=400, detail="Meetings not defined !") if not indexing_locks["tdoc"].acquire(blocking=False): raise HTTPException(status_code=409, detail="TDoc indexing already in progress") category, wg_number = get_folder_name(req.wg) folder = meetings_mapping[category][wg_number] url = f"https://www.3gpp.org/ftp/{meetings_mapping[category][0]}/{folder}" indexer = get_tdoc_indexer() def generate_events(): try: yield f"event: get-maximum\ndata: {len(req.meetings)}\n\n" for i, meet in enumerate(req.meetings): yield f"event: info\ndata: {req.wg}-{meet}\n\n" indexer.process_meeting(meet, url) yield f"event: progress\ndata: {i+1}\n\n" indexer.save_indexer() yield "event: end\ndata: Indexation ended successfully !\n\n" finally: indexing_locks["tdoc"].release() return StreamingResponse(generate_events(), media_type="text/event-stream") @app.post("/index_tdocs/all") def index_all_tdocs_progress(): if not indexing_locks["tdoc"].acquire(blocking=False): raise HTTPException(status_code=409, detail="TDoc indexing already in progress") indexer = get_tdoc_indexer() def generate_events(): try: for content in indexer.index_all_tdocs(): yield content indexer.save_indexer() yield "event: end\ndata: Indexation ended successfully !\n\n" finally: indexing_locks["tdoc"].release() return StreamingResponse(generate_events(), media_type="text/event-stream") @app.post("/index_specs/3gpp") def index_3gpp_specs_progress(): if not indexing_locks["3gpp"].acquire(blocking=False): raise HTTPException(status_code=409, detail="3GPP spec indexing already in progress") indexer = get_spec_3gpp_indexer() def generate_events(): try: for content in indexer.run(): yield content yield "event: info\ndata: Saving index ...\n\n" yield "event: get-maximum\ndata: 1\n\n" yield "event: progress\ndata: 1\n\n" indexer.save() yield "event: info\ndata: Creating BM25 models ...\n\n" yield "event: get-maximum\ndata: 1\n\n" yield "event: progress\ndata: 1\n\n" indexer.create_bm25_index() yield "event: end\ndata: Indexation ended successfully !\n\n" finally: indexing_locks["3gpp"].release() return StreamingResponse(generate_events(), media_type="text/event-stream") @app.post("/index_specs/etsi") def index_etsi_specs_progress(): if not indexing_locks["etsi"].acquire(blocking=False): raise HTTPException(status_code=409, detail="ETSI spec indexing already in progress") indexer = get_spec_etsi_indexer() def generate_events(): try: for content in indexer.run(): yield content yield "event: info\ndata: Saving index ...\n\n" yield "event: get-maximum\ndata: 1\n\n" yield "event: progress\ndata: 1\n\n" indexer.save() yield "event: info\ndata: Creating BM25 models ...\n\n" yield "event: get-maximum\ndata: 1\n\n" yield "event: progress\ndata: 1\n\n" indexer.create_bm25_index() yield "event: end\ndata: Indexation ended successfully !\n\n" finally: indexing_locks["etsi"].release() return StreamingResponse(generate_events(), media_type="text/event-stream")