Fix max_tokens calculation to respect model context window (5000 tokens) - Add dynamic max_tokens calculation based on input size - Add novita_model_context_window configuration - Prevents 400 errors when input tokens exceed available output space

.env

@@ -0,0 +1,163 @@
+# =============================================================================
+# Research AI Assistant API - Environment Configuration
+# =============================================================================
+# Copy this content to a file named .env and fill in your actual values
+# Never commit .env to version control!
+
+# =============================================================================
+# Novita AI Configuration (REQUIRED)
+# =============================================================================
+# Get your API key from: https://novita.ai
+NOVITA_API_KEY=sk_gaMaeJaUy-qQxms1NIgJuov_RotL_NZXMoQbJlNhS6M
+
+# Dedicated endpoint base URL (default for dedicated endpoints)
+NOVITA_BASE_URL=https://api.novita.ai/dedicated/v1/openai
+
+# Your dedicated endpoint model ID
+# Format: model-name:endpoint-id
+NOVITA_MODEL=deepseek-ai/DeepSeek-R1-Distill-Qwen-7B:de-1a706eeafbf3ebc2
+
+# =============================================================================
+# DeepSeek-R1 Optimized Settings
+# =============================================================================
+# Temperature: 0.5-0.7 range (0.6 recommended for DeepSeek-R1)
+DEEPSEEK_R1_TEMPERATURE=0.6
+
+# Force reasoning trigger: Enable to ensure DeepSeek-R1 uses reasoning pattern
+# Set to True to add `<think>` prefix for reasoning tasks
+DEEPSEEK_R1_FORCE_REASONING=True
+
+# =============================================================================
+# Token Allocation Configuration
+# =============================================================================
+# Maximum tokens dedicated for user input (prioritized over context)
+# Recommended: 8000 tokens for large queries
+USER_INPUT_MAX_TOKENS=8000
+
+# Maximum tokens for context preparation (includes user input + context)
+# Recommended: 28000 tokens for 32K context window models
+CONTEXT_PREPARATION_BUDGET=28000
+
+# Context pruning threshold (should match context_preparation_budget)
+CONTEXT_PRUNING_THRESHOLD=28000
+
+# Always prioritize user input over historical context
+PRIORITIZE_USER_INPUT=True
+
+# =============================================================================
+# Database Configuration
+# =============================================================================
+# SQLite database path (default: sessions.db)
+# Use /tmp/ for Docker/containerized environments
+DB_PATH=sessions.db
+
+# FAISS index path for embeddings (default: embeddings.faiss)
+FAISS_INDEX_PATH=embeddings.faiss
+
+# =============================================================================
+# Cache Configuration
+# =============================================================================
+# HuggingFace cache directory (for any remaining model downloads)
+HF_HOME=~/.cache/huggingface
+TRANSFORMERS_CACHE=~/.cache/huggingface
+
+# HuggingFace token (optional - only needed if using gated models)
+HF_TOKEN=
+
+# Cache TTL in seconds (default: 3600 = 1 hour)
+CACHE_TTL=3600
+
+# =============================================================================
+# Session Configuration
+# =============================================================================
+# Session timeout in seconds (default: 3600 = 1 hour)
+SESSION_TIMEOUT=3600
+
+# Maximum session size in megabytes (default: 10 MB)
+MAX_SESSION_SIZE_MB=10
+
+# =============================================================================
+# Performance Configuration
+# =============================================================================
+# Maximum worker threads for parallel processing (default: 4)
+MAX_WORKERS=4
+
+# =============================================================================
+# Mobile Optimization
+# =============================================================================
+# Maximum tokens for mobile responses (default: 1200)
+# Increased from 800 to allow better responses on mobile
+MOBILE_MAX_TOKENS=1200
+
+# Mobile request timeout in milliseconds (default: 15000)
+MOBILE_TIMEOUT=15000
+
+# =============================================================================
+# API Configuration
+# =============================================================================
+# Flask/Gradio server port (default: 7860)
+GRADIO_PORT=7860
+
+# Server host (default: 0.0.0.0 for all interfaces)
+GRADIO_HOST=0.0.0.0
+
+# =============================================================================
+# Logging Configuration
+# =============================================================================
+# Logging level: DEBUG, INFO, WARNING, ERROR, CRITICAL (default: INFO)
+LOG_LEVEL=INFO
+
+# Log format: json or text (default: json)
+LOG_FORMAT=json
+
+# Log directory (default: /tmp/logs)
+LOG_DIR=/tmp/logs
+
+# =============================================================================
+# Context Configuration
+# =============================================================================
+# Maximum context tokens (default: 4000)
+# Note: This is overridden by CONTEXT_PREPARATION_BUDGET if set
+MAX_CONTEXT_TOKENS=4000
+
+# Cache TTL for context in seconds (default: 300 = 5 minutes)
+CACHE_TTL_SECONDS=300
+
+# Maximum cache size (default: 100)
+MAX_CACHE_SIZE=100
+
+# Enable parallel processing (default: True)
+PARALLEL_PROCESSING=True
+
+# Context decay factor (default: 0.8)
+CONTEXT_DECAY_FACTOR=0.8
+
+# Maximum interactions to keep in context (default: 10)
+MAX_INTERACTIONS_TO_KEEP=10
+
+# Enable metrics collection (default: True)
+ENABLE_METRICS=True
+
+# Enable context compression (default: True)
+COMPRESSION_ENABLED=True
+
+# Summarization threshold in tokens (default: 2000)
+SUMMARIZATION_THRESHOLD=2000
+
+# =============================================================================
+# Model Selection (for context operations - if still using local models)
+# =============================================================================
+# These are optional and only used if local models are still needed
+# for context summarization or other operations
+CONTEXT_SUMMARIZATION_MODEL=Qwen/Qwen2.5-7B-Instruct
+CONTEXT_INTENT_MODEL=Qwen/Qwen2.5-7B-Instruct
+CONTEXT_SYNTHESIS_MODEL=Qwen/Qwen2.5-7B-Instruct
+
+# =============================================================================
+# Security Notes
+# =============================================================================
+# - Never commit .env file to version control
+# - Keep API keys secret and rotate them regularly
+# - Use environment variables in production (not .env files)
+# - Set proper file permissions: chmod 600 .env
+

ENV_EXAMPLE_CONTENT.txt

@@ -44,6 +44,11 @@ CONTEXT_PRUNING_THRESHOLD=28000
 # Always prioritize user input over historical context
 PRIORITIZE_USER_INPUT=True
 
+# Model context window (actual limit for your deployed model)
+# Default: 5000 tokens (adjust based on your Novita AI deployment)
+# This is the maximum total tokens (input + output) the model can handle
+NOVITA_MODEL_CONTEXT_WINDOW=5000
+
 # =============================================================================
 # Database Configuration
 # =============================================================================

src/config.py

@@ -232,6 +232,13 @@ class Settings(BaseSettings):
         env="PRIORITIZE_USER_INPUT"
     )
     
+    # Model Context Window Configuration
+    novita_model_context_window: int = Field(
+        default=5000,
+        description="Maximum context window for Novita AI model (input + output tokens)",
+        env="NOVITA_MODEL_CONTEXT_WINDOW"
+    )
+    
     @validator("novita_api_key", pre=True)
     def validate_novita_api_key(cls, v):
         """Validate and clean Novita API key"""
@@ -266,6 +273,12 @@ class Settings(BaseSettings):
         val = int(v) if v else 28000
         return max(4000, min(120000, val))
     
+    @validator("novita_model_context_window", pre=True)
+    def validate_context_window(cls, v):
+        """Validate context window size"""
+        val = int(v) if v else 5000
+        return max(1000, min(200000, val))  # Reasonable bounds
+    
     # ==================== Model Configuration ====================
     
     default_model: str = Field(

src/llm_router.py

@@ -108,7 +108,7 @@ class LLMRouter:
         model_name = kwargs.get('model', self.settings.novita_model)
         
         # Get optimized parameters
-        max_tokens = kwargs.get('max_tokens', model_config.get('max_tokens', 4096))
+        requested_max_tokens = kwargs.get('max_tokens', model_config.get('max_tokens', 4096))
         temperature = kwargs.get('temperature', 
             model_config.get('temperature', self.settings.deepseek_r1_temperature))
         top_p = kwargs.get('top_p', model_config.get('top_p', 0.95))
@@ -117,6 +117,9 @@ class LLMRouter:
         # Format prompt according to DeepSeek-R1 best practices
         formatted_prompt = self._format_deepseek_r1_prompt(prompt, task_type, model_config)
         
+        # IMPORTANT: Calculate safe max_tokens based on input size
+        max_tokens = self._calculate_safe_max_tokens(formatted_prompt, requested_max_tokens)
+        
         # IMPORTANT: No system prompt - all instructions in user prompt
         messages = [{"role": "user", "content": formatted_prompt}]
         
@@ -164,6 +167,41 @@ class LLMRouter:
             logger.error(f"Error calling Novita AI API: {e}", exc_info=True)
                         raise
             
+    def _calculate_safe_max_tokens(self, prompt: str, requested_max_tokens: int) -> int:
+        """
+        Calculate safe max_tokens based on input token count and model context window.
+        
+        Args:
+            prompt: Input prompt text
+            requested_max_tokens: Desired max_tokens value
+            
+        Returns:
+            int: Adjusted max_tokens that fits within context window
+        """
+        # Estimate input tokens (rough: 1 token ≈ 4 characters)
+        # For more accuracy, you could use tiktoken if available
+        input_tokens = len(prompt) // 4
+        
+        # Get model context window
+        context_window = self.settings.novita_model_context_window
+        
+        # Reserve minimum 100 tokens for safety margin
+        available_tokens = context_window - input_tokens - 100
+        
+        # Use the smaller of requested or available
+        safe_max_tokens = min(requested_max_tokens, available_tokens)
+        
+        # Ensure minimum of 50 tokens for output
+        safe_max_tokens = max(50, safe_max_tokens)
+        
+        if safe_max_tokens < requested_max_tokens:
+            logger.warning(
+                f"Reduced max_tokens from {requested_max_tokens} to {safe_max_tokens} "
+                f"(input: ~{input_tokens} tokens, context window: {context_window} tokens)"
+            )
+        
+        return safe_max_tokens
+            
     def _format_deepseek_r1_prompt(self, prompt: str, task_type: str, model_config: dict) -> str:
         """
         Format prompt according to DeepSeek-R1 best practices: