Refactor: Remove performance optimizer and server startup scripts

- Deleted `performance-optimizer.ts` which contained the PerformanceOptimizer class and related functionality for query optimization, caching, and metrics tracking.
- Removed `start-server.ts` script responsible for initializing the Enhanced MCP server with command line options and event listeners.
- Updated test script `test_mcp_tools.py` to reflect changes in tool calls and configuration.
- Added new Rust dependency `semchunk-rs` for lightweight semantic chunking, including its Cargo.toml and implementation in `src/lib.rs`.

Author: jruokola

.codegraph.toml.example

@@ -42,11 +42,12 @@ batch_size = 64
 # Set to false for maximum speed if using an external agent
 enabled = false
 
-# LLM provider: "ollama", "lmstudio", "anthropic", "openai", or "openai-compatible"
+# LLM provider: "ollama", "lmstudio", "anthropic", "openai", "xai", or "openai-compatible"
 # - "lmstudio": Local LLMs via LM Studio (recommended for MLX + Flash Attention 2 on macOS)
 # - "ollama": Local LLMs via Ollama
 # - "anthropic": Anthropic Claude API (requires API key)
 # - "openai": OpenAI GPT API (requires API key)
+# - "xai": xAI Grok API (2M context, $0.50-$1.50/M tokens, requires API key)
 # - "openai-compatible": Any OpenAI-compatible API endpoint
 provider = "lmstudio"
 
@@ -55,6 +56,7 @@ provider = "lmstudio"
 # For Ollama: Model name (e.g., "qwen2.5-coder:14b", "codellama:13b")
 # For Anthropic: Model name (e.g., "claude-3-5-sonnet-20241022", "claude-3-5-haiku-20241022")
 # For OpenAI: Model name (e.g., "gpt-4o", "gpt-4o-mini", "gpt-4-turbo")
+# For xAI: Model name (e.g., "grok-4-fast", "grok-4-turbo")
 # For OpenAI-compatible: Custom model name
 # Recommended: DeepSeek Coder v2 Lite Instruct Q4_K_M (local), or Claude 3.5 Sonnet (cloud)
 model = "lmstudio-community/DeepSeek-Coder-V2-Lite-Instruct-GGUF"
@@ -77,21 +79,30 @@ ollama_url = "http://localhost:11434"
 # Can also be set via OPENAI_API_KEY environment variable
 # openai_api_key = "sk-..."
 
+# xAI API key (only used if provider is "xai")
+# Can also be set via XAI_API_KEY environment variable
+# xai_api_key = "xai-..."
+
+# xAI base URL (only used if provider is "xai")
+# Default: "https://api.x.ai/v1"
+# xai_base_url = "https://api.x.ai/v1"
+
 # Context window size (tokens)
 # DeepSeek Coder v2 Lite: 32768 tokens
 # Claude 3.5 Sonnet: 200000 tokens
 # GPT-4o: 128000 tokens
+# Grok-4 (xAI): 2000000 tokens (2M!)
 context_window = 32000
 
 # Temperature for generation (0.0 = deterministic, 2.0 = very creative)
 temperature = 0.1
 
-# Maximum tokens to generate in responses (legacy parameter, use max_output_tokens for Responses API)
+# Maximum tokens to generate in responses (legacy parameter, use max_completion_token for Responses API)
 max_tokens = 4096
 
 # Maximum output tokens for Responses API and reasoning models
 # If not set, falls back to max_tokens
-# max_output_tokens = 4096
+# max_completion_token = 4096
 
 # Reasoning effort for reasoning models (o1, o3, o4-mini, GPT-5)
 # Options: "minimal", "low", "medium", "high"

.env.example

@@ -35,8 +35,16 @@ CODEGRAPH_EMBEDDING_PROVIDER=auto
 
 # Jina AI: Cloud embeddings with reranking (requires JINA_API_KEY)
 # CODEGRAPH_EMBEDDING_PROVIDER=jina
+# JINA_EMBEDDING_MODEL=jina-code-embeddings-1.5b
 # JINA_API_KEY=your-jina-api-key-here
-
+# JINA_MAX_TEXTS=512 # Leverage Jina API Batch functionality max 512 documents with 8192 tokens each remember to set --max-concurrent 1 when indexing
+# JINA_MAX_TOKENS=7000
+# CODEGRAPH_RERANK_CANDIDATES=512 # Leverage Jina API Batch functionality also here
+# JINA_API_BASE=https://api.jina.ai/v1
+# JINA_ENABLE_RERANKING=true
+# JINA_RERANKING_MODEL=jina-reranker-v3
+# JINA_REQUEST_DELAY_MS=600 # small delay not to throttle the API when batching
+# JINA_TRUNCATE=true # truncate texts and embeddings if over limit
 # ============================================================================
 # Dual-Mode Search Configuration
 # ============================================================================
@@ -54,7 +62,8 @@ CODEGRAPH_EMBEDDING_PROVIDER=auto
 # Cloud Mode (SurrealDB HNSW + Jina embeddings + reranking)
 # ----------------------------------------------------------
 # - Uses SurrealDB HNSW indexes for scalable vector search
-# - Embeddings: Jina AI (2048 dimensions)
+# - Embeddings: Jina AI (Variable Matryosha dimensions - check what the model outputs and adjust schema/codegraph.surql HNSW vector dims)
+# - Supported Jina AI embedding models are jina-code-embeddings-1.5b and jina-code-embeddings-0.5b
 # - Reranking: Jina reranker-v3 for improved relevance
 # - Best for: Cloud deployments, multi-user systems, scalability
 # - Requires: SurrealDB instance, Jina API key
@@ -70,7 +79,8 @@ CODEGRAPH_EMBEDDING_PROVIDER=auto
 # SURREALDB_PASSWORD=root
 #
 # Important: HNSW index dimension must match embedding provider
-# - Jina v4: 2048 dimensions
+# - Jina: Variable Matryoska dimensions depending on model
+# - OpenAI: Small 1536 dimensions, Large 3072 dimensions
 # - Local ONNX: typically 384 or 768 dimensions
 # - Update schema/codegraph.surql if changing providers
 
@@ -79,25 +89,41 @@ CODEGRAPH_EMBEDDING_PROVIDER=auto
 # Leave empty to use context-only mode (fastest, recommended for agents like Claude/GPT-4)
 # Set to enable local LLM insights generation
 
-# LM Studio with DeepSeek Coder v2 Lite Instruct (recommended)
+# LM Studio with DeepSeek Coder v2 Lite Instruct (or what ever fits in your vGPU memory)
 # Supported LLM provider options: "auto", "onnx", "lmstudio", "openai", "claude" or "ollama"
-# Superior MLX support and Flash Attention 2 on macOS
+# Superior MLX support, Flash Attention 2, KV-cache and Distillation model support on macOS
 # CODEGRAPH_LLM_PROVIDER=lmstudio
 # CODEGRAPH_MODEL=lmstudio-community/DeepSeek-Coder-V2-Lite-Instruct-GGUF/DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf
 # CODEGRAPH_LMSTUDIO_URL=http://localhost:1234
 # CODEGRAPH_CONTEXT_WINDOW=32000
 # CODEGRAPH_TEMPERATURE=0.1
 
 # Ollama (alternative)
-# LLM model (e.g., "qwen2.5-coder:14b", "codellama:13b")
+# LLM model (e.g., "qwen2.5-coder:14b", "Kimi-K2-Instruct")
 # CODEGRAPH_MODEL=qwen2.5-coder:14b
 # CODEGRAPH_OLLAMA_URL=http://localhost:11434
-
-# LLM context window size (tokens)
-# CODEGRAPH_CONTEXT_WINDOW=32000
-
-# LLM temperature (0.0 = deterministic, 1.0 = creative)
-# CODEGRAPH_TEMPERATURE=0.1
+# CODEGRAPH_CONTEXT_WINDOW=252000 # Max for Ollama depends on the model though
+
+# Anthropic (cloud - 200K/1M tokens)
+# CODEGRAPH_LLM_PROVIDER=anthropic
+# CODEGRAPH_MODEL=sonnet[1m]
+# ANTHROPIC_API_KEY=sk-ant-your-key-here
+# CODEGRAPH_CONTEXT_WINDOW=1000000  # 200K/1M tokens
+
+# OpenAI (cloud - 200K/400K tokens)
+# CODEGRAPH_LLM_PROVIDER=openai
+# CODEGRAPH_MODEL=gpt-5-codex-mini
+# OPENAI_API_KEY=sk-your-key-here
+# OPENAI_ORG_ID=your_fabulous_org
+# CODEGRAPH_CONTEXT_WINDOW=200000  # 400K tokens
+# CODEGRAPH_REASONING_BUDGET=medium
+
+# xAI (cloud - 252K/2M context window, $0.50-$1.50/M tokens!)
+# CODEGRAPH_LLM_PROVIDER=xai
+# CODEGRAPH_MODEL=grok-4-fast  # or grok-code-fast-1 252K tokens the price is same
+# XAI_API_KEY=xai-your-key-here
+# CODEGRAPH_CONTEXT_WINDOW=2000000  # 2M tokens!
+# CODEGRAPH_REASONING_BUDGET=high
 
 # Logging
 # -------
@@ -151,10 +177,6 @@ LOG_FORMAT=json
 METRICS_ENABLED=true
 PROMETHEUS_PORT=9090
 
-# OpenAI API (if using external AI services)
-# OPENAI_API_KEY=your_openai_api_key_here
-# OPENAI_ORG_ID=your_org_id_here
-
 # External Services
 # SENTRY_DSN=https://your-sentry-dsn
 # ANALYTICS_KEY=your_analytics_key