feat: Add SurrealDB as selectable database backend with flexible schema

This commit adds comprehensive SurrealDB support as an alternative to RocksDB:

## Configuration System
- Added DatabaseBackend enum to select between RocksDB and SurrealDB
- Created SurrealDbConfig with support for:
  - Multiple connection types (file://, mem://, http://, ws://)
  - Namespace and database selection for multi-tenancy
  - Optional authentication (username/password)
  - Strict mode for schema enforcement
  - Auto-migration on startup
- Updated Settings struct to use new DatabaseConfig structure
- Maintained backward compatibility with legacy rocksdb config

## Storage Implementation (surrealdb_storage.rs)
- Implemented GraphStore trait for full CRUD operations
- Features:
  - In-memory caching with DashMap for performance
  - Flexible JSON-based node representation
  - Automatic schema initialization
  - Built-in migration runner
  - Conversion between CodeNode and SurrealDB format
  - Support for embeddings, metadata, and all node attributes

## Schema Manager (surrealdb_schema.rs)
- Flexible schema definition system with:
  - TableSchema, FieldDefinition, and IndexDefinition types
  - Type-safe field type mappings to SurrealDB types
  - Dynamic field and index addition without downtime
  - Schema export/import functionality (JSON format)
  - Helper functions for standard node/edge schemas
- Designed for easy schema evolution and modifications

## Migration System (surrealdb_migrations.rs)
- Versioned migration framework with:
  - UP/DOWN migration support for rollbacks
  - Migration checksum verification for integrity
  - Migration status tracking and reporting
  - Automatic migration application
  - Template generation for new migrations
- Includes default migrations for initial schema
- Migration files stored in migrations/ directory

## Feature Flag & Dependencies
- Added 'surrealdb' feature flag to Cargo.toml
- Optional dependency on surrealdb v2.2
- Conditional compilation for zero overhead when not used

## Documentation & Examples
- Comprehensive SURREALDB_GUIDE.md covering:
  - Installation and configuration
  - Schema management best practices
  - Migration creation and management
  - Usage examples and advanced queries
  - Performance optimization tips
  - Migration path from RocksDB
  - Troubleshooting guide
- Example configuration file (surrealdb_example.toml)
- Updated default.toml with new database structure

## Migration SQL
- Initial schema migration (001_initial_schema.sql)
- Creates nodes, edges, schema_versions, and metadata tables
- Includes all necessary indexes for performance

This implementation is designed to be:
- Flexible: Easy to add new fields without migrations
- Maintainable: Clear migration system for schema evolution
- Production-ready: Strict mode, authentication, and validation
- Performant: Caching, indexes, and batch operations
- Well-documented: Comprehensive guide and examples

Author: claude

config/default.toml

@@ -4,10 +4,29 @@ env = "development"
 host = "0.0.0.0"
 port = 3000
 
-[rocksdb]
+# Database configuration
+[database]
+# Backend options: "rocksdb" (default), "surrealdb"
+backend = "rocksdb"
+
+[database.rocksdb]
 path = "data/graph.db"
 read_only = false
 
+[database.surrealdb]
+# Example SurrealDB configuration (uncomment to use)
+# connection = "file://data/surrealdb/graph.db"
+# namespace = "codegraph"
+# database = "graph"
+# auto_migrate = true
+# strict_mode = false
+
+# Deprecated: Legacy rocksdb configuration (use database.rocksdb instead)
+# This is kept for backward compatibility
+# [rocksdb]
+# path = "data/graph.db"
+# read_only = false
+
 [vector]
 dimension = 384
 index = "ivf_flat"

config/surrealdb_example.toml

@@ -0,0 +1,64 @@
+# CodeGraph Configuration with SurrealDB
+
+[database]
+# Select SurrealDB as the database backend
+backend = "surrealdb"
+
+# RocksDB configuration (not used when backend is surrealdb, but kept for backward compatibility)
+[database.rocksdb]
+path = "data/graph.db"
+read_only = false
+
+# SurrealDB configuration
+[database.surrealdb]
+# Connection string options:
+# - Local file: "file://data/graph.db"
+# - Memory (testing): "mem://"
+# - Remote HTTP: "http://localhost:8000"
+# - Remote HTTPS: "https://example.com:8000"
+# - WebSocket: "ws://localhost:8000"
+connection = "file://data/surrealdb/graph.db"
+
+# Namespace for multi-tenancy (default: "codegraph")
+namespace = "codegraph"
+
+# Database name (default: "graph")
+database = "graph"
+
+# Optional: Authentication credentials
+# username = "root"
+# password = "root"  # Can also be set via CODEGRAPH__DATABASE__SURREALDB__PASSWORD env var
+
+# Enable strict schema validation (default: false)
+# When true, SurrealDB will enforce the defined schema strictly
+# When false, allows for schema flexibility and easier migrations
+strict_mode = false
+
+# Auto-apply migrations on startup (default: true)
+# When true, automatically runs pending migrations when connecting
+auto_migrate = true
+
+# Example: Remote SurrealDB server configuration
+# [database.surrealdb]
+# connection = "https://your-surrealdb-server.com:8000"
+# namespace = "production"
+# database = "codegraph"
+# username = "admin"
+# # Password should be set via environment variable:
+# # CODEGRAPH__DATABASE__SURREALDB__PASSWORD=your_password
+# strict_mode = true
+# auto_migrate = false
+
+[server]
+host = "0.0.0.0"
+port = 3000
+
+[vector]
+dimension = 1024
+
+[logging]
+level = "info"
+
+[security]
+require_auth = false
+rate_limit_per_minute = 1200

crates/codegraph-core/src/config.rs

@@ -50,6 +50,96 @@ impl Default for RocksDbConfig {
     }
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+pub struct SurrealDbConfig {
+    /// Connection string for SurrealDB (e.g., "file://data/graph.db" or "http://localhost:8000")
+    pub connection: String,
+    /// Namespace for multi-tenancy
+    #[serde(default = "SurrealDbConfig::default_namespace")]
+    pub namespace: String,
+    /// Database name
+    #[serde(default = "SurrealDbConfig::default_database")]
+    pub database: String,
+    /// Optional username for authentication
+    #[serde(default)]
+    pub username: Option<String>,
+    /// Optional password for authentication
+    #[serde(default, skip_serializing)]
+    #[schemars(skip)]
+    pub password: Option<SecretString>,
+    /// Enable strict schema validation
+    #[serde(default = "SurrealDbConfig::default_strict_mode")]
+    pub strict_mode: bool,
+    /// Auto-apply migrations on startup
+    #[serde(default = "SurrealDbConfig::default_auto_migrate")]
+    pub auto_migrate: bool,
+}
+
+impl SurrealDbConfig {
+    fn default_namespace() -> String {
+        "codegraph".to_string()
+    }
+
+    fn default_database() -> String {
+        "graph".to_string()
+    }
+
+    fn default_strict_mode() -> bool {
+        false
+    }
+
+    fn default_auto_migrate() -> bool {
+        true
+    }
+}
+
+impl Default for SurrealDbConfig {
+    fn default() -> Self {
+        Self {
+            connection: "file://data/graph.db".into(),
+            namespace: Self::default_namespace(),
+            database: Self::default_database(),
+            username: None,
+            password: None,
+            strict_mode: Self::default_strict_mode(),
+            auto_migrate: Self::default_auto_migrate(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "snake_case")]
+pub enum DatabaseBackend {
+    RocksDb,
+    SurrealDb,
+}
+
+impl Default for DatabaseBackend {
+    fn default() -> Self {
+        Self::RocksDb
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+pub struct DatabaseConfig {
+    #[serde(default)]
+    pub backend: DatabaseBackend,
+    #[serde(default)]
+    pub rocksdb: RocksDbConfig,
+    #[serde(default)]
+    pub surrealdb: SurrealDbConfig,
+}
+
+impl Default for DatabaseConfig {
+    fn default() -> Self {
+        Self {
+            backend: DatabaseBackend::default(),
+            rocksdb: RocksDbConfig::default(),
+            surrealdb: SurrealDbConfig::default(),
+        }
+    }
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
 pub struct VectorConfig {
     pub dimension: usize,
@@ -118,7 +208,10 @@ pub struct Settings {
     #[serde(default)]
     pub server: ServerConfig,
     #[serde(default)]
-    pub rocksdb: RocksDbConfig,
+    pub database: DatabaseConfig,
+    /// Deprecated: Use database.rocksdb instead
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub rocksdb: Option<RocksDbConfig>,
     #[serde(default)]
     pub vector: VectorConfig,
     #[serde(default)]
@@ -134,7 +227,8 @@ impl Default for Settings {
         Self {
             env: Self::default_env(),
             server: ServerConfig::default(),
-            rocksdb: RocksDbConfig::default(),
+            database: DatabaseConfig::default(),
+            rocksdb: None,
             vector: VectorConfig::default(),
             logging: LoggingConfig::default(),
             security: SecurityConfig::default(),
@@ -161,6 +255,31 @@ impl Settings {
             self.vector.dimension > 0 && self.vector.dimension <= 8192,
             "vector.dimension must be 1..=8192"
         );
+
+        // Validate database configuration
+        match self.database.backend {
+            DatabaseBackend::RocksDb => {
+                anyhow::ensure!(
+                    !self.database.rocksdb.path.is_empty(),
+                    "database.rocksdb.path cannot be empty"
+                );
+            }
+            DatabaseBackend::SurrealDb => {
+                anyhow::ensure!(
+                    !self.database.surrealdb.connection.is_empty(),
+                    "database.surrealdb.connection cannot be empty"
+                );
+                anyhow::ensure!(
+                    !self.database.surrealdb.namespace.is_empty(),
+                    "database.surrealdb.namespace cannot be empty"
+                );
+                anyhow::ensure!(
+                    !self.database.surrealdb.database.is_empty(),
+                    "database.surrealdb.database cannot be empty"
+                );
+            }
+        }
+
         Ok(())
     }
 }

crates/codegraph-graph/Cargo.toml

@@ -31,6 +31,13 @@ num_cpus = "1.16"
 crossbeam-channel = "0.5"
 notify = { workspace = true }
 
+# SurrealDB support (optional)
+surrealdb = { version = "2.2", optional = true }
+
+[features]
+default = []
+surrealdb = ["dep:surrealdb"]
+
 [dev-dependencies]
 tokio-test = { workspace = true }
 tempfile = { workspace = true }

crates/codegraph-graph/migrations/001_initial_schema.sql

@@ -0,0 +1,57 @@
+-- Migration 001: Initial Schema
+-- This migration creates the core tables for CodeGraph with SurrealDB
+
+-- Nodes table: Stores code entities (functions, classes, variables, etc.)
+DEFINE TABLE IF NOT EXISTS nodes SCHEMAFULL;
+DEFINE FIELD IF NOT EXISTS id ON TABLE nodes TYPE string;
+DEFINE FIELD IF NOT EXISTS name ON TABLE nodes TYPE string;
+DEFINE FIELD IF NOT EXISTS node_type ON TABLE nodes TYPE option<string>;
+DEFINE FIELD IF NOT EXISTS language ON TABLE nodes TYPE option<string>;
+DEFINE FIELD IF NOT EXISTS content ON TABLE nodes TYPE option<string>;
+DEFINE FIELD IF NOT EXISTS file_path ON TABLE nodes TYPE option<string>;
+DEFINE FIELD IF NOT EXISTS start_line ON TABLE nodes TYPE option<number>;
+DEFINE FIELD IF NOT EXISTS end_line ON TABLE nodes TYPE option<number>;
+DEFINE FIELD IF NOT EXISTS embedding ON TABLE nodes TYPE option<array<float>>;
+DEFINE FIELD IF NOT EXISTS complexity ON TABLE nodes TYPE option<float>;
+DEFINE FIELD IF NOT EXISTS metadata ON TABLE nodes TYPE option<object>;
+DEFINE FIELD IF NOT EXISTS created_at ON TABLE nodes TYPE datetime DEFAULT time::now();
+DEFINE FIELD IF NOT EXISTS updated_at ON TABLE nodes TYPE datetime DEFAULT time::now();
+
+-- Indexes for efficient queries on nodes
+DEFINE INDEX IF NOT EXISTS idx_nodes_id ON TABLE nodes COLUMNS id UNIQUE;
+DEFINE INDEX IF NOT EXISTS idx_nodes_name ON TABLE nodes COLUMNS name;
+DEFINE INDEX IF NOT EXISTS idx_nodes_type ON TABLE nodes COLUMNS node_type;
+DEFINE INDEX IF NOT EXISTS idx_nodes_language ON TABLE nodes COLUMNS language;
+DEFINE INDEX IF NOT EXISTS idx_nodes_file_path ON TABLE nodes COLUMNS file_path;
+
+-- Edges table: Stores relationships between nodes
+DEFINE TABLE IF NOT EXISTS edges SCHEMAFULL;
+DEFINE FIELD IF NOT EXISTS id ON TABLE edges TYPE string;
+DEFINE FIELD IF NOT EXISTS from ON TABLE edges TYPE record(nodes);
+DEFINE FIELD IF NOT EXISTS to ON TABLE edges TYPE record(nodes);
+DEFINE FIELD IF NOT EXISTS edge_type ON TABLE edges TYPE string;
+DEFINE FIELD IF NOT EXISTS weight ON TABLE edges TYPE float DEFAULT 1.0;
+DEFINE FIELD IF NOT EXISTS metadata ON TABLE edges TYPE option<object>;
+DEFINE FIELD IF NOT EXISTS created_at ON TABLE edges TYPE datetime DEFAULT time::now();
+
+-- Indexes for graph traversal on edges
+DEFINE INDEX IF NOT EXISTS idx_edges_from ON TABLE edges COLUMNS from;
+DEFINE INDEX IF NOT EXISTS idx_edges_to ON TABLE edges COLUMNS to;
+DEFINE INDEX IF NOT EXISTS idx_edges_type ON TABLE edges COLUMNS edge_type;
+
+-- Schema versions table: Tracks applied migrations
+DEFINE TABLE IF NOT EXISTS schema_versions SCHEMAFULL;
+DEFINE FIELD IF NOT EXISTS version ON TABLE schema_versions TYPE number;
+DEFINE FIELD IF NOT EXISTS name ON TABLE schema_versions TYPE string;
+DEFINE FIELD IF NOT EXISTS applied_at ON TABLE schema_versions TYPE datetime DEFAULT time::now();
+DEFINE FIELD IF NOT EXISTS checksum ON TABLE schema_versions TYPE string;
+
+DEFINE INDEX IF NOT EXISTS idx_schema_version ON TABLE schema_versions COLUMNS version UNIQUE;
+
+-- Metadata table: Stores system-level metadata
+DEFINE TABLE IF NOT EXISTS metadata SCHEMAFULL;
+DEFINE FIELD IF NOT EXISTS key ON TABLE metadata TYPE string;
+DEFINE FIELD IF NOT EXISTS value ON TABLE metadata TYPE option<string | number | bool | object | array>;
+DEFINE FIELD IF NOT EXISTS updated_at ON TABLE metadata TYPE datetime DEFAULT time::now();
+
+DEFINE INDEX IF NOT EXISTS idx_metadata_key ON TABLE metadata COLUMNS key UNIQUE;

crates/codegraph-graph/src/lib.rs

@@ -19,6 +19,13 @@ pub mod traversal;
 pub mod update_scheduler;
 pub mod versioned_storage;
 
+#[cfg(feature = "surrealdb")]
+pub mod surrealdb_storage;
+#[cfg(feature = "surrealdb")]
+pub mod surrealdb_schema;
+#[cfg(feature = "surrealdb")]
+pub mod surrealdb_migrations;
+
 pub use cache::*;
 pub use delta::*;
 pub use delta_processor::*;
@@ -39,3 +46,10 @@ pub use transactional_graph::*;
 pub use traversal::*;
 pub use update_scheduler::*;
 pub use versioned_storage::*;
+
+#[cfg(feature = "surrealdb")]
+pub use surrealdb_storage::*;
+#[cfg(feature = "surrealdb")]
+pub use surrealdb_schema::*;
+#[cfg(feature = "surrealdb")]
+pub use surrealdb_migrations::*;