feat: Add repository estimation and embedding time calculation utilities

- Introduced `estimation.rs` for repository counting and embedding time estimation.
- Implemented `RepositoryEstimator` to analyze repositories and estimate indexing costs.
- Added CLI command for estimating indexing costs without writing to the database.
- Enhanced symbol indexing with `build_symbol_index` and `extend_symbol_index` functions.
- Refactored file parsing logic to use shared utilities for unified extraction.
- Added tests for embedding time estimation and symbol indexing.
- Updated dependencies in `Cargo.toml` to include `serde_yaml`.
- Cleaned up imports and organized code structure for better readability.

Author: jruokola

Cargo.lock

@@ -83,11 +83,12 @@ pub async fn auth_middleware(
     mut req: Request,
     next: Next,
 ) -> Result<Response, StatusCode> {
-    let api_key = req.headers()
+    let api_key = req
+        .headers()
         .get("X-API-KEY")
         .and_then(|v| v.to_str().ok())
         .map(|s| s.to_string());
-    
+
     if let Some(api_key) = api_key {
         return api_key_auth(&api_key, req, next).await;
     }
@@ -137,11 +138,7 @@ pub async fn auth_middleware(
     Ok(next.run(req).await)
 }
 
-async fn api_key_auth(
-    api_key: &str,
-    mut req: Request,
-    next: Next,
-) -> Result<Response, StatusCode> {
+async fn api_key_auth(api_key: &str, mut req: Request, next: Next) -> Result<Response, StatusCode> {
     // In a real application, you would look up the API key in a database.
     // For this example, we'll use a hardcoded key.
     if api_key == "test-api-key" {

crates/codegraph-api/src/auth.rs

@@ -1,6 +1,6 @@
-use crate::{ApiError, ApiResult, AppState};
 use crate::parser_ext::TreeSitterParserExt;
 use crate::semantic_search_ext::SemanticSearchExt;
+use crate::{ApiError, ApiResult, AppState};
 use axum::{extract::State, Json};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -150,7 +150,10 @@ async fn check_database_health_enhanced(state: &AppState) -> ComponentStatus {
             let mut details = HashMap::new();
             details.insert("total_nodes".to_string(), stats.total_nodes.to_string());
             details.insert("total_edges".to_string(), stats.total_edges.to_string());
-            details.insert("connection_test".to_string(), connection_ok.unwrap_or(false).to_string());
+            details.insert(
+                "connection_test".to_string(),
+                connection_ok.unwrap_or(false).to_string(),
+            );
 
             // Calculate health score based on response time and stats
             let health_score = calculate_health_score(response_time, Some(&details));
@@ -212,7 +215,10 @@ async fn check_vector_search_health_enhanced(state: &AppState) -> ComponentStatu
             );
             details.insert("index_type".to_string(), format!("{:?}", stats.index_type));
             details.insert("dimension".to_string(), stats.dimension.to_string());
-            details.insert("search_test".to_string(), test_ok.unwrap_or(false).to_string());
+            details.insert(
+                "search_test".to_string(),
+                test_ok.unwrap_or(false).to_string(),
+            );
 
             let health_score = calculate_health_score(response_time, Some(&details));
 

crates/codegraph-api/src/enhanced_health.rs

@@ -23,7 +23,7 @@ pub enum ApiError {
 
     #[error("Bad request: {0}")]
     BadRequest(String),
-    
+
     #[error("Service unavailable: {0}")]
     ServiceUnavailable(String),
 }

crates/codegraph-api/src/error.rs

@@ -1,12 +1,12 @@
 // Temporary stub types for codegraph-graph until it's fixed
 // These are placeholder implementations to allow compilation
 
+use chrono::{DateTime, Utc};
 use codegraph_core::{CodeNode, NodeId, Result};
 use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::RwLock;
 use uuid::Uuid;
-use chrono::{DateTime, Utc};
 
 /// Temporary stub for CodeGraph from codegraph-graph crate
 #[derive(Debug, Clone, Default)]
@@ -38,8 +38,13 @@ impl CodeGraph {
     pub fn node_count(&self) -> usize {
         self.nodes.len()
     }
-    
-    pub async fn astar_shortest_path<F>(&self, from: NodeId, to: NodeId, _heuristic: F) -> Result<Vec<NodeId>>
+
+    pub async fn astar_shortest_path<F>(
+        &self,
+        from: NodeId,
+        to: NodeId,
+        _heuristic: F,
+    ) -> Result<Vec<NodeId>>
     where
         F: Fn(&NodeId) -> f64,
     {
@@ -50,7 +55,7 @@ impl CodeGraph {
             Ok(vec![])
         }
     }
-    
+
     pub async fn shortest_path(&self, from: NodeId, to: NodeId) -> Result<Vec<NodeId>> {
         // Simple stub implementation - just return direct path
         if self.nodes.contains_key(&from) && self.nodes.contains_key(&to) {
@@ -82,8 +87,8 @@ impl TransactionalGraph {
     /// Create a new TransactionalGraph with real storage-backed managers
     pub async fn with_storage(storage_path: &str) -> Result<Self> {
         use codegraph_graph::VersionedRocksDbStorage;
-        use tokio::sync::RwLock as TokioRwLock;
         use std::sync::Arc;
+        use tokio::sync::RwLock as TokioRwLock;
 
         // Initialize storage
         let storage = VersionedRocksDbStorage::new(storage_path).await?;
@@ -266,13 +271,18 @@ impl ConcurrentTransactionManager {
         Self { storage: None }
     }
 
-    pub fn with_storage(storage: Arc<tokio::sync::RwLock<codegraph_graph::VersionedRocksDbStorage>>) -> Self {
+    pub fn with_storage(
+        storage: Arc<tokio::sync::RwLock<codegraph_graph::VersionedRocksDbStorage>>,
+    ) -> Self {
         Self {
             storage: Some(storage),
         }
     }
 
-    pub async fn begin_transaction(&self, _isolation_level: IsolationLevel) -> Result<TransactionId> {
+    pub async fn begin_transaction(
+        &self,
+        _isolation_level: IsolationLevel,
+    ) -> Result<TransactionId> {
         // Stub implementation - just generate a transaction ID
         Ok(Uuid::new_v4())
     }
@@ -311,7 +321,13 @@ impl GitLikeVersionManager {
         }
     }
 
-    pub async fn create_version(&self, _name: String, _description: String, _author: String, _parent_versions: Vec<VersionId>) -> Result<VersionId> {
+    pub async fn create_version(
+        &self,
+        _name: String,
+        _description: String,
+        _author: String,
+        _parent_versions: Vec<VersionId>,
+    ) -> Result<VersionId> {
         // Stub implementation - just generate a version ID
         Ok(Uuid::new_v4())
     }
@@ -415,4 +431,4 @@ impl RecoveryManager {
     pub async fn restore_from_backup(&self, _path: String) -> Result<()> {
         Ok(())
     }
-}
+}

crates/codegraph-api/src/graph_stub.rs

@@ -23,7 +23,10 @@ impl Loader<NodeId> for NodeLoader {
     type Error = CodeGraphError;
 
     #[instrument(skip(self, keys), fields(batch_size = keys.len()))]
-    async fn load(&self, keys: &[NodeId]) -> std::result::Result<HashMap<NodeId, GraphQLCodeNode>, CodeGraphError> {
+    async fn load(
+        &self,
+        keys: &[NodeId],
+    ) -> std::result::Result<HashMap<NodeId, GraphQLCodeNode>, CodeGraphError> {
         let start_time = Instant::now();
         debug!("Loading batch of {} nodes", keys.len());
 
@@ -111,7 +114,10 @@ impl Loader<NodeId> for EdgesBySourceLoader {
     type Error = CodeGraphError;
 
     #[instrument(skip(self, keys), fields(batch_size = keys.len()))]
-    async fn load(&self, keys: &[NodeId]) -> std::result::Result<HashMap<NodeId, Vec<GraphQLEdge>>, CodeGraphError> {
+    async fn load(
+        &self,
+        keys: &[NodeId],
+    ) -> std::result::Result<HashMap<NodeId, Vec<GraphQLEdge>>, CodeGraphError> {
         let start_time = Instant::now();
         debug!("Loading edges for {} source nodes", keys.len());
 
@@ -177,7 +183,10 @@ impl Loader<String> for SemanticSearchLoader {
     type Error = CodeGraphError;
 
     #[instrument(skip(self, keys), fields(batch_size = keys.len()))]
-    async fn load(&self, keys: &[String]) -> std::result::Result<HashMap<String, Vec<RetrievalResult>>, CodeGraphError> {
+    async fn load(
+        &self,
+        keys: &[String],
+    ) -> std::result::Result<HashMap<String, Vec<RetrievalResult>>, CodeGraphError> {
         let start_time = Instant::now();
         debug!("Semantic search batch for {} queries", keys.len());
 
@@ -209,7 +218,8 @@ impl Loader<String> for SemanticSearchLoader {
                         node_id: res.node_id,
                         node: res.node,
                         relevance_score: res.score,
-                        retrieval_method: codegraph_vector::rag::RetrievalMethod::SemanticSimilarity,
+                        retrieval_method:
+                            codegraph_vector::rag::RetrievalMethod::SemanticSimilarity,
                         context_snippet: snippet,
                     }
                 })

crates/codegraph-api/src/graphql/loaders.rs

@@ -1,6 +1,6 @@
 use async_graphql::{dataloader::DataLoader, Context, Object, Result, ID};
 use async_trait::async_trait;
-use codegraph_core::{CodeGraphError, NodeId, GraphStore};
+use codegraph_core::{CodeGraphError, GraphStore, NodeId};
 use std::collections::HashMap;
 use std::str::FromStr;
 use std::time::Instant;
@@ -14,13 +14,13 @@ use crate::graphql::loaders::{
     TraversalKey,
 };
 use crate::graphql::types::GraphQLEdge;
+use crate::graphql::types::UpdateNodeInput;
 use crate::graphql::types::{
     CodeSearchInput, CodeSearchResult, GraphQLCodeNode, GraphTraversalInput, GraphTraversalResult,
     PageInfo, ScoredNode, SearchMetadata, SearchSortBy, SemanticSearchInput,
     SemanticSearchMetadata, SemanticSearchResult, SubgraphExtractionInput, SubgraphMetadata,
     SubgraphResult, TraversalMetadata,
 };
-use crate::graphql::types::UpdateNodeInput;
 use crate::state::AppState;
 
 pub struct QueryRoot;
@@ -68,26 +68,29 @@ impl QueryRoot {
         if let Some(ref language_filters) = input.language_filter {
             nodes.retain(|node| {
                 node.language.as_ref().map_or(false, |lang| {
-                    language_filters.iter().any(|filter_lang| lang == filter_lang)
+                    language_filters
+                        .iter()
+                        .any(|filter_lang| lang == filter_lang)
                 })
             });
         }
         if let Some(ref node_type_filters) = input.node_type_filter {
             nodes.retain(|node| {
-                node.node_type.as_ref().map_or(false, |nt| {
-                    node_type_filters.iter().any(|f| nt == f)
-                })
+                node.node_type
+                    .as_ref()
+                    .map_or(false, |nt| node_type_filters.iter().any(|f| nt == f))
             });
         }
         if let Some(ref file_pattern) = input.file_path_pattern {
             nodes.retain(|node| node.location.file_path.contains(file_pattern));
         }
         if let Some(ref content_filter) = input.content_filter {
             let needle = content_filter.to_lowercase();
-            nodes.retain(|node| node
-                .content
-                .as_ref()
-                .map_or(false, |c| c.to_lowercase().contains(&needle)));
+            nodes.retain(|node| {
+                node.content
+                    .as_ref()
+                    .map_or(false, |c| c.to_lowercase().contains(&needle))
+            });
         }
 
         // Sorting

crates/codegraph-api/src/graphql/resolvers.rs

@@ -1,9 +1,9 @@
 use async_graphql::{value, Request, Variables};
+use codegraph_core::ConfigManager;
 use serde_json::json;
 use std::sync::Arc;
 use tokio_test;
 use uuid::Uuid;
-use codegraph_core::ConfigManager;
 
 use crate::graphql::types::*;
 use crate::schema::create_schema;
@@ -12,7 +12,9 @@ use crate::state::AppState;
 /// Helper function to create test app state
 async fn create_test_state() -> AppState {
     let config = Arc::new(ConfigManager::new().expect("Failed to create test config"));
-    AppState::new(config).await.expect("Failed to create test state")
+    AppState::new(config)
+        .await
+        .expect("Failed to create test state")
 }
 
 /// Helper function to create test schema with state

crates/codegraph-api/src/graphql/tests.rs

@@ -1,6 +1,6 @@
-use crate::{ApiError, ApiResult, AppState};
-use crate::semantic_search_ext::SemanticSearchExt;
 use crate::parser_ext::TreeSitterParserExt;
+use crate::semantic_search_ext::SemanticSearchExt;
+use crate::{ApiError, ApiResult, AppState};
 use axum::{extract::State, Json};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;

crates/codegraph-api/src/health.rs

@@ -20,6 +20,7 @@ pub mod metrics;
 // pub mod middleware;
 // Legacy mutations module is disabled to avoid conflicts; mutations live in GraphQL resolvers
 // pub mod mutations;
+pub mod parser_ext;
 pub mod performance;
 #[cfg(feature = "graphql")]
 pub mod queries;
@@ -28,16 +29,15 @@ pub mod rest;
 pub mod routes;
 #[cfg(feature = "graphql")]
 pub mod schema;
-pub mod server;
-pub mod parser_ext;
 pub mod semantic_search_ext;
-pub mod vector_store_ext;
+pub mod server;
 pub mod service_registry;
 pub mod state;
 pub mod streaming_handlers;
 #[cfg(feature = "graphql")]
 pub mod subscriptions;
 pub mod vector_handlers;
+pub mod vector_store_ext;
 pub mod versioning_handlers;
 
 #[cfg(test)]