diff --git a/code2llm/analysis/call_graph_engine.py b/code2llm/analysis/call_graph_engine.py index 67a11fc..b4ae6bb 100644 --- a/code2llm/analysis/call_graph_engine.py +++ b/code2llm/analysis/call_graph_engine.py @@ -204,6 +204,36 @@ def calculate_call_metrics( metrics: Dict[str, Any] | None = None, ) -> Dict[str, Dict[str, Any]]: """Calculate fan-in, fan-out and populate called_by for all functions.""" + from code2llm.analysis.native_bridge import native_calculate_call_metrics + + res_metrics = metrics if metrics is not None else {} + + # Fast path: native Rust + fn_tuples = [] + for name, fi in functions.items(): + calls = list(getattr(fi, "calls", [])) + called_by = list(getattr(fi, "called_by", [])) + raw_cc = getattr(fi, "complexity", 1.0) + cc_val = float(raw_cc.get("cyclomatic", 1.0) if isinstance(raw_cc, dict) else (raw_cc or 1.0)) + fn_tuples.append((name, calls, called_by, cc_val)) + + native_res = native_calculate_call_metrics(fn_tuples) + if native_res is not None: + computed_metrics, called_by_map = native_res + for name, callers in called_by_map.items(): + if name in functions: + fi = functions[name] + if hasattr(fi, "called_by"): + fi.called_by = callers + for name, (fan_in, fan_out, cc) in computed_metrics.items(): + res_metrics[name] = { + "fan_in": fan_in, + "fan_out": fan_out, + "complexity": cc, + } + return res_metrics + + # Fallback: Python implementation for caller_name, caller_info in functions.items(): calls = getattr(caller_info, "calls", []) for callee_name in calls: @@ -213,7 +243,6 @@ def calculate_call_metrics( if called_by is not None and caller_name not in called_by: called_by.append(caller_name) - res_metrics = metrics if metrics is not None else {} for func_name, func_info in functions.items(): calls = getattr(func_info, "calls", []) called_by = getattr(func_info, "called_by", []) diff --git a/code2llm/analysis/native_bridge.py b/code2llm/analysis/native_bridge.py index bcc9fc5..8af6df4 100644 --- a/code2llm/analysis/native_bridge.py +++ b/code2llm/analysis/native_bridge.py @@ -254,4 +254,32 @@ def native_resolve_call_graph( return None +def native_find_pipeline_paths( + nodes: List[str], + edges: List[Tuple[str, str]], + min_length: int = 3, + max_pipelines: int = 12, +) -> Optional[List[List[str]]]: + """Find pipeline paths using native Rust.""" + if _HAS_NATIVE_RUST and hasattr(code2llm_rust, "find_pipeline_paths"): + try: + return code2llm_rust.find_pipeline_paths(nodes, edges, min_length, max_pipelines) + except Exception as e: + logger.debug("code2llm_rust.find_pipeline_paths failed: %s", e) + return None + + +def native_calculate_call_metrics( + functions: List[Tuple[str, List[str], List[str], float]], +) -> Optional[Tuple[Dict[str, Tuple[int, int, float]], Dict[str, List[str]]]]: + """Calculate call metrics (fan_in, fan_out, complexity) and callers using native Rust.""" + if _HAS_NATIVE_RUST and hasattr(code2llm_rust, "calculate_call_metrics"): + try: + return code2llm_rust.calculate_call_metrics(functions) + except Exception as e: + logger.debug("code2llm_rust.calculate_call_metrics failed: %s", e) + return None + + + diff --git a/code2llm/analysis/pipeline_detector.py b/code2llm/analysis/pipeline_detector.py index a688376..954780a 100644 --- a/code2llm/analysis/pipeline_detector.py +++ b/code2llm/analysis/pipeline_detector.py @@ -150,13 +150,28 @@ def detect( if se_info is None: se_info = self._se_detector.analyze_all(funcs) - # Build networkx DiGraph - graph = self._build_graph(funcs) - if graph.number_of_nodes() == 0: - return [] + # Fast path: native Rust pipeline path finding + from code2llm.analysis.native_bridge import native_find_pipeline_paths - # Find pipeline candidates (longest paths in DAG) - paths = self._find_pipeline_paths(graph) + nodes = list(funcs.keys()) + edges: List[Tuple[str, str]] = [] + for qname, fi in funcs.items(): + for callee in fi.calls: + resolved = self._resolver.resolve(callee, funcs, caller=fi) + if resolved and resolved != qname: + edges.append((qname, resolved)) + + native_paths = native_find_pipeline_paths( + nodes, edges, MIN_PIPELINE_LENGTH, MAX_PIPELINES + ) + if native_paths is not None: + paths = native_paths + else: + # Build networkx DiGraph + graph = self._build_graph(funcs) + if graph.number_of_nodes() == 0: + return [] + paths = self._find_pipeline_paths(graph) # Build Pipeline objects with stages, purity, domain pipelines = self._build_pipelines(paths, funcs, se_info) diff --git a/docs/analysis/rust-optimization-breakdown.md b/docs/analysis/rust-optimization-breakdown.md index 2990e52..0bb499d 100644 --- a/docs/analysis/rust-optimization-breakdown.md +++ b/docs/analysis/rust-optimization-breakdown.md @@ -65,6 +65,14 @@ Through systematic profiling and code analysis of `code2llm` during multi-langua - **Problem**: In `ProjectAnalyzer._build_call_graph`, iterating through every function's call list in Python, executing string `rsplit` operations to find module matches, and building `called_by` with quadratic member checks adds significant runtime on large repositories. - **Native Rust Solution**: `calls::resolve_call_graph` resolving simple-to-full name candidate lookups with same-module affinity in a single linear pass in native Rust, returning resolved forward calls, backward caller maps, and root entry points. +### Bottleneck L: Pipeline Path Finding & Component Exploration +- **Problem**: In `PipelineDetector._find_pipeline_paths`, executing depth-bounded DFS with Python set union copies (`visited | {successor}`) and NetworkX DAG longest path algorithms on large call graphs creates massive interpreter overhead. +- **Native Rust Solution**: `pipelines::find_pipeline_paths` searching longest paths and weakly connected components directly in native Rust graph structures. + +### Bottleneck M: Call Graph Fan-In / Fan-Out Metrics +- **Problem**: In `calculate_call_metrics`, calculating pairwise caller/callee metrics across thousands of functions in Python dictionaries causes high GC churn. +- **Native Rust Solution**: `calls::calculate_call_metrics` vectorizing fan-in, fan-out, and caller maps in a single linear-time native pass. + ## 3. Architecture & Refactoring Strategy To prepare `code2llm` for clean separation and extraction into the `code2llm-rust` package, the following refactoring was implemented: diff --git a/packages/code2llm-rust/src/calls.rs b/packages/code2llm-rust/src/calls.rs index 64873cb..bb2338b 100644 --- a/packages/code2llm-rust/src/calls.rs +++ b/packages/code2llm-rust/src/calls.rs @@ -164,6 +164,57 @@ pub fn resolve_call_graph( } } +/// Result of call metrics calculation. +pub struct CallMetricsResult { + pub metrics: HashMap, // fan_in, fan_out, complexity + pub called_by: HashMap>, +} + +/// Fast native calculation of fan-in, fan-out and callers. +pub fn calculate_call_metrics( + functions: &[(String, Vec, Vec, f64)], +) -> CallMetricsResult { + let known_funcs: HashSet<&str> = functions.iter().map(|(n, _, _, _)| n.as_str()).collect(); + + let mut callers_map: HashMap> = HashMap::with_capacity(functions.len()); + let mut callers_set: HashMap<&str, HashSet<&str>> = HashMap::with_capacity(functions.len()); + + for (name, _, called_by, _) in functions { + let entry_set = callers_set.entry(name.as_str()).or_default(); + let entry_vec = callers_map.entry(name.clone()).or_default(); + for caller in called_by { + if entry_set.insert(caller.as_str()) { + entry_vec.push(caller.clone()); + } + } + } + + for (caller_name, calls, _, _) in functions { + for callee in calls { + let callee_str = callee.as_str(); + if known_funcs.contains(callee_str) { + let entry_set = callers_set.entry(callee_str).or_default(); + if entry_set.insert(caller_name.as_str()) { + callers_map.entry(callee.clone()).or_default().push(caller_name.clone()); + } + } + } + } + + let mut metrics = HashMap::with_capacity(functions.len()); + for (name, calls, _, complexity) in functions { + let unique_calls: HashSet<&str> = calls.iter().map(|s| s.as_str()).collect(); + let fan_out = unique_calls.len(); + let fan_in = callers_set.get(name.as_str()).map(|s| s.len()).unwrap_or(0); + metrics.insert(name.clone(), (fan_in, fan_out, *complexity)); + } + + CallMetricsResult { + metrics, + called_by: callers_map, + } +} + #[cfg(test)] mod tests { use super::*; @@ -183,5 +234,18 @@ mod tests { assert_eq!(res.called_by["db.query"], vec!["app.helper"]); assert_eq!(res.entry_points, vec!["app.main"]); } + + #[test] + fn test_calculate_call_metrics() { + let input = vec![ + ("app.main".to_string(), vec!["app.helper".to_string()], vec![], 2.0), + ("app.helper".to_string(), vec![], vec![], 5.0), + ]; + + let res = calculate_call_metrics(&input); + assert_eq!(res.metrics["app.main"], (0, 1, 2.0)); + assert_eq!(res.metrics["app.helper"], (1, 0, 5.0)); + assert_eq!(res.called_by["app.helper"], vec!["app.main"]); + } } diff --git a/packages/code2llm-rust/src/lib.rs b/packages/code2llm-rust/src/lib.rs index 5426c69..0d20c05 100644 --- a/packages/code2llm-rust/src/lib.rs +++ b/packages/code2llm-rust/src/lib.rs @@ -13,6 +13,7 @@ pub mod reachability; pub mod smells; pub mod cache; pub mod toon; +pub mod pipelines; /// Extract the body of a function between braces starting at `start_line` (1-indexed). #[pyfunction] @@ -239,6 +240,35 @@ fn resolve_call_graph( }) } +/// Detect pipeline paths from call graph. +#[pyfunction] +#[pyo3(signature = (nodes, edges, min_length = 3, max_pipelines = 12))] +fn find_pipeline_paths( + py: Python<'_>, + nodes: Vec, + edges: Vec<(String, String)>, + min_length: usize, + max_pipelines: usize, +) -> Vec> { + py.allow_threads(|| pipelines::find_pipeline_paths(&nodes, &edges, min_length, max_pipelines)) +} + +/// Calculate call metrics (fan_in, fan_out, complexity) and callers. +#[pyfunction] +#[pyo3(signature = (functions))] +fn calculate_call_metrics( + py: Python<'_>, + functions: Vec<(String, Vec, Vec, f64)>, +) -> ( + HashMap, + HashMap>, +) { + py.allow_threads(|| { + let res = calls::calculate_call_metrics(&functions); + (res.metrics, res.called_by) + }) +} + /// Module initialization #[pymodule] fn code2llm_rust(m: &Bound<'_, PyModule>) -> PyResult<()> { @@ -258,6 +288,8 @@ fn code2llm_rust(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(check_changed_files, m)?)?; m.add_function(wrap_pyfunction!(format_toon_header, m)?)?; m.add_function(wrap_pyfunction!(resolve_call_graph, m)?)?; + m.add_function(wrap_pyfunction!(find_pipeline_paths, m)?)?; + m.add_function(wrap_pyfunction!(calculate_call_metrics, m)?)?; m.add("__version__", env!("CARGO_PKG_VERSION"))?; Ok(()) } diff --git a/packages/code2llm-rust/src/pipelines.rs b/packages/code2llm-rust/src/pipelines.rs new file mode 100644 index 0000000..1af21b9 --- /dev/null +++ b/packages/code2llm-rust/src/pipelines.rs @@ -0,0 +1,201 @@ +//! Call graph pipeline path detection. + +use std::collections::{HashMap, HashSet}; + +/// Detect candidate pipeline paths in a directed call graph. +pub fn find_pipeline_paths( + nodes: &[String], + edges: &[(String, String)], + min_length: usize, + max_pipelines: usize, +) -> Vec> { + let mut adj: HashMap<&str, Vec<&str>> = HashMap::with_capacity(nodes.len()); + let mut in_degrees: HashMap<&str, usize> = HashMap::with_capacity(nodes.len()); + + for node in nodes { + adj.entry(node.as_str()).or_default(); + in_degrees.entry(node.as_str()).or_insert(0); + } + + for (src, dst) in edges { + adj.entry(src.as_str()).or_default().push(dst.as_str()); + *in_degrees.entry(dst.as_str()).or_insert(0) += 1; + } + + // Sources: nodes with in-degree == 0 + let mut sources: Vec<&str> = nodes + .iter() + .map(|s| s.as_str()) + .filter(|&n| in_degrees.get(n).copied().unwrap_or(0) == 0) + .collect(); + + if sources.is_empty() { + let mut sorted_nodes: Vec<&str> = nodes.iter().map(|s| s.as_str()).collect(); + sorted_nodes.sort_by_key(|&n| in_degrees.get(n).copied().unwrap_or(0)); + sources = sorted_nodes.into_iter().take(5).collect(); + } + + let mut paths: Vec> = Vec::new(); + let mut used_nodes: HashSet<&str> = HashSet::new(); + + // 1. Longest paths from sources + for source in sources { + let best_path = longest_path_from(&adj, source, 10); + if best_path.len() >= min_length { + used_nodes.extend(best_path.iter().copied()); + paths.push(best_path.into_iter().map(|s| s.to_string()).collect()); + } + } + + // 2. Weakly connected components + let components = find_weakly_connected_components(nodes, edges); + for comp in components { + if comp.len() < min_length { + continue; + } + + let used_overlap = comp.iter().filter(|&&n| used_nodes.contains(n)).count(); + if used_overlap > comp.len() / 2 { + continue; + } + + let comp_set: HashSet<&str> = comp.into_iter().collect(); + let mut comp_adj: HashMap<&str, Vec<&str>> = HashMap::new(); + for &n in &comp_set { + if let Some(succs) = adj.get(n) { + let filtered: Vec<&str> = succs.iter().copied().filter(|s| comp_set.contains(s)).collect(); + comp_adj.insert(n, filtered); + } + } + + let mut comp_sources: Vec<&str> = comp_set + .iter() + .copied() + .filter(|&n| in_degrees.get(n).copied().unwrap_or(0) == 0) + .collect(); + if comp_sources.is_empty() { + comp_sources = comp_set.iter().copied().take(3).collect(); + } + + let mut best: Vec<&str> = Vec::new(); + for src in comp_sources { + let p = longest_path_from(&comp_adj, src, 10); + if p.len() > best.len() { + best = p; + } + } + + if best.len() >= min_length { + let overlap = best.iter().filter(|&&n| used_nodes.contains(n)).count(); + if overlap <= best.len() / 2 { + used_nodes.extend(best.iter().copied()); + paths.push(best.into_iter().map(|s| s.to_string()).collect()); + } + } + } + + // Sort by path length descending + paths.sort_by(|a, b| b.len().cmp(&a.len())); + if paths.len() > max_pipelines { + paths.truncate(max_pipelines); + } + + paths +} + +fn longest_path_from<'a>( + adj: &HashMap<&'a str, Vec<&'a str>>, + source: &'a str, + max_depth: usize, +) -> Vec<&'a str> { + let mut best: Vec<&'a str> = vec![source]; + let mut stack: Vec<(&'a str, Vec<&'a str>)> = vec![(source, vec![source])]; + + while let Some((curr, path)) = stack.pop() { + if path.len() > best.len() { + best = path.clone(); + } + + if path.len() >= max_depth { + continue; + } + + if let Some(succs) = adj.get(curr) { + for &succ in succs { + if !path.contains(&succ) { + let mut next_path = path.clone(); + next_path.push(succ); + stack.push((succ, next_path)); + } + } + } + } + + best +} + +fn find_weakly_connected_components<'a>( + nodes: &'a [String], + edges: &'a [(String, String)], +) -> Vec> { + let mut undirected_adj: HashMap<&str, Vec<&str>> = HashMap::with_capacity(nodes.len()); + for node in nodes { + undirected_adj.entry(node.as_str()).or_default(); + } + for (src, dst) in edges { + undirected_adj.entry(src.as_str()).or_default().push(dst.as_str()); + undirected_adj.entry(dst.as_str()).or_default().push(src.as_str()); + } + + let mut visited: HashSet<&str> = HashSet::new(); + let mut components = Vec::new(); + + for node in nodes { + let node_str = node.as_str(); + if !visited.contains(node_str) { + let mut component = Vec::new(); + let mut queue = vec![node_str]; + visited.insert(node_str); + + while let Some(curr) = queue.pop() { + component.push(curr); + if let Some(neighbors) = undirected_adj.get(curr) { + for &next in neighbors { + if !visited.contains(next) { + visited.insert(next); + queue.push(next); + } + } + } + } + components.push(component); + } + } + + components +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_find_pipeline_paths() { + let nodes = vec![ + "stage_a".to_string(), + "stage_b".to_string(), + "stage_c".to_string(), + "stage_d".to_string(), + "isolated".to_string(), + ]; + let edges = vec![ + ("stage_a".to_string(), "stage_b".to_string()), + ("stage_b".to_string(), "stage_c".to_string()), + ("stage_c".to_string(), "stage_d".to_string()), + ]; + + let paths = find_pipeline_paths(&nodes, &edges, 3, 5); + assert_eq!(paths.len(), 1); + assert_eq!(paths[0], vec!["stage_a", "stage_b", "stage_c", "stage_d"]); + } +} diff --git a/tests/test_native_rust_bridge.py b/tests/test_native_rust_bridge.py index dc8994d..59c9fad 100644 --- a/tests/test_native_rust_bridge.py +++ b/tests/test_native_rust_bridge.py @@ -323,4 +323,35 @@ def test_native_resolve_call_graph(): assert entry_points == ["app.cli.run"] +def test_native_find_pipeline_paths(): + """Verify native pipeline path finding.""" + from code2llm.analysis.native_bridge import native_find_pipeline_paths + + nodes = ["step1", "step2", "step3", "step4", "unrelated"] + edges = [("step1", "step2"), ("step2", "step3"), ("step3", "step4")] + + paths = native_find_pipeline_paths(nodes, edges, min_length=3, max_pipelines=5) + assert paths is not None + assert len(paths) >= 1 + assert paths[0] == ["step1", "step2", "step3", "step4"] + + +def test_native_calculate_call_metrics(): + """Verify native call metrics calculation.""" + from code2llm.analysis.native_bridge import native_calculate_call_metrics + + functions = [ + ("caller", ["callee"], [], 3.0), + ("callee", [], [], 2.0), + ] + + res = native_calculate_call_metrics(functions) + assert res is not None + metrics, called_by = res + assert metrics["caller"] == (0, 1, 3.0) + assert metrics["callee"] == (1, 0, 2.0) + assert called_by["callee"] == ["caller"] + + +