diff --git a/.gitignore b/.gitignore index 8c117953..8fd22017 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ htmlcov/ tests/* !tests/test_gitignore_filtering.py !tests/test_module_tree_validation.py +!tests/test_ruby_analyzer.py # Jupyter *.ipynb diff --git a/README.md b/README.md index 3cbb1778..ea2fc808 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ codewiki generate --github-pages --create-branch ## What is CodeWiki? -CodeWiki is an open-source framework for **automated repository-level documentation** across nine programming languages. It generates holistic, architecture-aware documentation that captures not only individual functions but also their cross-file, cross-module, and system-level interactions. +CodeWiki is an open-source framework for **automated repository-level documentation** across ten programming languages. It generates holistic, architecture-aware documentation that captures not only individual functions but also their cross-file, cross-module, and system-level interactions. ### Key Innovations @@ -149,7 +149,7 @@ CodeWiki is an open-source framework for **automated repository-level documentat ### Supported Languages -**🐍 Python** • **☕ Java** • **🟨 JavaScript** • **🔷 TypeScript** • **⚙️ C** • **🔧 C++** • **🪟 C#** • **🎯 Kotlin** • **🐘 PHP** +**🐍 Python** • **☕ Java** • **🟨 JavaScript** • **🔷 TypeScript** • **⚙️ C** • **🔧 C++** • **🪟 C#** • **🎯 Kotlin** • **🐘 PHP** • **💎 Ruby** --- diff --git a/codewiki/cli/utils/repo_validator.py b/codewiki/cli/utils/repo_validator.py index 12e9f952..46a5fcfe 100644 --- a/codewiki/cli/utils/repo_validator.py +++ b/codewiki/cli/utils/repo_validator.py @@ -2,147 +2,142 @@ Repository validation utilities for documentation generation. """ -from pathlib import Path -from typing import Tuple, List import os +from pathlib import Path from codewiki.cli.utils.errors import RepositoryError -from codewiki.cli.utils.validation import validate_repository_path, detect_supported_languages - +from codewiki.cli.utils.validation import detect_supported_languages, validate_repository_path # Supported file extensions by language SUPPORTED_EXTENSIONS = { - '.py', # Python - '.java', # Java - '.js', # JavaScript - '.jsx', # JavaScript (React) - '.ts', # TypeScript - '.tsx', # TypeScript (React) - '.c', # C - '.h', # C headers - '.cpp', # C++ - '.hpp', # C++ headers - '.cc', # C++ - '.hh', # C++ headers - '.cxx', # C++ - '.hxx', # C++ headers - '.cs', # C# - '.php', # PHP - '.phtml', # PHP templates - '.inc', # PHP includes - '.kt', # Kotlin - '.kts', # Kotlin Scripts + ".py", # Python + ".java", # Java + ".js", # JavaScript + ".jsx", # JavaScript (React) + ".ts", # TypeScript + ".tsx", # TypeScript (React) + ".c", # C + ".h", # C headers + ".cpp", # C++ + ".hpp", # C++ headers + ".cc", # C++ + ".hh", # C++ headers + ".cxx", # C++ + ".hxx", # C++ headers + ".cs", # C# + ".php", # PHP + ".phtml", # PHP templates + ".inc", # PHP includes + ".kt", # Kotlin + ".kts", # Kotlin Scripts + ".rb", # Ruby } -def validate_repository(repo_path: Path) -> Tuple[Path, List[Tuple[str, int]]]: +def validate_repository(repo_path: Path) -> tuple[Path, list[tuple[str, int]]]: """ Validate repository for documentation generation. - + Checks: - Path exists and is a directory - Contains supported code files - Has sufficient files for meaningful documentation - + Args: repo_path: Path to repository - + Returns: Tuple of (validated_path, language_counts) - + Raises: RepositoryError: If validation fails """ # Validate path exists repo_path = validate_repository_path(repo_path) - + # Detect languages languages = detect_supported_languages(repo_path) - + if not languages: raise RepositoryError( f"No supported code files found in {repo_path}\n\n" - "CodeWiki supports: Python, Java, JavaScript, TypeScript, C, C++, C#, PHP\n\n" + "CodeWiki supports: Python, Java, JavaScript, TypeScript, C, C++, C#, PHP, Ruby\n\n" "Please navigate to a code repository or specify a custom directory:\n" " cd /path/to/your/project\n" " codewiki generate" ) - + return repo_path, languages def check_writable_output(output_dir: Path) -> Path: """ Check if output directory is writable. - + Args: output_dir: Output directory path - + Returns: Validated output directory path - + Raises: RepositoryError: If output directory is not writable """ output_dir = Path(output_dir).expanduser().resolve() - + # Check if output directory exists if output_dir.exists(): if not output_dir.is_dir(): - raise RepositoryError( - f"Output path exists but is not a directory: {output_dir}" - ) - + raise RepositoryError(f"Output path exists but is not a directory: {output_dir}") + # Check if writable if not os.access(output_dir, os.W_OK): raise RepositoryError( - f"Output directory is not writable: {output_dir}\n\n" - f"Try: chmod u+w {output_dir}" + f"Output directory is not writable: {output_dir}\n\nTry: chmod u+w {output_dir}" ) else: # Check if parent is writable parent = output_dir.parent if not parent.exists(): - raise RepositoryError( - f"Parent directory does not exist: {parent}" - ) - + raise RepositoryError(f"Parent directory does not exist: {parent}") + if not os.access(parent, os.W_OK): raise RepositoryError( f"Cannot create output directory (parent not writable): {parent}\n\n" f"Try: chmod u+w {parent}" ) - + return output_dir def _get_git_repo(repo_path: Path): """ Find a git repository starting at repo_path and searching parent directories. - + Args: repo_path: Path to start searching from - + Returns: git.Repo instance or None if no repository found """ try: import git + return git.Repo(repo_path, search_parent_directories=True) - except Exception: + except Exception: # noqa: BLE001 — not a git repo is a normal outcome return None def is_git_repository(repo_path: Path) -> bool: """ Check if path is inside a git repository. - + Searches parent directories if .git is not directly at repo_path, supporting monorepo subdirectories. - + Args: repo_path: Path to check - + Returns: True if inside a git repository, False otherwise """ @@ -152,54 +147,54 @@ def is_git_repository(repo_path: Path) -> bool: def get_git_commit_hash(repo_path: Path) -> str: """ Get current git commit hash. - + Searches parent directories to support monorepo subdirectories. - + Args: repo_path: Path inside a git repository - + Returns: Commit hash or empty string if not in a git repo """ repo = _get_git_repo(repo_path) if repo is None: return "" - + try: return repo.head.commit.hexsha - except Exception: + except Exception: # noqa: BLE001 — commit info is optional metadata return "" def get_git_branch(repo_path: Path) -> str: """ Get current git branch name. - + Searches parent directories to support monorepo subdirectories. - + Args: repo_path: Path inside a git repository - + Returns: Branch name or empty string if not in a git repo """ repo = _get_git_repo(repo_path) if repo is None: return "" - + try: return repo.active_branch.name - except Exception: + except Exception: # noqa: BLE001 — detached HEAD has no branch name return "" def count_code_files(repo_path: Path) -> int: """ Count supported code files in repository. - + Args: repo_path: Repository path - + Returns: Number of code files """ @@ -207,4 +202,3 @@ def count_code_files(repo_path: Path) -> int: for ext in SUPPORTED_EXTENSIONS: count += len(list(repo_path.rglob(f"*{ext}"))) return count - diff --git a/codewiki/cli/utils/validation.py b/codewiki/cli/utils/validation.py index 9711ba33..5056be7c 100644 --- a/codewiki/cli/utils/validation.py +++ b/codewiki/cli/utils/validation.py @@ -2,9 +2,7 @@ Validation utilities for CLI inputs and configuration. """ -import re from pathlib import Path -from typing import Optional, List, Tuple from urllib.parse import urlparse from codewiki.cli.utils.errors import ConfigurationError, RepositoryError @@ -13,40 +11,39 @@ def validate_url(url: str, require_https: bool = False, allow_localhost: bool = True) -> str: """ Validate URL format. - + Args: url: URL to validate require_https: Require HTTPS scheme (except localhost) allow_localhost: Allow localhost URLs - + Returns: Validated URL - + Raises: ConfigurationError: If URL is invalid """ try: parsed = urlparse(url) - + # Check scheme if not parsed.scheme: raise ConfigurationError(f"Invalid URL (missing scheme): {url}") - + # Check HTTPS requirement - if require_https and parsed.scheme != 'https': + if require_https and parsed.scheme != "https": # Allow HTTP for localhost - if allow_localhost and parsed.hostname in ['localhost', '127.0.0.1', '::1']: + if allow_localhost and parsed.hostname in ["localhost", "127.0.0.1", "::1"]: pass else: raise ConfigurationError( - f"URL must use HTTPS: {url}\n" - f"HTTP is only allowed for localhost" + f"URL must use HTTPS: {url}\nHTTP is only allowed for localhost" ) - + # Check hostname if not parsed.hostname: raise ConfigurationError(f"Invalid URL (missing hostname): {url}") - + return url except ValueError as e: raise ConfigurationError(f"Invalid URL format: {url}\nError: {e}") @@ -55,154 +52,169 @@ def validate_url(url: str, require_https: bool = False, allow_localhost: bool = def validate_api_key(api_key: str, min_length: int = 10) -> str: """ Validate API key format. - + Args: api_key: API key to validate min_length: Minimum key length - + Returns: Validated API key - + Raises: ConfigurationError: If API key is invalid """ if not api_key or not api_key.strip(): raise ConfigurationError("API key cannot be empty") - + api_key = api_key.strip() - + if len(api_key) < min_length: - raise ConfigurationError( - f"API key too short (minimum {min_length} characters)" - ) - + raise ConfigurationError(f"API key too short (minimum {min_length} characters)") + return api_key def validate_model_name(model: str) -> str: """ Validate model name format. - + Args: model: Model name to validate - + Returns: Validated model name - + Raises: ConfigurationError: If model name is invalid """ if not model or not model.strip(): raise ConfigurationError("Model name cannot be empty") - + return model.strip() def validate_output_directory(path: str) -> Path: """ Validate output directory path. - + Args: path: Directory path to validate - + Returns: Validated Path object - + Raises: ConfigurationError: If path is invalid """ if not path or not path.strip(): raise ConfigurationError("Output directory cannot be empty") - + try: resolved_path = Path(path).expanduser().resolve() - + # Check if path is writable (or parent is writable if path doesn't exist) - if resolved_path.exists(): - if not resolved_path.is_dir(): - raise ConfigurationError( - f"Output path exists but is not a directory: {path}" - ) - + if resolved_path.exists() and not resolved_path.is_dir(): + raise ConfigurationError(f"Output path exists but is not a directory: {path}") + return resolved_path - except Exception as e: + except Exception as e: # noqa: BLE001 — any path failure becomes a ConfigurationError raise ConfigurationError(f"Invalid output directory path: {path}\nError: {e}") def validate_repository_path(path: Path) -> Path: """ Validate repository path exists and contains code files. - + Args: path: Repository path to validate - + Returns: Validated Path object - + Raises: RepositoryError: If repository is invalid """ path = Path(path).expanduser().resolve() - + if not path.exists(): raise RepositoryError(f"Repository path does not exist: {path}") - + if not path.is_dir(): raise RepositoryError(f"Repository path is not a directory: {path}") - + return path -def detect_supported_languages(directory: Path) -> List[Tuple[str, int]]: +def detect_supported_languages(directory: Path) -> list[tuple[str, int]]: """ Detect supported programming languages in a directory. - + Args: directory: Directory to scan - + Returns: List of (language, file_count) tuples """ language_extensions = { - 'Python': ['.py'], - 'Java': ['.java'], - 'JavaScript': ['.js', '.jsx'], - 'TypeScript': ['.ts', '.tsx'], - 'C': ['.c', '.h'], - 'C++': ['.cpp', '.hpp', '.cc', '.hh', '.cxx', '.hxx'], - 'C#': ['.cs'], - 'PHP': ['.php', '.phtml', '.inc'], - 'Kotlin': ['.kt', '.kts'], + "Python": [".py"], + "Java": [".java"], + "JavaScript": [".js", ".jsx"], + "TypeScript": [".ts", ".tsx"], + "C": [".c", ".h"], + "C++": [".cpp", ".hpp", ".cc", ".hh", ".cxx", ".hxx"], + "C#": [".cs"], + "PHP": [".php", ".phtml", ".inc"], + "Kotlin": [".kt", ".kts"], + "Ruby": [".rb"], } - + # Directories to exclude from counting excluded_dirs = { - 'node_modules', '__pycache__', '.git', 'build', 'dist', - '.venv', 'venv', 'env', '.env', 'target', 'bin', 'obj', - '.pytest_cache', '.mypy_cache', '.tox', 'coverage', - 'htmlcov', '.eggs', '*.egg-info', 'vendor', 'bower_components', - '.idea', '.vscode', '.gradle', '.mvn' + "node_modules", + "__pycache__", + ".git", + "build", + "dist", + ".venv", + "venv", + "env", + ".env", + "target", + "bin", + "obj", + ".pytest_cache", + ".mypy_cache", + ".tox", + "coverage", + "htmlcov", + ".eggs", + "*.egg-info", + "vendor", + "bower_components", + ".idea", + ".vscode", + ".gradle", + ".mvn", } - + def should_exclude_file(file_path: Path) -> bool: """Check if file is in an excluded directory.""" parts = file_path.parts return any(excluded_dir in parts for excluded_dir in excluded_dirs) - + language_counts = {} - + for language, extensions in language_extensions.items(): count = 0 for ext in extensions: # Filter out files in excluded directories count += sum( - 1 for f in directory.rglob(f"*{ext}") - if f.is_file() and not should_exclude_file(f) + 1 for f in directory.rglob(f"*{ext}") if f.is_file() and not should_exclude_file(f) ) - + if count > 0: language_counts[language] = count - + # Sort by count descending return sorted(language_counts.items(), key=lambda x: x[1], reverse=True) @@ -210,21 +222,21 @@ def should_exclude_file(file_path: Path) -> bool: def is_top_tier_model(model: str) -> bool: """ Check if a model is considered top-tier for clustering. - + Args: model: Model name - + Returns: True if top-tier, False otherwise """ top_tier_models = [ - 'claude-opus', - 'claude-sonnet', - 'gpt-4', - 'gpt-5', - 'gemini-2.5', + "claude-opus", + "claude-sonnet", + "gpt-4", + "gpt-5", + "gemini-2.5", ] - + model_lower = model.lower() return any(tier in model_lower for tier in top_tier_models) @@ -232,20 +244,19 @@ def is_top_tier_model(model: str) -> bool: def mask_api_key(api_key: str, visible_chars: int = 4) -> str: """ Mask API key for display, showing only first and last few characters. - + Args: api_key: API key to mask visible_chars: Number of visible characters at start and end - + Returns: Masked API key (e.g., "sk-1234...5678") """ if not api_key: return "Not set" - + if len(api_key) <= visible_chars * 2: # Key too short, mask everything except edges return f"{api_key[:2]}...{api_key[-2:]}" - - return f"{api_key[:visible_chars]}...{api_key[-visible_chars:]}" + return f"{api_key[:visible_chars]}...{api_key[-visible_chars:]}" diff --git a/codewiki/mcp/tools/analysis.py b/codewiki/mcp/tools/analysis.py index 70a1eb6b..a08623eb 100644 --- a/codewiki/mcp/tools/analysis.py +++ b/codewiki/mcp/tools/analysis.py @@ -13,9 +13,9 @@ import logging import os from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any -from codewiki.mcp.session import SessionState, SessionStore +from codewiki.mcp.session import SessionStore from codewiki.mcp.workspace import SessionWorkspace logger = logging.getLogger(__name__) @@ -25,10 +25,11 @@ # Incremental update: detect changes since last generation # --------------------------------------------------------------------------- + def _detect_changes( repo_path: Path, output_dir: Path, -) -> Optional[Dict[str, Any]]: +) -> dict[str, Any] | None: """Detect changes since last documentation generation. Returns a changes dict with affected modules, or None if no previous @@ -89,9 +90,9 @@ def _detect_changes( def _detect_via_git( repo_path: Path, - metadata: Dict[str, Any], + metadata: dict[str, Any], output_dir: Path | None = None, -) -> Optional[Dict[str, Any]]: +) -> dict[str, Any] | None: """Detect changes via git. Returns None if not in a git repo or if no previous commit is recorded (so the caller can fall through to mtime). @@ -100,8 +101,9 @@ def _detect_via_git( """ try: import git + repo = git.Repo(repo_path, search_parent_directories=True) - except Exception: + except Exception: # noqa: BLE001 — not a git repo is a normal outcome return None prev_commit = metadata.get("generation_info", {}).get("commit_id") @@ -110,7 +112,7 @@ def _detect_via_git( try: current_commit = repo.head.commit.hexsha - except Exception: + except Exception: # noqa: BLE001 — empty repo has no HEAD commit return None # Compute subpath prefix for monorepo support. @@ -136,12 +138,12 @@ def _detect_via_git( except (ValueError, TypeError): pass - def _normalize(p: str) -> Optional[str]: + def _normalize(p: str) -> str | None: """Strip the monorepo subpath and drop generated/non-source paths.""" if subpath: if not p.startswith(subpath + "/"): return None # outside target subdirectory - p = p[len(subpath) + 1:] + p = p[len(subpath) + 1 :] if p.startswith(".codewiki/"): return None if output_dir_rel and (p == output_dir_rel or p.startswith(output_dir_rel + "/")): @@ -151,7 +153,7 @@ def _normalize(p: str) -> Optional[str]: changed: list[str] = [] seen: set[str] = set() - def _add(raw: Optional[str]) -> None: + def _add(raw: str | None) -> None: if raw: p = _normalize(raw) if p and p not in seen: @@ -162,14 +164,15 @@ def _add(raw: Optional[str]) -> None: if prev_commit != current_commit: try: diff_index = repo.commit(prev_commit).diff(current_commit) - except Exception: + except Exception: # noqa: BLE001 — see fallback note below # Baseline commit unreachable (shallow clone, rebase, gc). # Committed changes can't be enumerated, and returning an empty # list here would falsely report "up to date" on a clean tree — # fall back to mtime detection instead. logger.warning( "Stored commit %s is unreachable in %s; falling back to mtime detection", - prev_commit, repo_path, + prev_commit, + repo_path, ) return None for diff in diff_index: @@ -183,7 +186,7 @@ def _add(raw: Optional[str]) -> None: _add(d.b_path) for item in repo.untracked_files: _add(item) - except Exception: + except Exception: # noqa: BLE001, S110 — uncommitted-change listing is best-effort pass return {"changed_files": changed, "method": "git"} @@ -191,8 +194,8 @@ def _add(raw: Optional[str]) -> None: def _detect_via_mtime( repo_path: Path, - metadata: Dict[str, Any], -) -> Optional[Dict[str, Any]]: + metadata: dict[str, Any], +) -> dict[str, Any] | None: """Fallback: detect changed files by comparing mtime with generation timestamp.""" timestamp_str = metadata.get("generation_info", {}).get("timestamp") if not timestamp_str: @@ -200,22 +203,37 @@ def _detect_via_mtime( try: from datetime import datetime + prev_time = datetime.fromisoformat(timestamp_str).timestamp() except (ValueError, TypeError): return None # Language extensions recognized by CodeWiki source_extensions = { - ".py", ".java", ".js", ".jsx", ".ts", ".tsx", - ".c", ".h", ".cpp", ".hpp", ".cc", ".hh", - ".cs", ".kt", ".kts", + ".py", + ".java", + ".js", + ".jsx", + ".ts", + ".tsx", + ".c", + ".h", + ".cpp", + ".hpp", + ".cc", + ".hh", + ".cs", + ".kt", + ".kts", + ".rb", } changed: list[str] = [] for dirpath, dirnames, filenames in os.walk(repo_path): # Skip hidden dirs and common non-source dirs dirnames[:] = [ - d for d in dirnames + d + for d in dirnames if not d.startswith(".") and d not in ("node_modules", "__pycache__", "venv", ".venv") ] for filename in filenames: @@ -233,9 +251,9 @@ def _detect_via_mtime( def _find_affected_modules( - module_tree: Dict[str, Any], - changed_files: List[str], -) -> Tuple[set, set]: + module_tree: dict[str, Any], + changed_files: list[str], +) -> tuple[set, set]: """Map changed files to affected modules using module_tree.json. Uses substring matching (same as the CLI ``_invalidate_affected_modules``). @@ -244,7 +262,7 @@ def _find_affected_modules( affected: set[str] = set() cascade: set[str] = set() - def _walk(tree: Dict, parents: list[str] | None = None): + def _walk(tree: dict, parents: list[str] | None = None): if parents is None: parents = [] for mod_name, mod_info in tree.items(): @@ -253,7 +271,11 @@ def _walk(tree: Dict, parents: list[str] | None = None): for comp in components: comp_file = comp.split("::")[0] for cf in changed_files: - if comp_file == cf or comp_file.endswith("/" + cf) or cf.endswith("/" + comp_file): + if ( + comp_file == cf + or comp_file.endswith("/" + cf) + or cf.endswith("/" + comp_file) + ): hit = True break # Changed dir contains the component file, or vice versa @@ -280,7 +302,7 @@ def _walk(tree: Dict, parents: list[str] | None = None): def handle_analyze_repo( - arguments: Dict[str, Any], + arguments: dict[str, Any], store: SessionStore, ) -> str: """Run the dependency analysis, write results to workspace files, @@ -294,6 +316,7 @@ def handle_analyze_repo( # Build a minimal Config for the dependency analyzer (no LLM fields used) from codewiki.src.config import Config + config = Config( repo_path=str(repo_path), output_dir=str(output_dir / "temp"), @@ -311,7 +334,7 @@ def handle_analyze_repo( include = arguments.get("include_patterns") exclude = arguments.get("exclude_patterns") if include or exclude: - agent_instructions: Dict[str, Any] = {} + agent_instructions: dict[str, Any] = {} if include: agent_instructions["include_patterns"] = [p.strip() for p in include.split(",")] if exclude: @@ -319,6 +342,7 @@ def handle_analyze_repo( config.agent_instructions = agent_instructions from codewiki.src.be.dependency_analyzer import DependencyGraphBuilder + builder = DependencyGraphBuilder(config) components, leaf_nodes = builder.build_dependency_graph() @@ -333,6 +357,7 @@ def handle_analyze_repo( # Record the analyzed commit now — close_session uses it as the # incremental-update baseline in metadata.json. from codewiki.cli.utils.repo_validator import get_git_commit_hash + session.analyzed_commit = get_git_commit_hash(repo_path) or None # Create the workspace with the real session_id @@ -344,18 +369,20 @@ def handle_analyze_repo( # 1. Full component index (no pagination) component_index: list[dict] = [] for comp_id, node in components.items(): - component_index.append({ - "id": comp_id, - "type": getattr(node, "component_type", "unknown"), - "file": getattr(node, "relative_path", ""), - }) + component_index.append( + { + "id": comp_id, + "type": getattr(node, "component_type", "unknown"), + "file": getattr(node, "relative_path", ""), + } + ) workspace.write_json("component_index.json", component_index) # 2. Full leaf nodes list workspace.write_json("leaf_nodes.json", leaf_nodes) # 3. Language stats - languages: Dict[str, int] = {} + languages: dict[str, int] = {} for node in components.values(): lang = getattr(node, "language", "unknown") languages[lang] = languages.get(lang, 0) + 1 diff --git a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py index 0e31db3d..b3aaf625 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py +++ b/codewiki/src/be/dependency_analyzer/analysis/analysis_service.py @@ -8,15 +8,19 @@ import logging import traceback -from typing import Dict, List, Optional, Any from pathlib import Path -from codewiki.src.be.dependency_analyzer.utils.security import safe_open_text, assert_safe_path -from codewiki.src.be.dependency_analyzer.analysis.repo_analyzer import RepoAnalyzer +from typing import Any + from codewiki.src.be.dependency_analyzer.analysis.call_graph_analyzer import CallGraphAnalyzer -from codewiki.src.be.dependency_analyzer.analysis.cloning import clone_repository, cleanup_repository, parse_github_url +from codewiki.src.be.dependency_analyzer.analysis.cloning import ( + cleanup_repository, + clone_repository, + parse_github_url, +) +from codewiki.src.be.dependency_analyzer.analysis.repo_analyzer import RepoAnalyzer from codewiki.src.be.dependency_analyzer.models.analysis import AnalysisResult from codewiki.src.be.dependency_analyzer.models.core import Repository - +from codewiki.src.be.dependency_analyzer.utils.security import assert_safe_path, safe_open_text logger = logging.getLogger(__name__) @@ -42,64 +46,64 @@ def analyze_local_repository( self, repo_path: str, max_files: int = 100, - languages: Optional[List[str]] = None, + languages: list[str] | None = None, use_gitignore: bool = True, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """ Analyze a local repository folder. - + Args: repo_path: Path to local repository folder max_files: Maximum number of files to analyze languages: List of languages to include (e.g., ['python', 'javascript']) use_gitignore: Whether to apply Git ignore rules - + Returns: Dict with analysis results including nodes and relationships """ try: logger.debug(f"Analyzing local repository at {repo_path}") - + # Get repo analyzer to find files repo_analyzer = RepoAnalyzer(use_gitignore=use_gitignore) structure_result = repo_analyzer.analyze_repository_structure(repo_path) - + # Extract code files code_files = self.call_graph_analyzer.extract_code_files(structure_result["file_tree"]) - + # Filter by languages if specified if languages: code_files = [f for f in code_files if f.get("language") in languages] - + # Limit number of files if len(code_files) > max_files: code_files = code_files[:max_files] logger.debug(f"Limited analysis to {max_files} files") - + logger.debug(f"Analyzing {len(code_files)} files") - + # Analyze files result = self.call_graph_analyzer.analyze_code_files(code_files, repo_path) - + return { "nodes": result.get("functions", {}), "relationships": result.get("relationships", []), "summary": { "total_files": len(code_files), "total_nodes": len(result.get("functions", {})), - "total_relationships": len(result.get("relationships", [])) - } + "total_relationships": len(result.get("relationships", [])), + }, } - + except Exception as e: - logger.error(f"Local repository analysis failed: {str(e)}", exc_info=True) - raise RuntimeError(f"Analysis failed: {str(e)}") + logger.exception("Local repository analysis failed") + raise RuntimeError(f"Analysis failed: {e!s}") def analyze_repository_full( self, github_url: str, - include_patterns: Optional[List[str]] = None, - exclude_patterns: Optional[List[str]] = None, + include_patterns: list[str] | None = None, + exclude_patterns: list[str] | None = None, use_gitignore: bool = True, ) -> AnalysisResult: """ @@ -168,18 +172,18 @@ def analyze_repository_full( return analysis_result except Exception as e: - logger.error(f"Analysis failed: {str(e)}", exc_info=True) + logger.exception("Analysis failed") if "temp_dir" in locals() and Path(temp_dir).exists(): self._cleanup_repository(temp_dir) - raise RuntimeError(f"Repository analysis failed: {str(e)}") + raise RuntimeError(f"Repository analysis failed: {e!s}") def analyze_repository_structure_only( self, github_url: str, - include_patterns: Optional[List[str]] = None, - exclude_patterns: Optional[List[str]] = None, + include_patterns: list[str] | None = None, + exclude_patterns: list[str] | None = None, use_gitignore: bool = True, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """ Perform lightweight structure-only analysis without call graph generation. @@ -222,9 +226,9 @@ def analyze_repository_structure_only( except Exception as e: if temp_dir: self._cleanup_repository(temp_dir) - logger.error(f"Structure analysis failed for {github_url}: {str(e)}") + logger.error(f"Structure analysis failed for {github_url}: {e!s}") logger.error(f"Traceback: {traceback.format_exc()}") - raise RuntimeError(f"Structure analysis failed: {str(e)}") from e + raise RuntimeError(f"Structure analysis failed: {e!s}") from e def _clone_repository(self, github_url: str) -> str: """Clone repository and return temp dir path.""" @@ -234,17 +238,17 @@ def _clone_repository(self, github_url: str) -> str: self._temp_directories.append(temp_dir) return temp_dir - def _parse_repository_info(self, github_url: str) -> Dict[str, str]: + def _parse_repository_info(self, github_url: str) -> dict[str, str]: """Parse GitHub URL and extract repository metadata.""" return parse_github_url(github_url) def _analyze_structure( self, repo_dir: str, - include_patterns: Optional[List[str]], - exclude_patterns: Optional[List[str]], + include_patterns: list[str] | None, + exclude_patterns: list[str] | None, use_gitignore: bool = True, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """Analyze repository file structure with filtering.""" logger.debug( "Initializing RepoAnalyzer with include: %s, exclude: %s, use_gitignore: %s", @@ -255,7 +259,7 @@ def _analyze_structure( repo_analyzer = RepoAnalyzer(include_patterns, exclude_patterns, use_gitignore) return repo_analyzer.analyze_repository_structure(repo_dir) - def _read_readme_file(self, repo_dir: str) -> Optional[str]: + def _read_readme_file(self, repo_dir: str) -> str | None: """Find and read the README file from the repository root.""" # possible_readme_names = ["README.md", "README", "readme.md", "README.txt"] # for name in possible_readme_names: @@ -278,13 +282,13 @@ def _read_readme_file(self, repo_dir: str) -> Optional[str]: assert_safe_path(base, p) logger.debug(f"Found README file at {p}") return safe_open_text(base, p, encoding="utf-8") - except Exception as e: + except Exception as e: # noqa: BLE001 — README loading is best-effort logger.warning(f"Skipping unsafe/ unreadable README at {p}: {e}") return None logger.debug("No README file found in repository root.") return None - def _analyze_call_graph(self, file_tree: Dict[str, Any], repo_dir: str) -> Dict[str, Any]: + def _analyze_call_graph(self, file_tree: dict[str, Any], repo_dir: str) -> dict[str, Any]: """ Perform multi-language call graph analysis. @@ -296,7 +300,9 @@ def _analyze_call_graph(self, file_tree: Dict[str, Any], repo_dir: str) -> Dict[ logger.debug("Extracting code files from file tree...") code_files = self.call_graph_analyzer.extract_code_files(file_tree) - logger.debug(f"Found {len(code_files)} total code files. Filtering for supported languages.") + logger.debug( + f"Found {len(code_files)} total code files. Filtering for supported languages." + ) supported_files = self._filter_supported_languages(code_files) logger.debug(f"Analyzing {len(supported_files)} supported files.") @@ -307,11 +313,11 @@ def _analyze_call_graph(self, file_tree: Dict[str, Any], repo_dir: str) -> Dict[ return result - def _filter_supported_languages(self, code_files: List[Dict]) -> List[Dict]: + def _filter_supported_languages(self, code_files: list[dict]) -> list[dict]: """ Filter code files to only include supported languages. - Supports Python, JavaScript, TypeScript, Java, C#, C, C++, PHP, Go, and Rust. + Supports Python, JavaScript, TypeScript, Java, C#, C, C++, PHP, Ruby, Go, and Rust. """ supported_languages = { "python", @@ -322,6 +328,7 @@ def _filter_supported_languages(self, code_files: List[Dict]) -> List[Dict]: "c", "cpp", "php", + "ruby", "go", "rust", "kotlin", @@ -333,9 +340,20 @@ def _filter_supported_languages(self, code_files: List[Dict]) -> List[Dict]: if file_info.get("language") in supported_languages ] - def _get_supported_languages(self) -> List[str]: + def _get_supported_languages(self) -> list[str]: """Get list of currently supported languages for analysis.""" - return ["python", "javascript", "typescript", "java", "csharp", "c", "cpp", "php", "kotlin"] + return [ + "python", + "javascript", + "typescript", + "java", + "csharp", + "c", + "cpp", + "php", + "ruby", + "kotlin", + ] def _cleanup_repository(self, temp_dir: str): """Clean up cloned repository.""" @@ -372,7 +390,7 @@ def analyze_repository( def analyze_repository_structure_only( github_url: str, include_patterns=None, exclude_patterns=None, use_gitignore=True -) -> tuple[Dict, None]: +) -> tuple[dict, None]: """ Backward compatibility function. diff --git a/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py index e2fc61da..dfcd7340 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py @@ -6,38 +6,38 @@ across different programming languages in a repository. """ -from typing import Dict, List, Optional import logging -import traceback -import time -import signal import re +import signal +import time +import traceback from collections import defaultdict -from pathlib import Path from contextlib import contextmanager -from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship -from codewiki.src.be.dependency_analyzer.utils.patterns import CODE_EXTENSIONS -from codewiki.src.be.dependency_analyzer.utils.security import safe_open_text +from pathlib import Path + +from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node from codewiki.src.be.dependency_analyzer.utils.external_symbols import ( CPP_STANDARD_HEADERS, is_external_symbol, is_macro_name, ) +from codewiki.src.be.dependency_analyzer.utils.patterns import CODE_EXTENSIONS +from codewiki.src.be.dependency_analyzer.utils.security import safe_open_text logger = logging.getLogger(__name__) class TimeoutError(Exception): """Raised when file parsing exceeds timeout.""" - pass @contextmanager def timeout(seconds): """Context manager for timeout on file parsing.""" + def signal_handler(signum, frame): raise TimeoutError(f"File parsing exceeded {seconds}s timeout") - + # Only use signal on Unix systems (not Windows) try: old_handler = signal.signal(signal.SIGALRM, signal_handler) @@ -57,21 +57,21 @@ def signal_handler(signum, frame): class CallGraphAnalyzer: def __init__(self): """Initialize the call graph analyzer.""" - self.functions: Dict[str, Node] = {} - self.call_relationships: List[CallRelationship] = [] + self.functions: dict[str, Node] = {} + self.call_relationships: list[CallRelationship] = [] self._python_project_modules: set = set() self._python_external_import_roots: set = set() logger.debug("CallGraphAnalyzer initialized.") - def analyze_code_files(self, code_files: List[Dict], base_dir: str) -> Dict: + def analyze_code_files(self, code_files: list[dict], base_dir: str) -> dict: """ Complete analysis: Analyze all files to build complete call graph with all nodes. This approach: - 1. Analyzes all code files + 1. Analyzes all code files 2. Extracts all functions and relationships 3. Builds complete call graph - 4. Returns all nodes and relationships + 4. Returns all nodes and relationships """ logger.debug(f"Starting analysis of {len(code_files)} files") logger.info(f"📊 Parsing {len(code_files)} source files (this may take a few minutes)...") @@ -85,23 +85,27 @@ def analyze_code_files(self, code_files: List[Dict], base_dir: str) -> Dict: files_analyzed = 0 files_failed = 0 start_time = time.time() - + for idx, file_info in enumerate(code_files, 1): - file_path = file_info['path'] + file_path = file_info["path"] try: # Log progress every file with elapsed time if idx % max(1, len(code_files) // 10) == 0 or idx <= 5: elapsed = time.time() - start_time rate = idx / elapsed if elapsed > 0 else 0 remaining = (len(code_files) - idx) / rate if rate > 0 else 0 - logger.info(f" [{idx}/{len(code_files)}] {file_path} ({elapsed:.1f}s elapsed, ~{remaining:.1f}s remaining)") - + logger.info( + f" [{idx}/{len(code_files)}] {file_path} ({elapsed:.1f}s elapsed, ~{remaining:.1f}s remaining)" + ) + self._analyze_code_file(base_dir, file_info) files_analyzed += 1 - except Exception as e: + except Exception as e: # noqa: BLE001 — one unparsable file must not abort the sweep files_failed += 1 - logger.warning(f" ⚠️ [{idx}/{len(code_files)}] Failed to analyze {file_path}: {str(e)[:100]}") - + logger.warning( + f" ⚠️ [{idx}/{len(code_files)}] Failed to analyze {file_path}: {str(e)[:100]}" + ) + elapsed_time = time.time() - start_time logger.info( f"✓ Analysis complete: {files_analyzed}/{len(code_files)} files analyzed, " @@ -117,7 +121,7 @@ def analyze_code_files(self, code_files: List[Dict], base_dir: str) -> Dict: "call_graph": { "total_functions": len(self.functions), "total_calls": len(self.call_relationships), - "languages_found": list(set(f.get("language") for f in code_files)), + "languages_found": list({f.get("language") for f in code_files}), "files_analyzed": files_analyzed, "analysis_approach": "complete_unlimited", }, @@ -126,7 +130,7 @@ def analyze_code_files(self, code_files: List[Dict], base_dir: str) -> Dict: "visualization": viz_data, } - def extract_code_files(self, file_tree: Dict) -> List[Dict]: + def extract_code_files(self, file_tree: dict) -> list[dict]: """ Extract code files from file tree structure. @@ -161,7 +165,7 @@ def traverse(tree): traverse(file_tree) return code_files - def _route_contextual_headers(self, code_files: List[Dict], base_dir: str) -> List[Dict]: + def _route_contextual_headers(self, code_files: list[dict], base_dir: str) -> list[dict]: """Route ambiguous .h headers per file. A header is parsed as C++ when its own content shows C++ signals, or @@ -181,11 +185,12 @@ def _route_contextual_headers(self, code_files: List[Dict], base_dir: str) -> Li routed_files = [] for file_info in code_files: routed = dict(file_info) - if routed.get("extension", "").lower() == ".h": - if self._header_has_cpp_signal(base_dir, routed["path"]): - routed["language"] = "cpp" - elif has_cpp_files and not has_c_files: - routed["language"] = "cpp" + if routed.get("extension", "").lower() == ".h" and ( + self._header_has_cpp_signal(base_dir, routed["path"]) + or has_cpp_files + and not has_c_files + ): + routed["language"] = "cpp" routed_files.append(routed) return routed_files @@ -193,7 +198,7 @@ def _header_has_cpp_signal(self, base_dir: str, relative_path: str) -> bool: base = Path(base_dir) try: content = safe_open_text(base, base / relative_path) - except Exception: + except Exception: # noqa: BLE001 — unreadable file simply is not an entry point return False if re.search( @@ -209,7 +214,7 @@ def _header_has_cpp_signal(self, base_dir: str, relative_path: str) -> bool: return True return False - def _analyze_code_file(self, repo_dir: str, file_info: Dict): + def _analyze_code_file(self, repo_dir: str, file_info: dict): """ Analyze a single code file based on its language. @@ -246,15 +251,17 @@ def _analyze_code_file(self, repo_dir: str, file_info: Dict): self._analyze_cpp_file(file_path, content, repo_dir) elif language == "php": self._analyze_php_file(file_path, content, repo_dir) + elif language == "ruby": + self._analyze_ruby_file(file_path, content, repo_dir) # else: # logger.warning( # f"Unsupported language for call graph analysis: {language} for file {file_path}" # ) except TimeoutError as e: - logger.warning(f"⏱️ Timeout analyzing {file_path}: {str(e)}") - except Exception as e: - logger.debug(f"Error analyzing {file_path}: {str(e)}") + logger.warning(f"⏱️ Timeout analyzing {file_path}: {e!s}") + except Exception as e: # noqa: BLE001 — per-file analysis is best-effort + logger.debug(f"Error analyzing {file_path}: {e!s}") logger.debug(f"Traceback: {traceback.format_exc()}") def _analyze_python_file(self, file_path: str, content: str, base_dir: str): @@ -282,11 +289,11 @@ def _analyze_python_file(self, file_path: str, content: str, base_dir: str): self.call_relationships.extend(relationships) self._python_external_import_roots.update(external_import_roots) - except Exception as e: - logger.error(f"Failed to analyze Python file {file_path}: {e}", exc_info=True) + except Exception: + logger.exception(f"Failed to analyze Python file {file_path}") @staticmethod - def _collect_python_modules(code_files: List[Dict]) -> set: + def _collect_python_modules(code_files: list[dict]) -> set: """Dotted module paths for every Python file in the repository.""" modules = set() for file_info in code_files: @@ -298,8 +305,7 @@ def _collect_python_modules(code_files: List[Dict]) -> set: path = path[: -len(ext)] break module = path.replace("/", ".").replace("\\", ".") - if module.endswith(".__init__"): - module = module[: -len(".__init__")] + module = module.removesuffix(".__init__") if module: modules.add(module) return modules @@ -314,8 +320,9 @@ def _analyze_javascript_file(self, file_path: str, content: str, repo_dir: str): repo_dir: Repository base directory """ try: - - from codewiki.src.be.dependency_analyzer.analyzers.javascript import analyze_javascript_file_treesitter + from codewiki.src.be.dependency_analyzer.analyzers.javascript import ( + analyze_javascript_file_treesitter, + ) functions, relationships = analyze_javascript_file_treesitter( file_path, content, repo_path=repo_dir @@ -327,20 +334,21 @@ def _analyze_javascript_file(self, file_path: str, content: str, repo_dir: str): self.call_relationships.extend(relationships) - except Exception as e: - logger.error(f"Failed to analyze JavaScript file {file_path}: {e}", exc_info=True) + except Exception: + logger.exception(f"Failed to analyze JavaScript file {file_path}") def _analyze_typescript_file(self, file_path: str, content: str, repo_dir: str): """ - Analyze TypeScript file using tree-sitter based AST analyzer + Analyze TypeScript file using tree-sitter based AST analyzer Args: file_path: Relative path to the TypeScript file content: File content string """ try: - - from codewiki.src.be.dependency_analyzer.analyzers.typescript import analyze_typescript_file_treesitter + from codewiki.src.be.dependency_analyzer.analyzers.typescript import ( + analyze_typescript_file_treesitter, + ) functions, relationships = analyze_typescript_file_treesitter( file_path, content, repo_path=repo_dir @@ -352,10 +360,8 @@ def _analyze_typescript_file(self, file_path: str, content: str, repo_dir: str): self.call_relationships.extend(relationships) - except Exception as e: - logger.error(f"Failed to analyze TypeScript file {file_path}: {e}", exc_info=True) - - + except Exception: + logger.exception(f"Failed to analyze TypeScript file {file_path}") def _analyze_c_file(self, file_path: str, content: str, repo_dir: str): """ @@ -386,9 +392,7 @@ def _analyze_cpp_file(self, file_path: str, content: str, repo_dir: str): """ from codewiki.src.be.dependency_analyzer.analyzers.cpp import analyze_cpp_file - functions, relationships = analyze_cpp_file( - file_path, content, repo_path=repo_dir - ) + functions, relationships = analyze_cpp_file(file_path, content, repo_path=repo_dir) for func in functions: func_id = func.id if func.id else f"{file_path}:{func.name}" @@ -414,8 +418,8 @@ def _analyze_java_file(self, file_path: str, content: str, repo_dir: str): self.functions[func_id] = func self.call_relationships.extend(relationships) - except Exception as e: - logger.error(f"Failed to analyze Java file {file_path}: {e}", exc_info=True) + except Exception: + logger.exception(f"Failed to analyze Java file {file_path}") def _analyze_kotlin_file(self, file_path: str, content: str, repo_dir: str): """ @@ -435,8 +439,8 @@ def _analyze_kotlin_file(self, file_path: str, content: str, repo_dir: str): self.functions[func_id] = func self.call_relationships.extend(relationships) - except Exception as e: - logger.error(f"Failed to analyze Kotlin file {file_path}: {e}", exc_info=True) + except Exception: + logger.exception(f"Failed to analyze Kotlin file {file_path}") def _analyze_csharp_file(self, file_path: str, content: str, repo_dir: str): """ @@ -457,8 +461,8 @@ def _analyze_csharp_file(self, file_path: str, content: str, repo_dir: str): self.functions[func_id] = func self.call_relationships.extend(relationships) - except Exception as e: - logger.error(f"Failed to analyze C# file {file_path}: {e}", exc_info=True) + except Exception: + logger.exception(f"Failed to analyze C# file {file_path}") def _analyze_php_file(self, file_path: str, content: str, repo_dir: str): """ @@ -479,8 +483,30 @@ def _analyze_php_file(self, file_path: str, content: str, repo_dir: str): self.functions[func_id] = func self.call_relationships.extend(relationships) - except Exception as e: - logger.error(f"Failed to analyze PHP file {file_path}: {e}", exc_info=True) + except Exception: + logger.exception(f"Failed to analyze PHP file {file_path}") + + def _analyze_ruby_file(self, file_path: str, content: str, repo_dir: str): + """ + Analyze Ruby file using tree-sitter based analyzer. + + Args: + file_path: Relative path to the Ruby file + content: File content string + repo_dir: Repository base directory + """ + from codewiki.src.be.dependency_analyzer.analyzers.ruby import analyze_ruby_file + + try: + functions, relationships = analyze_ruby_file(file_path, content, repo_path=repo_dir) + + for func in functions: + func_id = func.id if func.id else f"{file_path}:{func.name}" + self.functions[func_id] = func + + self.call_relationships.extend(relationships) + except Exception: + logger.exception(f"Failed to analyze Ruby file {file_path}") def _resolve_call_relationships(self): """ @@ -489,7 +515,7 @@ def _resolve_call_relationships(self): Attempts to match function calls to actual function definitions, handling cross-language calls where possible. """ - for func_id, func_info in self.functions.items(): + for func_info in self.functions.values(): if not func_info.language: file_ext = Path(func_info.file_path).suffix.lower() func_info.language = CODE_EXTENSIONS.get(file_ext) @@ -519,12 +545,12 @@ def _resolve_call_relationships(self): ) ] - def _dotted_project_packages(self) -> Dict[str, set]: + def _dotted_project_packages(self) -> dict[str, set]: """Project packages/namespaces, partitioned by language. Java and C# share the namespace-origin rule: a dotted callee qualified to a package with no prefix relation to any project package came from a third-party import.""" - packages: Dict[str, set] = defaultdict(set) + packages: dict[str, set] = defaultdict(set) for func_info in self.functions.values(): if func_info.language in ("java", "csharp"): package = self._dotted_package_for_node(func_info) @@ -532,7 +558,9 @@ def _dotted_project_packages(self) -> Dict[str, set]: packages[func_info.language].add(package) return packages - def _is_external_callee(self, language: Optional[str], callee: str, dotted_packages: Dict[str, set]) -> bool: + def _is_external_callee( + self, language: str | None, callee: str, dotted_packages: dict[str, set] + ) -> bool: """Classify a still-unresolved callee as external, after project resolution has had its chance. @@ -582,19 +610,20 @@ def _is_external_python_callee(self, callee: str) -> bool: return True return parts[-1] in PYTHON_OBJECT_METHODS - def _build_resolution_indexes(self) -> Dict[str, Dict]: + def _build_resolution_indexes(self) -> dict[str, dict]: """Build exact/simple-name lookup indexes, both globally and per language. Resolution prefers the caller's own language partition: a name that is unique within the caller's language resolves even when another language defines the same name, and names made ambiguous only by foreign-language components keep resolving as before.""" - def make() -> Dict[str, Dict[str, List[str]]]: + + def make() -> dict[str, dict[str, list[str]]]: return {"exact": defaultdict(list), "simple": defaultdict(list)} global_indexes = make() - by_lang: Dict[str, Dict[str, Dict[str, List[str]]]] = defaultdict(make) + by_lang: dict[str, dict[str, dict[str, list[str]]]] = defaultdict(make) - def add(index: Dict[str, List[str]], key: Optional[str], func_id: str) -> None: + def add(index: dict[str, list[str]], key: str | None, func_id: str) -> None: if key and func_id not in index[key]: index[key].append(func_id) @@ -633,13 +662,17 @@ def add(index: Dict[str, List[str]], key: Optional[str], func_id: str) -> None: "by_lang": dict(by_lang), } - def _resolve_callee(self, relationship: CallRelationship, indexes: Dict[str, Dict]) -> Optional[str]: + def _resolve_callee( + self, relationship: CallRelationship, indexes: dict[str, dict] + ) -> str | None: caller = self.functions.get(relationship.caller) caller_language = caller.language if caller else None lang_indexes = indexes["by_lang"].get(caller_language) if caller_language else None if lang_indexes: - match = self._resolve_callee_in(relationship, lang_indexes["exact"], lang_indexes["simple"]) + match = self._resolve_callee_in( + relationship, lang_indexes["exact"], lang_indexes["simple"] + ) if match: return match @@ -648,9 +681,9 @@ def _resolve_callee(self, relationship: CallRelationship, indexes: Dict[str, Dic def _resolve_callee_in( self, relationship: CallRelationship, - exact: Dict[str, List[str]], - simple: Dict[str, List[str]], - ) -> Optional[str]: + exact: dict[str, list[str]], + simple: dict[str, list[str]], + ) -> str | None: callee_name = relationship.callee exact_match = self._unique_match(exact, callee_name) @@ -684,7 +717,7 @@ def _resolve_callee_in( return self._unique_match(simple, callee_name) - def _unique_match(self, index: Dict[str, List[str]], key: str) -> Optional[str]: + def _unique_match(self, index: dict[str, list[str]], key: str) -> str | None: matches = index.get(key, []) return matches[0] if len(matches) == 1 else None @@ -701,7 +734,7 @@ def _dotted_package_for_node(self, node: Node) -> str: return ".".join(parts[:-2]) return ".".join(parts[:-1]) - def _caller_language(self, caller_id: str) -> Optional[str]: + def _caller_language(self, caller_id: str) -> str | None: caller = self.functions.get(caller_id) if caller and caller.language: return caller.language @@ -727,7 +760,7 @@ def _deduplicate_relationships(self): self.call_relationships = unique_relationships - def _generate_visualization_data(self) -> Dict: + def _generate_visualization_data(self) -> dict: """ Generate visualization data for graph rendering. @@ -755,12 +788,22 @@ def _generate_visualization_data(self) -> Dict: node_classes.append("lang-typescript") elif language == "c": node_classes.append("lang-c") - elif language == "cpp" or file_ext in [".cpp", ".cc", ".cxx", ".c++", ".hpp", ".hxx", ".h++"]: + elif language == "cpp" or file_ext in [ + ".cpp", + ".cc", + ".cxx", + ".c++", + ".hpp", + ".hxx", + ".h++", + ]: node_classes.append("lang-cpp") elif file_ext in [".kt", ".kts"]: node_classes.append("lang-kotlin") elif file_ext in [".php", ".phtml", ".inc"]: node_classes.append("lang-php") + elif file_ext == ".rb": + node_classes.append("lang-ruby") cytoscape_elements.append( { @@ -800,7 +843,7 @@ def _generate_visualization_data(self) -> Dict: "summary": summary, } - def generate_llm_format(self) -> Dict: + def generate_llm_format(self) -> dict: """Generate clean format optimized for LLM consumption.""" return { "functions": [ @@ -853,31 +896,29 @@ def _select_most_connected_nodes(self, target_count: int): graph = {} for rel in self.call_relationships: - if rel.caller in self.functions: - if rel.caller not in graph: - graph[rel.caller] = set() - if rel.callee in self.functions: - if rel.callee not in graph: - graph[rel.callee] = set() + if rel.caller in self.functions and rel.caller not in graph: + graph[rel.caller] = set() + if rel.callee in self.functions and rel.callee not in graph: + graph[rel.callee] = set() if rel.caller in graph and rel.callee in graph: graph[rel.caller].add(rel.callee) graph[rel.callee].add(rel.caller) degree_centrality = {} - for func_id in self.functions.keys(): + for func_id in self.functions: degree_centrality[func_id] = len(graph.get(func_id, set())) sorted_func_ids = sorted(degree_centrality, key=degree_centrality.get, reverse=True) selected_func_ids = sorted_func_ids[:target_count] - original_func_count = len(self.functions) + len(self.functions) self.functions = { fid: func for fid, func in self.functions.items() if fid in selected_func_ids } - original_rel_count = len(self.call_relationships) + len(self.call_relationships) self.call_relationships = [ rel for rel in self.call_relationships diff --git a/codewiki/src/be/dependency_analyzer/analyzers/ruby.py b/codewiki/src/be/dependency_analyzer/analyzers/ruby.py new file mode 100644 index 00000000..0268b447 --- /dev/null +++ b/codewiki/src/be/dependency_analyzer/analyzers/ruby.py @@ -0,0 +1,908 @@ +import logging +import os + +import tree_sitter_ruby +from tree_sitter import Language, Parser + +from codewiki.src.be.dependency_analyzer.models.core import CallRelationship, Node + +logger = logging.getLogger(__name__) + +MAX_RECURSION_DEPTH = 100 + +# Kernel/Object/Enumerable methods and common metaprogramming DSL calls that +# never point at a project component. Calls whose only signal is one of these +# names are dropped instead of emitted as unresolved relationships. +RUBY_CORE_METHODS = frozenset( + { + # Kernel / Object + "puts", + "print", + "p", + "pp", + "warn", + "raise", + "fail", + "throw", + "catch", + "require", + "require_relative", + "load", + "autoload", + "loop", + "sleep", + "exit", + "exit!", + "abort", + "at_exit", + "rand", + "srand", + "format", + "sprintf", + "printf", + "gets", + "binding", + "caller", + "system", + "exec", + "spawn", + "fork", + "freeze", + "frozen?", + "dup", + "clone", + "tap", + "then", + "send", + "public_send", + "__send__", + "object_id", + "hash", + "inspect", + "to_s", + "to_str", + "to_a", + "to_ary", + "to_h", + "to_i", + "to_int", + "to_f", + "to_sym", + "to_proc", + "to_r", + "nil?", + "is_a?", + "kind_of?", + "instance_of?", + "respond_to?", + "equal?", + "eql?", + "instance_variable_get", + "instance_variable_set", + "instance_variables", + "method", + "methods", + "class", + "singleton_class", + "display", + "itself", + "yield_self", + "lambda", + "proc", + "block_given?", + "iterator?", + "instance_exec", + "instance_eval", + "class_eval", + "module_eval", + "define_method", + "define_singleton_method", + "alias_method", + "raise_error", + # Module / class-body DSL + "attr_accessor", + "attr_reader", + "attr_writer", + "attr", + "private", + "public", + "protected", + "module_function", + "private_constant", + "public_constant", + "private_class_method", + "public_class_method", + "def_delegator", + "def_delegators", + "delegate", + "refine", + "using", + # Enumerable / collection + "each", + "each_with_index", + "each_with_object", + "each_pair", + "each_key", + "each_value", + "each_slice", + "each_cons", + "each_char", + "each_line", + "each_byte", + "map", + "map!", + "flat_map", + "collect", + "collect!", + "select", + "select!", + "filter", + "filter_map", + "reject", + "reject!", + "detect", + "find", + "find_all", + "find_index", + "reduce", + "inject", + "sum", + "min", + "max", + "min_by", + "max_by", + "sort", + "sort!", + "sort_by", + "group_by", + "partition", + "chunk_while", + "slice_when", + "zip", + "take", + "take_while", + "drop", + "drop_while", + "first", + "last", + "count", + "size", + "length", + "empty?", + "any?", + "all?", + "none?", + "one?", + "include?", + "member?", + "index", + "rindex", + "push", + "pop", + "shift", + "unshift", + "append", + "prepend_element", + "concat", + "insert", + "delete", + "delete_at", + "delete_if", + "clear", + "compact", + "compact!", + "flatten", + "flatten!", + "uniq", + "uniq!", + "reverse", + "reverse!", + "join", + "split", + "keys", + "values", + "fetch", + "store", + "merge", + "merge!", + "update", + "key?", + "has_key?", + "value?", + "has_value?", + "dig", + "sample", + "shuffle", + "each_entry", + "entries", + "tally", + "cycle", + "lazy", + "force", + # String + "strip", + "strip!", + "lstrip", + "rstrip", + "chomp", + "chomp!", + "chop", + "chars", + "bytes", + "lines", + "upcase", + "downcase", + "capitalize", + "swapcase", + "sub", + "sub!", + "gsub", + "gsub!", + "tr", + "squeeze", + "start_with?", + "end_with?", + "match", + "match?", + "scan", + "slice", + "slice!", + "center", + "ljust", + "rjust", + "encode", + "force_encoding", + "unpack", + "pack", + "hex", + "ord", + "chr", + "succ", + "next", + "between?", + "clamp", + "floor", + "ceil", + "round", + "abs", + "zero?", + "positive?", + "negative?", + "even?", + "odd?", + "times", + "upto", + "downto", + "step", + "divmod", + "modulo", + "pow", + "gcd", + "lcm", + # Comparison / misc operators frequently parsed as plain calls + "call", + "yield", + "new_ostruct_member", + "synchronize", + # Test DSL (specs are usually excluded, but stay quiet if they leak in) + "describe", + "context", + "it", + "expect", + "before", + "after", + "let", + "subject", + "allow", + "double", + "shared_examples", + "it_behaves_like", + } +) + +# Receivers that are Ruby/stdlib constants rather than project classes. +RUBY_BUILTIN_CONSTANTS = frozenset( + { + "Array", + "Hash", + "String", + "Symbol", + "Integer", + "Float", + "Numeric", + "Rational", + "Complex", + "Range", + "Regexp", + "MatchData", + "Proc", + "Method", + "Object", + "BasicObject", + "Class", + "Module", + "Kernel", + "Comparable", + "Enumerable", + "Enumerator", + "Struct", + "OpenStruct", + "Set", + "Time", + "Date", + "DateTime", + "File", + "Dir", + "IO", + "StringIO", + "Pathname", + "Process", + "Thread", + "ThreadGroup", + "Mutex", + "Monitor", + "Queue", + "SizedQueue", + "ConditionVariable", + "Fiber", + "Signal", + "Marshal", + "JSON", + "YAML", + "CSV", + "URI", + "Net", + "Socket", + "TCPSocket", + "TCPServer", + "OpenSSL", + "Digest", + "SecureRandom", + "Base64", + "Zlib", + "Logger", + "ENV", + "ARGV", + "STDIN", + "STDOUT", + "STDERR", + "Math", + "GC", + "ObjectSpace", + "Exception", + "StandardError", + "RuntimeError", + "ArgumentError", + "TypeError", + "NameError", + "NoMethodError", + "KeyError", + "IndexError", + "RangeError", + "IOError", + "EOFError", + "Errno", + "SystemExit", + "NotImplementedError", + "FrozenError", + "StopIteration", + "Interrupt", + "LoadError", + "SyntaxError", + "SecurityError", + "ScriptError", + "EncodingError", + "FloatDomainError", + "ZeroDivisionError", + "LocalJumpError", + "SystemCallError", + "SystemStackError", + "NilClass", + "TrueClass", + "FalseClass", + "Data", + "Random", + "Ractor", + "Warning", + "Binding", + "TracePoint", + "Gem", + "RbConfig", + "FileUtils", + "Tempfile", + "Timeout", + "Forwardable", + "Singleton", + "Observable", + "MonitorMixin", + "Etc", + "Fcntl", + "ERB", + "OptionParser", + "Shellwords", + "Open3", + "PTY", + "Benchmark", + "Coverage", + "Ripper", + "WeakRef", + "GetText", + } +) + +MIXIN_METHODS = frozenset({"include", "extend", "prepend"}) + + +class TreeSitterRubyAnalyzer: + def __init__(self, file_path: str, content: str, repo_path: str | None = None): + self.file_path = file_path + self.content = content + self.repo_path = repo_path or "" + self.nodes: list[Node] = [] + self.call_relationships: list[CallRelationship] = [] + # Same-file symbol table keyed by logical name ("Foo", "Foo.bar"). + self.top_level_nodes = {} + self.seen_relationships = set() + self._analyze() + + def _get_relative_path(self) -> str: + """Get relative path from repo root.""" + if self.repo_path: + try: + return os.path.relpath(str(self.file_path), self.repo_path) + except ValueError: + return str(self.file_path) + return str(self.file_path) + + def _get_component_id(self, name: str, parent_class: str | None = None) -> str: + rel_path = self._get_relative_path() + if parent_class: + return f"{rel_path}::{parent_class}.{name}" + return f"{rel_path}::{name}" + + def _analyze(self): + try: + language_capsule = tree_sitter_ruby.language() + ruby_language = Language(language_capsule) + parser = Parser(ruby_language) + tree = parser.parse(bytes(self.content, "utf8")) + root = tree.root_node + lines = self.content.splitlines() + + self._extract_nodes(root, [], lines, 0) + self._extract_relationships(root, None, [], 0) + except RecursionError: + logger.error(f"Recursion limit hit parsing Ruby file {self.file_path}") + except Exception as e: # noqa: BLE001 — a broken file must not abort the sweep + logger.error(f"Error parsing Ruby file {self.file_path}: {e}") + + # ------------------------------------------------------------------ + # Pass 1: components + # ------------------------------------------------------------------ + + def _extract_nodes(self, node, scope: list[str], lines, depth: int): + if depth > MAX_RECURSION_DEPTH: + return + + if node.type in ("class", "module"): + name = self._constant_text(node.child_by_field_name("name")) + if name: + bare_name = name.split(".")[-1] + component_type = node.type + base_classes = None + if node.type == "class": + superclass = self._superclass_name(node) + if superclass: + base_classes = [superclass] + self._add_node( + node, + bare_name, + component_type, + lines, + base_classes=base_classes, + qualified_name=".".join(scope + [name]), + ) + body = node.child_by_field_name("body") + if body is not None: + for child in body.children: + self._extract_nodes(child, scope + [bare_name], lines, depth + 1) + return + + elif node.type == "singleton_class": + # `class << self` — its methods are singleton methods of the + # enclosing class, so keep the current scope. + for child in node.children: + self._extract_nodes(child, scope, lines, depth + 1) + return + + elif node.type == "method": + name_node = node.child_by_field_name("name") + method_name = name_node.text.decode() if name_node is not None else None + if method_name: + self._add_method_node(node, method_name, scope, lines) + return + + elif node.type == "singleton_method": + name_node = node.child_by_field_name("name") + object_node = node.child_by_field_name("object") + method_name = name_node.text.decode() if name_node is not None else None + if method_name: + owner_scope = scope + if object_node is not None and object_node.type in ("constant", "scope_resolution"): + owner = self._constant_text(object_node) + if owner: + owner_scope = [owner.split(".")[-1]] + self._add_method_node(node, method_name, owner_scope, lines) + return + + for child in node.children: + self._extract_nodes(child, scope, lines, depth + 1) + + def _add_method_node(self, node, method_name: str, scope: list[str], lines): + if scope: + owner = scope[-1] + logical_name = f"{owner}.{method_name}" + component_type = "method" + class_name = owner + else: + logical_name = method_name + component_type = "function" + class_name = None + + parameters = self._method_parameters(node) + # Constructors are registered for call resolution but are not + # documentable components themselves. + include_in_nodes = method_name != "initialize" + self._add_node( + node, + logical_name, + component_type, + lines, + parameters=parameters, + class_name=class_name, + include_in_nodes=include_in_nodes, + ) + + def _add_node( + self, + node, + logical_name: str, + component_type: str, + lines, + parameters: list[str] | None = None, + base_classes: list[str] | None = None, + class_name: str | None = None, + qualified_name: str | None = None, + include_in_nodes: bool = True, + ): + component_id = self._get_component_id(logical_name) + relative_path = self._get_relative_path() + + docstring = "" + comment = node.prev_sibling + if comment is None and node.parent is not None and node.parent.type == "body_statement": + # A comment before the first statement of a class/module body is + # attached as a sibling of the body itself. + comment = node.parent.prev_sibling + if comment is not None and comment.type == "comment": + docstring = comment.text.decode().strip() + + start_line_idx = node.start_point[0] + end_line_idx = node.end_point[0] + 1 + code_snippet = ( + "\n".join(lines[start_line_idx:end_line_idx]) if start_line_idx < len(lines) else "" + ) + + node_obj = Node( + id=component_id, + name=logical_name, + component_type=component_type, + file_path=str(self.file_path), + relative_path=relative_path, + source_code=code_snippet, + start_line=node.start_point[0] + 1, + end_line=node.end_point[0] + 1, + has_docstring=bool(docstring), + docstring=docstring, + parameters=parameters, + node_type=component_type, + base_classes=base_classes, + class_name=class_name, + display_name=f"{component_type} {logical_name}", + component_id=component_id, + language="ruby", + qualified_name=qualified_name or logical_name, + ) + if include_in_nodes: + self.nodes.append(node_obj) + self.top_level_nodes[logical_name] = node_obj + + def _method_parameters(self, method_node) -> list[str] | None: + params_node = method_node.child_by_field_name("parameters") + if params_node is None: + return None + params = [] + for child in params_node.children: + if child.type in ( + "identifier", + "optional_parameter", + "keyword_parameter", + "splat_parameter", + "hash_splat_parameter", + "block_parameter", + ): + params.append(child.text.decode()) + return params or None + + # ------------------------------------------------------------------ + # Pass 2: relationships + # ------------------------------------------------------------------ + + def _extract_relationships(self, node, caller: str | None, scope: list[str], depth: int): + if depth > MAX_RECURSION_DEPTH: + return + + if node.type in ("class", "module"): + name = self._constant_text(node.child_by_field_name("name")) + if name: + bare_name = name.split(".")[-1] + if node.type == "class": + superclass = self._superclass_name(node) + if superclass and superclass not in RUBY_BUILTIN_CONSTANTS: + self._add_relationship( + caller=self._get_component_id(bare_name), + callee_name=superclass, + call_line=node.start_point[0] + 1, + ) + body = node.child_by_field_name("body") + if body is not None: + for child in body.children: + self._extract_relationships( + child, bare_name, scope + [bare_name], depth + 1 + ) + return + + elif node.type in ("method", "singleton_method"): + name_node = node.child_by_field_name("name") + method_name = name_node.text.decode() if name_node is not None else None + if method_name: + owner = scope[-1] if scope else None + if node.type == "singleton_method": + object_node = node.child_by_field_name("object") + if object_node is not None and object_node.type in ( + "constant", + "scope_resolution", + ): + named_owner = self._constant_text(object_node) + if named_owner: + owner = named_owner.split(".")[-1] + logical_name = f"{owner}.{method_name}" if owner else method_name + for child in node.children: + self._extract_relationships(child, logical_name, scope, depth + 1) + return + + elif node.type == "call": + self._extract_call(node, caller, scope) + + for child in node.children: + self._extract_relationships(child, caller, scope, depth + 1) + + def _extract_call(self, node, caller: str | None, scope: list[str]): + if caller is None: + return + + method_node = node.child_by_field_name("method") + receiver = node.child_by_field_name("receiver") + method_name = method_node.text.decode() if method_node is not None else None + if not method_name: + return + + caller_id = self._get_component_id(caller) + call_line = node.start_point[0] + 1 + current_class = scope[-1] if scope else None + + # Mixins: `include Foo` / `extend Foo` / `prepend Foo`. + if receiver is None and method_name in MIXIN_METHODS: + args = node.child_by_field_name("arguments") + if args is not None: + for arg in args.children: + if arg.type in ("constant", "scope_resolution"): + mixin = self._constant_text(arg) + if mixin and mixin.split(".")[-1] not in RUBY_BUILTIN_CONSTANTS: + self._add_relationship(caller_id, mixin, call_line) + return + + # `require_relative "path/to/file"` — best-effort unresolved edge on + # the basename; plain `require` of gems is dropped with the core set. + if receiver is None and method_name == "require_relative": + target = self._string_argument(node) + if target: + self._add_relationship(caller_id, os.path.basename(target), call_line) + return + + if receiver is None: + # Implicit-self call: try the enclosing class's methods, then + # same-file top-level definitions. + if method_name in RUBY_CORE_METHODS: + return + self._add_relationship(caller_id, method_name, call_line, owner_class=current_class) + return + + if receiver.type == "self": + if method_name in RUBY_CORE_METHODS: + return + self._add_relationship(caller_id, method_name, call_line, owner_class=current_class) + return + + if receiver.type in ("constant", "scope_resolution"): + const_name = self._constant_text(receiver) + if not const_name: + return + bare_const = const_name.split(".")[-1] + if bare_const in RUBY_BUILTIN_CONSTANTS: + return + if method_name == "new": + # Instantiation is an edge to the class itself. + if bare_const in self.top_level_nodes: + self._add_relationship_raw( + caller_id, self.top_level_nodes[bare_const].id, call_line, True + ) + else: + self._add_relationship_raw(caller_id, const_name, call_line, False) + return + logical = f"{bare_const}.{method_name}" + if logical in self.top_level_nodes: + self._add_relationship_raw( + caller_id, self.top_level_nodes[logical].id, call_line, True + ) + else: + self._add_relationship_raw( + caller_id, f"{const_name}.{method_name}", call_line, False + ) + return + + if receiver.type == "identifier": + receiver_name = receiver.text.decode() + receiver_class = self._receiver_class(node, receiver_name) + if receiver_class: + logical = f"{receiver_class}.{method_name}" + if logical in self.top_level_nodes: + self._add_relationship_raw( + caller_id, self.top_level_nodes[logical].id, call_line, True + ) + return + self._add_relationship_raw(caller_id, logical, call_line, False) + return + if method_name in RUBY_CORE_METHODS: + return + self._add_relationship_raw( + caller_id, f"{receiver_name}.{method_name}", call_line, False + ) + return + + # Composite receiver (a call chain, literal, ...): keep only the bare + # method name, and only when it can't be core-library noise. + if method_name not in RUBY_CORE_METHODS: + self._add_relationship_raw(caller_id, method_name, call_line, False) + + def _add_relationship( + self, + caller: str, + callee_name: str, + call_line: int, + owner_class: str | None = None, + ): + """Emit an edge, resolving against the same-file symbol table.""" + candidates = [] + if owner_class: + candidates.append(f"{owner_class}.{callee_name}") + candidates.append(callee_name) + bare = callee_name.split(".")[-1] + if bare != callee_name: + candidates.append(bare) + + for candidate in candidates: + if candidate in self.top_level_nodes: + self._add_relationship_raw( + caller, self.top_level_nodes[candidate].id, call_line, True + ) + return + # Unresolved callees stay bare logical names — the global resolver + # matches them against name indexes afterwards. + self._add_relationship_raw(caller, callee_name, call_line, False) + + def _add_relationship_raw(self, caller: str, callee: str, call_line: int, resolved: bool): + key = (caller, callee, call_line) + if key in self.seen_relationships or caller == callee: + return + self.seen_relationships.add(key) + self.call_relationships.append( + CallRelationship( + caller=caller, + callee=callee, + call_line=call_line, + is_resolved=resolved, + ) + ) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _constant_text(self, node) -> str | None: + """Flatten a constant / Foo::Bar scope_resolution to dotted text.""" + if node is None: + return None + if node.type == "constant": + return node.text.decode() + if node.type == "scope_resolution": + return node.text.decode().replace("::", ".").lstrip(".") + return None + + def _superclass_name(self, class_node) -> str | None: + superclass_node = class_node.child_by_field_name("superclass") + if superclass_node is None: + return None + for child in superclass_node.children: + name = self._constant_text(child) + if name: + return name.split(".")[-1] + return None + + def _string_argument(self, call_node) -> str | None: + args = call_node.child_by_field_name("arguments") + if args is None: + return None + for arg in args.children: + if arg.type == "string": + for part in arg.children: + if part.type == "string_content": + return part.text.decode() + return None + + def _receiver_class(self, call_node, receiver_name: str) -> str | None: + """Walk the enclosing method for `receiver_name = Const.new` to infer + the receiver's class.""" + scope_node = call_node.parent + while scope_node is not None and scope_node.type not in ("method", "singleton_method"): + scope_node = scope_node.parent + if scope_node is None: + return None + return self._find_new_assignment(scope_node, receiver_name, 0) + + def _find_new_assignment(self, node, variable_name: str, depth: int) -> str | None: + if depth > MAX_RECURSION_DEPTH: + return None + if node.type == "assignment": + left = node.child_by_field_name("left") + right = node.child_by_field_name("right") + if ( + left is not None + and left.type == "identifier" + and left.text.decode() == variable_name + and right is not None + and right.type == "call" + ): + receiver = right.child_by_field_name("receiver") + method = right.child_by_field_name("method") + if ( + receiver is not None + and receiver.type in ("constant", "scope_resolution") + and method is not None + and method.text.decode() == "new" + ): + const_name = self._constant_text(receiver) + if const_name: + return const_name.split(".")[-1] + for child in node.children: + result = self._find_new_assignment(child, variable_name, depth + 1) + if result: + return result + return None + + +def analyze_ruby_file( + file_path: str, content: str, repo_path: str | None = None +) -> tuple[list[Node], list[CallRelationship]]: + analyzer = TreeSitterRubyAnalyzer(file_path, content, repo_path) + return analyzer.nodes, analyzer.call_relationships diff --git a/codewiki/src/be/dependency_analyzer/ast_parser.py b/codewiki/src/be/dependency_analyzer/ast_parser.py index 1a402582..a0aaffa6 100644 --- a/codewiki/src/be/dependency_analyzer/ast_parser.py +++ b/codewiki/src/be/dependency_analyzer/ast_parser.py @@ -1,33 +1,27 @@ -import os import json import logging -import argparse -from dataclasses import dataclass, field -from typing import Dict, List, Set, Tuple, Optional, Any, Union -from pathlib import Path -import re +import os from codewiki.src.be.dependency_analyzer.analysis.analysis_service import AnalysisService from codewiki.src.be.dependency_analyzer.models.core import Node - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) class DependencyParser: """Parser for extracting code components from multi-language repositories.""" - + def __init__( self, repo_path: str, - include_patterns: List[str] = None, - exclude_patterns: List[str] = None, + include_patterns: list[str] | None = None, + exclude_patterns: list[str] | None = None, use_gitignore: bool = True, ): """ Initialize the dependency parser. - + Args: repo_path: Path to the repository include_patterns: File patterns to include (e.g., ["*.cs", "*.py"]) @@ -35,55 +29,56 @@ def __init__( use_gitignore: Whether to apply Git ignore rules """ self.repo_path = os.path.abspath(repo_path) - self.components: Dict[str, Node] = {} - self.modules: Set[str] = set() + self.components: dict[str, Node] = {} + self.modules: set[str] = set() self.include_patterns = include_patterns self.exclude_patterns = exclude_patterns self.use_gitignore = use_gitignore - + self.analysis_service = AnalysisService() - def parse_repository(self, filtered_folders: List[str] = None) -> Dict[str, Node]: + def parse_repository(self, filtered_folders: list[str] | None = None) -> dict[str, Node]: logger.debug(f"Parsing repository at {self.repo_path}") - + # Log custom patterns if set if self.include_patterns: logger.info(f"Using custom include patterns: {self.include_patterns}") if self.exclude_patterns: logger.info(f"Using custom exclude patterns: {self.exclude_patterns}") - + structure_result = self.analysis_service._analyze_structure( - self.repo_path, + self.repo_path, include_patterns=self.include_patterns, exclude_patterns=self.exclude_patterns, use_gitignore=self.use_gitignore, ) - + call_graph_result = self.analysis_service._analyze_call_graph( - structure_result["file_tree"], - self.repo_path + structure_result["file_tree"], self.repo_path ) - + self._build_components_from_analysis(call_graph_result) - + logger.debug(f"Found {len(self.components)} components across {len(self.modules)} modules") return self.components - - def _build_components_from_analysis(self, call_graph_result: Dict): + + def _build_components_from_analysis(self, call_graph_result: dict): functions = call_graph_result.get("functions", []) relationships = call_graph_result.get("relationships", []) - + component_id_mapping = {} - + for func_dict in functions: component_id = func_dict.get("id", "") if not component_id: continue - + node = Node( id=component_id, name=func_dict.get("name", ""), - component_type=func_dict.get("component_type", func_dict.get("node_type", "function")), + component_type=func_dict.get( + "component_type", func_dict.get("node_type", "function") + ), file_path=func_dict.get("file_path", ""), relative_path=func_dict.get("relative_path", ""), source_code=func_dict.get("source_code", func_dict.get("code_snippet", "")), @@ -96,16 +91,16 @@ def _build_components_from_analysis(self, call_graph_result: Dict): base_classes=func_dict.get("base_classes"), class_name=func_dict.get("class_name"), display_name=func_dict.get("display_name", ""), - component_id=component_id + component_id=component_id, ) - + self.components[component_id] = node - + component_id_mapping[component_id] = component_id legacy_id = f"{func_dict.get('file_path', '')}:{func_dict.get('name', '')}" if legacy_id and legacy_id != component_id: component_id_mapping[legacy_id] = component_id - + if "::" in component_id: file_path_part = component_id.split("::")[0] if file_path_part: @@ -115,60 +110,91 @@ def _build_components_from_analysis(self, call_graph_result: Dict): module_path = ".".join(module_parts) if module_path: self.modules.add(module_path) - + processed_relationships = 0 for rel_dict in relationships: caller_id = rel_dict.get("caller", "") callee_id = rel_dict.get("callee", "") - is_resolved = rel_dict.get("is_resolved", False) - + caller_component_id = component_id_mapping.get(caller_id) - + callee_component_id = component_id_mapping.get(callee_id) if not callee_component_id: for comp_id, comp_node in self.components.items(): if comp_node.name == callee_id: callee_component_id = comp_id break - - if caller_component_id and caller_component_id in self.components: - if callee_component_id: - self.components[caller_component_id].depends_on.add(callee_component_id) - processed_relationships += 1 - - def _determine_component_type(self, func_dict: Dict) -> str: + + if ( + caller_component_id + and caller_component_id in self.components + and callee_component_id + ): + self.components[caller_component_id].depends_on.add(callee_component_id) + processed_relationships += 1 + + def _determine_component_type(self, func_dict: dict) -> str: if func_dict.get("is_method", False): return "method" - + node_type = func_dict.get("node_type", "") - if node_type in ["class", "interface", "struct", "enum", "record", "abstract class", "annotation", "delegate"]: + if node_type in [ + "class", + "interface", + "struct", + "enum", + "record", + "abstract class", + "annotation", + "delegate", + ]: return node_type - + return "function" - + def _file_to_module_path(self, file_path: str) -> str: path = file_path - extensions = ['.py', '.js', '.ts', '.java', '.cs', '.cpp', '.hpp', '.h', '.c', '.tsx', '.jsx', '.cc', '.mjs', '.cxx', '.cc', '.cjs', '.kt', '.kts'] + extensions = [ + ".py", + ".js", + ".ts", + ".java", + ".cs", + ".cpp", + ".hpp", + ".h", + ".c", + ".tsx", + ".jsx", + ".cc", + ".mjs", + ".cxx", + ".cc", + ".cjs", + ".kt", + ".kts", + ".rb", + ] for ext in extensions: if path.endswith(ext): - path = path[:-len(ext)] + path = path[: -len(ext)] break return path.replace(os.path.sep, ".") - + def save_dependency_graph(self, output_path: str): result = {} for component_id, component in self.components.items(): component_dict = component.model_dump() - if 'depends_on' in component_dict and isinstance(component_dict['depends_on'], set): - component_dict['depends_on'] = list(component_dict['depends_on']) + if "depends_on" in component_dict and isinstance(component_dict["depends_on"], set): + component_dict["depends_on"] = list(component_dict["depends_on"]) result[component_id] = component_dict - + dir_name = os.path.dirname(output_path) if dir_name: os.makedirs(dir_name, exist_ok=True) - - with open(output_path, 'w', encoding='utf-8') as f: + + with open(output_path, "w", encoding="utf-8") as f: json.dump(result, f, indent=2, ensure_ascii=False) - + logger.debug(f"Saved {len(self.components)} components to {output_path}") return result diff --git a/codewiki/src/be/dependency_analyzer/utils/patterns.py b/codewiki/src/be/dependency_analyzer/utils/patterns.py index 0e96652e..32472dd0 100644 --- a/codewiki/src/be/dependency_analyzer/utils/patterns.py +++ b/codewiki/src/be/dependency_analyzer/utils/patterns.py @@ -5,15 +5,12 @@ and function definitions across multiple programming languages. """ -from typing import List, Dict, Optional - DEFAULT_IGNORE_PATTERNS = { ".github", ".vscode", ".git", ".gitignore", ".gitmodules", - ".gitignore", "examples", # Python "*.pyc", @@ -74,12 +71,9 @@ # Go / .NET / C# "bin/", # Version control - ".git", ".svn", ".hg", - ".gitignore", ".gitattributes", - ".gitmodules", # Images and media "*.svg", "*.png", @@ -100,7 +94,6 @@ "virtualenv", # IDEs and editors ".idea", - ".vscode", ".vs", "*.swo", "*.swn", @@ -140,7 +133,6 @@ "test", "Tests", "Test", - "examples", "Examples", } @@ -274,6 +266,12 @@ "console", # Symfony CLI "server.php", "start.php", + # Ruby + "main.rb", + "app.rb", + "application.rb", + "config.ru", + "Rakefile", } # Additional entry point path patterns (for when filename patterns fail) @@ -418,8 +416,20 @@ "rust": ["fn {name}", "pub fn {name}"], "c": ["void {name}", "int {name}", "{name}("], "cpp": ["void {name}", "int {name}", "{name}("], - "php": ["function {name}", "public function {name}", "private function {name}", "protected function {name}"], - "kotlin": ["fun {name}", "private fun {name}", "public fun {name}", "internal fun {name}", "protected fun {name}"], + "php": [ + "function {name}", + "public function {name}", + "private function {name}", + "protected function {name}", + ], + "kotlin": [ + "fun {name}", + "private fun {name}", + "public fun {name}", + "internal fun {name}", + "protected fun {name}", + ], + "ruby": ["def {name}", "def self.{name}"], "general": ["{name}("], # Fallback pattern } @@ -540,13 +550,10 @@ def has_high_connectivity_potential(filename: str, filepath: str) -> bool: return True # Check source directory patterns - if any(pattern in filepath_lower for pattern in SOURCE_DIRECTORY_PATTERNS): - return True - - return False + return bool(any(pattern in filepath_lower for pattern in SOURCE_DIRECTORY_PATTERNS)) -def is_critical_function(func_name: str, code_snippet: Optional[str] = None) -> bool: +def is_critical_function(func_name: str, code_snippet: str | None = None) -> bool: """ Check if a function is critical based on name and code patterns. @@ -570,7 +577,7 @@ def is_critical_function(func_name: str, code_snippet: Optional[str] = None) -> return False -def find_fallback_entry_points(code_files: List[Dict], max_files: int = 5) -> List[Dict]: +def find_fallback_entry_points(code_files: list[dict], max_files: int = 5) -> list[dict]: """ Find fallback entry points when standard patterns don't match. @@ -589,11 +596,9 @@ def find_fallback_entry_points(code_files: List[Dict], max_files: int = 5) -> Li filepath = file_info["path"].lower() # Check for any main-like files - if any(pattern in filename for pattern in ["main", "app", "server", "start", "index"]): - fallback_files.append(file_info) - - # Check for entry point paths - elif is_entry_point_path(filepath): + if any( + pattern in filename for pattern in ["main", "app", "server", "start", "index"] + ) or is_entry_point_path(filepath): fallback_files.append(file_info) # If still nothing, try files in root or common directories @@ -625,7 +630,7 @@ def fallback_priority(file_info): return fallback_files[:max_files] -def find_fallback_connectivity_files(code_files: List[Dict], max_files: int = 10) -> List[Dict]: +def find_fallback_connectivity_files(code_files: list[dict], max_files: int = 10) -> list[dict]: """ Find fallback high-connectivity files when standard patterns don't match. @@ -652,9 +657,10 @@ def find_fallback_connectivity_files(code_files: List[Dict], max_files: int = 10 if file_info not in fallback_files: name = file_info["name"].lower() # Include common source file extensions - if any(ext in name for ext in [".py", ".js", ".ts", ".go", ".rs", ".c", ".cpp"]): - # Skip test files - if not any(test_pattern in name for test_pattern in ["test", "spec", "_test"]): - fallback_files.append(file_info) + # Include common source file extensions, skipping test files + if any( + ext in name for ext in [".py", ".js", ".ts", ".go", ".rs", ".c", ".cpp"] + ) and not any(test_pattern in name for test_pattern in ["test", "spec", "_test"]): + fallback_files.append(file_info) return fallback_files[:max_files] diff --git a/codewiki/src/be/prompt_template.py b/codewiki/src/be/prompt_template.py index 95e0b593..10f70bcb 100644 --- a/codewiki/src/be/prompt_template.py +++ b/codewiki/src/be/prompt_template.py @@ -308,6 +308,7 @@ ".php": "php", ".phtml": "php", ".inc": "php", + ".rb": "ruby", } diff --git a/pyproject.toml b/pyproject.toml index 93150ecf..2675acfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "tree-sitter-c-sharp>=0.23.1", "tree-sitter-php>=0.23.0", "tree-sitter-kotlin>=1.1.0", + "tree-sitter-ruby>=0.23.1", "openai>=1.107.0", "litellm>=1.77.0", "pydantic>=2.11.7", diff --git a/requirements.txt b/requirements.txt index e2dce481..a38713ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -150,6 +150,7 @@ tree-sitter-javascript==0.21.4 tree-sitter-kotlin==1.1.0 tree-sitter-language-pack==0.8.0 tree-sitter-python==0.23.6 +tree-sitter-ruby==0.23.1 tree-sitter-typescript==0.21.2 tree-sitter-yaml==0.7.1 types-protobuf==6.30.2.20250822 diff --git a/tests/test_ruby_analyzer.py b/tests/test_ruby_analyzer.py new file mode 100644 index 00000000..3e6653f1 --- /dev/null +++ b/tests/test_ruby_analyzer.py @@ -0,0 +1,127 @@ +"""Tests for the tree-sitter based Ruby analyzer.""" + +from pathlib import Path + +import pytest + +pytest.importorskip("tree_sitter_ruby") + +from codewiki.src.be.dependency_analyzer.analyzers.ruby import analyze_ruby_file +from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser + +SAMPLE = """\ +require "json" +require_relative "helpers/formatter" + +module Pipeline + # Buffers events before flushing them downstream. + class Buffer < BaseBuffer + include Enumerable + include Flushable + + def initialize(size) + @size = size + @items = [] + end + + def push(event) + validate(event) + @items.push(event) + Formatter.render(event) + end + + def validate(event) + raise ArgumentError if event.nil? + end + + def self.build(size) + Buffer.new(size) + end + end + + def self.default_buffer + Buffer.new(10) + end +end + +def standalone_helper(value) + value.to_s +end +""" + + +def _analyze(tmp_path: Path): + file_path = tmp_path / "buffer.rb" + file_path.write_text(SAMPLE, encoding="utf-8") + return analyze_ruby_file(str(file_path), SAMPLE, repo_path=str(tmp_path)) + + +def test_extracts_modules_classes_and_methods(tmp_path: Path) -> None: + nodes, _ = _analyze(tmp_path) + by_name = {node.name: node for node in nodes} + + assert by_name["Pipeline"].component_type == "module" + assert by_name["Buffer"].component_type == "class" + assert by_name["Buffer"].base_classes == ["BaseBuffer"] + assert by_name["Buffer.push"].component_type == "method" + assert by_name["Buffer.push"].class_name == "Buffer" + assert by_name["Buffer.build"].component_type == "method" + assert by_name["Pipeline.default_buffer"].component_type == "method" + assert by_name["standalone_helper"].component_type == "function" + + # Constructors are not documentable components. + assert "Buffer.initialize" not in by_name + + assert by_name["Buffer"].id == "buffer.rb::Buffer" + assert by_name["Buffer.push"].id == "buffer.rb::Buffer.push" + assert all(node.language == "ruby" for node in nodes) + + +def test_extracts_docstring_and_parameters(tmp_path: Path) -> None: + nodes, _ = _analyze(tmp_path) + by_name = {node.name: node for node in nodes} + + assert by_name["Buffer"].has_docstring + assert "Buffers events" in by_name["Buffer"].docstring + assert by_name["Buffer.push"].parameters == ["event"] + + +def test_extracts_call_relationships(tmp_path: Path) -> None: + _, relationships = _analyze(tmp_path) + edges = {(rel.caller, rel.callee, rel.is_resolved) for rel in relationships} + + # Inheritance and mixins (BaseBuffer / Flushable live in another file). + assert ("buffer.rb::Buffer", "BaseBuffer", False) in edges + assert ("buffer.rb::Buffer", "Flushable", False) in edges + # Builtin mixins are dropped. + assert not any(rel.callee == "Enumerable" for rel in relationships) + + # Intra-class implicit-self call resolves to the sibling method. + assert ("buffer.rb::Buffer.push", "buffer.rb::Buffer.validate", True) in edges + # Constant-receiver call to an out-of-file class stays a bare logical name. + assert ("buffer.rb::Buffer.push", "Formatter.render", False) in edges + # Instantiation points at the class, resolved within the same file. + assert ("buffer.rb::Buffer.build", "buffer.rb::Buffer", True) in edges + assert ("buffer.rb::Pipeline.default_buffer", "buffer.rb::Buffer", True) in edges + + # Kernel noise like `raise` and `to_s` is not emitted. + assert not any(rel.callee.endswith("raise") for rel in relationships) + assert not any(rel.callee.endswith("to_s") for rel in relationships) + + +def test_dependency_parser_end_to_end(tmp_path: Path) -> None: + (tmp_path / "base_buffer.rb").write_text( + "class BaseBuffer\n def flush\n end\nend\n", encoding="utf-8" + ) + (tmp_path / "buffer.rb").write_text(SAMPLE, encoding="utf-8") + + components = DependencyParser(str(tmp_path)).parse_repository() + + assert "buffer.rb::Buffer" in components + assert "buffer.rb::Buffer.push" in components + assert "base_buffer.rb::BaseBuffer" in components + + # The cross-file inheritance edge resolves during global resolution. + assert "base_buffer.rb::BaseBuffer" in components["buffer.rb::Buffer"].depends_on + # The intra-file call edge survives into depends_on. + assert "buffer.rb::Buffer.validate" in components["buffer.rb::Buffer.push"].depends_on