feat(slidev): update 3 4
This commit is contained in:
smart
+313
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a source-grounded project inventory for knowledge-base work."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
IGNORE_DIRS = {
|
||||
".git",
|
||||
".idea",
|
||||
".vscode",
|
||||
"node_modules",
|
||||
"dist",
|
||||
"build",
|
||||
"target",
|
||||
".gradle",
|
||||
".mvn/wrapper",
|
||||
"coverage",
|
||||
".next",
|
||||
".nuxt",
|
||||
"__pycache__",
|
||||
}
|
||||
|
||||
MANIFEST_NAMES = {
|
||||
"package.json",
|
||||
"pnpm-lock.yaml",
|
||||
"yarn.lock",
|
||||
"package-lock.json",
|
||||
"pom.xml",
|
||||
"build.gradle",
|
||||
"settings.gradle",
|
||||
"Cargo.toml",
|
||||
"pyproject.toml",
|
||||
"requirements.txt",
|
||||
"go.mod",
|
||||
"composer.json",
|
||||
"Gemfile",
|
||||
}
|
||||
|
||||
CONFIG_PATTERNS = (
|
||||
re.compile(r".*\.env(\..*)?$"),
|
||||
re.compile(r".*application[-\w]*\.(yml|yaml|properties)$"),
|
||||
re.compile(r".*\.(ya?ml|toml|ini|properties|conf)$"),
|
||||
)
|
||||
|
||||
HTTP_MAPPING_RE = re.compile(
|
||||
r"@(RequestMapping|GetMapping|PostMapping|PutMapping|DeleteMapping|PatchMapping)\s*(?:\((.*?)\))?"
|
||||
)
|
||||
PERMISSION_RE = re.compile(r"@(?:RequiresPermissions|RequiresRoles)\s*\((.*?)\)")
|
||||
TABLE_NAME_RE = re.compile(
|
||||
r"\bCREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?`?([A-Za-z0-9_.$-]+)`?",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
VUE_ROUTE_RE = re.compile(r"\b(path|name|component)\s*:\s*['\"]([^'\"]+)['\"]")
|
||||
|
||||
|
||||
def is_ignored(path: Path, root: Path) -> bool:
|
||||
rel_parts = path.relative_to(root).parts
|
||||
for i in range(len(rel_parts)):
|
||||
joined = "/".join(rel_parts[: i + 1])
|
||||
if joined in IGNORE_DIRS or rel_parts[i] in IGNORE_DIRS:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def iter_files(root: Path, max_files: int) -> Iterable[Path]:
|
||||
count = 0
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
current = Path(dirpath)
|
||||
dirnames[:] = [d for d in dirnames if not is_ignored(current / d, root)]
|
||||
for name in filenames:
|
||||
path = current / name
|
||||
if is_ignored(path, root):
|
||||
continue
|
||||
count += 1
|
||||
if count > max_files:
|
||||
return
|
||||
yield path
|
||||
|
||||
|
||||
def rel(path: Path, root: Path) -> str:
|
||||
return path.relative_to(root).as_posix()
|
||||
|
||||
|
||||
def read_text(path: Path, limit: int = 400_000) -> str:
|
||||
try:
|
||||
data = path.read_bytes()
|
||||
except OSError:
|
||||
return ""
|
||||
if len(data) > limit:
|
||||
data = data[:limit]
|
||||
try:
|
||||
return data.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
try:
|
||||
return data.decode("utf-8", errors="ignore")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def detect_config(path: Path) -> bool:
|
||||
name = path.name
|
||||
return any(pattern.fullmatch(name) for pattern in CONFIG_PATTERNS)
|
||||
|
||||
|
||||
def extract_mapping_value(raw: str | None) -> str:
|
||||
if not raw:
|
||||
return ""
|
||||
value_match = re.search(r"['\"]([^'\"]+)['\"]", raw)
|
||||
if value_match:
|
||||
return value_match.group(1)
|
||||
return raw.strip().replace("\n", " ")[:120]
|
||||
|
||||
|
||||
def scan(root: Path, max_files: int) -> dict:
|
||||
files = list(iter_files(root, max_files))
|
||||
ext_counts = Counter(path.suffix.lower() or "[none]" for path in files)
|
||||
manifests = []
|
||||
configs = []
|
||||
docker = []
|
||||
tests = []
|
||||
java_endpoints = []
|
||||
permission_codes = []
|
||||
vue_routes = []
|
||||
views = []
|
||||
sql_tables = []
|
||||
major_dirs = defaultdict(int)
|
||||
|
||||
for path in files:
|
||||
r = rel(path, root)
|
||||
parts = Path(r).parts
|
||||
if parts:
|
||||
major_dirs[parts[0]] += 1
|
||||
if path.name in MANIFEST_NAMES:
|
||||
manifests.append(r)
|
||||
lower = path.name.lower()
|
||||
if lower.startswith("dockerfile") or "docker-compose" in lower:
|
||||
docker.append(r)
|
||||
if detect_config(path):
|
||||
configs.append(r)
|
||||
if re.search(r"(^|/)(test|tests|__tests__)(/|$)", r) or re.search(r"\.(test|spec)\.[A-Za-z0-9]+$", r):
|
||||
tests.append(r)
|
||||
if "/src/views/" in r and path.suffix.lower() in {".vue", ".tsx", ".ts"}:
|
||||
views.append(r)
|
||||
|
||||
suffix = path.suffix.lower()
|
||||
if suffix not in {".java", ".ts", ".tsx", ".vue", ".js", ".jsx", ".sql", ".yml", ".yaml", ".properties"}:
|
||||
continue
|
||||
text = read_text(path)
|
||||
if not text:
|
||||
continue
|
||||
|
||||
if suffix == ".java":
|
||||
lines = text.splitlines()
|
||||
class_mapping = ""
|
||||
class_name = ""
|
||||
pending_annotations: list[tuple[int, str, str]] = []
|
||||
for idx, line in enumerate(lines, start=1):
|
||||
class_match = re.search(r"\bclass\s+([A-Za-z0-9_]+)", line)
|
||||
if class_match:
|
||||
class_name = class_match.group(1)
|
||||
for match in HTTP_MAPPING_RE.finditer(line):
|
||||
value = extract_mapping_value(match.group(2))
|
||||
annotation = match.group(1)
|
||||
pending_annotations.append((idx, annotation, value))
|
||||
if annotation == "RequestMapping" and not class_mapping and ("class " not in line):
|
||||
class_mapping = value
|
||||
if re.search(r"\b(public|private|protected)\b.*\(", line) and pending_annotations:
|
||||
method = line.strip()[:160]
|
||||
for ann_line, annotation, value in pending_annotations:
|
||||
java_endpoints.append(
|
||||
{
|
||||
"file": r,
|
||||
"line": ann_line,
|
||||
"class": class_name,
|
||||
"annotation": annotation,
|
||||
"class_mapping": class_mapping,
|
||||
"mapping": value,
|
||||
"method_signature": method,
|
||||
}
|
||||
)
|
||||
pending_annotations = []
|
||||
for perm in PERMISSION_RE.finditer(line):
|
||||
permission_codes.append({"file": r, "line": idx, "value": perm.group(1).strip()})
|
||||
|
||||
if suffix in {".ts", ".tsx", ".vue", ".js", ".jsx"}:
|
||||
for idx, line in enumerate(text.splitlines(), start=1):
|
||||
if "path:" in line or "component:" in line or "name:" in line:
|
||||
for match in VUE_ROUTE_RE.finditer(line):
|
||||
vue_routes.append({"file": r, "line": idx, "key": match.group(1), "value": match.group(2)})
|
||||
|
||||
if suffix == ".sql":
|
||||
for match in TABLE_NAME_RE.finditer(text):
|
||||
sql_tables.append({"file": r, "table": match.group(1)})
|
||||
|
||||
return {
|
||||
"root": str(root),
|
||||
"file_count": len(files),
|
||||
"extensions": dict(ext_counts.most_common(40)),
|
||||
"major_dirs": dict(sorted(major_dirs.items(), key=lambda item: (-item[1], item[0]))),
|
||||
"manifests": sorted(manifests),
|
||||
"configs": sorted(configs),
|
||||
"docker": sorted(docker),
|
||||
"tests": sorted(tests),
|
||||
"views": sorted(views),
|
||||
"java_endpoints": java_endpoints,
|
||||
"permission_codes": permission_codes,
|
||||
"vue_routes": vue_routes,
|
||||
"sql_tables": sql_tables,
|
||||
}
|
||||
|
||||
|
||||
def write_md(data: dict, out_dir: Path) -> None:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def bullet(items: list[str], limit: int | None = None) -> str:
|
||||
selected = items if limit is None else items[:limit]
|
||||
if not selected:
|
||||
return "- None found\n"
|
||||
body = "".join(f"- `{item}`\n" for item in selected)
|
||||
if limit is not None and len(items) > limit:
|
||||
body += f"- ... {len(items) - limit} more\n"
|
||||
return body
|
||||
|
||||
inventory = [
|
||||
"# Project Inventory\n\n",
|
||||
f"Root: `{data['root']}`\n\n",
|
||||
f"Files scanned: {data['file_count']}\n\n",
|
||||
"## Major Directories\n\n",
|
||||
]
|
||||
inventory.extend(f"- `{name}`: {count} files\n" for name, count in data["major_dirs"].items())
|
||||
inventory.extend(["\n## Extensions\n\n"])
|
||||
inventory.extend(f"- `{name}`: {count}\n" for name, count in data["extensions"].items())
|
||||
inventory.extend(["\n## Manifests\n\n", bullet(data["manifests"])])
|
||||
inventory.extend(["\n## Config Files\n\n", bullet(data["configs"], 200)])
|
||||
inventory.extend(["\n## Docker Files\n\n", bullet(data["docker"])])
|
||||
inventory.extend(["\n## Test Files\n\n", bullet(data["tests"], 200)])
|
||||
(out_dir / "project-inventory.md").write_text("".join(inventory), encoding="utf-8")
|
||||
|
||||
endpoints = ["# Backend Endpoints\n\n"]
|
||||
if data["java_endpoints"]:
|
||||
endpoints.append("| File | Line | Class | Annotation | Class Mapping | Mapping | Method |\n")
|
||||
endpoints.append("| --- | ---: | --- | --- | --- | --- | --- |\n")
|
||||
for item in data["java_endpoints"]:
|
||||
method_signature = item["method_signature"].replace("|", "\\|")
|
||||
endpoints.append(
|
||||
f"| `{item['file']}` | {item['line']} | `{item['class']}` | `{item['annotation']}` | "
|
||||
f"`{item['class_mapping']}` | `{item['mapping']}` | `{method_signature}` |\n"
|
||||
)
|
||||
else:
|
||||
endpoints.append("No Java endpoint annotations found.\n")
|
||||
(out_dir / "backend-endpoints.md").write_text("".join(endpoints), encoding="utf-8")
|
||||
|
||||
routes = ["# Frontend Routes And Views\n\n", "## Views\n\n", bullet(data["views"], 400), "\n## Route-like Literals\n\n"]
|
||||
if data["vue_routes"]:
|
||||
routes.append("| File | Line | Key | Value |\n| --- | ---: | --- | --- |\n")
|
||||
for item in data["vue_routes"][:1000]:
|
||||
routes.append(f"| `{item['file']}` | {item['line']} | `{item['key']}` | `{item['value']}` |\n")
|
||||
if len(data["vue_routes"]) > 1000:
|
||||
routes.append(f"\nTruncated: {len(data['vue_routes']) - 1000} more route-like literals.\n")
|
||||
else:
|
||||
routes.append("No route-like literals found.\n")
|
||||
(out_dir / "frontend-routes.md").write_text("".join(routes), encoding="utf-8")
|
||||
|
||||
configs = ["# Config Reference\n\n", bullet(data["configs"], 500)]
|
||||
(out_dir / "config-reference.md").write_text("".join(configs), encoding="utf-8")
|
||||
|
||||
tables = ["# Database Dictionary\n\n"]
|
||||
if data["sql_tables"]:
|
||||
tables.append("| Table | Source |\n| --- | --- |\n")
|
||||
for item in sorted(data["sql_tables"], key=lambda x: (x["table"], x["file"])):
|
||||
tables.append(f"| `{item['table']}` | `{item['file']}` |\n")
|
||||
else:
|
||||
tables.append("No SQL CREATE TABLE statements found in scanned files.\n")
|
||||
(out_dir / "database-dictionary.md").write_text("".join(tables), encoding="utf-8")
|
||||
|
||||
perms = ["# Permission Codes\n\n"]
|
||||
if data["permission_codes"]:
|
||||
perms.append("| File | Line | Value |\n| --- | ---: | --- |\n")
|
||||
for item in data["permission_codes"]:
|
||||
permission_value = item["value"].replace("|", "\\|")
|
||||
perms.append(f"| `{item['file']}` | {item['line']} | `{permission_value}` |\n")
|
||||
else:
|
||||
perms.append("No permission annotations found.\n")
|
||||
(out_dir / "permission-codes.md").write_text("".join(perms), encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", default=".", help="Repository root to scan")
|
||||
parser.add_argument("--out", required=True, help="Output directory for generated Markdown")
|
||||
parser.add_argument("--json", action="store_true", help="Also write inventory.json")
|
||||
parser.add_argument("--max-files", type=int, default=30000, help="Maximum files to scan")
|
||||
args = parser.parse_args()
|
||||
|
||||
root = Path(args.root).resolve()
|
||||
out_dir = Path(args.out).resolve()
|
||||
data = scan(root, args.max_files)
|
||||
write_md(data, out_dir)
|
||||
if args.json:
|
||||
(out_dir / "inventory.json").write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(f"Wrote project inventory to {out_dir}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user