Update pre-commit to fixed versions, run remod for ck_tile (#2895)

* Fix ruff linter errors

* Fix remod dos2unix command

* Clang format

* Ignore utility in remod

* Run remod

* Specify clang-format version in pre-commit

* Specify ruff version

* Include PoolKernelArgs in reference_pool

* Add calculate_total_elements to reference batched contraction

* Fix calculate_total_elements declaration

* Refactor remod pre-commit hook

* Fix Aquant tests

---------

Co-authored-by: Illia Silin <98187287+illsilin@users.noreply.github.com>
This commit is contained in:
Johannes Graner
2025-10-17 00:29:17 +02:00
committed by GitHub
parent 440358c168
commit d40b50b9d5
77 changed files with 21671 additions and 9858 deletions

View File

@@ -14,43 +14,69 @@ Features:
import argparse
import sys
import os
def run_dependency_parser(args):
from src.enhanced_ninja_parser import main as ninja_main
sys.argv = ["enhanced_ninja_parser.py"] + args
ninja_main()
def run_selective_test_filter(args):
from src.selective_test_filter import main as filter_main
sys.argv = ["selective_test_filter.py"] + args
filter_main()
def main():
parser = argparse.ArgumentParser(description="Unified Ninja Dependency & Selective Testing Tool")
parser = argparse.ArgumentParser(
description="Unified Ninja Dependency & Selective Testing Tool"
)
subparsers = parser.add_subparsers(dest="command", required=True)
# Dependency parsing
parser_parse = subparsers.add_parser("parse", help="Parse build.ninja and generate dependency mapping")
parser_parse = subparsers.add_parser(
"parse", help="Parse build.ninja and generate dependency mapping"
)
parser_parse.add_argument("build_ninja", help="Path to build.ninja")
parser_parse.add_argument("--ninja", help="Path to ninja executable", default="ninja")
parser_parse.add_argument("--workspace-root", help="Path to workspace root", default=None)
parser_parse.add_argument(
"--ninja", help="Path to ninja executable", default="ninja"
)
parser_parse.add_argument(
"--workspace-root", help="Path to workspace root", default=None
)
# Selective testing
parser_test = subparsers.add_parser("select", help="Selective test filtering between git refs")
parser_test = subparsers.add_parser(
"select", help="Selective test filtering between git refs"
)
parser_test.add_argument("depmap_json", help="Path to dependency mapping JSON")
parser_test.add_argument("ref1", help="Source git ref")
parser_test.add_argument("ref2", help="Target git ref")
parser_test.add_argument("--all", action="store_true", help="Include all executables")
parser_test.add_argument("--test-prefix", action="store_true", help="Only include executables starting with 'test_'")
parser_test.add_argument("--output", help="Output JSON file", default="tests_to_run.json")
parser_test.add_argument(
"--all", action="store_true", help="Include all executables"
)
parser_test.add_argument(
"--test-prefix",
action="store_true",
help="Only include executables starting with 'test_'",
)
parser_test.add_argument(
"--output", help="Output JSON file", default="tests_to_run.json"
)
# Code auditing
parser_audit = subparsers.add_parser("audit", help="List all files and their dependent executables")
parser_audit = subparsers.add_parser(
"audit", help="List all files and their dependent executables"
)
parser_audit.add_argument("depmap_json", help="Path to dependency mapping JSON")
# Build optimization
parser_opt = subparsers.add_parser("optimize", help="List affected executables for changed files")
parser_opt = subparsers.add_parser(
"optimize", help="List affected executables for changed files"
)
parser_opt.add_argument("depmap_json", help="Path to dependency mapping JSON")
parser_opt.add_argument("changed_files", nargs="+", help="List of changed files")
@@ -73,9 +99,12 @@ def main():
elif args.command == "audit":
run_selective_test_filter([args.depmap_json, "--audit"])
elif args.command == "optimize":
run_selective_test_filter([args.depmap_json, "--optimize-build"] + args.changed_files)
run_selective_test_filter(
[args.depmap_json, "--optimize-build"] + args.changed_files
)
else:
parser.print_help()
if __name__ == "__main__":
main()

View File

@@ -14,96 +14,100 @@ import re
import os
import sys
import subprocess
from pathlib import Path
from collections import defaultdict
import json
from concurrent.futures import ThreadPoolExecutor, as_completed
import threading
class EnhancedNinjaDependencyParser:
def __init__(self, build_file_path, ninja_executable="ninja"):
self.build_file_path = build_file_path
self.build_dir = os.path.dirname(build_file_path)
self.ninja_executable = ninja_executable
# Core data structures
self.executable_to_objects = {} # exe -> [object_files]
self.object_to_source = {} # object -> primary_source
self.object_to_all_deps = {} # object -> [all_dependencies]
self.object_to_source = {} # object -> primary_source
self.object_to_all_deps = {} # object -> [all_dependencies]
self.file_to_executables = defaultdict(set) # file -> {executables}
# Thread safety
self.lock = threading.Lock()
def parse_dependencies(self):
"""Main method to parse all dependencies."""
print(f"Parsing ninja dependencies from: {self.build_file_path}")
# Step 1: Parse build file for executable -> object mappings
self._parse_build_file()
# Step 2: Get all object files and their dependencies
print(f"Found {len(self.object_to_source)} object files")
print("Extracting detailed dependencies for all object files...")
self._extract_object_dependencies()
# Step 3: Build the final file -> executables mapping
self._build_file_to_executable_mapping()
def _parse_build_file(self):
"""Parse the ninja build file to extract executable -> object mappings."""
print("Parsing ninja build file...")
with open(self.build_file_path, 'r') as f:
with open(self.build_file_path, "r") as f:
content = f.read()
# Parse executable build rules
exe_pattern = r'^build (bin/[^:]+):\s+\S+\s+([^|]+)'
obj_pattern = r'^build ([^:]+\.(?:cpp|cu|hip)\.o):\s+\S+\s+([^\s|]+)'
lines = content.split('\n')
# Parse executable build rules
exe_pattern = r"^build (bin/[^:]+):\s+\S+\s+([^|]+)"
obj_pattern = r"^build ([^:]+\.(?:cpp|cu|hip)\.o):\s+\S+\s+([^\s|]+)"
lines = content.split("\n")
for line in lines:
# Match executable rules
exe_match = re.match(exe_pattern, line)
if exe_match and ('EXECUTABLE' in line or 'test_' in exe_match.group(1) or 'example_' in exe_match.group(1)):
if exe_match and (
"EXECUTABLE" in line
or "test_" in exe_match.group(1)
or "example_" in exe_match.group(1)
):
exe = exe_match.group(1)
deps_part = exe_match.group(2).strip()
object_files = []
for dep in deps_part.split():
if dep.endswith('.o') and not dep.startswith('/'):
if dep.endswith(".o") and not dep.startswith("/"):
object_files.append(dep)
self.executable_to_objects[exe] = object_files
continue
# Match object compilation rules
obj_match = re.match(obj_pattern, line)
if obj_match:
object_file = obj_match.group(1)
source_file = obj_match.group(2)
self.object_to_source[object_file] = source_file
print(f"Found {len(self.executable_to_objects)} executables")
print(f"Found {len(self.object_to_source)} object-to-source mappings")
def _extract_object_dependencies(self):
"""Extract detailed dependencies for all object files using ninja -t deps."""
object_files = list(self.object_to_source.keys())
# Process object files in parallel for better performance
# Process object files in parallel for better performance
if not object_files:
print("No object files found - skipping dependency extraction")
return
max_workers = min(16, len(object_files)) # Limit concurrent processes
with ThreadPoolExecutor(max_workers=max_workers) as executor:
# Submit all object files for processing
future_to_obj = {
executor.submit(self._get_object_dependencies, obj): obj
executor.submit(self._get_object_dependencies, obj): obj
for obj in object_files
}
# Process completed futures
# Process completed futures
completed = 0
for future in as_completed(future_to_obj):
obj_file = future_to_obj[future]
@@ -113,52 +117,52 @@ class EnhancedNinjaDependencyParser:
self.object_to_all_deps[obj_file] = dependencies
completed += 1
if completed % 100 == 0:
print(f"Processed {completed}/{len(object_files)} object files...")
print(
f"Processed {completed}/{len(object_files)} object files..."
)
except Exception as e:
print(f"Error processing {obj_file}: {e}")
print(f"Completed dependency extraction for {len(self.object_to_all_deps)} object files")
print(
f"Completed dependency extraction for {len(self.object_to_all_deps)} object files"
)
def _get_object_dependencies(self, object_file):
"""Get all dependencies for a single object file using ninja -t deps."""
try:
# Run ninja -t deps for this object file
cmd = [self.ninja_executable, "-t", "deps", object_file]
result = subprocess.run(
cmd,
cwd=self.build_dir,
capture_output=True,
text=True,
timeout=30
cmd, cwd=self.build_dir, capture_output=True, text=True, timeout=30
)
if result.returncode != 0:
return []
dependencies = []
lines = result.stdout.strip().split('\n')
lines = result.stdout.strip().split("\n")
for line in lines[1:]: # Skip first line with metadata
line = line.strip()
if line and not line.startswith('#'):
if line and not line.startswith("#"):
# Convert absolute paths to relative paths from workspace root
dep_file = line
ws_root = getattr(self, "workspace_root", "..")
ws_prefix = ws_root.rstrip("/") + "/"
if dep_file.startswith(ws_prefix):
dep_file = dep_file[len(ws_prefix):]
dep_file = dep_file[len(ws_prefix) :]
dependencies.append(dep_file)
return dependencies
except Exception as e:
print(f"Error getting dependencies for {object_file}: {e}")
return []
def _build_file_to_executable_mapping(self):
"""Build the final mapping from files to executables."""
print("Building file-to-executable mapping...")
for exe, object_files in self.executable_to_objects.items():
for obj_file in object_files:
# Add all dependencies of this object file
@@ -167,106 +171,135 @@ class EnhancedNinjaDependencyParser:
# Filter out system files and focus on project files
if self._is_project_file(dep_file):
self.file_to_executables[dep_file].add(exe)
print(f"Built mapping for {len(self.file_to_executables)} files")
# Show statistics
multi_exe_files = {f: exes for f, exes in self.file_to_executables.items() if len(exes) > 1}
multi_exe_files = {
f: exes for f, exes in self.file_to_executables.items() if len(exes) > 1
}
print(f"Files used by multiple executables: {len(multi_exe_files)}")
if multi_exe_files:
print("Sample files with multiple dependencies:")
for f, exes in sorted(multi_exe_files.items())[:5]:
print(f" {f}: {len(exes)} executables")
def _is_project_file(self, file_path):
"""Determine if a file is part of the project (not system files)."""
# Include files that are clearly part of the project
if any(file_path.startswith(prefix) for prefix in [
'include/', 'library/', 'test/', 'example/', 'src/', 'profiler/',
'build/include/', 'build/_deps/gtest', 'client_example', 'codegen', 'tile_engine'
]):
if any(
file_path.startswith(prefix)
for prefix in [
"include/",
"library/",
"test/",
"example/",
"src/",
"profiler/",
"build/include/",
"build/_deps/gtest",
"client_example",
"codegen",
"tile_engine",
]
):
return True
# Exclude system files
if any(file_path.startswith(prefix) for prefix in [
'/usr/', '/opt/rocm', '/lib/', '/system/', '/local/'
]):
if any(
file_path.startswith(prefix)
for prefix in ["/usr/", "/opt/rocm", "/lib/", "/system/", "/local/"]
):
return False
# Include files with common source/header extensions
if file_path.endswith(('.cpp', '.hpp', '.h', '.c', '.cc', '.cxx', '.cu', '.hip', '.inc')):
if file_path.endswith(
(".cpp", ".hpp", ".h", ".c", ".cc", ".cxx", ".cu", ".hip", ".inc")
):
return True
return False
def export_to_csv(self, output_file):
"""Export the file-to-executable mapping to CSV with proper comma separation."""
print(f"Exporting mapping to {output_file}")
with open(output_file, 'w') as f:
with open(output_file, "w") as f:
f.write("source_file,executables\n")
for file_path in sorted(self.file_to_executables.keys()):
executables = sorted(self.file_to_executables[file_path])
# Use semicolon to separate multiple executables within the field
exe_list = ';'.join(executables)
exe_list = ";".join(executables)
f.write(f'"{file_path}","{exe_list}"\n')
def export_to_json(self, output_file):
"""Export the complete mapping to JSON."""
print(f"Exporting complete mapping to {output_file}")
# Build reverse mapping (executable -> files)
exe_to_files = defaultdict(set)
for file_path, exes in self.file_to_executables.items():
for exe in exes:
exe_to_files[exe].add(file_path)
mapping_data = {
'file_to_executables': {
file_path: list(exes) for file_path, exes in self.file_to_executables.items()
"file_to_executables": {
file_path: list(exes)
for file_path, exes in self.file_to_executables.items()
},
'executable_to_files': {
"executable_to_files": {
exe: sorted(files) for exe, files in exe_to_files.items()
},
'statistics': {
'total_files': len(self.file_to_executables),
'total_executables': len(self.executable_to_objects),
'total_object_files': len(self.object_to_source),
'files_with_multiple_executables': len([f for f, exes in self.file_to_executables.items() if len(exes) > 1])
}
"statistics": {
"total_files": len(self.file_to_executables),
"total_executables": len(self.executable_to_objects),
"total_object_files": len(self.object_to_source),
"files_with_multiple_executables": len(
[f for f, exes in self.file_to_executables.items() if len(exes) > 1]
),
},
}
with open(output_file, 'w') as f:
with open(output_file, "w") as f:
json.dump(mapping_data, f, indent=2)
def print_summary(self):
"""Print a summary of the parsed dependencies."""
"""Print a summary of the parsed dependencies."""
print("\n=== Enhanced Dependency Mapping Summary ===")
print(f"Total executables: {len(self.executable_to_objects)}")
print(f"Total files mapped: {len(self.file_to_executables)}")
print(f"Total object files processed: {len(self.object_to_all_deps)}")
# Files by type
cpp_files = sum(1 for f in self.file_to_executables.keys() if f.endswith('.cpp'))
hpp_files = sum(1 for f in self.file_to_executables.keys() if f.endswith('.hpp'))
h_files = sum(1 for f in self.file_to_executables.keys() if f.endswith('.h'))
print(f"\nFile types:")
cpp_files = sum(
1 for f in self.file_to_executables.keys() if f.endswith(".cpp")
)
hpp_files = sum(
1 for f in self.file_to_executables.keys() if f.endswith(".hpp")
)
h_files = sum(1 for f in self.file_to_executables.keys() if f.endswith(".h"))
print("\nFile types:")
print(f" .cpp files: {cpp_files}")
print(f" .hpp files: {hpp_files}")
print(f" .h files: {h_files}")
# Multi-executable files
multi_exe_files = {f: exes for f, exes in self.file_to_executables.items() if len(exes) > 1}
multi_exe_files = {
f: exes for f, exes in self.file_to_executables.items() if len(exes) > 1
}
print(f"\nFiles used by multiple executables: {len(multi_exe_files)}")
if multi_exe_files:
print("\nTop files with most dependencies:")
sorted_multi = sorted(multi_exe_files.items(), key=lambda x: len(x[1]), reverse=True)
sorted_multi = sorted(
multi_exe_files.items(), key=lambda x: len(x[1]), reverse=True
)
for file_path, exes in sorted_multi[:10]:
print(f" {file_path}: {len(exes)} executables")
def main():
# Accept: build_file, ninja_path, workspace_root
default_workspace_root = ".."
@@ -304,15 +337,16 @@ def main():
# Export results
output_dir = os.path.dirname(build_file)
csv_file = os.path.join(output_dir, 'enhanced_file_executable_mapping.csv')
json_file = os.path.join(output_dir, 'enhanced_dependency_mapping.json')
csv_file = os.path.join(output_dir, "enhanced_file_executable_mapping.csv")
json_file = os.path.join(output_dir, "enhanced_dependency_mapping.json")
parser.export_to_csv(csv_file)
parser.export_to_json(json_file)
print(f"\nResults exported to:")
print("\nResults exported to:")
print(f" CSV: {csv_file}")
print(f" JSON: {json_file}")
if __name__ == "__main__":
main()

View File

@@ -30,12 +30,15 @@ import subprocess
import json
import os
def get_changed_files(ref1, ref2):
"""Return a set of files changed between two git refs."""
try:
result = subprocess.run(
["git", "diff", "--name-only", ref1, ref2],
capture_output=True, text=True, check=True
capture_output=True,
text=True,
check=True,
)
files = set(line.strip() for line in result.stdout.splitlines() if line.strip())
return files
@@ -43,6 +46,7 @@ def get_changed_files(ref1, ref2):
print(f"Error running git diff: {e}")
sys.exit(1)
def load_depmap(depmap_json):
"""Load the dependency mapping JSON."""
with open(depmap_json, "r") as f:
@@ -52,6 +56,7 @@ def load_depmap(depmap_json):
return data["file_to_executables"]
return data
def select_tests(file_to_executables, changed_files, filter_mode):
"""Return a set of test executables affected by changed files."""
affected = set()
@@ -64,6 +69,7 @@ def select_tests(file_to_executables, changed_files, filter_mode):
affected.add(exe)
return sorted(affected)
def main():
if "--audit" in sys.argv:
if len(sys.argv) < 2:
@@ -81,7 +87,9 @@ def main():
if "--optimize-build" in sys.argv:
if len(sys.argv) < 3:
print("Usage: python selective_test_filter.py <depmap_json> --optimize-build <changed_file1> [<changed_file2> ...]")
print(
"Usage: python selective_test_filter.py <depmap_json> --optimize-build <changed_file1> [<changed_file2> ...]"
)
sys.exit(1)
depmap_json = sys.argv[1]
changed_files = set(sys.argv[sys.argv.index("--optimize-build") + 1 :])
@@ -100,7 +108,9 @@ def main():
sys.exit(0)
if len(sys.argv) < 4:
print("Usage: python selective_test_filter.py <depmap_json> <ref1> <ref2> [--all | --test-prefix] [--output <output_json>]")
print(
"Usage: python selective_test_filter.py <depmap_json> <ref1> <ref2> [--all | --test-prefix] [--output <output_json>]"
)
sys.exit(1)
depmap_json = sys.argv[1]
@@ -131,9 +141,12 @@ def main():
tests = select_tests(file_to_executables, changed_files, filter_mode)
with open(output_json, "w") as f:
json.dump({"tests_to_run": tests, "changed_files": sorted(changed_files)}, f, indent=2)
json.dump(
{"tests_to_run": tests, "changed_files": sorted(changed_files)}, f, indent=2
)
print(f"Exported {len(tests)} tests to run to {output_json}")
if __name__ == "__main__":
main()