From 9503dc6af01153e11eb3714340a3757faade7161 Mon Sep 17 00:00:00 2001 From: Geo Min Date: Mon, 18 Aug 2025 14:16:31 -0700 Subject: [PATCH] [TheRock CI] Adding presubmit check for CK (#2688) * Adding presubmit check for CK * Adding exclusion * Enable forks [ROCm/composable_kernel commit: b4f3487d8423a0e14bbb90e1cf8305d3560f3d17] --- .github/scripts/therock_configure_ci.py | 112 ++++++++++++++++++++ .github/workflows/therock-ci-linux.yml | 8 +- .github/workflows/therock-ci.yml | 31 ++++++ .github/workflows/therock-test-packages.yml | 1 + 4 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 .github/scripts/therock_configure_ci.py diff --git a/.github/scripts/therock_configure_ci.py b/.github/scripts/therock_configure_ci.py new file mode 100644 index 0000000000..557afe2d84 --- /dev/null +++ b/.github/scripts/therock_configure_ci.py @@ -0,0 +1,112 @@ +import fnmatch +import json +import os +from pathlib import Path +import subprocess +import sys +from typing import Iterable, Optional, Mapping + +def gha_set_output(vars: Mapping[str, str | Path]): + """Sets values in a step's output parameters. + + This appends to the file located at the $GITHUB_OUTPUT environment variable. + + See + * https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-output-parameter + * https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/passing-information-between-jobs + """ + print(f"Setting github output:\n{vars}") + + step_output_file = os.getenv("GITHUB_OUTPUT") + if not step_output_file: + print(" Warning: GITHUB_OUTPUT env var not set, can't set github outputs") + return + + with open(step_output_file, "a") as f: + f.writelines(f"{k}={str(v)}" + "\n" for k, v in vars.items()) + +def get_modified_paths(base_ref: str) -> Optional[Iterable[str]]: + """Returns the paths of modified files relative to the base reference.""" + try: + return subprocess.run( + ["git", "diff", "--name-only", base_ref], + stdout=subprocess.PIPE, + check=True, + text=True, + timeout=60, + ).stdout.splitlines() + except TimeoutError: + print( + "Computing modified files timed out. Not using PR diff to determine" + " jobs to run.", + file=sys.stderr, + ) + return None + +# Paths matching any of these patterns are considered to have no influence over +# build or test workflows so any related jobs can be skipped if all paths +# modified by a commit/PR match a pattern in this list. +SKIPPABLE_PATH_PATTERNS = [ + "docs/*", + "*.gitignore", + "*.md", + "*.pre-commit-config.*", + "*LICENSE", + 'Jenkinsfile', + '.github/ISSUE_TEMPLATE/*', + '.github/CODEOWNERS', + '.github/*.md', + '.github/dependabot.yml', +] + +def is_path_skippable(path: str) -> bool: + """Determines if a given relative path to a file matches any skippable patterns.""" + return any(fnmatch.fnmatch(path, pattern) for pattern in SKIPPABLE_PATH_PATTERNS) + +def check_for_non_skippable_path(paths: Optional[Iterable[str]]) -> bool: + """Returns true if at least one path is not in the skippable set.""" + if paths is None: + return False + return any(not is_path_skippable(p) for p in paths) + +def should_ci_run_given_modified_paths(paths: Optional[Iterable[str]]) -> bool: + """Returns true if CI workflows should run given a list of modified paths.""" + + if paths is None: + print("No files were modified, skipping TheRock CI jobs") + return False + + paths_set = set(paths) + github_workflows_paths = set( + [p for p in paths if p.startswith(".github/workflows")] + ) + other_paths = paths_set - github_workflows_paths + + contains_other_non_skippable_files = check_for_non_skippable_path(other_paths) + + print("should_ci_run_given_modified_paths findings:") + print(f" contains_other_non_skippable_files: {contains_other_non_skippable_files}") + + if contains_other_non_skippable_files: + print("Enabling TheRock CI jobs since a non-skippable path was modified") + return True + else: + print( + "Only unrelated and/or skippable paths were modified, skipping TheRock CI jobs" + ) + return False + +def main(args): + base_ref = args.get("base_ref") + modified_paths = get_modified_paths(base_ref) + print("modified_paths (max 200):", modified_paths[:200]) + enable_jobs = should_ci_run_given_modified_paths(modified_paths) + output = { + 'enable_therock_ci': json.dumps(enable_jobs) + } + gha_set_output(output) + +if __name__ == "__main__": + args = {} + args["base_ref"] = os.environ.get("BASE_REF", "HEAD^1") + main(args) diff --git a/.github/workflows/therock-ci-linux.yml b/.github/workflows/therock-ci-linux.yml index 645a91c030..7db124d2a1 100644 --- a/.github/workflows/therock-ci-linux.yml +++ b/.github/workflows/therock-ci-linux.yml @@ -21,9 +21,11 @@ jobs: id-token: write container: image: ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:044b113562629f4bd2ec5d2e64b32eee11562d48fb1a75d7493daec9dd8d8292 + options: -v /runner/config:/home/awsconfig/ env: AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} TEATIME_FORCE_INTERACTIVE: 0 + AWS_SHARED_CREDENTIALS_FILE: /home/awsconfig/credentials.ini steps: - name: Checkout composable_kernel repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -83,9 +85,9 @@ jobs: echo "----------" du -h -d 1 TheRock/build/artifacts - - name: Configure AWS Credentials - if: always() - uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 + - name: Configure AWS Credentials for non-forked repos + if: ${{ always() && !github.event.pull_request.head.repo.fork }} + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts-external diff --git a/.github/workflows/therock-ci.yml b/.github/workflows/therock-ci.yml index 18411baa09..3232652b6b 100644 --- a/.github/workflows/therock-ci.yml +++ b/.github/workflows/therock-ci.yml @@ -5,6 +5,15 @@ on: branches: - develop workflow_dispatch: + pull_request: + types: + - opened + - synchronize + branches: + - mainline + - release/* + - release-staging/* + - develop permissions: contents: read @@ -18,8 +27,29 @@ concurrency: cancel-in-progress: true jobs: + setup: + runs-on: ubuntu-24.04 + env: + # The commit being checked out is the merge commit for a PR. Its first + # parent will be the tip of the base branch. + BASE_REF: HEAD^ + outputs: + enable_therock_ci: ${{ steps.configure.outputs.enable_therock_ci }} + steps: + - name: "Checking out repository" + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + # We need the parent commit to do a diff + fetch-depth: 2 + + - name: "Configuring CI options" + id: configure + run: python .github/scripts/therock_configure_ci.py + therock-ci-linux: name: TheRock CI Linux + needs: setup + if: ${{ needs.setup.outputs.enable_therock_ci == 'true' }} permissions: contents: read id-token: write @@ -34,6 +64,7 @@ jobs: name: TheRock CI Summary if: always() needs: + - setup - therock-ci-linux runs-on: ubuntu-24.04 steps: diff --git a/.github/workflows/therock-test-packages.yml b/.github/workflows/therock-test-packages.yml index 439135743c..37ddd399ad 100644 --- a/.github/workflows/therock-test-packages.yml +++ b/.github/workflows/therock-test-packages.yml @@ -68,6 +68,7 @@ jobs: VENV_DIR: ${{ env.VENV_DIR }} FETCH_ARTIFACT_ARGS: ${{ matrix.components.fetch_artifact_args }} PLATFORM: ${{ inputs.platform }} + IS_PR_FROM_FORK: ${{ github.event.pull_request.head.repo.fork }} - name: Test timeout-minutes: ${{ matrix.components.timeout_minutes }}