mscclpp/.github/workflows/ut-backup.yml

name: UnitTest

on: workflow_dispatch

jobs:
  UnitTest:
    runs-on: [ self-hosted, A100 ]
    defaults:
      run:
        shell: bash
    timeout-minutes: 30
    strategy:
      matrix:
        cuda: [ cuda11.8, cuda12.2 ]

    container:
      image: "ghcr.io/microsoft/mscclpp/mscclpp:base-dev-${{ matrix.cuda }}"
      options: --privileged --ipc=host --gpus=all --ulimit memlock=-1:-1

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Build
        run: |
          mkdir build && cd build
          cmake -DCMAKE_BUILD_TYPE=Release ..
          make -j
        working-directory: ${{ github.workspace }}

      - name: LockGPUClock
        run: |
          sudo nvidia-smi -pm 1
          for i in $(seq 0 $(( $(nvidia-smi -L | wc -l) - 1 ))); do
            sudo nvidia-smi -ac $(nvidia-smi --query-gpu=clocks.max.memory,clocks.max.sm --format=csv,noheader,nounits -i $i | sed 's/\ //') -i $i
          done

      - name: UnitTests
        run: |
          ./build/test/unit_tests

      - name: MpUnitTests
        run: |
          set -e
          mpirun --allow-run-as-root -tag-output -np 2 ./build/test/mp_unit_tests
          mpirun --allow-run-as-root -tag-output -np 4 ./build/test/mp_unit_tests
          mpirun --allow-run-as-root -tag-output -np 8 ./build/test/mp_unit_tests

      - name: PyTests
        run: |
          set -e
          mpirun --allow-run-as-root -tag-output -np 8 $(which pytest) ./python/test/test_mscclpp.py -x