[workflow] only report coverage for changed files (#2524)

* [workflow] only report coverage for changed files

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

* polish file

.github/workflows/build.yml

@@ -6,15 +6,17 @@ on:
 
 jobs:
   detect:
-    name: Detect kernel-related file change
+    name: Detect file change
     if: |
         github.event.pull_request.draft == false &&
         github.base_ref == 'main' &&
         github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI' &&
         contains( github.event.pull_request.labels.*.name, 'Run Build and Test')
     outputs:
-      changedFiles: ${{ steps.find-changed-files.outputs.changedFiles }}
+      changedExtenisonFiles: ${{ steps.find-extension-change.outputs.all_changed_files }}
-      anyChanged: ${{ steps.find-changed-files.outputs.any_changed }}
+      anyExtensionFileChanged: ${{ steps.find-extension-change.outputs.any_changed }}
+      changedLibraryFiles: ${{ steps.find-lib-change.outputs.all_changed_files }}
+      anyLibraryFileChanged: ${{ steps.find-lib-change.outputs.any_changed }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -30,8 +32,8 @@ jobs:
             echo $commonCommit
             echo "baseSHA=$commonCommit" >> $GITHUB_OUTPUT
 
-      - name: Find the changed files
+      - name: Find the changed extension-related files
-        id: find-changed-files
+        id: find-extension-change
         uses: tj-actions/changed-files@v35
         with:
           base_sha: ${{ steps.locate-base-sha.outputs.baseSHA }}
@@ -40,9 +42,23 @@ jobs:
             colossalai/kernel/**
             setup.py
 
+      - name: Find the changed library-related files
+        id: find-lib-change
+        uses: tj-actions/changed-files@v35
+        with:
+          base_sha: ${{ steps.locate-base-sha.outputs.baseSHA }}
+          files: |
+            **/*.py
+            **/*.h
+            **/*.cpp
+            **/*.cu
+
       - name: List changed files
         run: |
-          for file in ${{ steps.find-changed-files.outputs.all_changed_files }}; do
+          for file in ${{ steps.find-extension-change.outputs.all_changed_files }}; do
+            echo "$file was changed"
+          done
+          for file in ${{ steps.find-lib-change.outputs.all_changed_files }}; do
             echo "$file was changed"
           done
 
@@ -55,38 +71,58 @@ jobs:
       image: hpcaitech/pytorch-cuda:1.11.0-11.3.0
       options: --gpus all --rm -v /data/scratch/cifar-10:/data/scratch/cifar-10
     timeout-minutes: 40
+    defaults:
+      run:
+        shell: bash
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout TensorNVMe
+        uses: actions/checkout@v2
         with:
           repository: hpcaitech/TensorNVMe
           ssh-key: ${{ secrets.SSH_KEY_FOR_CI }}
           path: TensorNVMe
 
-      - name: Install tensornvme
+      - name: Restore TensorNVMe Cache
+        run: |
+          [ ! -z "$(ls -A /github/home/tensornvme_cache/)" ] && cp -p -r /github/home/tensornvme_cache/* /__w/ColossalAI/ColossalAI/TensorNVMe
+
+      - name: Install TensorNVMe
         run: |
           cd TensorNVMe
           conda install cmake
           pip install -r requirements.txt
           pip install -v .
 
-      - uses: actions/checkout@v2
+      - name: Store TensorNVMe Cache
+        run: |
+          cp -p -r ./build /github/home/tensornvme_cache/
+
+      - name: Checkout Colossal-AI
+        uses: actions/checkout@v2
         with:
           ssh-key: ${{ secrets.SSH_KEY_FOR_CI }}
 
-      - name: Restore cache
+      - name: Restore Colossal-AI Cache
-        if: needs.detect.outputs.anyChanged != 'true'
+        if: needs.detect.outputs.anyExtensionFileChanged != 'true'
         run: |
           # -p flag is required to preserve the file timestamp to avoid ninja rebuild
           [ ! -z "$(ls -A /github/home/cuda_ext_cache/)" ] && cp -p -r /github/home/cuda_ext_cache/* /__w/ColossalAI/ColossalAI/
 
       - name: Install Colossal-AI
+        if: needs.detect.outputs.anyLibraryFileChanged == 'true'
         run: |
           CUDA_EXT=1 pip install -v -e .
           pip install -r requirements/requirements-test.txt
 
-      - name: Unit Testing
+      - name: Store Colossal-AI Cache
         run: |
-          PYTHONPATH=$PWD pytest --cov=. --cov-report xml tests
+          # -p flag is required to preserve the file timestamp to avoid ninja rebuild
+          cp -p -r /__w/ColossalAI/ColossalAI/build /github/home/cuda_ext_cache/
+
+      - name: Execute Unit Testing
+        if: needs.detect.outputs.anyLibraryFileChanged == 'true'
+        run: |
+          PYTHONPATH=$PWD pytest --cov=. --cov-report xml tests/
         env:
           DATA: /data/scratch/cifar-10
           NCCL_SHM_DISABLE: 1
@@ -95,18 +131,36 @@ jobs:
       - name: Collate artifact
         env:
           PR_NUMBER: ${{ github.event.number }}
+          changedLibraryFiles: ${{ needs.detect.outputs.changedLibraryFiles }}
+          anyLibraryFileChanged: ${{ needs.detect.outputs.anyLibraryFileChanged }}
+          changedExtenisonFiles: ${{ needs.detect.outputs.changedExtenisonFiles }}
         run: |
           mkdir report
           echo $PR_NUMBER > ./report/pr_number
-          mv coverage.xml ./report
+
+          # generate coverage.xml if any
+          if [ "$anyLibraryFileChanged" == "true" ]; then
+            allFiles=""
+            for file in $changedLibraryFiles; do
+              if [ "$allFiles" == "" ]; then
+                allFiles=$file
+              else
+                allFiles=$allFiles,$file
+              fi
+            done
+
+            coverage report --data-file .coverage --include $allFiles > ./coverage.txt
+
+            covPercentage=$(tail -n 1 coverage.txt  | grep -o '[1-9]*%$')
+            covNum=${covPercentage::-1}
+            mv coverage.txt ./report
+            echo $covNum > ./report/cov_number
+          else
+            echo "No coverage report is generated"
+          fi
 
       - name: Upload test coverage artifact
         uses: actions/upload-artifact@v3
         with:
           name: report
           path: report/
-
-      - name: Store Cache
-        run: |
-          # -p flag is required to preserve the file timestamp to avoid ninja rebuild
-          cp -p -r /__w/ColossalAI/ColossalAI/build /github/home/cuda_ext_cache/

.github/workflows/report_test_coverage.yml

@@ -32,28 +32,31 @@ jobs:
             fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/report.zip`, Buffer.from(download.data));
 
       - name: 'Unzip artifact'
+        id: unzip
         run: |
           unzip report.zip
-
+          if [ -f "coverage.txt" ]; then
-      - name: Code Coverage Report
+            echo "hasReport=true" >> $GITHUB_OUTPUT
-        uses: irongut/CodeCoverageSummary@v1.3.0
+          else
-        with:
+            echo "hasReport=false" >> $GITHUB_OUTPUT
-          filename: coverage.xml
+          fi
-          badge: true
-          format: markdown
-          hide_branch_rate: false
-          hide_complexity: false
-          indicators: true
-          output: both
-          thresholds: '80 90'
 
       - name: Make Coverage Report Collapsable
+        if: steps.unzip.outputs.hasReport == "true"
         run: |
-          sed -i '2 i <details>' code-coverage-results.md
+          covNum=$(cat cov_number)
-          sed -i '3 i <summary>Click me to view the complete report</summary>' code-coverage-results.md
+          title="The code coverage for the changed files is ${covNum}%."
-          echo "</details>" >> code-coverage-results.md
+          (echo $title; cat coverage.txt) > coverage_tmp.txt
+          mv coverage_tmp.txt coverage.txt
+          sed -i '2 i <details>' coverage.txt
+          sed -i '3 i <summary>Click me to view the complete report</summary>' coverage.txt
+          sed -i '4 i \n' coverage.txt
+          sed -i '5 i \`\`\`text' coverage.txt
+          echo "\`\`\`" >> coverage.txt
+          echo "</details>" >> coverage.txt
 
       - name: 'Comment on PR'
+        if: steps.unzip.outputs.hasReport == "true"
         uses: actions/github-script@v6
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -64,7 +67,7 @@ jobs:
             let repo = context.repo.repo;
             let run_id = context.payload.workflow_run.id;
             let run_url = `https://github.com/${owner}/${repo}/actions/runs/${run_id}`
-            let body = fs.readFileSync('./code-coverage-results.md', {encoding:'utf8', flag:'r'})
+            let body = fs.readFileSync('./coverage.txt', {encoding:'utf8', flag:'r'})
 
             await github.rest.issues.createComment({
               owner: owner,

tests/test_amp/test_naive_fp16.py

@@ -24,7 +24,6 @@ def run_naive_amp():
     In this test, we compare the naive fp16 optimizer implemented in colossalai
     and fp32 torch optimizer
     """
-
     torch.backends.cudnn.benchmark = False
     torch.backends.cudnn.deterministic = True