From 0c03802bff8484660f3d16c62a8d03d4dbe96899 Mon Sep 17 00:00:00 2001
From: Frank Lee <somerlee.9@gmail.com>
Date: Mon, 6 Feb 2023 16:29:04 +0800
Subject: [PATCH] [workflow] hooked pypi release with lark (#2596)
---
.github/workflows/auto_example_check.yml | 143 ------------------
.github/workflows/auto_release_bdist.yml | 70 ---------
...draft_github_release_post_after_merge.yml} | 0
.../workflows/release_nightly_on_schedule.yml | 36 +++++
.../workflows/release_pypi_after_merge.yml | 36 +++++
5 files changed, 72 insertions(+), 213 deletions(-)
delete mode 100644 .github/workflows/auto_example_check.yml
delete mode 100644 .github/workflows/auto_release_bdist.yml
rename .github/workflows/{draft_github_release_post.yml => draft_github_release_post_after_merge.yml} (100%)
diff --git a/.github/workflows/auto_example_check.yml b/.github/workflows/auto_example_check.yml
deleted file mode 100644
index df413f646..000000000
--- a/.github/workflows/auto_example_check.yml
+++ /dev/null
@@ -1,143 +0,0 @@
-name: Test Example
-on:
- pull_request:
- # any change in the examples folder will trigger check for the corresponding example.
- paths:
- - 'examples/**'
- # run at 00:00 of every Sunday(singapore time) so here is UTC time Saturday 16:00
- schedule:
- - cron: '0 16 * * 6'
-
-jobs:
- # This is for changed example files detect and output a matrix containing all the corresponding directory name.
- detect-changed-example:
- if: |
- github.event.pull_request.draft == false &&
- github.base_ref == 'main' &&
- github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI' && github.event_name == 'pull_request'
- runs-on: ubuntu-latest
- outputs:
- matrix: ${{ steps.setup-matrix.outputs.matrix }}
- anyChanged: ${{ steps.setup-matrix.outputs.anyChanged }}
- name: Detect changed example files
- steps:
- - uses: actions/checkout@v3
- with:
- fetch-depth: 0
- ref: ${{ github.event.pull_request.head.sha }}
-
- - name: Locate base commit
- id: locate-base-sha
- run: |
- curBranch=$(git rev-parse --abbrev-ref HEAD)
- commonCommit=$(git merge-base origin/main $curBranch)
- echo $commonCommit
- echo "baseSHA=$commonCommit" >> $GITHUB_OUTPUT
-
- - name: Get all changed example files
- id: changed-files
- uses: tj-actions/changed-files@v35
- with:
- base_sha: ${{ steps.locate-base-sha.outputs.baseSHA }}
-
- - name: setup matrix
- id: setup-matrix
- run: |
- changedFileName=""
- for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
- changedFileName="${file}:${changedFileName}"
- done
- echo "$changedFileName was changed"
- res=`python .github/workflows/scripts/example_checks/detect_changed_example.py --fileNameList $changedFileName`
- echo "All changed examples are $res"
-
- if [ "$res" = "[]" ]; then
- echo "anyChanged=false" >> $GITHUB_OUTPUT
- echo "matrix=null" >> $GITHUB_OUTPUT
- else
- dirs=$( IFS=',' ; echo "${res[*]}" )
- echo "anyChanged=true" >> $GITHUB_OUTPUT
- echo "matrix={\"directory\":$(echo "$dirs")}" >> $GITHUB_OUTPUT
- fi
-
- # If no file is changed, it will prompt an error and shows the matrix do not have value.
- check-changed-example:
- # Add this condition to avoid executing this job if the trigger event is workflow_dispatch.
- if: |
- github.event.pull_request.draft == false &&
- github.base_ref == 'main' &&
- github.event.pull_request.base.repo.full_name == 'hpcaitech/ColossalAI' && github.event_name == 'pull_request' &&
- needs.detect-changed-example.outputs.anyChanged == 'true'
- name: Test the changed example
- needs: detect-changed-example
- runs-on: [self-hosted, gpu]
- strategy:
- matrix: ${{fromJson(needs.detect-changed-example.outputs.matrix)}}
- container:
- image: hpcaitech/pytorch-cuda:1.12.0-11.3.0
- options: --gpus all --rm -v /data/scratch/examples-data:/data/
- timeout-minutes: 10
- steps:
- - uses: actions/checkout@v3
-
- - name: Install Colossal-AI
- run: |
- pip install -v .
-
- - name: Test the example
- run: |
- example_dir=${{ matrix.directory }}
- cd "${PWD}/examples/${example_dir}"
- bash test_ci.sh
- env:
- NCCL_SHM_DISABLE: 1
-
- # This is for all files' weekly check. Specifically, this job is to find all the directories.
- matrix_preparation:
- if: |
- github.repository == 'hpcaitech/ColossalAI' &&
- github.event_name == 'schedule'
- name: Prepare matrix for weekly check
- runs-on: ubuntu-latest
- outputs:
- matrix: ${{ steps.setup-matrix.outputs.matrix }}
- steps:
- - name: 📚 Checkout
- uses: actions/checkout@v3
-
- - name: setup matrix
- id: setup-matrix
- run: |
- res=`python .github/workflows/scripts/example_checks/check_example_weekly.py`
- all_loc=$( IFS=',' ; echo "${res[*]}" )
- echo "Found the examples: $all_loc"
- echo "matrix={\"directory\":$(echo "$all_loc")}" >> $GITHUB_OUTPUT
-
- weekly_check:
- if: |
- github.repository == 'hpcaitech/ColossalAI' &&
- github.event_name == 'schedule'
- name: Weekly check all examples
- needs: matrix_preparation
- runs-on: [self-hosted, gpu]
- strategy:
- matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}}
- container:
- image: hpcaitech/pytorch-cuda:1.12.0-11.3.0
- timeout-minutes: 10
- steps:
- - name: 📚 Checkout
- uses: actions/checkout@v3
-
- - name: Install Colossal-AI
- run: |
- pip install -v .
-
- - name: Traverse all files
- run: |
- example_dir=${{ matrix.diretory }}
- echo "Testing ${example_dir} now"
- cd "${PWD}/examples/${example_dir}"
- bash test_ci.sh
- env:
- NCCL_SHM_DISABLE: 1
diff --git a/.github/workflows/auto_release_bdist.yml b/.github/workflows/auto_release_bdist.yml
deleted file mode 100644
index 56a3036f8..000000000
--- a/.github/workflows/auto_release_bdist.yml
+++ /dev/null
@@ -1,70 +0,0 @@
-name: Auto Release bdist wheel
-
-on:
- workflow_dispatch:
- pull_request:
- paths:
- - 'version.txt'
- types:
- - closed
-
-jobs:
- matrix_preparation:
- name: Prepare Container List
- if: ( github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true ) && github.repository == 'hpcaitech/ColossalAI'
- runs-on: ubuntu-latest
- outputs:
- matrix: ${{ steps.set-matrix.outputs.matrix }}
- steps:
- - uses: actions/checkout@v3
- - id: set-matrix
- run: |
- bdist=$(cat .bdist.json | tr '\n' ' ')
- echo "matrix=${bdist}" >> $GITHUB_OUTPUT
-
- build:
- name: Release bdist wheels
- needs: matrix_preparation
- runs-on: [self-hosted, gpu]
- strategy:
- fail-fast: false
- matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}}
- container:
- image: ${{ matrix.build.cuda_image }}
- options: --gpus all --rm
- steps:
- - uses: actions/checkout@v2
- with:
- fetch-depth: 0
- # cub is for cuda 10.2
- - name: Copy scripts
- run: |
- cp -r ./.github/workflows/scripts/* ./
-
- # link the cache diretories to current path
- ln -s /github/home/conda_pkgs ./conda_pkgs
- ln -s /github/home/pip_wheels ./pip_wheels
-
- # set the conda package path
- echo "pkgs_dirs:\n - $PWD/conda_pkgs" > ~/.condarc
-
- # set safe directory
- git config --global --add safe.directory /__w/ColossalAI/ColossalAI
-
- # get cub package for cuda 10.2
- wget https://github.com/NVIDIA/cub/archive/refs/tags/1.8.0.zip
- unzip 1.8.0.zip
- - name: Build bdist wheel
- run: |
- pip install beautifulsoup4 requests packaging
- python ./build_colossalai_wheel.py --torch_version $TORCH_VERSIONS
- env:
- TORCH_VERSIONS: ${{ matrix.build.torch_version }}
- - name: 🚀 Deploy
- uses: garygrossgarten/github-action-scp@release
- with:
- local: all_dist
- remote: ${{ secrets.PRIVATE_PYPI_DIR }}
- host: ${{ secrets.PRIVATE_PYPI_HOST }}
- username: ${{ secrets.PRIVATE_PYPI_USER }}
- password: ${{ secrets.PRIVATE_PYPI_PASSWD }}
diff --git a/.github/workflows/draft_github_release_post.yml b/.github/workflows/draft_github_release_post_after_merge.yml
similarity index 100%
rename from .github/workflows/draft_github_release_post.yml
rename to .github/workflows/draft_github_release_post_after_merge.yml
diff --git a/.github/workflows/release_nightly_on_schedule.yml b/.github/workflows/release_nightly_on_schedule.yml
index 8aa48b8ed..aab42e1d7 100644
--- a/.github/workflows/release_nightly_on_schedule.yml
+++ b/.github/workflows/release_nightly_on_schedule.yml
@@ -23,7 +23,43 @@ jobs:
# publish to PyPI if executed on the main branch
- name: Publish package to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
+ id: publish
with:
user: __token__
password: ${{ secrets.PYPI_API_TOKEN }}
verbose: true
+
+ notify:
+ name: Notify Lark via webhook
+ needs: release
+ runs-on: ubuntu-latest
+ if: ${{ always() }}
+ steps:
+ - uses: actions/checkout@v2
+
+ - uses: actions/setup-python@v2
+ with:
+ python-version: '3.8.14'
+
+ - name: Install requests
+ run: pip install requests
+
+ - name: Notify Lark
+ id: message-preparation
+ run: |
+ url=$SERVER_URL/$REPO/actions/runs/$RUN_ID
+
+ if [ $STATUS == 'success' ]
+ then
+ msg="The Colossal-AI nightly version has been successfully released to PyPI."
+ else
+ msg="Failed to release Colossal-AI nightly version to PyPI, please visit $url for details."
+ fi
+ echo $msg
+ python .github/workflows/scripts/send_message_to_lark.py -m "$msg" -u $WEBHOOK_URL
+ env:
+ SERVER_URL: ${{github.server_url }}
+ REPO: ${{ github.repository }}
+ RUN_ID: ${{ github.run_id }}
+ WEBHOOK_URL: ${{ secrets.LARK_NOTIFICATION_WEBHOOK_URL }}
+ STATUS: ${{ steps.publish.outcome }}
diff --git a/.github/workflows/release_pypi_after_merge.yml b/.github/workflows/release_pypi_after_merge.yml
index 797cd69f3..7fa4bdd03 100644
--- a/.github/workflows/release_pypi_after_merge.yml
+++ b/.github/workflows/release_pypi_after_merge.yml
@@ -25,8 +25,44 @@ jobs:
# publish to PyPI if executed on the main branch
- name: Publish package to PyPI
+ id: publish
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.PYPI_API_TOKEN }}
verbose: true
+
+ notify:
+ name: Notify Lark via webhook
+ needs: release
+ runs-on: ubuntu-latest
+ if: ${{ always() }}
+ steps:
+ - uses: actions/checkout@v2
+
+ - uses: actions/setup-python@v2
+ with:
+ python-version: '3.8.14'
+
+ - name: Install requests
+ run: pip install requests
+
+ - name: Notify Lark
+ id: message-preparation
+ run: |
+ url=$SERVER_URL/$REPO/actions/runs/$RUN_ID
+
+ if [ $STATUS == 'success' ]
+ then
+ msg="The Colossal-AI latest version has been successfully released to PyPI."
+ else
+ msg="Failed to release Colossal-AI to PyPI, please visit $url for details."
+ fi
+ echo $msg
+ python .github/workflows/scripts/send_message_to_lark.py -m "$msg" -u $WEBHOOK_URL
+ env:
+ SERVER_URL: ${{github.server_url }}
+ REPO: ${{ github.repository }}
+ RUN_ID: ${{ github.run_id }}
+ WEBHOOK_URL: ${{ secrets.LARK_NOTIFICATION_WEBHOOK_URL }}
+ STATUS: ${{ steps.publish.outcome }}