diff --git a/.github/workflows/release_bdist.yml b/.github/workflows/release_bdist.yml
index 79b17c64a..ab60ecc63 100644
--- a/.github/workflows/release_bdist.yml
+++ b/.github/workflows/release_bdist.yml
@@ -3,18 +3,21 @@ name: Release bdist wheel
 on:
   workflow_dispatch:
     inputs:
-      version:
+      cuda_version:
         type: choice
-        description: version for testing
+        description: CUDA Version
         default: 'all'
         required: true
         options:
         - all
-        - pytorch-cuda:1.9.0-11.1.1 # python 3.8
-        - pytorch-cuda:1.8.1-11.1.1 # python 3.8
-        - pytorch-cuda:1.7.1-11.0.3 # python 3.8
-        - pytorch-cuda:1.6.0-10.2 # python 3.6
-        
+        - "11.3"
+        - "11.1"
+        - "10.2"
+      github_ref:
+        type: string
+        description: Branch or Tag
+        default: 'main'
+        required: true
 
 jobs:
   matrix_preparation:
@@ -25,9 +28,9 @@ jobs:
     steps:
     - id: set-matrix
       run: |
-        [ "${{github.event.inputs.version}}" != "" ] && matrix="[\"frankleeeee/${{github.event.inputs.version}}\"]"
-        [ "${{github.event.inputs.version}}" == "" ] || [ "${{github.event.inputs.version}}" == "all" ] && \
-          matrix="[\"frankleeeee/pytorch-cuda:1.9.0-11.1.1\", \"frankleeeee/pytorch-cuda:1.8.1-11.1.1\", \"frankleeeee/pytorch-cuda:1.7.1-11.0.3\", \"frankleeeee/pytorch-cuda:1.6.0-10.2\"]"
+        [ "${{github.event.inputs.cuda_version}}" != "" ] && matrix="[\"frankleeeee/cuda-conda:${{github.event.inputs.cuda_version}}\"]"
+        [ "${{github.event.inputs.cuda_version}}" == "" ] || [ "${{github.event.inputs.version}}" == "all" ] && \
+          matrix="[\"frankleeeee/cuda-conda:11.3\", \"frankleeeee/cuda-conda:11.1\", \"frankleeeee/cuda-conda:10.2\"]"
         echo $matrix
         echo "::set-output name=matrix::{\"container\":$(echo $matrix)}"
 
@@ -42,20 +45,28 @@ jobs:
     container:
       image: ${{ matrix.container }}
       options: --gpus all --rm
-    timeout-minutes: 120
     steps:
-      - name: Install dependencies
-        run: |
-          pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
-          pip install -U pip setuptools wheel --user
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      # cub is for cuda 10.2
+      - name: Copy scripts and checkout
+        run: |
+          cp -r ./.github/workflows/scripts/* ./
+          ln -s /github/home/pip_wheels ./pip_wheels
+          git checkout $git_ref
+          wget https://github.com/NVIDIA/cub/archive/refs/tags/1.8.0.zip
+          unzip 1.8.0.zip
+        env:
+          git_ref: ${{ github.event.inputs.github_ref }}
       - name: Build bdist wheel
         run: |
-          python setup.py bdist_wheel
+          pip install beautifulsoup4 requests
+          python ./build_colossalai_wheel.py
       - name: 🚀 Deploy
         uses: garygrossgarten/github-action-scp@release
         with:
-          local: dist
+          local: all_dist
           remote: ${{ secrets.PRIVATE_PYPI_DIR }}
           host: ${{ secrets.PRIVATE_PYPI_HOST }}
           username: ${{ secrets.PRIVATE_PYPI_USER }}
diff --git a/.github/workflows/scripts/build_colossalai_wheel.py b/.github/workflows/scripts/build_colossalai_wheel.py
new file mode 100644
index 000000000..c6b7acc25
--- /dev/null
+++ b/.github/workflows/scripts/build_colossalai_wheel.py
@@ -0,0 +1,82 @@
+import requests
+from bs4 import BeautifulSoup
+import re
+import os
+import subprocess
+
+
+WHEEL_TEXT_ROOT_URL = 'https://github.com/hpcaitech/public_assets/tree/main/colossalai/torch_build/torch_wheels'
+RAW_TEXT_FILE_PREFIX = 'https://raw.githubusercontent.com/hpcaitech/public_assets/main/colossalai/torch_build/torch_wheels'
+CUDA_HOME = os.environ['CUDA_HOME']
+
+def get_cuda_bare_metal_version():
+    raw_output = subprocess.check_output([CUDA_HOME + "/bin/nvcc", "-V"], universal_newlines=True)
+    output = raw_output.split()
+    release_idx = output.index("release") + 1
+    release = output[release_idx].split(".")
+    bare_metal_major = release[0]
+    bare_metal_minor = release[1][0]
+
+    return bare_metal_major, bare_metal_minor
+
+def all_wheel_info():
+    page_text = requests.get(WHEEL_TEXT_ROOT_URL).text
+    soup = BeautifulSoup(page_text)
+
+    all_a_links = soup.find_all('a')
+
+    wheel_info = dict()
+
+    for a_link in all_a_links:
+        if 'cuda' in a_link.text and '.txt' in a_link.text:
+            filename = a_link.text
+            torch_version, cuda_version = filename.rstrip('.txt').split('-')
+            cuda_version = cuda_version.lstrip('cuda')
+
+            if torch_version not in wheel_info:
+                wheel_info[torch_version] = dict()
+            wheel_info[torch_version][cuda_version] = dict()
+
+            file_text = requests.get(f'{RAW_TEXT_FILE_PREFIX}/{filename}').text
+            lines = file_text.strip().split('\n')
+
+            for line in lines:
+                parts = line.split('\t')
+                method, url, python_version = parts[:3]
+
+                if len(parts) > 3:
+                    flags = parts[3]
+                    flags = ' '.join(flags.split('+'))
+                else:
+                    flags = ''
+                wheel_info[torch_version][cuda_version][python_version] = dict(method=method, url=url, flags=flags)
+    return wheel_info
+
+
+def build_colossalai(wheel_info):
+    cuda_version_major, cuda_version_minor = get_cuda_bare_metal_version()
+    cuda_version_on_host = f'{cuda_version_major}.{cuda_version_minor}'
+
+    for torch_version, cuda_versioned_wheel_info in wheel_info.items():
+        for cuda_version, python_versioned_wheel_info in cuda_versioned_wheel_info.items():
+            if cuda_version_on_host == cuda_version:
+                for python_version, wheel_info in python_versioned_wheel_info.items():
+                    url = wheel_info['url']
+                    method = wheel_info['method']
+                    flags = wheel_info['flags']
+                    filename = url.split('/')[-1].replace('%2B', '+')
+                    cmd = f'bash ./build_colossalai_wheel.sh {method} {url} {filename} {cuda_version} {python_version} {torch_version} {flags}'
+                    os.system(cmd)
+
+def main():
+    wheel_info = all_wheel_info()
+    build_colossalai(wheel_info)
+
+if __name__ == '__main__':
+    main()
+
+
+
+
+
+        
\ No newline at end of file
diff --git a/.github/workflows/scripts/build_colossalai_wheel.sh b/.github/workflows/scripts/build_colossalai_wheel.sh
new file mode 100644
index 000000000..3a6eecbc8
--- /dev/null
+++ b/.github/workflows/scripts/build_colossalai_wheel.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+method=${1}
+url=${2}
+filename=${3}
+cuda_version=${4}
+python_version=${5}
+torch_version=${6}
+flags=${@:7}
+
+git reset --hard HEAD
+mkdir -p ./all_dist
+source activate base
+conda create -n $python_version -y python=$python_version
+source activate $python_version
+
+if [ $1 == "pip" ]
+then
+    wget -nc -q -O ./pip_wheels/$filename $url
+    pip install ./pip_wheels/$filename
+    
+elif [ $1 == 'conda' ]
+then
+    conda install pytorch==$torch_version cudatoolkit=$cuda_version $flags
+else
+    echo Invalid installation method
+    exit
+fi
+
+if [ $cuda_version == "10.2" ]
+then
+    cp -r cub-1.8.0/cub/ colossalai/kernel/cuda_native/csrc/kernels/include/
+fi
+
+python setup.py bdist_wheel
+mv ./dist/* ./all_dist
+python setup.py clean
+conda env remove -n $python_version
+
+