[ci] added wheel build scripts (#910)

* [ci] added wheel build scripts

* polish code and workflow

* polish code and workflow

* polish code and workflow

* polish code and workflow

* polish code and workflow

* polish code and workflow

* polish code and workflow

* polish code and workflow

* polish code and workflow

* polish code and workflow

* polish code and workflow

* [ci] polish wheel build scripts

.github/workflows/scripts/build_colossalai_wheel.py

@@ -0,0 +1,82 @@
+import requests
+from bs4 import BeautifulSoup
+import re
+import os
+import subprocess
+
+
+WHEEL_TEXT_ROOT_URL = 'https://github.com/hpcaitech/public_assets/tree/main/colossalai/torch_build/torch_wheels'
+RAW_TEXT_FILE_PREFIX = 'https://raw.githubusercontent.com/hpcaitech/public_assets/main/colossalai/torch_build/torch_wheels'
+CUDA_HOME = os.environ['CUDA_HOME']
+
+def get_cuda_bare_metal_version():
+    raw_output = subprocess.check_output([CUDA_HOME + "/bin/nvcc", "-V"], universal_newlines=True)
+    output = raw_output.split()
+    release_idx = output.index("release") + 1
+    release = output[release_idx].split(".")
+    bare_metal_major = release[0]
+    bare_metal_minor = release[1][0]
+
+    return bare_metal_major, bare_metal_minor
+
+def all_wheel_info():
+    page_text = requests.get(WHEEL_TEXT_ROOT_URL).text
+    soup = BeautifulSoup(page_text)
+
+    all_a_links = soup.find_all('a')
+
+    wheel_info = dict()
+
+    for a_link in all_a_links:
+        if 'cuda' in a_link.text and '.txt' in a_link.text:
+            filename = a_link.text
+            torch_version, cuda_version = filename.rstrip('.txt').split('-')
+            cuda_version = cuda_version.lstrip('cuda')
+
+            if torch_version not in wheel_info:
+                wheel_info[torch_version] = dict()
+            wheel_info[torch_version][cuda_version] = dict()
+
+            file_text = requests.get(f'{RAW_TEXT_FILE_PREFIX}/{filename}').text
+            lines = file_text.strip().split('\n')
+
+            for line in lines:
+                parts = line.split('\t')
+                method, url, python_version = parts[:3]
+
+                if len(parts) > 3:
+                    flags = parts[3]
+                    flags = ' '.join(flags.split('+'))
+                else:
+                    flags = ''
+                wheel_info[torch_version][cuda_version][python_version] = dict(method=method, url=url, flags=flags)
+    return wheel_info
+
+
+def build_colossalai(wheel_info):
+    cuda_version_major, cuda_version_minor = get_cuda_bare_metal_version()
+    cuda_version_on_host = f'{cuda_version_major}.{cuda_version_minor}'
+
+    for torch_version, cuda_versioned_wheel_info in wheel_info.items():
+        for cuda_version, python_versioned_wheel_info in cuda_versioned_wheel_info.items():
+            if cuda_version_on_host == cuda_version:
+                for python_version, wheel_info in python_versioned_wheel_info.items():
+                    url = wheel_info['url']
+                    method = wheel_info['method']
+                    flags = wheel_info['flags']
+                    filename = url.split('/')[-1].replace('%2B', '+')
+                    cmd = f'bash ./build_colossalai_wheel.sh {method} {url} {filename} {cuda_version} {python_version} {torch_version} {flags}'
+                    os.system(cmd)
+
+def main():
+    wheel_info = all_wheel_info()
+    build_colossalai(wheel_info)
+
+if __name__ == '__main__':
+    main()
+
+
+
+
+
+        

.github/workflows/scripts/build_colossalai_wheel.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+method=${1}
+url=${2}
+filename=${3}
+cuda_version=${4}
+python_version=${5}
+torch_version=${6}
+flags=${@:7}
+
+git reset --hard HEAD
+mkdir -p ./all_dist
+source activate base
+conda create -n $python_version -y python=$python_version
+source activate $python_version
+
+if [ $1 == "pip" ]
+then
+    wget -nc -q -O ./pip_wheels/$filename $url
+    pip install ./pip_wheels/$filename
+    
+elif [ $1 == 'conda' ]
+then
+    conda install pytorch==$torch_version cudatoolkit=$cuda_version $flags
+else
+    echo Invalid installation method
+    exit
+fi
+
+if [ $cuda_version == "10.2" ]
+then
+    cp -r cub-1.8.0/cub/ colossalai/kernel/cuda_native/csrc/kernels/include/
+fi
+
+python setup.py bdist_wheel
+mv ./dist/* ./all_dist
+python setup.py clean
+conda env remove -n $python_version
+
+