From d10d1844f4a563894213fae214ed654df7b508ad Mon Sep 17 00:00:00 2001 From: Dejan Jesenko Date: Wed, 15 Dec 2021 19:18:10 +0100 Subject: [PATCH] gitlab-ci: reduce the number of deploy stages Merge major, minor, patch and patch-date deploy stages in a single stage. With this commit also the "deploy multiarch" job will start automatically if the previous job is successful. Related-to: TOR-1471 Signed-off-by: Dejan Jesenko --- .gitlab-ci.yml | 342 +++++++++---------------------------------------- 1 file changed, 59 insertions(+), 283 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fd659f2..9d51ea8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,14 +31,8 @@ before_script: stages: - build-rt-tests - build-rt-tests-multiarch - - deploy-major - - deploy-minor - - deploy-patch - - deploy-patch-date - - deploy-multiarch-major - - deploy-multiarch-minor - - deploy-multiarch-patch - - deploy-multiarch-patch-date + - deploy + - deploy-multiarch # Docker image builds # @@ -162,276 +156,95 @@ docker-build-stress-tests-multiarch: # # Make sure DOCKER_HUB_USER/DOCKER_HUB_PW is properly set in the project # specific Gitlab CI settings. -.docker-deploy-major: +.docker-deploy: + when: manual + allow_failure: false + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY when: manual script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker pull ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${GITLAB_DOCKERREGISTRY_SUFFIX} - - docker tag ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${GITLAB_DOCKERREGISTRY_SUFFIX} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR} - - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PW - - docker push ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR} + - *do_docker_experimental + - export DATE=$(date +%Y%m%d) + - export DOCKER_TAGS_CHECK="${MAJOR}.${MINOR}.${PATCH} ${MAJOR}.${MINOR}.${PATCH}-${DATE}" + - export DOCKER_TAGS="${MAJOR} ${MAJOR}.${MINOR}" + - docker pull ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${GITLAB_DOCKERREGISTRY_SUFFIX} + - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PW; + - for tag in ${DOCKER_TAGS_CHECK}; + do + docker manifest inspect ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${tag} > /dev/null && true; RESULT=$?; + (if [ $RESULT == 0 ] ; then echo "Image with ${tag} tag already exists." && exit 1 ; fi); + done + - for tag in ${DOCKER_TAGS} ${DOCKER_TAGS_CHECK}; + do + docker tag ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${GITLAB_DOCKERREGISTRY_SUFFIX} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${tag}; + docker push ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${tag}; + done -docker-deploy-rt-tests-major: - extends: .docker-deploy-major - before_script: - - export MAJOR="${RT_TESTS_MAJOR}" - variables: - IMAGE_NAME: arm32v7-rt-tests - stage: deploy-major - -docker-deploy-stress-tests-major: - extends: .docker-deploy-major - before_script: - - export MAJOR="${STRESS_TESTS_MAJOR}" - variables: - IMAGE_NAME: arm32v7-stress-tests - stage: deploy-major - -docker-deploy-rt-tests-arm64-major: - extends: .docker-deploy-major - before_script: - - export MAJOR="${RT_TESTS_MAJOR}" - variables: - IMAGE_NAME: arm64v8-rt-tests - stage: deploy-major - -docker-deploy-stress-tests-arm64-major: - extends: .docker-deploy-major - before_script: - - export MAJOR="${STRESS_TESTS_MAJOR}" - variables: - IMAGE_NAME: arm64v8-stress-tests - stage: deploy-major - -.docker-deploy-minor: - when: manual - script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker pull ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${GITLAB_DOCKERREGISTRY_SUFFIX} - - docker tag ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${GITLAB_DOCKERREGISTRY_SUFFIX} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR} - - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PW - - docker push ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR} - -docker-deploy-rt-tests-minor: - extends: .docker-deploy-minor - before_script: - - export MAJOR="${RT_TESTS_MAJOR}" - - export MINOR="${RT_TESTS_MINOR}" - variables: - IMAGE_NAME: arm32v7-rt-tests - stage: deploy-minor - -docker-deploy-stress-tests-minor: - extends: .docker-deploy-minor - before_script: - - export MAJOR="${STRESS_TESTS_MAJOR}" - - export MINOR="${STRESS_TESTS_MINOR}" - variables: - IMAGE_NAME: arm32v7-stress-tests - stage: deploy-minor - -docker-deploy-rt-tests-arm64-minor: - extends: .docker-deploy-minor - before_script: - - export MAJOR="${RT_TESTS_MAJOR}" - - export MINOR="${RT_TESTS_MINOR}" - variables: - IMAGE_NAME: arm64v8-rt-tests - stage: deploy-minor - -docker-deploy-stress-tests-arm64-minor: - extends: .docker-deploy-minor - before_script: - - export MAJOR="${STRESS_TESTS_MAJOR}" - - export MINOR="${STRESS_TESTS_MINOR}" - variables: - IMAGE_NAME: arm64v8-stress-tests - stage: deploy-minor - -.docker-deploy-patch: - when: manual - script: - - *do_docker_experimental - - docker manifest inspect ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH} > /dev/null && true; RESULT=$? - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker pull ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${GITLAB_DOCKERREGISTRY_SUFFIX} - - docker tag ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${GITLAB_DOCKERREGISTRY_SUFFIX} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH} - - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PW - - (if [ ${RESULT} == 1 ] ; then docker push ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH} ; else echo Image with this tag already exists. ; fi) - -docker-deploy-rt-tests-patch: - extends: .docker-deploy-patch +docker-deploy-rt-tests: + extends: .docker-deploy before_script: - export MAJOR="${RT_TESTS_MAJOR}" - export MINOR="${RT_TESTS_MINOR}" - export PATCH="${RT_TESTS_PATCH}" variables: IMAGE_NAME: arm32v7-rt-tests - stage: deploy-patch + stage: deploy -docker-deploy-stress-tests-patch: - extends: .docker-deploy-patch +docker-deploy-stress-tests: + extends: .docker-deploy before_script: - export MAJOR="${STRESS_TESTS_MAJOR}" - export MINOR="${STRESS_TESTS_MINOR}" - export PATCH="${STRESS_TESTS_PATCH}" variables: IMAGE_NAME: arm32v7-stress-tests - stage: deploy-patch + stage: deploy -docker-deploy-rt-tests-arm64-patch: - extends: .docker-deploy-patch +docker-deploy-rt-tests-arm64: + extends: .docker-deploy before_script: - export MAJOR="${RT_TESTS_MAJOR}" - export MINOR="${RT_TESTS_MINOR}" - export PATCH="${RT_TESTS_PATCH}" variables: IMAGE_NAME: arm64v8-rt-tests - stage: deploy-patch + stage: deploy -docker-deploy-stress-tests-arm64-patch: - extends: .docker-deploy-patch +docker-deploy-stress-tests-arm64: + extends: .docker-deploy before_script: - export MAJOR="${STRESS_TESTS_MAJOR}" - export MINOR="${STRESS_TESTS_MINOR}" - export PATCH="${STRESS_TESTS_PATCH}" variables: IMAGE_NAME: arm64v8-stress-tests - stage: deploy-patch + stage: deploy -.docker-deploy-patch-date: - when: manual +.docker-deploy-multiarch: script: - *do_docker_experimental - export DATE=$(date +%Y%m%d) - - docker manifest inspect ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH}-${DATE} > /dev/null && true; RESULT=$? - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker pull ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${GITLAB_DOCKERREGISTRY_SUFFIX} - - docker tag ${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${GITLAB_DOCKERREGISTRY_SUFFIX} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH}-${DATE} - - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PW - - (if [ ${RESULT} == 1 ] ; then docker push ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH}-${DATE} ; else echo Image with this tag already exists. ; fi) - -docker-deploy-rt-tests-patch-date: - extends: .docker-deploy-patch-date - before_script: - - export MAJOR="${RT_TESTS_MAJOR}" - - export MINOR="${RT_TESTS_MINOR}" - - export PATCH="${RT_TESTS_PATCH}" - variables: - IMAGE_NAME: arm32v7-rt-tests - stage: deploy-patch-date - -docker-deploy-stress-tests-patch-date: - extends: .docker-deploy-patch-date - before_script: - - export MAJOR="${STRESS_TESTS_MAJOR}" - - export MINOR="${STRESS_TESTS_MINOR}" - - export PATCH="${STRESS_TESTS_PATCH}" - variables: - IMAGE_NAME: arm32v7-stress-tests - stage: deploy-patch-date - -docker-deploy-rt-tests-arm64-patch-date: - extends: .docker-deploy-patch-date - before_script: - - export MAJOR="${RT_TESTS_MAJOR}" - - export MINOR="${RT_TESTS_MINOR}" - - export PATCH="${RT_TESTS_PATCH}" - variables: - IMAGE_NAME: arm64v8-rt-tests - stage: deploy-patch-date - -docker-deploy-stress-tests-arm64-patch-date: - extends: .docker-deploy-patch-date - before_script: - - export MAJOR="${STRESS_TESTS_MAJOR}" - - export MINOR="${STRESS_TESTS_MINOR}" - - export PATCH="${STRESS_TESTS_PATCH}" - variables: - IMAGE_NAME: arm64v8-stress-tests - stage: deploy-patch-date - -.docker-deploy-multiarch-major: - when: manual - script: - - *do_docker_experimental + - export DOCKER_TAGS_CHECK="${MAJOR}.${MINOR}.${PATCH} ${MAJOR}.${MINOR}.${PATCH}-${DATE}" + - export DOCKER_TAGS="${MAJOR} ${MAJOR}.${MINOR}" - rm -rf ~/.docker/manifests/* - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PW - - docker manifest create ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_32}:${MAJOR} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_64}:${MAJOR} - - docker manifest annotate ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_32}:${MAJOR} --os linux --arch arm - - docker manifest annotate ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_64}:${MAJOR} --os linux --arch arm64 - - docker manifest inspect -v ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR} - - docker manifest push ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR} + - for tag in ${DOCKER_TAGS_CHECK}; + do + docker manifest inspect ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${tag} > /dev/null && true; RESULT=$?; + (if [ $RESULT == 0 ] ; then echo "Image with ${tag} tag already exists." && exit 1 ; fi); + done + - for tag in ${DOCKER_TAGS} ${DOCKER_TAGS_CHECK}; + do + docker manifest create ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${tag} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_32}:${tag} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_64}:${tag}; + docker manifest annotate ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${tag} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_32}:${tag} --os linux --arch arm; + docker manifest annotate ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${tag} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_64}:${tag} --os linux --arch arm64; + docker manifest inspect -v ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${tag}; + docker manifest push ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${tag}; + done -docker-deploy-rt-tests-multiarch-major: - extends: .docker-deploy-multiarch-major - before_script: - - export MAJOR="${RT_TESTS_MAJOR}" - variables: - IMAGE_NAME: rt-tests - IMAGE_NAME_32: arm32v7-rt-tests - IMAGE_NAME_64: arm64v8-rt-tests - stage: deploy-multiarch-major - -docker-deploy-stress-tests-multiarch-major: - extends: .docker-deploy-multiarch-major - before_script: - - export MAJOR="${STRESS_TESTS_MAJOR}" - variables: - IMAGE_NAME: stress-tests - IMAGE_NAME_32: arm32v7-stress-tests - IMAGE_NAME_64: arm64v8-stress-tests - stage: deploy-multiarch-major - -.docker-deploy-multiarch-minor: - when: manual - script: - - *do_docker_experimental - - rm -rf ~/.docker/manifests/* - - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PW - - docker manifest create ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_32}:${MAJOR}.${MINOR} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_64}:${MAJOR}.${MINOR} - - docker manifest annotate ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_32}:${MAJOR}.${MINOR} --os linux --arch arm - - docker manifest annotate ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_64}:${MAJOR}.${MINOR} --os linux --arch arm64 - - docker manifest inspect -v ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR} - - docker manifest push ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR} - -docker-deploy-rt-tests-multiarch-minor: - extends: .docker-deploy-multiarch-minor - before_script: - - export MAJOR="${RT_TESTS_MAJOR}" - - export MINOR="${RT_TESTS_MINOR}" - variables: - IMAGE_NAME: rt-tests - IMAGE_NAME_32: arm32v7-rt-tests - IMAGE_NAME_64: arm64v8-rt-tests - stage: deploy-multiarch-minor - -docker-deploy-stress-tests-multiarch-minor: - extends: .docker-deploy-multiarch-minor - before_script: - - export MAJOR="${STRESS_TESTS_MAJOR}" - - export MINOR="${STRESS_TESTS_MINOR}" - variables: - IMAGE_NAME: stress-tests - IMAGE_NAME_32: arm32v7-stress-tests - IMAGE_NAME_64: arm64v8-stress-tests - stage: deploy-multiarch-minor - -.docker-deploy-multiarch-patch: - when: manual - script: - - *do_docker_experimental - - rm -rf ~/.docker/manifests/* - - docker manifest inspect ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH} > /dev/null && true; RESULT=$? - - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PW - - docker manifest create ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_32}:${MAJOR}.${MINOR}.${PATCH} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_64}:${MAJOR}.${MINOR}.${PATCH} - - docker manifest annotate ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_32}:${MAJOR}.${MINOR}.${PATCH} --os linux --arch arm - - docker manifest annotate ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_64}:${MAJOR}.${MINOR}.${PATCH} --os linux --arch arm64 - - docker manifest inspect -v ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH} - - (if [ ${RESULT} == 1 ] ; then docker manifest push ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH} ; else echo Image with this tag already exists. ; fi) - -docker-deploy-rt-tests-multiarch-patch: - extends: .docker-deploy-multiarch-patch +docker-deploy-rt-tests-multiarch: + extends: .docker-deploy-multiarch + needs: ["docker-deploy-rt-tests", "docker-deploy-rt-tests-arm64"] before_script: - export MAJOR="${RT_TESTS_MAJOR}" - export MINOR="${RT_TESTS_MINOR}" @@ -440,10 +253,11 @@ docker-deploy-rt-tests-multiarch-patch: IMAGE_NAME: rt-tests IMAGE_NAME_32: arm32v7-rt-tests IMAGE_NAME_64: arm64v8-rt-tests - stage: deploy-multiarch-patch + stage: deploy-multiarch -docker-deploy-stress-tests-multiarch-patch: - extends: .docker-deploy-multiarch-patch +docker-deploy-stress-tests-multiarch: + extends: .docker-deploy-multiarch + needs: ["docker-deploy-stress-tests", "docker-deploy-stress-tests-arm64"] before_script: - export MAJOR="${STRESS_TESTS_MAJOR}" - export MINOR="${STRESS_TESTS_MINOR}" @@ -452,42 +266,4 @@ docker-deploy-stress-tests-multiarch-patch: IMAGE_NAME: stress-tests IMAGE_NAME_32: arm32v7-stress-tests IMAGE_NAME_64: arm64v8-stress-tests - stage: deploy-multiarch-patch - -.docker-deploy-multiarch-patch-date: - when: manual - script: - - *do_docker_experimental - - rm -rf ~/.docker/manifests/* - - export DATE=$(date +%Y%m%d) - - docker manifest inspect ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH}-${DATE} > /dev/null && true; RESULT=$? - - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PW - - docker manifest create ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH}-${DATE} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_32}:${MAJOR}.${MINOR}.${PATCH}-${DATE} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_64}:${MAJOR}.${MINOR}.${PATCH}-${DATE} - - docker manifest annotate ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH}-${DATE} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_32}:${MAJOR}.${MINOR}.${PATCH}-${DATE} --os linux --arch arm - - docker manifest annotate ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH}-${DATE} ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME_64}:${MAJOR}.${MINOR}.${PATCH}-${DATE} --os linux --arch arm64 - - docker manifest inspect -v ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH}-${DATE} - - (if [ ${RESULT} == 1 ] ; then docker manifest push ${DOCKER_HUB_REPOSITORY}/${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH}-${DATE} ; else echo Image with this tag already exists. ; fi) - -docker-deploy-rt-tests-multiarch-patch-date: - extends: .docker-deploy-multiarch-patch-date - before_script: - - export MAJOR="${RT_TESTS_MAJOR}" - - export MINOR="${RT_TESTS_MINOR}" - - export PATCH="${RT_TESTS_PATCH}" - variables: - IMAGE_NAME: rt-tests - IMAGE_NAME_32: arm32v7-rt-tests - IMAGE_NAME_64: arm64v8-rt-tests - stage: deploy-multiarch-patch-date - -docker-deploy-stress-tests-multiarch-patch-date: - extends: .docker-deploy-multiarch-patch-date - before_script: - - export MAJOR="${STRESS_TESTS_MAJOR}" - - export MINOR="${STRESS_TESTS_MINOR}" - - export PATCH="${STRESS_TESTS_PATCH}" - variables: - IMAGE_NAME: stress-tests - IMAGE_NAME_32: arm32v7-stress-tests - IMAGE_NAME_64: arm64v8-stress-tests - stage: deploy-multiarch-patch-date + stage: deploy-multiarch