cleanup(ci): natively builds docker images for x86_64 and arm64 and then use docker manifest to combine them.

This allows for native-speed build of docker images, reducing CI time spent in
docker buildx qemu cross build for arm64 up to 10x.

Signed-off-by: Federico Di Pierro <nierro92@gmail.com>
This commit is contained in:
Federico Di Pierro 2022-07-14 16:19:49 +02:00 committed by poiana
parent b759e77fda
commit fb579615a3

View File

@ -316,6 +316,105 @@ jobs:
FALCO_VERSION=$(cat /build/release/userspace/falco/config_falco.h | grep 'FALCO_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
/source/falco/scripts/publish-deb -f /build/release/falco-${FALCO_VERSION}-x86_64.deb -f /build-arm64/release/falco-${FALCO_VERSION}-aarch64.deb -r deb-dev
"build-docker-dev":
docker:
- image: alpine:3.16
steps:
- attach_workspace:
at: /
- checkout
- setup_remote_docker:
version: 20.10.12
docker_layer_caching: true
- run:
name: Install deps
command: |
apk update
apk add make bash git docker docker-cli-buildx py3-pip
pip install awscli
- run:
name: Login to registries
command: |
echo ${DOCKERHUB_SECRET} | docker login -u ${DOCKERHUB_USER} --password-stdin
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/falcosecurity
- run:
name: Build and publish no-driver-dev
command: |
FALCO_VERSION=$(cat /build/release/userspace/falco/config_falco.h | grep 'FALCO_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
cd /source
docker buildx build --push --build-arg VERSION_BUCKET=bin-dev --build-arg FALCO_VERSION=${FALCO_VERSION} \
-t falcosecurity/falco-no-driver:x86_64-master \
-t falcosecurity/falco:x86_64-master-slim \
-t public.ecr.aws/falcosecurity/falco-no-driver:x86_64-master \
-t public.ecr.aws/falcosecurity/falco:x86_64-master-slim \
docker/no-driver
- run:
name: Build and publish falco-dev
command: |
FALCO_VERSION=$(cat /build/release/userspace/falco/config_falco.h | grep 'FALCO_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
cd /source
docker buildx build --push --build-arg VERSION_BUCKET=deb-dev --build-arg FALCO_VERSION=${FALCO_VERSION} \
-t falcosecurity/falco:x86_64-master \
-t public.ecr.aws/falcosecurity/falco:x86_64-master \
docker/falco
- run:
name: Build and publish falco-driver-loader-dev
command: |
cd /source
docker buildx build --push --build-arg FALCO_IMAGE_TAG=master \
-t falcosecurity/falco-driver-loader:x86_64-master \
-t public.ecr.aws/falcosecurity/falco-driver-loader:x86_64-master \
docker/driver-loader
"build-docker-dev-arm64":
machine:
enabled: true
image: ubuntu-2004:202101-01
docker_layer_caching: true
resource_class: arm.medium
steps:
- attach_workspace:
at: /tmp
- run:
name: Install deps
command: |
sudo apt update
sudo apt install groff less python3-pip
pip install awscli
- run:
name: Login to registries
command: |
echo ${DOCKERHUB_SECRET} | docker login -u ${DOCKERHUB_USER} --password-stdin
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/falcosecurity
- run:
name: Build and publish no-driver-dev
command: |
FALCO_VERSION=$(cat /tmp/build/release/userspace/falco/config_falco.h | grep 'FALCO_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
cd /tmp/source-arm64
docker buildx build --push --build-arg VERSION_BUCKET=bin-dev --build-arg FALCO_VERSION=${FALCO_VERSION} \
-t falcosecurity/falco-no-driver:aarch64-master \
-t falcosecurity/falco:aarch64-master-slim \
-t public.ecr.aws/falcosecurity/falco-no-driver:aarch64-master \
-t public.ecr.aws/falcosecurity/falco:aarch64-master-slim \
docker/no-driver
- run:
name: Build and publish falco-dev
command: |
FALCO_VERSION=$(cat /tmp/build/release/userspace/falco/config_falco.h | grep 'FALCO_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
cd /tmp/source-arm64
docker buildx build --push --build-arg VERSION_BUCKET=deb-dev --build-arg FALCO_VERSION=${FALCO_VERSION} \
-t falcosecurity/falco:aarch64-master \
-t public.ecr.aws/falcosecurity/falco:aarch64-master \
docker/falco
- run:
name: Build and publish falco-driver-loader-dev
command: |
cd /tmp/source-arm64
docker buildx build --push --build-arg FALCO_IMAGE_TAG=master \
-t falcosecurity/falco-driver-loader:aarch64-master \
-t public.ecr.aws/falcosecurity/falco-driver-loader:aarch64-master \
docker/driver-loader
# Publish docker packages
"publish-docker-dev":
docker:
@ -328,44 +427,62 @@ jobs:
- setup_remote_docker:
version: 20.10.12
- run:
name: Prepare env
name: Install deps
command: |
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
docker context create falco-env
docker buildx create falco-env --driver docker-container --use
echo ${DOCKERHUB_SECRET} | docker login -u ${DOCKERHUB_USER} --password-stdin
sudo apt update
sudo apt install groff less python3-pip
pip install awscli
- run:
name: Login to aws ECR
name: Login to registries
command: |
echo ${DOCKERHUB_SECRET} | docker login -u ${DOCKERHUB_USER} --password-stdin
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/falcosecurity
- run:
name: Build and publish no-driver-dev
name: Upload no-driver-dev manifest to registries
command: |
FALCO_VERSION=$(cat /build/release/userspace/falco/config_falco.h | grep 'FALCO_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
docker buildx build --build-arg VERSION_BUCKET=bin-dev --build-arg FALCO_VERSION=${FALCO_VERSION} --platform "arm64,amd64" --push \
-t falcosecurity/falco-no-driver:master \
-t falcosecurity/falco:master-slim \
-t public.ecr.aws/falcosecurity/falco-no-driver:master \
-t public.ecr.aws/falcosecurity/falco:master-slim \
docker/no-driver
docker manifest create falcosecurity/falco-no-driver:master \
falcosecurity/falco-no-driver:aarch64-master \
falcosecurity/falco-no-driver:x86_64-master
docker manifest push falcosecurity/falco-no-driver:master
docker manifest create falcosecurity/falco:master-slim \
falcosecurity/falco:aarch64-master-slim \
falcosecurity/falco:x86_64-master-slim
docker manifest push falcosecurity/falco:master-slim
docker manifest create public.ecr.aws/falcosecurity/falco-no-driver:master \
public.ecr.aws/falcosecurity/falco-no-driver:aarch64-master \
public.ecr.aws/falcosecurity/falco-no-driver:x86_64-master
docker manifest push public.ecr.aws/falcosecurity/falco-no-driver:master
docker manifest create public.ecr.aws/falcosecurity/falco:master-slim \
public.ecr.aws/falcosecurity/falco:aarch64-master-slim \
public.ecr.aws/falcosecurity/falco:x86_64-master-slim
docker manifest push public.ecr.aws/falcosecurity/falco:master-slim
- run:
name: Build and publish dev
name: Upload falco-dev manifest to registries
command: |
FALCO_VERSION=$(cat /build/release/userspace/falco/config_falco.h | grep 'FALCO_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
docker buildx build --build-arg VERSION_BUCKET=deb-dev --build-arg FALCO_VERSION=${FALCO_VERSION} --platform "arm64,amd64" --push \
-t falcosecurity/falco:master \
-t public.ecr.aws/falcosecurity/falco:master \
docker/falco
docker manifest create falcosecurity/falco:master \
falcosecurity/falco:aarch64-master \
falcosecurity/falco:x86_64-master
docker manifest push falcosecurity/falco:master
docker manifest create public.ecr.aws/falcosecurity/falco:master \
public.ecr.aws/falcosecurity/falco:aarch64-master \
public.ecr.aws/falcosecurity/falco:x86_64-master
docker manifest push public.ecr.aws/falcosecurity/falco:master
- run:
name: Build and publish dev falco-driver-loader-dev
name: Upload falco-driver-loader-dev manifest to registries
command: |
docker buildx build --build-arg FALCO_IMAGE_TAG=master --platform "arm64,amd64" --push \
-t falcosecurity/falco-driver-loader:master \
-t public.ecr.aws/falcosecurity/falco-driver-loader:master \
docker/driver-loader
docker manifest create falcosecurity/falco-driver-loader:master \
falcosecurity/falco-driver-loader:aarch64-master \
falcosecurity/falco-driver-loader:x86_64-master
docker manifest push falcosecurity/falco-driver-loader:master
docker manifest create public.ecr.aws/falcosecurity/falco-driver-loader:master \
public.ecr.aws/falcosecurity/falco-driver-loader:aarch64-master \
public.ecr.aws/falcosecurity/falco-driver-loader:x86_64-master
docker manifest push public.ecr.aws/falcosecurity/falco-driver-loader:master
# Publish the packages
"publish-packages":
@ -417,6 +534,118 @@ jobs:
command: |
FALCO_VERSION=$(cat /build/release/userspace/falco/config_falco.h | grep 'FALCO_VERSION ' | cut -d' ' -f3 | sed -e 's/^"//' -e 's/"$//')
/source/falco/scripts/publish-deb -f /build/release/falco-${FALCO_VERSION}-x86_64.deb -f /build-arm64/release/falco-${FALCO_VERSION}-aarch64.deb -r deb
"build-docker":
docker:
- image: alpine:3.16
steps:
- attach_workspace:
at: /
- checkout
- setup_remote_docker:
version: 20.10.12
docker_layer_caching: true
- run:
name: Install deps
command: |
apk update
apk add make bash git docker docker-cli-buildx py3-pip
pip install awscli
- run:
name: Login to registries
command: |
echo ${DOCKERHUB_SECRET} | docker login -u ${DOCKERHUB_USER} --password-stdin
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/falcosecurity
- run:
name: Build and publish no-driver
command: |
cd /source
docker buildx build --push --build-arg VERSION_BUCKET=bin-dev --build-arg FALCO_VERSION=${CIRCLE_TAG} \
-t "falcosecurity/falco-no-driver:x86_64-${CIRCLE_TAG}" \
-t falcosecurity/falco-no-driver:x86_64-latest \
-t "falcosecurity/falco:x86_64-${CIRCLE_TAG}-slim" \
-t "falcosecurity/falco:x86_64-latest-slim" \
-t "public.ecr.aws/falcosecurity/falco-no-driver:x86_64-${CIRCLE_TAG}" \
-t "public.ecr.aws/falcosecurity/falco-no-driver:x86_64-latest" \
-t "public.ecr.aws/falcosecurity/falco:x86_64-${CIRCLE_TAG}-slim" \
-t "public.ecr.aws/falcosecurity/falco:x86_64-latest-slim" \
docker/no-driver
- run:
name: Build and publish falco
command: |
cd /source
docker buildx build --push --build-arg VERSION_BUCKET=deb-dev --build-arg FALCO_VERSION=${CIRCLE_TAG} \
-t "falcosecurity/falco:x86_64-${CIRCLE_TAG}" \
-t "falcosecurity/falco:x86_64-latest" \
-t "public.ecr.aws/falcosecurity/falco:x86_64-${CIRCLE_TAG}" \
-t "public.ecr.aws/falcosecurity/falco:x86_64-latest" \
docker/falco
- run:
name: Build and publish falco-driver-loader
command: |
cd /source
docker buildx build --push --build-arg FALCO_IMAGE_TAG=${CIRCLE_TAG} \
-t "falcosecurity/falco-driver-loader:x86_64-${CIRCLE_TAG}" \
-t "falcosecurity/falco-driver-loader:x86_64-latest" \
-t "public.ecr.aws/falcosecurity/falco-driver-loader:x86_64-${CIRCLE_TAG}" \
-t "public.ecr.aws/falcosecurity/falco-driver-loader:x86_64-latest" \
docker/driver-loader
"build-docker-arm64":
machine:
enabled: true
image: ubuntu-2004:202101-01
docker_layer_caching: true
resource_class: arm.medium
steps:
- attach_workspace:
at: /tmp
- run:
name: Install deps
command: |
sudo apt update
sudo apt install groff less python3-pip
pip install awscli
- run:
name: Login to registries
command: |
echo ${DOCKERHUB_SECRET} | docker login -u ${DOCKERHUB_USER} --password-stdin
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/falcosecurity
- run:
name: Build and publish no-driver
command: |
cd /tmp/source-arm64
docker buildx build --push --build-arg VERSION_BUCKET=bin --build-arg FALCO_VERSION=${CIRCLE_TAG} \
-t falcosecurity/falco-no-driver:aarch64-${CIRCLE_TAG} \
-t falcosecurity/falco-no-driver:aarch64-latest \
-t falcosecurity/falco:aarch64-${CIRCLE_TAG}-slim \
-t "falcosecurity/falco:aarch64-latest-slim" \
-t public.ecr.aws/falcosecurity/falco-no-driver:aarch64-${CIRCLE_TAG} \
-t "public.ecr.aws/falcosecurity/falco-no-driver:aarch64-latest" \
-t public.ecr.aws/falcosecurity/falco:aarch64-${CIRCLE_TAG}-slim \
-t "public.ecr.aws/falcosecurity/falco:aarch64-latest-slim" \
docker/no-driver
- run:
name: Build and publish falco
command: |
cd /tmp/source-arm64
docker buildx build --push --build-arg VERSION_BUCKET=deb-dev --build-arg FALCO_VERSION=${CIRCLE_TAG} \
-t "falcosecurity/falco:aarch64-${CIRCLE_TAG}" \
-t "falcosecurity/falco:aarch64-latest" \
-t "public.ecr.aws/falcosecurity/falco:aarch64-${CIRCLE_TAG}" \
-t "public.ecr.aws/falcosecurity/falco:aarch64-latest" \
docker/falco
- run:
name: Build and publish falco-driver-loader
command: |
cd /tmp/source-arm64
docker buildx build --push --build-arg FALCO_IMAGE_TAG=${CIRCLE_TAG} \
-t "falcosecurity/falco-driver-loader:aarch64-${CIRCLE_TAG}" \
-t "falcosecurity/falco-driver-loader:aarch64-latest" \
-t "public.ecr.aws/falcosecurity/falco-driver-loader:aarch64-${CIRCLE_TAG}" \
-t "public.ecr.aws/falcosecurity/falco-driver-loader:aarch64-latest" \
docker/driver-loader
# Publish docker packages
"publish-docker":
docker:
@ -429,50 +658,102 @@ jobs:
- setup_remote_docker:
version: 20.10.12
- run:
name: Prepare env
name: Install deps
command: |
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
docker context create falco-env
docker buildx create falco-env --driver docker-container --use
echo ${DOCKERHUB_SECRET} | docker login -u ${DOCKERHUB_USER} --password-stdin
sudo apt update
sudo apt install groff less python3-pip
pip install awscli
- run:
name: Login to aws ECR
name: Login to registries
command: |
echo ${DOCKERHUB_SECRET} | docker login -u ${DOCKERHUB_USER} --password-stdin
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/falcosecurity
- run:
name: Build and publish no-driver
name: Upload no-driver manifest to registries
command: |
docker buildx build --build-arg VERSION_BUCKET=bin --build-arg FALCO_VERSION=${CIRCLE_TAG} --platform "arm64,amd64" --push \
-t "falcosecurity/falco-no-driver:${CIRCLE_TAG}" \
-t falcosecurity/falco-no-driver:latest \
-t "falcosecurity/falco:${CIRCLE_TAG}-slim" \
-t "falcosecurity/falco:latest-slim" \
-t "public.ecr.aws/falcosecurity/falco-no-driver:${CIRCLE_TAG}" \
-t "public.ecr.aws/falcosecurity/falco-no-driver:latest" \
-t "public.ecr.aws/falcosecurity/falco:${CIRCLE_TAG}-slim" \
-t "public.ecr.aws/falcosecurity/falco:latest-slim" \
docker/no-driver
docker manifest create falcosecurity/falco-no-driver:${CIRCLE_TAG} \
falcosecurity/falco-no-driver:aarch64-${CIRCLE_TAG} \
falcosecurity/falco-no-driver:x86_64-${CIRCLE_TAG}
docker manifest push falcosecurity/falco-no-driver:${CIRCLE_TAG}
docker manifest create falcosecurity/falco-no-driver:latest \
falcosecurity/falco-no-driver:aarch64-latest \
falcosecurity/falco-no-driver:x86_64-latest
docker manifest push falcosecurity/falco-no-driver:latest
docker manifest create falcosecurity/falco:${CIRCLE_TAG}-slim \
falcosecurity/falco:aarch64-${CIRCLE_TAG}-slim \
falcosecurity/falco:x86_64-${CIRCLE_TAG}-slim
docker manifest push falcosecurity/falco:${CIRCLE_TAG}-slim
docker manifest create falcosecurity/falco:latest-slim \
falcosecurity/falco:aarch64-latest-slim \
falcosecurity/falco:x86_64-latest-slim
docker manifest push falcosecurity/falco:latest-slim
docker manifest create public.ecr.aws/falcosecurity/falco-no-driver:${CIRCLE_TAG} \
public.ecr.aws/falcosecurity/falco-no-driver:aarch64-${CIRCLE_TAG} \
public.ecr.aws/falcosecurity/falco-no-driver:x86_64-${CIRCLE_TAG}
docker manifest push public.ecr.aws/falcosecurity/falco-no-driver:${CIRCLE_TAG}
docker manifest create public.ecr.aws/falcosecurity/falco-no-driver:latest \
public.ecr.aws/falcosecurity/falco-no-driver:aarch64-latest \
public.ecr.aws/falcosecurity/falco-no-driver:x86_64-latest
docker manifest push public.ecr.aws/falcosecurity/falco-no-driver:latest
docker manifest create public.ecr.aws/falcosecurity/falco:${CIRCLE_TAG}-slim \
public.ecr.aws/falcosecurity/falco:aarch64-${CIRCLE_TAG}-slim \
public.ecr.aws/falcosecurity/falco:x86_64-${CIRCLE_TAG}-slim
docker manifest push public.ecr.aws/falcosecurity/falco:${CIRCLE_TAG}-slim
docker manifest create public.ecr.aws/falcosecurity/falco:latest-slim \
public.ecr.aws/falcosecurity/falco:aarch64-latest-slim \
public.ecr.aws/falcosecurity/falco:x86_64-latest-slim
docker manifest push public.ecr.aws/falcosecurity/falco:latest-slim
- run:
name: Build and publish falco
name: Upload falco manifest to registries
command: |
docker buildx build --build-arg VERSION_BUCKET=deb --build-arg FALCO_VERSION=${CIRCLE_TAG} --platform "arm64,amd64" --push \
-t "falcosecurity/falco:${CIRCLE_TAG}" \
-t "falcosecurity/falco:latest" \
-t "public.ecr.aws/falcosecurity/falco:${CIRCLE_TAG}" \
-t "public.ecr.aws/falcosecurity/falco:latest" \
docker/falco
docker manifest create falcosecurity/falco:${CIRCLE_TAG} \
falcosecurity/falco:aarch64-${CIRCLE_TAG} \
falcosecurity/falco:x86_64-${CIRCLE_TAG}
docker manifest push falcosecurity/falco:${CIRCLE_TAG}
docker manifest create falcosecurity/falco:latest \
falcosecurity/falco:aarch64-latest \
falcosecurity/falco:x86_64-latest
docker manifest push falcosecurity/falco:latest
docker manifest create public.ecr.aws/falcosecurity/falco:${CIRCLE_TAG} \
public.ecr.aws/falcosecurity/falco:aarch64-${CIRCLE_TAG} \
public.ecr.aws/falcosecurity/falco:x86_64-${CIRCLE_TAG}
docker manifest push public.ecr.aws/falcosecurity/falco:${CIRCLE_TAG}
docker manifest create public.ecr.aws/falcosecurity/falco:latest \
public.ecr.aws/falcosecurity/falco:aarch64-latest \
public.ecr.aws/falcosecurity/falco:x86_64-latest
docker manifest push public.ecr.aws/falcosecurity/falco:latest
- run:
name: Build and publish falco-driver-loader
name: Upload falco-driver-loader manifest to registries
command: |
docker buildx build --build-arg FALCO_IMAGE_TAG=${CIRCLE_TAG} --platform "arm64,amd64" --push \
-t "falcosecurity/falco-driver-loader:${CIRCLE_TAG}" \
-t "falcosecurity/falco-driver-loader:latest" \
-t "public.ecr.aws/falcosecurity/falco-driver-loader:${CIRCLE_TAG}" \
-t "public.ecr.aws/falcosecurity/falco-driver-loader:latest" \
docker/driver-loader
docker manifest create falcosecurity/falco-driver-loader:${CIRCLE_TAG} \
falcosecurity/falco-driver-loader:aarch64-${CIRCLE_TAG} \
falcosecurity/falco-driver-loader:x86_64-${CIRCLE_TAG}
docker manifest push falcosecurity/falco-driver-loader:${CIRCLE_TAG}
docker manifest create falcosecurity/falco-driver-loader:latest \
falcosecurity/falco-driver-loader:aarch64-latest \
falcosecurity/falco-driver-loader:x86_64-latest
docker manifest push falcosecurity/falco-driver-loader:latest
docker manifest create public.ecr.aws/falcosecurity/falco-driver-loader:${CIRCLE_TAG} \
public.ecr.aws/falcosecurity/falco-driver-loader:aarch64-${CIRCLE_TAG} \
public.ecr.aws/falcosecurity/falco-driver-loader:x86_64-${CIRCLE_TAG}
docker manifest push public.ecr.aws/falcosecurity/falco-driver-loader:${CIRCLE_TAG}
docker manifest create public.ecr.aws/falcosecurity/falco-driver-loader:latest \
public.ecr.aws/falcosecurity/falco-driver-loader:aarch64-latest \
public.ecr.aws/falcosecurity/falco-driver-loader:x86_64-latest
docker manifest push public.ecr.aws/falcosecurity/falco-driver-loader:latest
workflows:
version: 2.1
@ -527,7 +808,7 @@ workflows:
requires:
- "tests-integration"
- "tests-integration-arm64"
- "publish-docker-dev":
- "build-docker-dev":
context:
- falco
- test-infra
@ -540,6 +821,31 @@ workflows:
- "publish-packages-dev"
- "publish-packages-deb-dev"
- "tests-driver-loader-integration"
- "build-docker-dev-arm64":
context:
- falco
- test-infra
filters:
tags:
ignore: /.*/
branches:
only: master
requires:
- "publish-packages-dev"
- "publish-packages-deb-dev"
- "tests-driver-loader-integration"
- "publish-docker-dev":
context:
- falco
- test-infra
filters:
tags:
ignore: /.*/
branches:
only: master
requires:
- "build-docker-dev"
- "build-docker-dev-arm64"
# - "quality/static-analysis" # This is temporarily disabled: https://github.com/falcosecurity/falco/issues/1526
release:
jobs:
@ -595,7 +901,7 @@ workflows:
only: /.*/
branches:
ignore: /.*/
- "publish-docker":
- "build-docker":
context:
- falco
- test-infra
@ -607,3 +913,27 @@ workflows:
only: /.*/
branches:
ignore: /.*/
- "build-docker-arm64":
context:
- falco
- test-infra
requires:
- "publish-packages"
- "publish-packages-deb"
filters:
tags:
only: /.*/
branches:
ignore: /.*/
- "publish-docker":
context:
- falco
- test-infra
requires:
- "build-docker"
- "build-docker-arm64"
filters:
tags:
only: /.*/
branches:
ignore: /.*/