From 909f6b9219879a9c932e7f4a52d72bfc5664b3c3 Mon Sep 17 00:00:00 2001
From: Helmut Buchsbaum <helmut.buchsbaum@opensource.tttech-industrial.com>
Date: Mon, 4 Apr 2022 12:15:05 +0200
Subject: [PATCH] debian: Add docker build helpers

To ease building ACRN with all its requirements, provide a distribution aware,
self-contained docker build environment to build regular Debian packages.

For Debian Buster(10) and Ubuntu Focal Fossa(20.04 LTS) some build tools are
grabbed from their respective backports repositories. This provides a much
easier alignment with the more resent distributions.

There are three packages to be re-packaged to accommodate the build needs as
well as the runtime needs of the ACRN packages:

 * python3-elementpath_2.4.0-1
 * python3-xmlschema_1.9.2-1
 * acpica-tools_20200925-1

These packages are built during docker image build time and provided within
a local APT repository for installation at build time, i.e. when running the
respective docker container.

The multistage Dockerfile separates the required pre-build stages and reduces
the resulting image footprint of the final docker image. There are separate
docker images for each distribution.

To summarize the build steps the script acrn-docker-build.sh is provided. To
control the vendor and the distribution for the build, just set the
environment variables VENDOR and DISTRO before calling the script from the
top directory of the ACRN hypervisor workspace, e.g.:

  VENDOR=debian DISTRO=bullseye debian/docker/acrn-docker-build.sh

or

  VENDOR=ubuntu DISTRO=focal debian/docker/acrn-docker-build.sh

Tracked-On: #6688
Signed-off-by: Helmut Buchsbaum <helmut.buchsbaum@opensource.tttech-industrial.com>
---
 debian/docker/Dockerfile           | 244 +++++++++++++++++++++++++++++
 debian/docker/acrn-docker-build.sh |  44 ++++++
 debian/docker/debian-pkg-build.sh  |  26 +++
 debian/docker/gbp.conf             |   5 +
 debian/docker/lintian.sh           |  28 ++++
 5 files changed, 347 insertions(+)
 create mode 100644 debian/docker/Dockerfile
 create mode 100755 debian/docker/acrn-docker-build.sh
 create mode 100755 debian/docker/debian-pkg-build.sh
 create mode 100644 debian/docker/gbp.conf
 create mode 100755 debian/docker/lintian.sh

diff --git a/debian/docker/Dockerfile b/debian/docker/Dockerfile
new file mode 100644
index 000000000..311d8e7db
--- /dev/null
+++ b/debian/docker/Dockerfile
@@ -0,0 +1,244 @@
+# use debian stable as default distribution
+ARG DISTRO=stable
+ARG VENDOR=debian
+
+###############################################################################
+# Prepare apt config and eventually repackage
+FROM ${VENDOR}:${DISTRO} AS tool-builder
+
+ARG DISTRO
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get -y update && \
+    apt-get install -y --no-install-recommends ca-certificates
+
+###############################################################################
+# For older distros: Add backports packages needed
+#  * debhelper dwz libdebhelper-perl: build tool releate update
+#  * lintian: to keep an acceptable recent version
+#  * linux-libc-dev: acrn-dm needs >= 4.20 for udmabuf UAPI
+RUN REPOBASE=$(cat /etc/apt/sources.list | grep -v -E '^#' | head -1 | awk '{print $3}') && \
+    eval $(cat /etc/os-release) && \
+    if [ -n "${VERSION_CODENAME}" ]; then \
+        DISTRONAME=${VERSION_CODENAME}; \
+    else \
+        DISTRONAME=${REPOBASE}; \
+    fi && \
+    case ${DISTRONAME} in \
+        buster) \
+            echo "deb https://deb.debian.org/debian ${DISTRONAME}-backports main" > /etc/apt/sources.list.d/${DISTRONAME}-backports.list; \
+            for p in debhelper dwz libdebhelper-perl lintian linux-libc-dev; do \
+                (echo "Package: $p"; \
+                 echo "Pin: release a=${DISTRONAME}-backports"; \
+                 echo "Pin-Priority: 900"; \
+                 echo "") >> /etc/apt/preferences.d/pin-${DISTRONAME}-backports; \
+            done; \
+            ;; \
+        focal) \
+            for p in debhelper dwz libdebhelper-perl lintian; do \
+                (echo "Package: $p"; \
+                 echo "Pin: release a=${DISTRONAME}-backports"; \
+                 echo "Pin-Priority: 900"; \
+                 echo "") >> /etc/apt/preferences.d/pin-${DISTRONAME}-backports; \
+            done; \
+            ;; \
+    esac
+
+###############################################################################
+# Install packages needed for git buildpackage based build
+RUN apt-get -y update && \
+    apt-get install -y --no-install-recommends \
+        build-essential git-buildpackage devscripts dpkg-dev equivs \
+        lintian sudo apt-utils pristine-tar
+
+
+###############################################################################
+# Repackage packages
+
+# prepare local apt repo for backported packages
+RUN mkdir -p /opt/apt && cd /opt/apt && \
+    echo "Origin: ACRN Local Build" > .Release.header && \
+    echo "Label: acrn-local-build" >> .Release.header && \
+    apt-ftparchive packages . > Packages && \
+    cp .Release.header Release && apt-ftparchive release . >> Release && \
+    echo "deb [trusted=yes] file:/opt/apt ./" > /etc/apt/sources.list.d/local-apt.list && \
+    touch /etc/apt/preferences.d/pin-acrn
+
+# setup git config for temporary use
+RUN git config --global user.name "ACRN Debian Package Build" && \
+    git config --global user.email "acrn-dev@lists.projectacrn.org"
+
+# elementpath 2.4.0
+# Attention: 2.4.0 not available at Debian package repos, so
+# import the upstream version and repackage
+RUN NEEDEDVERSION="2.4.0" && \
+    srcpkg="elementpath" && \
+    url="https://salsa.debian.org/debian/${srcpkg}.git" && \
+    upstream_tag="upstream/2.3.1" && \
+    debian_tag="debian/2.3.1-1" && \
+    debian_branch="master" && \
+    upstream_branch="upstream" && \
+    mkdir -p /usr/src/${srcpkg} && cd /usr/src/${srcpkg} && \
+    git init && git remote add origin ${url} && \
+    git fetch origin --depth 1 refs/tags/${upstream_tag}:refs/tags/${upstream_tag} && \
+    git fetch origin --depth 1 refs/tags/${debian_tag}:refs/tags/${debian_tag} && \
+    if git show ${debian_tag}:debian | grep -qw gbp.conf; then \
+        pristine_tar=$(git show ${debian_tag}:debian/gbp.conf | awk -F "=" '/pristine-tar/ {print $2}' | tr '[:upper:]' '[:lower:]' | xargs); \
+        if [ "${pristine_tar}" = "true" ]; then \
+            git fetch origin pristine-tar; git branch -t pristine-tar origin/pristine-tar; \
+        fi; \
+        debian_branch=$(git show ${debian_tag}:debian/gbp.conf | awk -F "=" '/debian-branch/ {print $2}' | xargs) && \
+        if [ -z "${debian_branch}" ]; then \
+            debian_branch="master"; \
+        fi; \
+        upstream_branch=$(git show ${debian_tag}:debian/gbp.conf | awk -F "=" '/upstream-branch/ {print $2}' | xargs) && \
+        if [ -z "${upstream_branch}" ]; then \
+            upstream_branch="upstream"; \
+        fi; \
+    fi && \
+    git checkout -b ${upstream_branch} ${upstream_tag} && \
+    git checkout -b ${debian_branch} ${debian_tag} && \
+    sed -i 's#/latest##g' debian/watch && git add -u && git commit -m "d/watch: Fix watch url to get arbitrary versions" && \
+    gbp import-orig --uscan --upstream-version ${NEEDEDVERSION} && \
+    EDITOR=true DEBEMAIL=$(git config user.email) DEBFULLNAME=$(git config user.name) gbp dch -R -N ${NEEDEDVERSION}-1  --commit && \
+    mk-build-deps --tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes' --install debian/control --remove && \
+    rm -f $(dpkg-parsechangelog -Ssource)-build-deps_$(dpkg-parsechangelog -Sversion)_*.* && \
+    DEB_BUILD_OPTIONS="nocheck" gbp buildpackage -b -us -uc  && \
+    for p in $(grep -E '^Package:' debian/control | awk '{print $2}'); do \
+        echo "Package: $p" >> /etc/apt/preferences.d/pin-acrn; \
+        echo "Pin: release l=acrn-local-build" >> /etc/apt/preferences.d/pin-acrn; \
+        echo "Pin-Priority: 900" >> /etc/apt/preferences.d/pin-acrn; \
+        echo "" >> /etc/apt/preferences.d/pin-acrn; \
+    done && \
+    cd /usr/src && mv *.deb /opt/apt && \
+    cd /opt/apt && apt-ftparchive packages . > Packages && cp .Release.header Release && apt-ftparchive release . >> Release && \
+    ls -al /opt/apt && \
+    cat /etc/apt/preferences.d/pin-acrn && \
+    apt-get update -y
+
+# xmlschema 1.9.2
+# Again we need to repackage. This the upstream version is already available, but no debian version.
+# Use 1.4.2-1 as base
+RUN NEEDEDVERSION="1.9.2"; \
+    srcpkg="xmlschema" && \
+    url="https://salsa.debian.org/python-team/packages/python-xmlschema.git" && \
+    upstream_tag="upstream/${NEEDEDVERSION}" && \
+    debian_tag="debian/1.4.2-1" && \
+    debian_branch="master" && \
+    upstream_branch="upstream" && \
+    mkdir -p /usr/src/${srcpkg} && cd /usr/src/${srcpkg} && \
+    git init && git remote add origin ${url} && \
+    git fetch origin refs/tags/${upstream_tag}:refs/tags/${upstream_tag} && \
+    git fetch origin refs/tags/${debian_tag}:refs/tags/${debian_tag} && \
+    if git show ${debian_tag}:debian | grep -qw gbp.conf; then \
+        pristine_tar=$(git show ${debian_tag}:debian/gbp.conf | awk -F "=" '/pristine-tar/ {print $2}' | tr '[:upper:]' '[:lower:]' | xargs); \
+        if [ "${pristine_tar}" = "true" ]; then \
+            git fetch origin pristine-tar; git branch -t pristine-tar origin/pristine-tar; \
+        fi; \
+        debian_branch=$(git show ${debian_tag}:debian/gbp.conf | awk -F "=" '/debian-branch/ {print $2}' | xargs) && \
+        if [ -z "${debian_branch}" ]; then \
+            debian_branch="master"; \
+        fi; \
+        upstream_branch=$(git show ${debian_tag}:debian/gbp.conf | awk -F "=" '/upstream-branch/ {print $2}' | xargs) && \
+        if [ -z "${upstream_branch}" ]; then \
+            upstream_branch="upstream"; \
+        fi; \
+    fi && \
+    git checkout -b ${upstream_branch} ${upstream_tag} && \
+    git checkout -b ${debian_branch} ${debian_tag} && \
+    git merge --no-edit ${upstream_tag} && \
+    EDITOR=true DEBEMAIL=$(git config user.email) DEBFULLNAME=$(git config user.name) gbp dch -R -N ${NEEDEDVERSION}-1  --commit && \
+    mk-build-deps --tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes' --install debian/control --remove && \
+    rm -f $(dpkg-parsechangelog -Ssource)-build-deps_$(dpkg-parsechangelog -Sversion)_*.* && \
+    DEB_BUILD_OPTIONS="nocheck" gbp buildpackage -b -us -uc  && \
+    for p in $(grep -E '^Package:' debian/control | awk '{print $2}'); do \
+        echo "Package: $p" >> /etc/apt/preferences.d/pin-acrn; \
+        echo "Pin: release l=acrn-local-build" >> /etc/apt/preferences.d/pin-acrn; \
+        echo "Pin-Priority: 900" >> /etc/apt/preferences.d/pin-acrn; \
+        echo "" >> /etc/apt/preferences.d/pin-acrn; \
+    done && \
+    cd /usr/src && mv *.deb /opt/apt && \
+    cd /opt/apt && apt-ftparchive packages . > Packages && cp .Release.header Release && apt-ftparchive release . >> Release && \
+    apt-get update
+
+# acpica-unix >= 20200925
+RUN NEEDEDVERSION="20200925"; \
+    PKGVERSION=$(apt-cache policy acpica-tools | grep "Candidate:" | awk '{ print $2}'); \
+    if [ -z "${PKGVERSION}" -o "${NEEDEDVERSION}" != "$(echo ${NEEDEDVERSION}\\n${PKGVERSION} | sort -V | head -n1)" ]; then \
+        srcpkg="acpica-unix" && \
+        url="https://github.com/ahs3/acpica-tools" && \
+        upstream_tag="upstream/${NEEDEDVERSION}" && \
+        debian_tag="debian/${NEEDEDVERSION}-1" && \
+        debian_branch="master" && \
+        upstream_branch="upstream" && \
+        mkdir -p /usr/src/${srcpkg} && cd /usr/src/${srcpkg} && \
+        git init && git remote add origin ${url} && \
+        git fetch origin --depth 1 refs/tags/${upstream_tag}:refs/tags/${upstream_tag} && \
+        git fetch origin --depth 1 refs/tags/${debian_tag}:refs/tags/${debian_tag} && \
+        if git show ${debian_tag}:debian | grep -qw gbp.conf; then \
+            pristine_tar=$(git show ${debian_tag}:debian/gbp.conf | awk -F "=" '/pristine-tar/ {print $2}' | tr '[:upper:]' '[:lower:]' | xargs); \
+            if [ "${pristine_tar}" = "true" ]; then \
+                git fetch origin pristine-tar; git branch -t pristine-tar origin/pristine-tar; \
+            fi; \
+            debian_branch=$(git show ${debian_tag}:debian/gbp.conf | awk -F "=" '/debian-branch/ {print $2}' | xargs) && \
+            if [ -z "${debian_branch}" ]; then \
+                debian_branch="master"; \
+            fi; \
+            upstream_branch=$(git show ${debian_tag}:debian/gbp.conf | awk -F "=" '/upstream-branch/ {print $2}' | xargs) && \
+            if [ -z "${upstream_branch}" ]; then \
+                upstream_branch="upstream"; \
+            fi; \
+        fi && \
+        git checkout -b ${upstream_branch} ${upstream_tag} && \
+        git checkout -b ${debian_branch} ${debian_tag} && \
+        mk-build-deps --tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes' --install debian/control --remove && \
+        rm -f $(dpkg-parsechangelog -Ssource)-build-deps_$(dpkg-parsechangelog -Sversion)_*.* && \
+        DEB_BUILD_OPTIONS="nocheck" gbp buildpackage -b -us -uc  && \
+        for p in $(grep -E '^Package:' debian/control | awk '{print $2}'); do \
+            echo "Package: $p" >> /etc/apt/preferences.d/pin-acrn; \
+            echo "Pin: release l=acrn-local-build" >> /etc/apt/preferences.d/pin-acrn; \
+            echo "Pin-Priority: 900" >> /etc/apt/preferences.d/pin-acrn; \
+            echo "" >> /etc/apt/preferences.d/pin-acrn; \
+        done && \
+        cd /usr/src && mv *.deb /opt/apt && \
+        cd /opt/apt && apt-ftparchive packages . > Packages && cp .Release.header Release && apt-ftparchive release . >> Release && \
+        apt-get update -y; \
+    fi
+
+###############################################################################
+# the final image
+FROM ${VENDOR}:${DISTRO}
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get -y update && \
+    apt-get install -y --no-install-recommends ca-certificates
+
+COPY --from=tool-builder /etc/apt/sources.list.d/* /etc/apt/sources.list.d/
+COPY --from=tool-builder /etc/apt/preferences.d/* /etc/apt/preferences.d/
+COPY --from=tool-builder /opt/apt /opt/apt
+
+###############################################################################
+# Install build script requirements
+RUN apt-get -y update && apt-get install -y --no-install-recommends \
+    devscripts \
+    equivs \
+    git-buildpackage \
+    lintian \
+    sudo
+
+
+###############################################################################
+# Mount the topdir of the Debian git repository at /source
+VOLUME /source/
+WORKDIR /source/
+
+###############################################################################
+# Get default settings and helper scripts
+ADD gbp.conf /etc/git-buildpackage/
+ADD debian-pkg-build.sh /usr/local/bin/debian-pkg-build.sh
+ADD lintian.sh /usr/local/bin/lintian.sh
+
+###############################################################################
+# build Debian packages
+ENTRYPOINT ["/usr/local/bin/debian-pkg-build.sh"]
diff --git a/debian/docker/acrn-docker-build.sh b/debian/docker/acrn-docker-build.sh
new file mode 100755
index 000000000..df33ca15d
--- /dev/null
+++ b/debian/docker/acrn-docker-build.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Helper script to build ACRN with docker
+# This also includes building packages required for ACRN build or runtime
+
+VENDOR=${VENDOR:-debian}
+DISTRO=${DISTRO:-stable}
+
+TOPDIR=$(git rev-parse --show-toplevel)
+DOCKER=$(which docker)
+
+if [ -z "${TOPDIR}" ]; then
+    echo "Run $0 from inside git repository!"
+    exit 1
+fi
+
+if [ -z "${DOCKER}" ]; then
+    echo "Cannot find docker binary, please install!"
+    exit 1
+fi
+
+pushd ${TOPDIR} >/dev/null
+
+if [ ! -f debian/docker/Dockerfile ]; then
+    echo "No Dockerfile available!"
+    exit 1
+fi
+
+set -e
+# create docker image for Debian package build
+${DOCKER} build \
+    -f debian/docker/Dockerfile \
+    --build-arg DISTRO=${DISTRO} \
+    --build-arg VENDOR=${VENDOR} \
+    -t acrn-pkg-builder:${DISTRO} debian/docker
+
+# build ACRN packages
+${DOCKER} run \
+    --rm \
+    -e UID=$(id -u) \
+    -e GID=$(id -g) \
+    -v $(pwd):/source acrn-pkg-builder:${DISTRO} -F --no-sign --git-export-dir=build/${DISTRO} "$@"
+
+popd >/dev/null
diff --git a/debian/docker/debian-pkg-build.sh b/debian/docker/debian-pkg-build.sh
new file mode 100755
index 000000000..01a5c63a6
--- /dev/null
+++ b/debian/docker/debian-pkg-build.sh
@@ -0,0 +1,26 @@
+#!/bin/bash -e
+
+if [ ! -f debian/control ]; then
+    echo "Cannot find debian/control" >&2
+    exit 1
+fi
+
+if [[ -n ${UID} && -n ${GID} ]]; then
+    addgroup --gid ${GID} docker-build
+    adduser --uid=${UID} --gid=${GID} --disabled-password --gecos '' docker-build
+else
+    echo "UID/GID not set. Use docker run -e UID=$(id -u) -e GID=$(id -g)" >&2
+    exit 1
+fi
+
+# install build dependencies using tmpdir to not interfer with parallel builds
+topdir=$(pwd)
+tmpdir=$(mktemp -d)
+pushd ${tmpdir} >/dev/null
+mk-build-deps --tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes' --install ${topdir}/debian/control
+popd >/dev/null
+rm -rf ${tmpdir}
+
+# start build
+export HOME=$(echo ~docker-build)
+sudo -E -u docker-build gbp buildpackage "$@"
diff --git a/debian/docker/gbp.conf b/debian/docker/gbp.conf
new file mode 100644
index 000000000..1a56ceb7e
--- /dev/null
+++ b/debian/docker/gbp.conf
@@ -0,0 +1,5 @@
+[DEFAULT]
+
+[buildpackage]
+# To make build fail on lintian errors
+builder = debuild --no-lintian -i -I --check-command=/usr/local/bin/lintian.sh
diff --git a/debian/docker/lintian.sh b/debian/docker/lintian.sh
new file mode 100755
index 000000000..4bfde865e
--- /dev/null
+++ b/debian/docker/lintian.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+# helper to compare version
+verlte() {
+    [  "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]
+}
+
+lintian_version=$(lintian --version | awk '{print $2}')
+
+# Explicitly use --fail-on error if available (since 2.77.0)
+# This circumvents a problem in bullseye lintian, where failing on error
+# is not the default action any more
+# Always use debian profile for lintian
+if $(verlte 2.77.0 ${lintian_version}); then
+    LINTIAN_ARGS="--profile debian --fail-on error"
+else
+    LINTIAN_ARGS="--profile debian"
+fi
+
+echo "lintian ${LINTIAN_ARGS} $1"
+lintian ${LINTIAN_ARGS} $1
+status=$?
+
+if [ $status -ne 0 ]; then
+    echo "+++ LINTIAN ERRORS DETECTED +++" >&2
+fi
+
+exit $status