From c67d2494108151215b30f18eaa4e5724c8f3b0b4 Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Fri, 5 Jan 2018 14:01:16 +0000 Subject: [PATCH] foreign-kernels: Rewrite scripts to only build one local image The scripts used to scrape the various repositories to build and push package for all foreign kernels. They were designed to be run periodically and provide "official" foreign kernel packages. Needless to say we did not run them periodically and the linuxkit packages became out-dated quickly. Now, we just provide users who are interested in using foreign kernels the means to build their own package from specific vendor kernels. Each script uses slightly different command line arguments as the location and naming of the kernel packages differ wildly. The help message provide a working example which has been tested with a minimal LinuxKit YAML file. Signed-off-by: Rolf Neugebauer --- scripts/kernels/centos.sh | 60 ++++++++++------------------- scripts/kernels/debian.sh | 46 +++++++++++----------- scripts/kernels/fedora.sh | 69 ++++++++++----------------------- scripts/kernels/mainline.sh | 76 ++++++++++++------------------------- scripts/kernels/ubuntu.sh | 56 +++++++++++---------------- 5 files changed, 110 insertions(+), 197 deletions(-) diff --git a/scripts/kernels/centos.sh b/scripts/kernels/centos.sh index 8be7d63f2..488790a52 100755 --- a/scripts/kernels/centos.sh +++ b/scripts/kernels/centos.sh @@ -1,47 +1,25 @@ #! /bin/sh -REPO="linuxkit/kernel-centos" -BASE_URL=http://mirror.centos.org/centos/ +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + echo + echo "Example:" + echo "$0 foobar/kernel-centos http://mirror.centos.org/centos/7/os/x86_64/Packages 3.10.0-693.el7" + echo + echo "This will create a local LinuxKit kernel package:" + echo "foobar/kernel-centos:3.10.0-693.el7" + echo "which you can then push to hub or just use locally" + exit 1 +fi -TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) +REPO=$1 +URL=$2 +VER=$3 +ARCH=x86_64 -LINKS=$(curl -s ${BASE_URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') -# Just get names for Centos 7 -RELEASES=$(echo $LINKS | grep -o "7\.[^ ]*") -RELEASES="7/ $RELEASES" +KERNEL_RPM="$URL/kernel-$VER.$ARCH.rpm" +HEADERS_RPM="$URL/kernel-headers-$VER.$ARCH.rpm" -# Add updates -URLS="" -for RELEASE in $RELEASES; do - URLS="$URLS ${BASE_URL}/${RELEASE}/os/x86_64/Packages/" -done -URLS="$URLS ${BASE_URL}/7/updates/x86_64/Packages/" +RPM_URLS="$KERNEL_RPM $HEADERS_RPM" -for URL in $URLS; do - PACKAGES=$(curl -s ${URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') - - KERNEL_RPMS=$(echo $PACKAGES | \ - grep -o "kernel-[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+\.[^ ]\+\.rpm") - for KERNEL_RPM in $KERNEL_RPMS; do - RPM_URLS="${URL}/${KERNEL_RPM}" - - VERSION=$(echo $KERNEL_RPM | \ - grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9\.]\+\.el[0-9]\+") - - if echo $TAGS | grep -q "\"${VERSION}\""; then - echo "${REPO}:${VERSION} exists" - continue - fi - - # Don't pull in the headers. This is mostly for testing - # HEADERS_RPM="kernel-headers-${VERSION}.x86_64.rpm" - # RPM_URLS="${RPM_URLS} ${URL}/${HEADERS_RPM}" - - docker build -t ${REPO}:${VERSION} -f Dockerfile.rpm --no-cache \ - --build-arg RPM_URLS="${RPM_URLS}" . && - DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION} - - docker rmi ${REPO}:${VERSION} - docker system prune -f - done -done +docker build -t "$REPO:$VER" -f Dockerfile.rpm --no-cache --build-arg RPM_URLS="$RPM_URLS" . diff --git a/scripts/kernels/debian.sh b/scripts/kernels/debian.sh index c5661750f..ff4099d57 100755 --- a/scripts/kernels/debian.sh +++ b/scripts/kernels/debian.sh @@ -1,32 +1,30 @@ #! /bin/sh -REPO="linuxkit/kernel-debian" -BASE_URL=http://mirrors.kernel.org/debian/pool/main/l/linux/ +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + echo + echo "Example:" + echo "$0 foobar/kernel-debian 4.14.0-2 4.14.7-1" + echo + echo "This will create a local LinuxKit kernel package:" + echo "foobar/kernel-debian:4.14.7-1" + echo "which you can then push to hub or just use locally" + exit 1 +fi -TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) +# List all available kernels with: +# curl -s http://mirrors.kernel.org/debian/pool/main/l/linux/ | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep -o "linux-image-[0-9]\.[0-9]\+\.[0-9]\+-[0-9]\+-amd64[^ ]\+_amd64\.deb +REPO=$1 +VER1=$2 +VER2=$3 +URL=http://mirrors.kernel.org/debian/pool/main/l/linux ARCH=amd64 -LINKS=$(curl -s ${BASE_URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') -# Just get names for 4.x kernels -KERNELS=$(echo $LINKS | \ - grep -o "linux-image-4\.[0-9]\+\.[0-9]\+-[0-9]\+-${ARCH}[^ ]\+_${ARCH}\.deb") -for KERN_DEB in $KERNELS; do - VERSION=$(echo $KERN_DEB | \ - grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+" | head -1) +KERNEL_DEB="${URL}/linux-image-${VER1}-${ARCH}_${VER2}_${ARCH}.deb" +HEADERS_DEB="${URL}/linux-headers-${VER1}-${ARCH}_${VER2}_${ARCH}.deb" +HEADERS_ALL_DEB="${URL}/linux-headers-${VER1}-all_${VER2}_${ARCH}.deb" - if echo $TAGS | grep -q "\"${VERSION}\""; then - echo "${REPO}:${VERSION} exists" - continue - fi +DEB_URLS="${KERNEL_DEB} ${HEADERS_DEB} ${HEADERS_ALL_DEB}" - URLS="${BASE_URL}/${KERN_DEB}" - - # Doesn't exist build and push - docker build -t ${REPO}:${VERSION} -f Dockerfile.deb --no-cache \ - --build-arg DEB_URLS="${URLS}" . && - DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION} - - docker rmi ${REPO}:${VERSION} - docker system prune -f -done +docker build -t "${REPO}:${VER2}" -f Dockerfile.deb --no-cache --build-arg DEB_URLS="${DEB_URLS}" . diff --git a/scripts/kernels/fedora.sh b/scripts/kernels/fedora.sh index 74fe3e5e9..1ea9957e5 100755 --- a/scripts/kernels/fedora.sh +++ b/scripts/kernels/fedora.sh @@ -1,55 +1,28 @@ #! /bin/sh -REPO="linuxkit/kernel-fedora" -BASE_URL=http://mirrors.kernel.org/fedora/ - -TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) - -LINKS=$(curl -s ${BASE_URL}/releases/ | sed -n 's/.*href="\([^"]*\).*/\1/p') -# Just get releases 20+ -RELEASES=$(echo $LINKS | grep -o "2[0-9]") +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + echo + echo "Example:" + echo "$0 foobar/kernel-fedora http://mirrors.kernel.org/fedora/releases/27/Everything/x86_64/os/Packages/k/ 4.13.9-300.fc27" + echo + echo "This will create a local LinuxKit kernel package:" + echo "foobar/kernel-fedora:4.13.9-300.fc27" + echo "which you can then push to hub or just use locally" + exit 1 +fi +REPO=$1 +URL=$2 +VER=$3 ARCH=x86_64 -URLS="" -for RELEASE in $RELEASES; do - URLS="$URLS ${BASE_URL}/releases/${RELEASE}/Everything/${ARCH}/os/Packages/k/" - URLS="$URLS ${BASE_URL}/updates/${RELEASE}/${ARCH}/k/" -done -for URL in $URLS; do - PACKAGES=$(curl -s ${URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') +KERNEL_RPM="$URL/kernel-$VER.$ARCH.rpm" +CORE_RPM="$URL/kernel-core-$VER.$ARCH.rpm" +MOD_RPM="$URL/kernel-modules-$VER.$ARCH.rpm" +MOD_EXTRA_RPM="$URL/kernel-modules-extra-$VER.$ARCH.rpm" +HEADERS_RPM="$URL/kernel-headers-$VER.$ARCH.rpm" - KERNEL_RPMS=$(echo $PACKAGES | \ - grep -o "kernel-[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+\.[^ ]\+\.rpm") - for KERNEL_RPM in $KERNEL_RPMS; do - RPM_URLS="${URL}/${KERNEL_RPM}" +RPM_URLS="$KERNEL_RPM $CORE_RPM $MOD_RPM $MOD_EXTRA_RPM $HEADERS_RPM" - VERSION=$(echo $KERNEL_RPM | \ - grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9\.]\+\.fc[0-9]\+") - - if echo $TAGS | grep -q "\"${VERSION}\""; then - echo "${REPO}:${VERSION} exists" - continue - fi - - CORE_RPM="kernel-core-${VERSION}.${ARCH}.rpm" - RPM_URLS="${RPM_URLS} ${URL}/${CORE_RPM}" - - MOD_RPM="kernel-modules-${VERSION}.${ARCH}.rpm" - RPM_URLS="${RPM_URLS} ${URL}/${MOD_RPM}" - - MOD_EXTRA_RPM="kernel-modules-extra-${VERSION}.${ARCH}.rpm" - RPM_URLS="${RPM_URLS} ${URL}/${MOD_EXTRA_RPM}" - - # Don't pull in the headers. This is mostly for testing - # HEADERS_RPM="kernel-headers-${VERSION}.x86_64.rpm" - # RPM_URLS="${RPM_URLS} ${URL}/${HEADERS_RPM}" - - docker build -t ${REPO}:${VERSION} -f Dockerfile.rpm --no-cache \ - --build-arg RPM_URLS="${RPM_URLS}" . && - DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION} - - docker rmi ${REPO}:${VERSION} - docker system prune -f - done -done +docker build -t "$REPO:$VER" -f Dockerfile.rpm --no-cache --build-arg RPM_URLS="$RPM_URLS" . diff --git a/scripts/kernels/mainline.sh b/scripts/kernels/mainline.sh index 5694eb6c7..780691ab9 100755 --- a/scripts/kernels/mainline.sh +++ b/scripts/kernels/mainline.sh @@ -1,57 +1,31 @@ #! /bin/sh -REPO="linuxkit/kernel-mainline" +if [ "$#" -ne 4 ]; then + echo "Usage: $0 " + echo + echo "Example:" + echo "$0 foobar/kernel-mainline v4.14.11 041411 201801022143" + echo + echo "This will create a local LinuxKit kernel package:" + echo "foobar/kernel-mainline:4.14.11" + echo "which you can then push to hub or just use locally" + exit 1 +fi + +REPO=$1 +VER=$2 +VER1=$3 +DATE=$4 BASE_URL=http://kernel.ubuntu.com/~kernel-ppa/mainline +ARCH=amd64 +# Strip leading 'v' +KVER=${VER:1} +URL="${BASE_URL}/${VER}" -TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) +KERNEL_DEB="${URL}/linux-image-${KVER}-${VER1}-generic_${KVER}-${VER1}.${DATE}_${ARCH}.deb" +HEADERS_DEB="${URL}/linux-headers-${KVER}-${VER1}-generic_${KVER}-${VER1}.${DATE}_${ARCH}.deb" +HEADERS_ALL_DEB="${URL}/linux-headers-${KVER}-${VER1}_${KVER}-${VER1}.${DATE}_all.deb" -build_image() { - VERSION=$1 - KDIR=$2 - ARCH=amd64 +DEB_URLS="${KERNEL_DEB} ${HEADERS_DEB} ${HEADERS_ALL_DEB}" - LINKS=$(curl -s ${BASE_URL}/${KDIR}/ | \ - sed -n 's/.*href="\([^"]*\).*/\1/p') - - IMAGE=$(echo $LINKS | \ - grep -o "linux-image[^ ]\+-generic[^ ]\+${ARCH}.deb" | head -1) - [ -z "${IMAGE}" ] && return 1 - HDR_GEN=$(echo $LINKS | grep -o "linux-headers[^ ]\+_all.deb" | head -1) - [ -z "${HDR_GEN}" ] && return 1 - HDR_ARCH=$(echo $LINKS | \ - grep -o "linux-headers[^ ]\+-generic[^ ]\+${ARCH}.deb" | head -1) - [ -z "${HDR_ARCH}" ] && return 1 - - DEB_URL=${BASE_URL}/${KDIR}/${IMAGE} - HDR_GEN_URL=${BASE_URL}/${KDIR}/${HDR_GEN} - HDR_ARCH_URL=${BASE_URL}/${KDIR}/${HDR_ARCH} - echo "Trying to fetch ${VERSION} from ${DEB_URL}" - - docker build -t ${REPO}:${VERSION} -f Dockerfile.deb --no-cache \ - --build-arg DEB_URLS="${DEB_URL} ${HDR_GEN_URL} ${HDR_ARCH_URL}" . -} - -LINKS=$(curl -s ${BASE_URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') -# Extract all kernel versions (drop RCs, ckt(?) and other links) -VERSIONS=$(echo $LINKS | grep -o "v[0-9]\+\.[0-9]\+\.[0-9]\+[^ ]*" | \ - grep -ve '-rc' | grep -ve '-ckt' | uniq) - -# Extract 3.16.x and 4.x.x -THREES=$(echo $VERSIONS | grep -o "v3\.16\.[0-9]\+[^ ]*") -FOURS=$(echo $VERSIONS | grep -o "v4\.[0-9]\+\.[0-9]\+[^ ]*") -KDIRS="${THREES} ${FOURS}" - -for KDIR in $KDIRS; do - # Strip the Ubuntu release name for the tag and also the 'v' like with - # the other kernel packages - VERSION=$(echo $KDIR | grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+") - if echo $TAGS | grep -q "\"${VERSION}\""; then - echo "${REPO}:${VERSION} exists" - continue - fi - build_image ${VERSION} ${KDIR} && \ - DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION} - - docker rmi ${REPO}:${VERSION} - docker system prune -f -done +docker build -t "${REPO}:${KVER}" -f Dockerfile.deb --no-cache --build-arg DEB_URLS="${DEB_URLS}" . diff --git a/scripts/kernels/ubuntu.sh b/scripts/kernels/ubuntu.sh index b5aa11c7f..0221824ae 100755 --- a/scripts/kernels/ubuntu.sh +++ b/scripts/kernels/ubuntu.sh @@ -1,41 +1,31 @@ #! /bin/sh -REPO="linuxkit/kernel-ubuntu" -BASE_URL=http://mirrors.kernel.org/ubuntu/pool/main/l/linux/ +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + echo + echo "Example:" + echo "$0 foobar/kernel-ubuntu 4.14.0-13 15" + echo + echo "This will create a local LinuxKit kernel package:" + echo "foobar/kernel-ubuntu:4.14.0-13.15" + echo "which you can then push to hub or just use locally" + exit 1 +fi -TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) +# List all available kernels with: +# curl -s http://mirrors.kernel.org/ubuntu/pool/main/l/linux/ | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep -o "linux-image-[0-9]\.[0-9]\+\.[0-9]\+-[0-9]\+-generic_[^ ]\+amd64\.deb" +REPO=$1 +VER1=$2 +VER2=$3 +URL=http://mirrors.kernel.org/ubuntu/pool/main/l/linux ARCH=amd64 -LINKS=$(curl -s ${BASE_URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') -# Just get names for 4.x kernels -KERNELS=$(echo $LINKS | \ - grep -o "linux-image-4\.[0-9]\+\.[0-9]\+-[0-9]\+-generic_[^ ]\+${ARCH}\.deb") -for KERN_DEB in $KERNELS; do - VERSION=$(echo $KERN_DEB | \ - grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+" | head -1) +KERNEL_DEB="${URL}/linux-image-${VER1}-generic_${VER1}.${VER2}_${ARCH}.deb" +KERNEL_EXTRA_DEB="${URL}/linux-image-extra-${VER1}-generic_${VER1}.${VER2}_${ARCH}.deb" +HEADERS_DEB="${URL}/linux-headers-${VER1}-generic_${VER1}.${VER2}_${ARCH}.deb" +HEADERS_ALL_DEB="${URL}/linux-headers-${VER1}_${VER1}.${VER2}_all.deb" - if echo $TAGS | grep -q "\"${VERSION}\""; then - echo "${REPO}:${VERSION} exists" - continue - fi +DEB_URLS="${KERNEL_DEB} ${KERNEL_EXTRA_DEB} ${HEADERS_DEB} ${HEADERS_ALL_DEB}" - EXTRA_DEB=$(echo $LINKS | \ - grep -o "linux-image-extra-${VERSION}-generic_[^ ]\+${ARCH}\.deb") - - URLS="${BASE_URL}/${KERN_DEB} ${BASE_URL}/${EXTRA_DEB}" - - # Don't pull in the headers. This is mostly for testing - # HDR_DEB=$(echo $LINKS | \ - # grep -o "linux-headers-${VERSION}_[^ ]\+_all\.deb") - # HDR_ARCH_DEB=$(echo $LINKS | \ - # grep -o "linux-headers-${VERSION}-generic_[^ ]\+_${ARCH}\.deb") - # URLS="${URLS} ${BASE_URL}/${HDR_DEB} ${BASE_URL}/${HDR_ARCH_DEB}" - - docker build -t ${REPO}:${VERSION} -f Dockerfile.deb --no-cache \ - --build-arg DEB_URLS="${URLS}" . && - DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION} - - docker rmi ${REPO}:${VERSION} - docker system prune -f -done +docker build -t "${REPO}:${VER1}.${VER2}" -f Dockerfile.deb --no-cache --build-arg DEB_URLS="${DEB_URLS}" .