foreign-kernels: Rewrite scripts to only build one local image

The scripts used to scrape the various repositories to build
and push package for all foreign kernels. They were designed
to be run periodically and provide "official" foreign kernel
packages. Needless to say we did not run them periodically
and the linuxkit packages became out-dated quickly.

Now, we just provide users who are interested in using foreign
kernels the means to build their own package from specific
vendor kernels.

Each script uses slightly different command line arguments
as the location and naming of the kernel packages differ
wildly. The help message provide a working example which
has been tested with a minimal LinuxKit YAML file.

Signed-off-by: Rolf Neugebauer <rolf.neugebauer@docker.com>
This commit is contained in:
Rolf Neugebauer 2018-01-05 14:01:16 +00:00
parent f696c5a253
commit c67d249410
5 changed files with 110 additions and 197 deletions

View File

@ -1,47 +1,25 @@
#! /bin/sh #! /bin/sh
REPO="linuxkit/kernel-centos" if [ "$#" -ne 3 ]; then
BASE_URL=http://mirror.centos.org/centos/ echo "Usage: $0 <org/repo> <base url> <kernel version>"
echo
echo "Example:"
echo "$0 foobar/kernel-centos http://mirror.centos.org/centos/7/os/x86_64/Packages 3.10.0-693.el7"
echo
echo "This will create a local LinuxKit kernel package:"
echo "foobar/kernel-centos:3.10.0-693.el7"
echo "which you can then push to hub or just use locally"
exit 1
fi
TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) REPO=$1
URL=$2
VER=$3
ARCH=x86_64
LINKS=$(curl -s ${BASE_URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') KERNEL_RPM="$URL/kernel-$VER.$ARCH.rpm"
# Just get names for Centos 7 HEADERS_RPM="$URL/kernel-headers-$VER.$ARCH.rpm"
RELEASES=$(echo $LINKS | grep -o "7\.[^ ]*")
RELEASES="7/ $RELEASES"
# Add updates RPM_URLS="$KERNEL_RPM $HEADERS_RPM"
URLS=""
for RELEASE in $RELEASES; do
URLS="$URLS ${BASE_URL}/${RELEASE}/os/x86_64/Packages/"
done
URLS="$URLS ${BASE_URL}/7/updates/x86_64/Packages/"
for URL in $URLS; do docker build -t "$REPO:$VER" -f Dockerfile.rpm --no-cache --build-arg RPM_URLS="$RPM_URLS" .
PACKAGES=$(curl -s ${URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p')
KERNEL_RPMS=$(echo $PACKAGES | \
grep -o "kernel-[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+\.[^ ]\+\.rpm")
for KERNEL_RPM in $KERNEL_RPMS; do
RPM_URLS="${URL}/${KERNEL_RPM}"
VERSION=$(echo $KERNEL_RPM | \
grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9\.]\+\.el[0-9]\+")
if echo $TAGS | grep -q "\"${VERSION}\""; then
echo "${REPO}:${VERSION} exists"
continue
fi
# Don't pull in the headers. This is mostly for testing
# HEADERS_RPM="kernel-headers-${VERSION}.x86_64.rpm"
# RPM_URLS="${RPM_URLS} ${URL}/${HEADERS_RPM}"
docker build -t ${REPO}:${VERSION} -f Dockerfile.rpm --no-cache \
--build-arg RPM_URLS="${RPM_URLS}" . &&
DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION}
docker rmi ${REPO}:${VERSION}
docker system prune -f
done
done

View File

@ -1,32 +1,30 @@
#! /bin/sh #! /bin/sh
REPO="linuxkit/kernel-debian" if [ "$#" -ne 3 ]; then
BASE_URL=http://mirrors.kernel.org/debian/pool/main/l/linux/ echo "Usage: $0 <org/repo> <ABI version> <kernel version>"
echo
echo "Example:"
echo "$0 foobar/kernel-debian 4.14.0-2 4.14.7-1"
echo
echo "This will create a local LinuxKit kernel package:"
echo "foobar/kernel-debian:4.14.7-1"
echo "which you can then push to hub or just use locally"
exit 1
fi
TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) # List all available kernels with:
# curl -s http://mirrors.kernel.org/debian/pool/main/l/linux/ | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep -o "linux-image-[0-9]\.[0-9]\+\.[0-9]\+-[0-9]\+-amd64[^ ]\+_amd64\.deb
REPO=$1
VER1=$2
VER2=$3
URL=http://mirrors.kernel.org/debian/pool/main/l/linux
ARCH=amd64 ARCH=amd64
LINKS=$(curl -s ${BASE_URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p')
# Just get names for 4.x kernels
KERNELS=$(echo $LINKS | \
grep -o "linux-image-4\.[0-9]\+\.[0-9]\+-[0-9]\+-${ARCH}[^ ]\+_${ARCH}\.deb")
for KERN_DEB in $KERNELS; do KERNEL_DEB="${URL}/linux-image-${VER1}-${ARCH}_${VER2}_${ARCH}.deb"
VERSION=$(echo $KERN_DEB | \ HEADERS_DEB="${URL}/linux-headers-${VER1}-${ARCH}_${VER2}_${ARCH}.deb"
grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+" | head -1) HEADERS_ALL_DEB="${URL}/linux-headers-${VER1}-all_${VER2}_${ARCH}.deb"
if echo $TAGS | grep -q "\"${VERSION}\""; then DEB_URLS="${KERNEL_DEB} ${HEADERS_DEB} ${HEADERS_ALL_DEB}"
echo "${REPO}:${VERSION} exists"
continue
fi
URLS="${BASE_URL}/${KERN_DEB}" docker build -t "${REPO}:${VER2}" -f Dockerfile.deb --no-cache --build-arg DEB_URLS="${DEB_URLS}" .
# Doesn't exist build and push
docker build -t ${REPO}:${VERSION} -f Dockerfile.deb --no-cache \
--build-arg DEB_URLS="${URLS}" . &&
DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION}
docker rmi ${REPO}:${VERSION}
docker system prune -f
done

View File

@ -1,55 +1,28 @@
#! /bin/sh #! /bin/sh
REPO="linuxkit/kernel-fedora" if [ "$#" -ne 3 ]; then
BASE_URL=http://mirrors.kernel.org/fedora/ echo "Usage: $0 <org/repo> <base url> <kernel version>"
echo
TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) echo "Example:"
echo "$0 foobar/kernel-fedora http://mirrors.kernel.org/fedora/releases/27/Everything/x86_64/os/Packages/k/ 4.13.9-300.fc27"
LINKS=$(curl -s ${BASE_URL}/releases/ | sed -n 's/.*href="\([^"]*\).*/\1/p') echo
# Just get releases 20+ echo "This will create a local LinuxKit kernel package:"
RELEASES=$(echo $LINKS | grep -o "2[0-9]") echo "foobar/kernel-fedora:4.13.9-300.fc27"
echo "which you can then push to hub or just use locally"
exit 1
fi
REPO=$1
URL=$2
VER=$3
ARCH=x86_64 ARCH=x86_64
URLS=""
for RELEASE in $RELEASES; do
URLS="$URLS ${BASE_URL}/releases/${RELEASE}/Everything/${ARCH}/os/Packages/k/"
URLS="$URLS ${BASE_URL}/updates/${RELEASE}/${ARCH}/k/"
done
for URL in $URLS; do KERNEL_RPM="$URL/kernel-$VER.$ARCH.rpm"
PACKAGES=$(curl -s ${URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') CORE_RPM="$URL/kernel-core-$VER.$ARCH.rpm"
MOD_RPM="$URL/kernel-modules-$VER.$ARCH.rpm"
MOD_EXTRA_RPM="$URL/kernel-modules-extra-$VER.$ARCH.rpm"
HEADERS_RPM="$URL/kernel-headers-$VER.$ARCH.rpm"
KERNEL_RPMS=$(echo $PACKAGES | \ RPM_URLS="$KERNEL_RPM $CORE_RPM $MOD_RPM $MOD_EXTRA_RPM $HEADERS_RPM"
grep -o "kernel-[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+\.[^ ]\+\.rpm")
for KERNEL_RPM in $KERNEL_RPMS; do
RPM_URLS="${URL}/${KERNEL_RPM}"
VERSION=$(echo $KERNEL_RPM | \ docker build -t "$REPO:$VER" -f Dockerfile.rpm --no-cache --build-arg RPM_URLS="$RPM_URLS" .
grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9\.]\+\.fc[0-9]\+")
if echo $TAGS | grep -q "\"${VERSION}\""; then
echo "${REPO}:${VERSION} exists"
continue
fi
CORE_RPM="kernel-core-${VERSION}.${ARCH}.rpm"
RPM_URLS="${RPM_URLS} ${URL}/${CORE_RPM}"
MOD_RPM="kernel-modules-${VERSION}.${ARCH}.rpm"
RPM_URLS="${RPM_URLS} ${URL}/${MOD_RPM}"
MOD_EXTRA_RPM="kernel-modules-extra-${VERSION}.${ARCH}.rpm"
RPM_URLS="${RPM_URLS} ${URL}/${MOD_EXTRA_RPM}"
# Don't pull in the headers. This is mostly for testing
# HEADERS_RPM="kernel-headers-${VERSION}.x86_64.rpm"
# RPM_URLS="${RPM_URLS} ${URL}/${HEADERS_RPM}"
docker build -t ${REPO}:${VERSION} -f Dockerfile.rpm --no-cache \
--build-arg RPM_URLS="${RPM_URLS}" . &&
DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION}
docker rmi ${REPO}:${VERSION}
docker system prune -f
done
done

View File

@ -1,57 +1,31 @@
#! /bin/sh #! /bin/sh
REPO="linuxkit/kernel-mainline" if [ "$#" -ne 4 ]; then
echo "Usage: $0 <org/repo> <base url> <kernel version> <version> <date>"
echo
echo "Example:"
echo "$0 foobar/kernel-mainline v4.14.11 041411 201801022143"
echo
echo "This will create a local LinuxKit kernel package:"
echo "foobar/kernel-mainline:4.14.11"
echo "which you can then push to hub or just use locally"
exit 1
fi
REPO=$1
VER=$2
VER1=$3
DATE=$4
BASE_URL=http://kernel.ubuntu.com/~kernel-ppa/mainline BASE_URL=http://kernel.ubuntu.com/~kernel-ppa/mainline
ARCH=amd64
# Strip leading 'v'
KVER=${VER:1}
URL="${BASE_URL}/${VER}"
TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) KERNEL_DEB="${URL}/linux-image-${KVER}-${VER1}-generic_${KVER}-${VER1}.${DATE}_${ARCH}.deb"
HEADERS_DEB="${URL}/linux-headers-${KVER}-${VER1}-generic_${KVER}-${VER1}.${DATE}_${ARCH}.deb"
HEADERS_ALL_DEB="${URL}/linux-headers-${KVER}-${VER1}_${KVER}-${VER1}.${DATE}_all.deb"
build_image() { DEB_URLS="${KERNEL_DEB} ${HEADERS_DEB} ${HEADERS_ALL_DEB}"
VERSION=$1
KDIR=$2
ARCH=amd64
LINKS=$(curl -s ${BASE_URL}/${KDIR}/ | \ docker build -t "${REPO}:${KVER}" -f Dockerfile.deb --no-cache --build-arg DEB_URLS="${DEB_URLS}" .
sed -n 's/.*href="\([^"]*\).*/\1/p')
IMAGE=$(echo $LINKS | \
grep -o "linux-image[^ ]\+-generic[^ ]\+${ARCH}.deb" | head -1)
[ -z "${IMAGE}" ] && return 1
HDR_GEN=$(echo $LINKS | grep -o "linux-headers[^ ]\+_all.deb" | head -1)
[ -z "${HDR_GEN}" ] && return 1
HDR_ARCH=$(echo $LINKS | \
grep -o "linux-headers[^ ]\+-generic[^ ]\+${ARCH}.deb" | head -1)
[ -z "${HDR_ARCH}" ] && return 1
DEB_URL=${BASE_URL}/${KDIR}/${IMAGE}
HDR_GEN_URL=${BASE_URL}/${KDIR}/${HDR_GEN}
HDR_ARCH_URL=${BASE_URL}/${KDIR}/${HDR_ARCH}
echo "Trying to fetch ${VERSION} from ${DEB_URL}"
docker build -t ${REPO}:${VERSION} -f Dockerfile.deb --no-cache \
--build-arg DEB_URLS="${DEB_URL} ${HDR_GEN_URL} ${HDR_ARCH_URL}" .
}
LINKS=$(curl -s ${BASE_URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p')
# Extract all kernel versions (drop RCs, ckt(?) and other links)
VERSIONS=$(echo $LINKS | grep -o "v[0-9]\+\.[0-9]\+\.[0-9]\+[^ ]*" | \
grep -ve '-rc' | grep -ve '-ckt' | uniq)
# Extract 3.16.x and 4.x.x
THREES=$(echo $VERSIONS | grep -o "v3\.16\.[0-9]\+[^ ]*")
FOURS=$(echo $VERSIONS | grep -o "v4\.[0-9]\+\.[0-9]\+[^ ]*")
KDIRS="${THREES} ${FOURS}"
for KDIR in $KDIRS; do
# Strip the Ubuntu release name for the tag and also the 'v' like with
# the other kernel packages
VERSION=$(echo $KDIR | grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+")
if echo $TAGS | grep -q "\"${VERSION}\""; then
echo "${REPO}:${VERSION} exists"
continue
fi
build_image ${VERSION} ${KDIR} && \
DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION}
docker rmi ${REPO}:${VERSION}
docker system prune -f
done

View File

@ -1,41 +1,31 @@
#! /bin/sh #! /bin/sh
REPO="linuxkit/kernel-ubuntu" if [ "$#" -ne 3 ]; then
BASE_URL=http://mirrors.kernel.org/ubuntu/pool/main/l/linux/ echo "Usage: $0 <org/repo> <kernel version> <sub version>"
echo
echo "Example:"
echo "$0 foobar/kernel-ubuntu 4.14.0-13 15"
echo
echo "This will create a local LinuxKit kernel package:"
echo "foobar/kernel-ubuntu:4.14.0-13.15"
echo "which you can then push to hub or just use locally"
exit 1
fi
TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) # List all available kernels with:
# curl -s http://mirrors.kernel.org/ubuntu/pool/main/l/linux/ | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep -o "linux-image-[0-9]\.[0-9]\+\.[0-9]\+-[0-9]\+-generic_[^ ]\+amd64\.deb"
REPO=$1
VER1=$2
VER2=$3
URL=http://mirrors.kernel.org/ubuntu/pool/main/l/linux
ARCH=amd64 ARCH=amd64
LINKS=$(curl -s ${BASE_URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p')
# Just get names for 4.x kernels
KERNELS=$(echo $LINKS | \
grep -o "linux-image-4\.[0-9]\+\.[0-9]\+-[0-9]\+-generic_[^ ]\+${ARCH}\.deb")
for KERN_DEB in $KERNELS; do KERNEL_DEB="${URL}/linux-image-${VER1}-generic_${VER1}.${VER2}_${ARCH}.deb"
VERSION=$(echo $KERN_DEB | \ KERNEL_EXTRA_DEB="${URL}/linux-image-extra-${VER1}-generic_${VER1}.${VER2}_${ARCH}.deb"
grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+" | head -1) HEADERS_DEB="${URL}/linux-headers-${VER1}-generic_${VER1}.${VER2}_${ARCH}.deb"
HEADERS_ALL_DEB="${URL}/linux-headers-${VER1}_${VER1}.${VER2}_all.deb"
if echo $TAGS | grep -q "\"${VERSION}\""; then DEB_URLS="${KERNEL_DEB} ${KERNEL_EXTRA_DEB} ${HEADERS_DEB} ${HEADERS_ALL_DEB}"
echo "${REPO}:${VERSION} exists"
continue
fi
EXTRA_DEB=$(echo $LINKS | \ docker build -t "${REPO}:${VER1}.${VER2}" -f Dockerfile.deb --no-cache --build-arg DEB_URLS="${DEB_URLS}" .
grep -o "linux-image-extra-${VERSION}-generic_[^ ]\+${ARCH}\.deb")
URLS="${BASE_URL}/${KERN_DEB} ${BASE_URL}/${EXTRA_DEB}"
# Don't pull in the headers. This is mostly for testing
# HDR_DEB=$(echo $LINKS | \
# grep -o "linux-headers-${VERSION}_[^ ]\+_all\.deb")
# HDR_ARCH_DEB=$(echo $LINKS | \
# grep -o "linux-headers-${VERSION}-generic_[^ ]\+_${ARCH}\.deb")
# URLS="${URLS} ${BASE_URL}/${HDR_DEB} ${BASE_URL}/${HDR_ARCH_DEB}"
docker build -t ${REPO}:${VERSION} -f Dockerfile.deb --no-cache \
--build-arg DEB_URLS="${URLS}" . &&
DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION}
docker rmi ${REPO}:${VERSION}
docker system prune -f
done