diff --git a/docs/kernels.md b/docs/kernels.md index f9cc71dc5..406548de9 100644 --- a/docs/kernels.md +++ b/docs/kernels.md @@ -22,20 +22,10 @@ containing the `perf` utility for debugging and performance tracing. The perf package is called `kernel-perf` and is tagged the same way as the kernel packages. -In addition to the official kernel images, LinuxKit offers the ability -to build bootable Linux images with kernels from various -distributions. We mostly offer this mostly for testing -purposes. "Foreign" kernel images are created by re-packing the native -kernel packages into hub images. The hub images are typically tagged -with the kernel version. - -In summary, LinuxKit offers a choice of the following kernels: -- [linuxkit/kernel](https://hub.docker.com/r/linuxkit/kernel/): Official LinuxKit kernels. -- [linuxkit/kernel-mainline](https://hub.docker.com/r/linuxkit/kernel-mainline/): Mainline [kernel.org](http://kernel.org) kernels from the [Ubuntu Mainline PPA](http://kernel.ubuntu.com/~kernel-ppa/mainline/). -- [linuxkit/kernel-ubuntu](https://hub.docker.com/r/linuxkit/kernel-ubuntu/): Selected Ubuntu kernels. -- [linuxkit/kernel-debian](https://hub.docker.com/r/linuxkit/kernel-debian/): Selected Debian kernels. -- [linuxkit/kernel-centos](https://hub.docker.com/r/linuxkit/kernel-centos/): Selected CentOS kernels. -- [linuxkit/kernel-fedora](https://hub.docker.com/r/linuxkit/kernel-fedora/): Selected Fedora kernels. +In addition to the official images, there are also some +[scripts](../scripts/kernels) which repackage kernels packages from +some Linux distributions into LinuxKit kernel packages. These are +mostly provided for testing purposes. ## Loading kernel modules diff --git a/scripts/kernels/Dockerfile.deb b/scripts/kernels/Dockerfile.deb index 9652d1daf..4b796ebee 100644 --- a/scripts/kernels/Dockerfile.deb +++ b/scripts/kernels/Dockerfile.deb @@ -1,4 +1,4 @@ -FROM alpine:3.5 AS extract +FROM alpine:3.7 AS extract ARG DEB_URLS @@ -20,7 +20,7 @@ RUN cp -a boot/System.map-* /out/System.map RUN tar cf /out/kernel.tar lib RUN tar cf /out/kernel-dev.tar usr || true -FROM linuxkit/toybox-media:b396a375852e5dffc002389d95e0658c8de72914@sha256:a317cc378946ee48cc011cdfc5aa08f0229f5bf10ff70e3690d8f60b36700033 +FROM scratch WORKDIR / ENTRYPOINT [] CMD [] diff --git a/scripts/kernels/Dockerfile.rpm b/scripts/kernels/Dockerfile.rpm index 332a90996..819117265 100644 --- a/scripts/kernels/Dockerfile.rpm +++ b/scripts/kernels/Dockerfile.rpm @@ -1,4 +1,4 @@ -FROM alpine:3.5 AS extract +FROM alpine:3.7 AS extract ARG RPM_URLS @@ -21,7 +21,7 @@ RUN cp -a boot/System.map-* /out/System.map || mv lib/modules/*/System.map /out/ RUN tar cf /out/kernel.tar lib RUN tar cf /out/kernel-dev.tar usr || true -FROM linuxkit/toybox-media:b396a375852e5dffc002389d95e0658c8de72914@sha256:a317cc378946ee48cc011cdfc5aa08f0229f5bf10ff70e3690d8f60b36700033 +FROM scratch WORKDIR / ENTRYPOINT [] CMD [] diff --git a/scripts/kernels/README.md b/scripts/kernels/README.md new file mode 100644 index 000000000..cea87d795 --- /dev/null +++ b/scripts/kernels/README.md @@ -0,0 +1,31 @@ +# Using "foreign" kernels + +This directory contains a number of scripts to re-package other +distributions kernels into a LinuxKit kernel package. The scripts +download the relevant `rpm`s or `deb`s and create a local docker image +which can be used in LinuxKit. You can optionally push the package to +hub, if you like. + +All scripts take slightly different command line arguments (which +could be improved) as each distribution uses different naming +conventions and repository layouts. + +## Example + +To build a package using the `4.14.11` from the mainline [ppa +repository](http://kernel.ubuntu.com/~kernel-ppa/mainline), first +build the package: + +```sh +./mainline.sh foobar/kernel-mainline v4.14.11 041411 201801022143 +``` + +Here `v4.14.11` is the sub-directory of the [ppa +repository](http://kernel.ubuntu.com/~kernel-ppa/mainline), `041411` +seems to be another version used in the name of the `deb`s, and +`201801022143` is the date. You can find the names by browsing the +[ppa repository](http://kernel.ubuntu.com/~kernel-ppa/mainline). + + +The result is a local image `foobar/kernel-mainline:4.14.11`, which +can be used in a YAML file like a normal LinuxKit kernel image. diff --git a/scripts/kernels/centos.sh b/scripts/kernels/centos.sh index 8be7d63f2..488790a52 100755 --- a/scripts/kernels/centos.sh +++ b/scripts/kernels/centos.sh @@ -1,47 +1,25 @@ #! /bin/sh -REPO="linuxkit/kernel-centos" -BASE_URL=http://mirror.centos.org/centos/ +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + echo + echo "Example:" + echo "$0 foobar/kernel-centos http://mirror.centos.org/centos/7/os/x86_64/Packages 3.10.0-693.el7" + echo + echo "This will create a local LinuxKit kernel package:" + echo "foobar/kernel-centos:3.10.0-693.el7" + echo "which you can then push to hub or just use locally" + exit 1 +fi -TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) +REPO=$1 +URL=$2 +VER=$3 +ARCH=x86_64 -LINKS=$(curl -s ${BASE_URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') -# Just get names for Centos 7 -RELEASES=$(echo $LINKS | grep -o "7\.[^ ]*") -RELEASES="7/ $RELEASES" +KERNEL_RPM="$URL/kernel-$VER.$ARCH.rpm" +HEADERS_RPM="$URL/kernel-headers-$VER.$ARCH.rpm" -# Add updates -URLS="" -for RELEASE in $RELEASES; do - URLS="$URLS ${BASE_URL}/${RELEASE}/os/x86_64/Packages/" -done -URLS="$URLS ${BASE_URL}/7/updates/x86_64/Packages/" +RPM_URLS="$KERNEL_RPM $HEADERS_RPM" -for URL in $URLS; do - PACKAGES=$(curl -s ${URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') - - KERNEL_RPMS=$(echo $PACKAGES | \ - grep -o "kernel-[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+\.[^ ]\+\.rpm") - for KERNEL_RPM in $KERNEL_RPMS; do - RPM_URLS="${URL}/${KERNEL_RPM}" - - VERSION=$(echo $KERNEL_RPM | \ - grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9\.]\+\.el[0-9]\+") - - if echo $TAGS | grep -q "\"${VERSION}\""; then - echo "${REPO}:${VERSION} exists" - continue - fi - - # Don't pull in the headers. This is mostly for testing - # HEADERS_RPM="kernel-headers-${VERSION}.x86_64.rpm" - # RPM_URLS="${RPM_URLS} ${URL}/${HEADERS_RPM}" - - docker build -t ${REPO}:${VERSION} -f Dockerfile.rpm --no-cache \ - --build-arg RPM_URLS="${RPM_URLS}" . && - DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION} - - docker rmi ${REPO}:${VERSION} - docker system prune -f - done -done +docker build -t "$REPO:$VER" -f Dockerfile.rpm --no-cache --build-arg RPM_URLS="$RPM_URLS" . diff --git a/scripts/kernels/debian.sh b/scripts/kernels/debian.sh index c5661750f..ff4099d57 100755 --- a/scripts/kernels/debian.sh +++ b/scripts/kernels/debian.sh @@ -1,32 +1,30 @@ #! /bin/sh -REPO="linuxkit/kernel-debian" -BASE_URL=http://mirrors.kernel.org/debian/pool/main/l/linux/ +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + echo + echo "Example:" + echo "$0 foobar/kernel-debian 4.14.0-2 4.14.7-1" + echo + echo "This will create a local LinuxKit kernel package:" + echo "foobar/kernel-debian:4.14.7-1" + echo "which you can then push to hub or just use locally" + exit 1 +fi -TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) +# List all available kernels with: +# curl -s http://mirrors.kernel.org/debian/pool/main/l/linux/ | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep -o "linux-image-[0-9]\.[0-9]\+\.[0-9]\+-[0-9]\+-amd64[^ ]\+_amd64\.deb +REPO=$1 +VER1=$2 +VER2=$3 +URL=http://mirrors.kernel.org/debian/pool/main/l/linux ARCH=amd64 -LINKS=$(curl -s ${BASE_URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') -# Just get names for 4.x kernels -KERNELS=$(echo $LINKS | \ - grep -o "linux-image-4\.[0-9]\+\.[0-9]\+-[0-9]\+-${ARCH}[^ ]\+_${ARCH}\.deb") -for KERN_DEB in $KERNELS; do - VERSION=$(echo $KERN_DEB | \ - grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+" | head -1) +KERNEL_DEB="${URL}/linux-image-${VER1}-${ARCH}_${VER2}_${ARCH}.deb" +HEADERS_DEB="${URL}/linux-headers-${VER1}-${ARCH}_${VER2}_${ARCH}.deb" +HEADERS_ALL_DEB="${URL}/linux-headers-${VER1}-all_${VER2}_${ARCH}.deb" - if echo $TAGS | grep -q "\"${VERSION}\""; then - echo "${REPO}:${VERSION} exists" - continue - fi +DEB_URLS="${KERNEL_DEB} ${HEADERS_DEB} ${HEADERS_ALL_DEB}" - URLS="${BASE_URL}/${KERN_DEB}" - - # Doesn't exist build and push - docker build -t ${REPO}:${VERSION} -f Dockerfile.deb --no-cache \ - --build-arg DEB_URLS="${URLS}" . && - DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION} - - docker rmi ${REPO}:${VERSION} - docker system prune -f -done +docker build -t "${REPO}:${VER2}" -f Dockerfile.deb --no-cache --build-arg DEB_URLS="${DEB_URLS}" . diff --git a/scripts/kernels/fedora.sh b/scripts/kernels/fedora.sh index 74fe3e5e9..1ea9957e5 100755 --- a/scripts/kernels/fedora.sh +++ b/scripts/kernels/fedora.sh @@ -1,55 +1,28 @@ #! /bin/sh -REPO="linuxkit/kernel-fedora" -BASE_URL=http://mirrors.kernel.org/fedora/ - -TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) - -LINKS=$(curl -s ${BASE_URL}/releases/ | sed -n 's/.*href="\([^"]*\).*/\1/p') -# Just get releases 20+ -RELEASES=$(echo $LINKS | grep -o "2[0-9]") +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + echo + echo "Example:" + echo "$0 foobar/kernel-fedora http://mirrors.kernel.org/fedora/releases/27/Everything/x86_64/os/Packages/k/ 4.13.9-300.fc27" + echo + echo "This will create a local LinuxKit kernel package:" + echo "foobar/kernel-fedora:4.13.9-300.fc27" + echo "which you can then push to hub or just use locally" + exit 1 +fi +REPO=$1 +URL=$2 +VER=$3 ARCH=x86_64 -URLS="" -for RELEASE in $RELEASES; do - URLS="$URLS ${BASE_URL}/releases/${RELEASE}/Everything/${ARCH}/os/Packages/k/" - URLS="$URLS ${BASE_URL}/updates/${RELEASE}/${ARCH}/k/" -done -for URL in $URLS; do - PACKAGES=$(curl -s ${URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') +KERNEL_RPM="$URL/kernel-$VER.$ARCH.rpm" +CORE_RPM="$URL/kernel-core-$VER.$ARCH.rpm" +MOD_RPM="$URL/kernel-modules-$VER.$ARCH.rpm" +MOD_EXTRA_RPM="$URL/kernel-modules-extra-$VER.$ARCH.rpm" +HEADERS_RPM="$URL/kernel-headers-$VER.$ARCH.rpm" - KERNEL_RPMS=$(echo $PACKAGES | \ - grep -o "kernel-[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+\.[^ ]\+\.rpm") - for KERNEL_RPM in $KERNEL_RPMS; do - RPM_URLS="${URL}/${KERNEL_RPM}" +RPM_URLS="$KERNEL_RPM $CORE_RPM $MOD_RPM $MOD_EXTRA_RPM $HEADERS_RPM" - VERSION=$(echo $KERNEL_RPM | \ - grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9\.]\+\.fc[0-9]\+") - - if echo $TAGS | grep -q "\"${VERSION}\""; then - echo "${REPO}:${VERSION} exists" - continue - fi - - CORE_RPM="kernel-core-${VERSION}.${ARCH}.rpm" - RPM_URLS="${RPM_URLS} ${URL}/${CORE_RPM}" - - MOD_RPM="kernel-modules-${VERSION}.${ARCH}.rpm" - RPM_URLS="${RPM_URLS} ${URL}/${MOD_RPM}" - - MOD_EXTRA_RPM="kernel-modules-extra-${VERSION}.${ARCH}.rpm" - RPM_URLS="${RPM_URLS} ${URL}/${MOD_EXTRA_RPM}" - - # Don't pull in the headers. This is mostly for testing - # HEADERS_RPM="kernel-headers-${VERSION}.x86_64.rpm" - # RPM_URLS="${RPM_URLS} ${URL}/${HEADERS_RPM}" - - docker build -t ${REPO}:${VERSION} -f Dockerfile.rpm --no-cache \ - --build-arg RPM_URLS="${RPM_URLS}" . && - DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION} - - docker rmi ${REPO}:${VERSION} - docker system prune -f - done -done +docker build -t "$REPO:$VER" -f Dockerfile.rpm --no-cache --build-arg RPM_URLS="$RPM_URLS" . diff --git a/scripts/kernels/mainline.sh b/scripts/kernels/mainline.sh index 5694eb6c7..780691ab9 100755 --- a/scripts/kernels/mainline.sh +++ b/scripts/kernels/mainline.sh @@ -1,57 +1,31 @@ #! /bin/sh -REPO="linuxkit/kernel-mainline" +if [ "$#" -ne 4 ]; then + echo "Usage: $0 " + echo + echo "Example:" + echo "$0 foobar/kernel-mainline v4.14.11 041411 201801022143" + echo + echo "This will create a local LinuxKit kernel package:" + echo "foobar/kernel-mainline:4.14.11" + echo "which you can then push to hub or just use locally" + exit 1 +fi + +REPO=$1 +VER=$2 +VER1=$3 +DATE=$4 BASE_URL=http://kernel.ubuntu.com/~kernel-ppa/mainline +ARCH=amd64 +# Strip leading 'v' +KVER=${VER:1} +URL="${BASE_URL}/${VER}" -TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) +KERNEL_DEB="${URL}/linux-image-${KVER}-${VER1}-generic_${KVER}-${VER1}.${DATE}_${ARCH}.deb" +HEADERS_DEB="${URL}/linux-headers-${KVER}-${VER1}-generic_${KVER}-${VER1}.${DATE}_${ARCH}.deb" +HEADERS_ALL_DEB="${URL}/linux-headers-${KVER}-${VER1}_${KVER}-${VER1}.${DATE}_all.deb" -build_image() { - VERSION=$1 - KDIR=$2 - ARCH=amd64 +DEB_URLS="${KERNEL_DEB} ${HEADERS_DEB} ${HEADERS_ALL_DEB}" - LINKS=$(curl -s ${BASE_URL}/${KDIR}/ | \ - sed -n 's/.*href="\([^"]*\).*/\1/p') - - IMAGE=$(echo $LINKS | \ - grep -o "linux-image[^ ]\+-generic[^ ]\+${ARCH}.deb" | head -1) - [ -z "${IMAGE}" ] && return 1 - HDR_GEN=$(echo $LINKS | grep -o "linux-headers[^ ]\+_all.deb" | head -1) - [ -z "${HDR_GEN}" ] && return 1 - HDR_ARCH=$(echo $LINKS | \ - grep -o "linux-headers[^ ]\+-generic[^ ]\+${ARCH}.deb" | head -1) - [ -z "${HDR_ARCH}" ] && return 1 - - DEB_URL=${BASE_URL}/${KDIR}/${IMAGE} - HDR_GEN_URL=${BASE_URL}/${KDIR}/${HDR_GEN} - HDR_ARCH_URL=${BASE_URL}/${KDIR}/${HDR_ARCH} - echo "Trying to fetch ${VERSION} from ${DEB_URL}" - - docker build -t ${REPO}:${VERSION} -f Dockerfile.deb --no-cache \ - --build-arg DEB_URLS="${DEB_URL} ${HDR_GEN_URL} ${HDR_ARCH_URL}" . -} - -LINKS=$(curl -s ${BASE_URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') -# Extract all kernel versions (drop RCs, ckt(?) and other links) -VERSIONS=$(echo $LINKS | grep -o "v[0-9]\+\.[0-9]\+\.[0-9]\+[^ ]*" | \ - grep -ve '-rc' | grep -ve '-ckt' | uniq) - -# Extract 3.16.x and 4.x.x -THREES=$(echo $VERSIONS | grep -o "v3\.16\.[0-9]\+[^ ]*") -FOURS=$(echo $VERSIONS | grep -o "v4\.[0-9]\+\.[0-9]\+[^ ]*") -KDIRS="${THREES} ${FOURS}" - -for KDIR in $KDIRS; do - # Strip the Ubuntu release name for the tag and also the 'v' like with - # the other kernel packages - VERSION=$(echo $KDIR | grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+") - if echo $TAGS | grep -q "\"${VERSION}\""; then - echo "${REPO}:${VERSION} exists" - continue - fi - build_image ${VERSION} ${KDIR} && \ - DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION} - - docker rmi ${REPO}:${VERSION} - docker system prune -f -done +docker build -t "${REPO}:${KVER}" -f Dockerfile.deb --no-cache --build-arg DEB_URLS="${DEB_URLS}" . diff --git a/scripts/kernels/ubuntu.sh b/scripts/kernels/ubuntu.sh index b5aa11c7f..0221824ae 100755 --- a/scripts/kernels/ubuntu.sh +++ b/scripts/kernels/ubuntu.sh @@ -1,41 +1,31 @@ #! /bin/sh -REPO="linuxkit/kernel-ubuntu" -BASE_URL=http://mirrors.kernel.org/ubuntu/pool/main/l/linux/ +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + echo + echo "Example:" + echo "$0 foobar/kernel-ubuntu 4.14.0-13 15" + echo + echo "This will create a local LinuxKit kernel package:" + echo "foobar/kernel-ubuntu:4.14.0-13.15" + echo "which you can then push to hub or just use locally" + exit 1 +fi -TAGS=$(curl --silent -f -lSL https://registry.hub.docker.com/v1/repositories/${REPO}/tags) +# List all available kernels with: +# curl -s http://mirrors.kernel.org/ubuntu/pool/main/l/linux/ | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep -o "linux-image-[0-9]\.[0-9]\+\.[0-9]\+-[0-9]\+-generic_[^ ]\+amd64\.deb" +REPO=$1 +VER1=$2 +VER2=$3 +URL=http://mirrors.kernel.org/ubuntu/pool/main/l/linux ARCH=amd64 -LINKS=$(curl -s ${BASE_URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p') -# Just get names for 4.x kernels -KERNELS=$(echo $LINKS | \ - grep -o "linux-image-4\.[0-9]\+\.[0-9]\+-[0-9]\+-generic_[^ ]\+${ARCH}\.deb") -for KERN_DEB in $KERNELS; do - VERSION=$(echo $KERN_DEB | \ - grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+" | head -1) +KERNEL_DEB="${URL}/linux-image-${VER1}-generic_${VER1}.${VER2}_${ARCH}.deb" +KERNEL_EXTRA_DEB="${URL}/linux-image-extra-${VER1}-generic_${VER1}.${VER2}_${ARCH}.deb" +HEADERS_DEB="${URL}/linux-headers-${VER1}-generic_${VER1}.${VER2}_${ARCH}.deb" +HEADERS_ALL_DEB="${URL}/linux-headers-${VER1}_${VER1}.${VER2}_all.deb" - if echo $TAGS | grep -q "\"${VERSION}\""; then - echo "${REPO}:${VERSION} exists" - continue - fi +DEB_URLS="${KERNEL_DEB} ${KERNEL_EXTRA_DEB} ${HEADERS_DEB} ${HEADERS_ALL_DEB}" - EXTRA_DEB=$(echo $LINKS | \ - grep -o "linux-image-extra-${VERSION}-generic_[^ ]\+${ARCH}\.deb") - - URLS="${BASE_URL}/${KERN_DEB} ${BASE_URL}/${EXTRA_DEB}" - - # Don't pull in the headers. This is mostly for testing - # HDR_DEB=$(echo $LINKS | \ - # grep -o "linux-headers-${VERSION}_[^ ]\+_all\.deb") - # HDR_ARCH_DEB=$(echo $LINKS | \ - # grep -o "linux-headers-${VERSION}-generic_[^ ]\+_${ARCH}\.deb") - # URLS="${URLS} ${BASE_URL}/${HDR_DEB} ${BASE_URL}/${HDR_ARCH_DEB}" - - docker build -t ${REPO}:${VERSION} -f Dockerfile.deb --no-cache \ - --build-arg DEB_URLS="${URLS}" . && - DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION} - - docker rmi ${REPO}:${VERSION} - docker system prune -f -done +docker build -t "${REPO}:${VER1}.${VER2}" -f Dockerfile.deb --no-cache --build-arg DEB_URLS="${DEB_URLS}" .