test images: Fixes echoserver s390x image

When trying to build the s390x image, it would fail when running the apk
command with the following error:

ERROR: Unable to open root: Bad address
ERROR: Failed to open apk database: Bad address

This can be fixed by updating the third_party/multiarch/qemu-user-static/register/register.sh
and third_party/multiarch/qemu-user-static/register/qemu-binfmt-conf.sh scripts
and their usage to a newer version [1].

Additionally, the packages nginx-mod-http-lua and nginx-mod-http-lua-upstream
cannot be found in the regular http://dl-cdn.alpinelinux.org/alpine/v3.9/main/s390x/
repository, but we can use an older one [2].

[1] https://github.com/qemu/qemu/blob/master/scripts/qemu-binfmt-conf.sh
[2] http://dl-cdn.alpinelinux.org/alpine/v3.8/main

test/images/echoserver/Dockerfile

@@ -17,7 +17,8 @@ FROM $BASEIMAGE
 
 CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
 
-RUN apk add --no-cache openssl nginx-mod-http-lua nginx-mod-http-lua-upstream
+COPY install-deps.sh /
+RUN ["sh", "-c", "/install-deps.sh"]
 RUN mkdir -p /run/nginx
 
 ADD nginx.conf /etc/nginx/nginx.conf

test/images/echoserver/install-deps.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARCH=$(uname -m)
+
+if [ "${ARCH}" = "s390x" ]; then
+    echo "http://dl-cdn.alpinelinux.org/alpine/v3.8/main" > /etc/apk/repositories
+    echo "http://dl-cdn.alpinelinux.org/alpine/v3.8/community" >> /etc/apk/repositories
+fi
+
+apk add --no-cache openssl nginx-mod-http-lua nginx-mod-http-lua-upstream
+
+exit 0

test/images/image-util.sh

@@ -140,7 +140,7 @@ build() {
         if [[ $(id -u) != 0 ]]; then
           sudo=sudo
         fi
-        ${sudo} "${KUBE_ROOT}/third_party/multiarch/qemu-user-static/register/register.sh" --reset
+        ${sudo} "${KUBE_ROOT}/third_party/multiarch/qemu-user-static/register/register.sh" --reset -p yes
         curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/"${QEMUVERSION}"/x86_64_qemu-"${QEMUARCHS[$arch]}"-static.tar.gz | tar -xz -C "${temp_dir}"
         # Ensure we don't get surprised by umask settings
         chmod 0755 "${temp_dir}/qemu-${QEMUARCHS[$arch]}-static"
@@ -256,8 +256,7 @@ if [[ "${WHAT}" == "all-conformance" ]]; then
   # no point in rebuilding all of them every time. This will only build the Conformance-related images.
   # Discussed during Conformance Office Hours Meeting (2019.12.17):
   # https://docs.google.com/document/d/1W31nXh9RYAb_VaYkwuPLd1hFxuRX3iU0DmaQ4lkCsX8/edit#heading=h.l87lu17xm9bh
-  # echoserver image not included: https://github.com/kubernetes/kubernetes/issues/84158
+  conformance_images=("busybox" "agnhost" "echoserver" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver")
-  conformance_images=("busybox" "agnhost" "jessie-dnsutils" "kitten" "nautilus" "nonewprivs" "resource-consumer" "sample-apiserver")
   for image in "${conformance_images[@]}"; do
     eval "${TASK}" "${image}"
   done

third_party/multiarch/qemu-user-static/register/qemu-binfmt-conf.sh

@@ -4,20 +4,27 @@
 #
 # downloaded from https://raw.githubusercontent.com/qemu/qemu/master/scripts/qemu-binfmt-conf.sh
 
-qemu_target_list="i386 i486 alpha arm armeb sparc32plus ppc ppc64 ppc64le m68k \
+# Enable automatic program execution by the kernel.
-mips mipsel mipsn32 mipsn32el mips64 mips64el \
+
-sh4 sh4eb s390x aarch64 aarch64_be hppa riscv32 riscv64 xtensa xtensaeb microblaze microblazeel"
+qemu_target_list="i386 i486 alpha arm armeb sparc sparc32plus sparc64 \
+ppc ppc64 ppc64le m68k mips mipsel mipsn32 mipsn32el mips64 mips64el \
+sh4 sh4eb s390x aarch64 aarch64_be hppa riscv32 riscv64 xtensa xtensaeb \
+microblaze microblazeel or1k x86_64"
 
 i386_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00'
-i386_mask='\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+i386_mask='\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
 i386_family=i386
 
 i486_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x06\x00'
-i486_mask='\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+i486_mask='\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
 i486_family=i386
 
+x86_64_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00'
+x86_64_mask='\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+x86_64_family=i386
+
 alpha_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x26\x90'
-alpha_mask='\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+alpha_mask='\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
 alpha_family=alpha
 
 arm_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00'
@@ -29,13 +36,17 @@ armeb_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff
 armeb_family=armeb
 
 sparc_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x02'
-sparc_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+sparc_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
 sparc_family=sparc
 
 sparc32plus_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x12'
-sparc32plus_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+sparc32plus_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
 sparc32plus_family=sparc
 
+sparc64_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2b'
+sparc64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+sparc64_family=sparc
+
 ppc_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x14'
 ppc_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
 ppc_family=ppc
@@ -49,7 +60,7 @@ ppc64le_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\x
 ppc64le_family=ppcle
 
 m68k_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x04'
-m68k_mask='\xff\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+m68k_mask='\xff\xff\xff\xff\xff\xff\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
 m68k_family=m68k
 
 # FIXME: We could use the other endianness on a MIPS host.
@@ -79,11 +90,11 @@ mips64el_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\
 mips64el_family=mips
 
 sh4_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a\x00'
-sh4_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+sh4_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
 sh4_family=sh4
 
 sh4eb_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a'
-sh4eb_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+sh4eb_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
 sh4eb_family=sh4
 
 s390x_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x16'
@@ -126,6 +137,10 @@ microblazeel_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\
 microblazeel_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
 microblazeel_family=microblazeel
 
+or1k_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x5c'
+or1k_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+or1k_family=or1k
+
 qemu_get_family() {
     cpu=${HOST_ARCH:-$(uname -m)}
     case "$cpu" in
@@ -163,21 +178,26 @@ usage() {
     cat <<EOF
 Usage: qemu-binfmt-conf.sh [--qemu-path PATH][--debian][--systemd CPU]
                            [--help][--credential yes|no][--exportdir PATH]
+                           [--persistent yes|no][--qemu-suffix SUFFIX]
 
        Configure binfmt_misc to use qemu interpreter
 
-       --help:       display this usage
+       --help:        display this usage
-       --qemu-path:  set path to qemu interpreter ($QEMU_PATH)
+       --qemu-path:   set path to qemu interpreter ($QEMU_PATH)
-       --debian:     don't write into /proc,
+       --qemu-suffix: add a suffix to the default interpreter name
-                     instead generate update-binfmts templates
+       --debian:      don't write into /proc,
-       --systemd:    don't write into /proc,
+                      instead generate update-binfmts templates
-                     instead generate file for systemd-binfmt.service
+       --systemd:     don't write into /proc,
-                     for the given CPU. If CPU is "ALL", generate a
+                      instead generate file for systemd-binfmt.service
-                     file for all known cpus
+                      for the given CPU. If CPU is "ALL", generate a
-       --exportdir:  define where to write configuration files
+                      file for all known cpus
-                     (default: $SYSTEMDDIR or $DEBIANDIR)
+       --exportdir:   define where to write configuration files
-       --credential: if yes, credential and security tokens are
+                      (default: $SYSTEMDDIR or $DEBIANDIR)
-                     calculated according to the binary to interpret
+       --credential:  if yes, credential and security tokens are
+                      calculated according to the binary to interpret
+       --persistent:  if yes, the interpreter is loaded when binfmt is
+                      configured and remains in memory. All future uses
+                      are cloned from the open file.
 
     To import templates with update-binfmts, use :
 
@@ -243,7 +263,15 @@ qemu_check_systemd() {
 }
 
 qemu_generate_register() {
-    echo ":qemu-$cpu:M::$magic:$mask:$qemu:$FLAGS"
+    flags=""
+    if [ "$CREDENTIAL" = "yes" ] ; then
+        flags="OC"
+    fi
+    if [ "$PERSISTENT" = "yes" ] ; then
+        flags="${flags}F"
+    fi
+
+    echo ":qemu-$cpu:M::$magic:$mask:$qemu:$flags"
 }
 
 qemu_register_interpreter() {
@@ -262,10 +290,8 @@ package qemu-$cpu
 interpreter $qemu
 magic $magic
 mask $mask
+credential $CREDENTIAL
 EOF
-    if [ "$FLAGS" = "OC" ] ; then
-        echo "credentials yes" >> "$EXPORTDIR/qemu-$cpu"
-    fi
 }
 
 qemu_set_binfmts() {
@@ -284,11 +310,12 @@ qemu_set_binfmts() {
             continue
         fi
 
-        qemu="$QEMU_PATH/qemu-$cpu-static"
+        qemu="$QEMU_PATH/qemu-$cpu"
         if [ "$cpu" = "i486" ] ; then
-            qemu="$QEMU_PATH/qemu-i386-static"
+            qemu="$QEMU_PATH/qemu-i386"
         fi
 
+        qemu="$qemu$QEMU_SUFFIX"
         if [ "$host_family" != "$family" ] ; then
             $BINFMT_SET
         fi
@@ -302,9 +329,11 @@ SYSTEMDDIR="/etc/binfmt.d"
 DEBIANDIR="/usr/share/binfmts"
 
 QEMU_PATH=/usr/local/bin
-FLAGS=""
+CREDENTIAL=no
+PERSISTENT=no
+QEMU_SUFFIX=""
 
-options=$(getopt -o ds:Q:e:hc: -l debian,systemd:,qemu-path:,exportdir:,help,credential: -- "$@")
+options=$(getopt -o ds:Q:S:e:hc:p: -l debian,systemd:,qemu-path:,qemu-suffix:,exportdir:,help,credential:,persistent: -- "$@")
 eval set -- "$options"
 
 while true ; do
@@ -340,6 +369,10 @@ while true ; do
         shift
         QEMU_PATH="$1"
         ;;
+    -F|--qemu-suffix)
+        shift
+        QEMU_SUFFIX="$1"
+        ;;
     -e|--exportdir)
         shift
         EXPORTDIR="$1"
@@ -350,11 +383,11 @@ while true ; do
         ;;
     -c|--credential)
         shift
-        if [ "$1" = "yes" ] ; then
+        CREDENTIAL="$1"
-            FLAGS="OC"
+        ;;
-        else
+    -p|--persistent)
-            FLAGS=""
+        shift
-        fi
+        PERSISTENT="$1"
         ;;
     *)
         break

third_party/multiarch/qemu-user-static/register/register.sh

@@ -14,18 +14,10 @@ if [ ! -f /proc/sys/fs/binfmt_misc/register ]; then
     mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc
 fi
 
-entries="aarch64 aarch64_be alpha arm armeb hppa m68k microblaze microblazeel mips mips64 mips64el mipsel mipsn32 mipsn32el ppc ppc64 ppc64le riscv32 riscv64 s390x sh4 sh4eb sparc sparc32plus sparc64 xtensa xtensaeb"
 
 if [ "${1}" = "--reset" ]; then
     shift
-    (
+    find /proc/sys/fs/binfmt_misc -type f -name 'qemu-*' -exec sh -c 'echo -1 > {}' \;
-    cd /proc/sys/fs/binfmt_misc || exit
-    for file in $entries; do
-        if [ -f "qemu-${file}" ]; then
-            echo -1 > "qemu-${file}"
-        fi
-    done
-    )
 fi
 
-exec $(dirname "${BASH_SOURCE[0]}")/qemu-binfmt-conf.sh --qemu-path="${QEMU_BIN_DIR}" "$@"
+exec $(dirname "${BASH_SOURCE[0]}")/qemu-binfmt-conf.sh --qemu-suffix "-static" --qemu-path="${QEMU_BIN_DIR}" "$@"