diff --git a/kata-deploy/Dockerfile b/kata-deploy/Dockerfile index 548dd8facc..543804bde2 100644 --- a/kata-deploy/Dockerfile +++ b/kata-deploy/Dockerfile @@ -1,21 +1,16 @@ FROM centos/systemd -ARG KATA_VER=1.0.0 +ARG KATA_VER=1.1.0 ARG KATA_URL=https://github.com/kata-containers/runtime/releases/download/${KATA_VER} - -RUN yum install -y wget -WORKDIR /tmp/kata/ -RUN wget -q ${KATA_URL}/{vmlinuz.container,kata-containers.img} - -WORKDIR /tmp/kata/bin/ -RUN wget -q ${KATA_URL}/{kata-runtime,kata-proxy,kata-shim} - ARG KUBECTL_VER=v1.10.2 -RUN wget -qO /bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VER}/bin/linux/amd64/kubectl && \ - chmod +x /bin/kubectl -COPY bin /tmp/kata/bin -COPY qemu-artifacts /tmp/kata/share/qemu +RUN \ +curl -sOL ${KATA_URL}/kata-release-binaries.tar.xz && \ +mkdir -p /opt/kata-artifacts && \ +tar xvf kata-release-binaries.tar.xz -C /opt/kata-artifacts && \ +rm kata-release-binaries.tar.xz -COPY configuration.toml /tmp/kata/ -COPY scripts /tmp/kata/scripts +RUN \ +curl -s -o /bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VER}/bin/linux/amd64/kubectl && \ +chmod +x /bin/kubectl +COPY scripts /opt/kata-artifacts/scripts diff --git a/kata-deploy/README.md b/kata-deploy/README.md index e0a4e1fc41..92bb68b4cc 100644 --- a/kata-deploy/README.md +++ b/kata-deploy/README.md @@ -14,8 +14,9 @@ + [Kata cleanup](#kata-cleanup-) -[kata-deploy](kata-deploy) provides a Dockerfile which contains all of the binaries -and artifacts required to run Kata Containers, as well as reference daemonsets which can be utilized to install Kata Containers on a running Kubernetes cluster. +[kata-deploy](kata-deploy) provides a Dockerfile, which contains all of the binaries +and artifacts required to run Kata Containers, as well as reference daemonsets, which can +be utilized to install Kata Containers on a running Kubernetes cluster. Note, installation through daemonsets only succesfully installs `kata-containers.io/kata-runtime` on a node if it uses either containerd or CRI-O CRI-shims. @@ -118,7 +119,7 @@ the cluster identifying the CRI shim in use. For example, Depending the value of `kata-containers.io/container-runtime` label on the node, either the CRI-O or containerd kata installation daemonset executes. These daemonsets install -the necessary kata binaries, configuration files and virtual machine artifacts on +the necessary kata binaries, configuration files, and virtual machine artifacts on the node. Once installed, the daemonset adds a node label `kata-containers.io/kata-runtime=true` and reconfigures either CRI-O or containerd to make use of Kata for untrusted workloads. As a final step the daemonset restarts either CRI-O or containerd and kubelet. Upon deletion, the daemonset removes the kata binaries diff --git a/kata-deploy/bin/qemu-system-x86_64 b/kata-deploy/bin/qemu-system-x86_64 deleted file mode 100755 index 00f7d97a44..0000000000 Binary files a/kata-deploy/bin/qemu-system-x86_64 and /dev/null differ diff --git a/kata-deploy/configuration.toml b/kata-deploy/configuration.toml deleted file mode 100755 index a8d9f5e46f..0000000000 --- a/kata-deploy/configuration.toml +++ /dev/null @@ -1,144 +0,0 @@ -# XXX: WARNING: this file is auto-generated. -# XXX: -# XXX: Source file: "cli/config/configuration.toml.in" -# XXX: Project: -# XXX: Name: Kata Containers -# XXX: Type: kata - -[hypervisor.qemu] -path = "/opt/kata/bin/qemu-system-x86_64" -kernel = "/opt/kata/vmlinuz.container" -# initrd = "/opt/kata/vm-artifacts/kata-containers-initrd.img" -image = "/opt/kata/kata-containers.img" -machine_type = "pc" - -# Optional space-separated list of options to pass to the guest kernel. -# For example, use `kernel_params = "vsyscall=emulate"` if you are having -# trouble running pre-2.15 glibc. -# -# WARNING: - any parameter specified here will take priority over the default -# parameter value of the same name used to start the virtual machine. -# Do not set values here unless you understand the impact of doing so as you -# may stop the virtual machine from booting. -# To see the list of default parameters, enable hypervisor debug, create a -# container and look for 'default-kernel-parameters' log entries. -kernel_params = "" -#kernel_params = " agent.log=debug" - -# Path to the firmware. -# If you want that qemu uses the default firmware leave this option empty -firmware = "" - -# Machine accelerators -# comma-separated list of machine accelerators to pass to the hypervisor. -# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"` -machine_accelerators="" - -# Default number of vCPUs per POD/VM: -# unspecified or 0 --> will be set to 1 -# < 0 --> will be set to the actual number of physical cores -# > 0 <= number of physical cores --> will be set to the specified number -# > number of physical cores --> will be set to the actual number of physical cores -default_vcpus = 1 - - -# Bridges can be used to hot plug devices. -# Limitations: -# * Currently only pci bridges are supported -# * Until 30 devices per bridge can be hot plugged. -# * Until 5 PCI bridges can be cold plugged per VM. -# This limitation could be a bug in qemu or in the kernel -# Default number of bridges per POD/VM: -# unspecified or 0 --> will be set to 1 -# > 1 <= 5 --> will be set to the specified number -# > 5 --> will be set to 5 -default_bridges = 1 - -# Default memory size in MiB for POD/VM. -# If unspecified then it will be set 2048 MiB. -#default_memory = 2048 - -# Disable block device from being used for a container's rootfs. -# In case of a storage driver like devicemapper where a container's -# root file system is backed by a block device, the block device is passed -# directly to the hypervisor for performance reasons. -# This flag prevents the block device from being passed to the hypervisor, -# 9pfs is used instead to pass the rootfs. -disable_block_device_use = false - -# Block storage driver to be used for the hypervisor in case the container -# rootfs is backed by a block device. This is either virtio-scsi or -# virtio-blk. -block_device_driver = "virtio-scsi" - -# Enable pre allocation of VM RAM, default false -# Enabling this will result in lower container density -# as all of the memory will be allocated and locked -# This is useful when you want to reserve all the memory -# upfront or in the cases where you want memory latencies -# to be very predictable -# Default false -#enable_mem_prealloc = true - -# Enable huge pages for VM RAM, default false -# Enabling this will result in the VM memory -# being allocated using huge pages. -# This is useful when you want to use vhost-user network -# stacks within the container. This will automatically -# result in memory pre allocation -#enable_hugepages = true - -# Enable swap of vm memory. Default false. -# The behaviour is undefined if mem_prealloc is also set to true -#enable_swap = true - -# This option changes the default hypervisor and kernel parameters -# to enable debug output where available. This extra output is added -# to the proxy logs, but only when proxy debug is also enabled. -# -# Default false -#enable_debug = true - -# Disable the customizations done in the runtime when it detects -# that it is running on top a VMM. This will result in the runtime -# behaving as it would when running on bare metal. -# -#disable_nesting_checks = true - -[proxy.kata] -path = "/opt/kata/bin/kata-proxy" - -# If enabled, proxy messages will be sent to the system log -# (default: disabled) -#enable_debug = true - -[shim.kata] -path = "/opt/kata/bin/kata-shim" - -# If enabled, shim messages will be sent to the system log -# (default: disabled) -#enable_debug = true - -[agent.kata] -# There is no field for this section. The goal is only to be able to -# specify which type of agent the user wants to use. - -[runtime] -# If enabled, the runtime will log additional debug messages to the -# system log -# (default: disabled) -#enable_debug = true -# -# Internetworking model -# Determines how the VM should be connected to the -# the container network interface -# Options: -# -# - bridged -# Uses a linux bridge to interconnect the container interface to -# the VM. Works for most cases except macvlan and ipvlan. -# -# - macvtap -# Used when the Container network interface can be bridged using -# macvtap. -internetworking_model="macvtap" diff --git a/kata-deploy/example/nginx-untrusted.yaml b/kata-deploy/example/nginx-untrusted.yaml deleted file mode 100644 index 714f0ebc1c..0000000000 --- a/kata-deploy/example/nginx-untrusted.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: v1 -kind: Pod -metadata: - annotations: - io.kubernetes.cri-o.TrustedSandbox: "false" - io.kubernetes.cri.untrusted-workload: "true" - name: nginx-untrusted -spec: - containers: - - name: nginx - image: nginx - nodeSelector: - kata-runtime: "true" diff --git a/kata-deploy/kata-cleanup.yaml b/kata-deploy/kata-cleanup.yaml index 722389e567..868ba05131 100644 --- a/kata-deploy/kata-cleanup.yaml +++ b/kata-deploy/kata-cleanup.yaml @@ -23,8 +23,11 @@ spec: command: [ "sh", "-c" ] args: - kubectl label node $NODE_NAME kata-containers.io/container-runtime- kata-containers.io/kata-runtime-; - systemctl daemon-reload && systemctl restart containerd && systemctl restart crio && systemctl restart kubelet; - tail -f /dev/null; + systemctl daemon-reload; + systemctl restart containerd; + systemctl restart crio; + systemctl restart kubelet; + sleep infinity; env: - name: NODE_NAME valueFrom: diff --git a/kata-deploy/kata-deploy.yaml b/kata-deploy/kata-deploy.yaml index aa078326f5..e7d78747e1 100644 --- a/kata-deploy/kata-deploy.yaml +++ b/kata-deploy/kata-deploy.yaml @@ -24,7 +24,7 @@ spec: kubectl get node $NODE_NAME --show-labels; kubectl label node $NODE_NAME kata-containers.io/container-runtime=$(kubectl describe node $NODE_NAME | awk -F'[:]' '/Container Runtime Version/ {print $2}' | tr -d ' '); kubectl get node $NODE_NAME --show-labels; - tail -f /dev/null; + sleep infinity; env: - name: NODE_NAME valueFrom: @@ -61,12 +61,12 @@ spec: lifecycle: preStop: exec: - command: ["sh", "-c", "/tmp/kata/scripts/remove-kata-crio.sh && kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup"] - command: [ "sh", "-c" ] + command: ["sh", "-c", "/opt/kata-artifacts/scripts/remove-kata-crio.sh && kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup"] + command: [ "sh", "-ce" ] args: - - /tmp/kata/scripts/install-kata-crio.sh && kubectl label node $NODE_NAME kata-containers.io/kata-runtime=true; + - /opt/kata-artifacts/scripts/install-kata-crio.sh && kubectl label node $NODE_NAME kata-containers.io/kata-runtime=true; kubectl get node $NODE_NAME --show-labels; - tail -f /dev/null; + sleep infinity; env: - name: NODE_NAME valueFrom: @@ -132,12 +132,12 @@ spec: lifecycle: preStop: exec: - command: ["sh", "-c", "/tmp/kata/scripts/remove-kata-containerd.sh && kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup"] + command: ["sh", "-c", "/opt/kata-artifacts/scripts/remove-kata-containerd.sh && kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup"] command: [ "sh", "-c" ] args: - - /tmp/kata/scripts/install-kata-containerd.sh && kubectl label node $NODE_NAME kata-containers.io/kata-runtime=true; + - /opt/kata-artifacts/scripts/install-kata-containerd.sh && kubectl label node $NODE_NAME kata-containers.io/kata-runtime=true; kubectl get node $NODE_NAME --show-labels; - tail -f /dev/null; + sleep infinity; env: - name: NODE_NAME valueFrom: diff --git a/kata-deploy/qemu-artifacts/bios-256k.bin b/kata-deploy/qemu-artifacts/bios-256k.bin deleted file mode 100644 index 18666c9f2f..0000000000 Binary files a/kata-deploy/qemu-artifacts/bios-256k.bin and /dev/null differ diff --git a/kata-deploy/qemu-artifacts/bios.bin b/kata-deploy/qemu-artifacts/bios.bin deleted file mode 100644 index a394411fe5..0000000000 Binary files a/kata-deploy/qemu-artifacts/bios.bin and /dev/null differ diff --git a/kata-deploy/qemu-artifacts/efi-virtio.rom b/kata-deploy/qemu-artifacts/efi-virtio.rom deleted file mode 100644 index 3563776dbd..0000000000 Binary files a/kata-deploy/qemu-artifacts/efi-virtio.rom and /dev/null differ diff --git a/kata-deploy/qemu-artifacts/linuxboot.bin b/kata-deploy/qemu-artifacts/linuxboot.bin deleted file mode 100644 index 923d1796fb..0000000000 Binary files a/kata-deploy/qemu-artifacts/linuxboot.bin and /dev/null differ diff --git a/kata-deploy/qemu-artifacts/linuxboot_dma.bin b/kata-deploy/qemu-artifacts/linuxboot_dma.bin deleted file mode 100644 index 218d3ab4a2..0000000000 Binary files a/kata-deploy/qemu-artifacts/linuxboot_dma.bin and /dev/null differ diff --git a/kata-deploy/scripts/install-kata-containerd.sh b/kata-deploy/scripts/install-kata-containerd.sh index e88f7585e2..cd2ed9d1f3 100755 --- a/kata-deploy/scripts/install-kata-containerd.sh +++ b/kata-deploy/scripts/install-kata-containerd.sh @@ -1,17 +1,28 @@ #!/bin/sh -echo "copying kata artifacts onto host" -cp -R /tmp/kata/* /opt/kata/ -chmod +x /opt/kata/bin/* -cp /opt/kata/configuration.toml /usr/share/defaults/kata-containers/configuration.toml -## Configure containerd to use Kata: +echo "copying kata artifacts onto host" +cp -R /opt/kata-artifacts/bin /opt/kata/ +mkdir /opt/kata/share +mv /opt/kata/bin/qemu /opt/kata/share/ +chmod +x /opt/kata/bin/* +cp /opt/kata-artifacts/configuration.toml /usr/share/defaults/kata-containers/configuration.toml + +# Update Kata configuration for /opt/kata path usage +sed -i 's!/usr.*kata-containers/!/opt/kata/bin/!' /usr/share/defaults/kata-containers/configuration.toml +sed -i 's!/usr/bin/!/opt/kata/bin/!' /usr/share/defaults/kata-containers/configuration.toml +sed -i 's!qemu-lite!qemu!' /usr/share/defaults/kata-containers/configuration.toml + +# Configure containerd to use Kata: echo "create containerd configuration for Kata" mkdir -p /etc/containerd/ +if [ -f /etc/containerd/config.toml ]; then + cp /etc/containerd/config.toml /etc/containerd/config.toml.bak +fi + cat << EOT | tee /etc/containerd/config.toml [plugins] [plugins.cri.containerd] - snapshotter = "overlayfs" [plugins.cri.containerd.untrusted_workload_runtime] runtime_type = "io.containerd.runtime.v1.linux" runtime_engine = "/opt/kata/bin/kata-runtime" diff --git a/kata-deploy/scripts/install-kata-crio.sh b/kata-deploy/scripts/install-kata-crio.sh index 0aef0df98f..4e5e7088e1 100755 --- a/kata-deploy/scripts/install-kata-crio.sh +++ b/kata-deploy/scripts/install-kata-crio.sh @@ -1,14 +1,19 @@ #!/bin/sh -echo "copying kata artifacts from /tmp to /opt" -cp -R /tmp/kata/* /opt/kata/ +echo "copying kata artifacts onto host" +cp -R /opt/kata-artifacts/bin /opt/kata/ +mkdir /opt/kata/share +mv /opt/kata/bin/qemu /opt/kata/share/ chmod +x /opt/kata/bin/* +cp /opt/kata-artifacts/configuration.toml /usr/share/defaults/kata-containers/configuration.toml -cp /opt/kata/configuration.toml /usr/share/defaults/kata-containers/configuration.toml - -cp /etc/crio/crio.conf /etc/crio/crio.conf.bak +# Update Kata configuration for /opt/kata path usage +sed -i 's!/usr.*kata-containers/!/opt/kata/bin/!' /usr/share/defaults/kata-containers/configuration.toml +sed -i 's!/usr/bin/!/opt/kata/bin/!' /usr/share/defaults/kata-containers/configuration.toml +sed -i 's!qemu-lite!qemu!' /usr/share/defaults/kata-containers/configuration.toml echo "Set Kata containers as default runtime in CRI-O for untrusted workloads" +cp /etc/crio/crio.conf /etc/crio/crio.conf.bak sed -i '/runtime_untrusted_workload = /c\runtime_untrusted_workload = "/opt/kata/bin/kata-runtime"' /etc/crio/crio.conf echo "Reload systemd services" diff --git a/kata-deploy/scripts/remove-kata-containerd.sh b/kata-deploy/scripts/remove-kata-containerd.sh index dbff017505..797e4df3a6 100755 --- a/kata-deploy/scripts/remove-kata-containerd.sh +++ b/kata-deploy/scripts/remove-kata-containerd.sh @@ -3,3 +3,8 @@ echo "delete kata artifacts" rm -rf /opt/kata rm -rf /usr/share/defaults/kata-containers rm -f /etc/containerd/config.toml + +if [ -f /etc/containerd/config.toml.bak ]; then + mv /etc/containerd/config.toml.bak /etc/containerd/config.toml +fi + diff --git a/kata-deploy/scripts/remove-kata-crio.sh b/kata-deploy/scripts/remove-kata-crio.sh index 2c8839884b..b23e40932d 100755 --- a/kata-deploy/scripts/remove-kata-crio.sh +++ b/kata-deploy/scripts/remove-kata-crio.sh @@ -1,5 +1,5 @@ #!/bin/sh echo "deleting kata artifacts" rm -rf /opt/kata/ -rm -rf /usr/sahre/defaults/kata-containers +rm -rf /usr/share/defaults/kata-containers mv /etc/crio/crio.conf.bak /etc/crio/crio.conf