From d901b625c9e2a8440a024ee8ecf1c285e22b8c07 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Wed, 23 Jan 2019 09:05:14 -0600 Subject: [PATCH 01/13] snap: get sources from git repositories get source from git repositories to show the right commit version of each component. fixes #317 Signed-off-by: Julio Montes --- snap/snapcraft.yaml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 33d988aecc..a391cf6343 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -43,8 +43,7 @@ parts: mkdir -p ${pkg_gopath} # download source - curl -LO https://github.com/kata-containers/${pkg_name}/archive/${SNAPCRAFT_PROJECT_VERSION}.tar.gz - tar -xf ${SNAPCRAFT_PROJECT_VERSION}.tar.gz --strip-components=1 -C ${pkg_gopath} + git clone -b ${SNAPCRAFT_PROJECT_VERSION} https://github.com/kata-containers/${pkg_name} ${pkg_gopath} cd ${pkg_gopath} # build and install @@ -76,8 +75,7 @@ parts: mkdir -p ${pkg_gopath} # download source - curl -LO https://github.com/kata-containers/${pkg_name}/archive/${SNAPCRAFT_PROJECT_VERSION}.tar.gz - tar -xf ${SNAPCRAFT_PROJECT_VERSION}.tar.gz --strip-components=1 -C ${pkg_gopath} + git clone -b ${SNAPCRAFT_PROJECT_VERSION} https://github.com/kata-containers/${pkg_name} ${pkg_gopath} cd ${pkg_gopath} # build and install @@ -100,8 +98,7 @@ parts: mkdir -p ${pkg_gopath} # download source - curl -LO https://github.com/kata-containers/${pkg_name}/archive/${SNAPCRAFT_PROJECT_VERSION}.tar.gz - tar -xf ${SNAPCRAFT_PROJECT_VERSION}.tar.gz --strip-components=1 -C ${pkg_gopath} + git clone -b ${SNAPCRAFT_PROJECT_VERSION} https://github.com/kata-containers/${pkg_name} ${pkg_gopath} cd ${pkg_gopath} # build and install @@ -156,8 +153,7 @@ parts: mkdir -p ${pkg_gopath} # download source - curl -LO https://github.com/kata-containers/${pkg_name}/archive/${SNAPCRAFT_PROJECT_VERSION}.tar.gz - tar -xf ${SNAPCRAFT_PROJECT_VERSION}.tar.gz --strip-components=1 -C ${pkg_gopath} + git clone -b ${SNAPCRAFT_PROJECT_VERSION} https://github.com/kata-containers/${pkg_name} ${pkg_gopath} cd ${pkg_gopath} # build and install From e7ff7e1c44df828bc208aa18bac6bf25800e2a57 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Fri, 25 Jan 2019 07:38:23 -0600 Subject: [PATCH 02/13] snap: fix qemu command name set qemu-system instead of qemu-lite as default qemu command Signed-off-by: Julio Montes --- snap/snapcraft.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index a391cf6343..345e829c72 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -49,11 +49,13 @@ parts: # build and install make \ PREFIX=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr \ - SKIP_GO_VERSION_CHECK=1 + SKIP_GO_VERSION_CHECK=1 \ + QEMUCMD=qemu-system-$(arch) make install \ PREFIX=/usr \ DESTDIR=${SNAPCRAFT_PART_INSTALL} \ - SKIP_GO_VERSION_CHECK=1 + SKIP_GO_VERSION_CHECK=1 \ + QEMUCMD=qemu-system-$(arch) # A snap is read-only squashfs, unfourtunately it's not possible to use the rootfs image with DAX # since rw access is required therefore initrd image must be used instead. From 2145fc1112db6ccc761379763f7133e75e986226 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Fri, 25 Jan 2019 07:50:36 -0600 Subject: [PATCH 03/13] snap: apply patch in runtime to fix DESTDIR This is a hotfix to fix https://github.com/kata-containers/runtime/pull/1162 Signed-off-by: Julio Montes --- snap/1162-runtime.patch | 27 +++++++++++++++++++++++++++ snap/snapcraft.yaml | 4 ++++ 2 files changed, 31 insertions(+) create mode 100644 snap/1162-runtime.patch diff --git a/snap/1162-runtime.patch b/snap/1162-runtime.patch new file mode 100644 index 0000000000..be87ee7051 --- /dev/null +++ b/snap/1162-runtime.patch @@ -0,0 +1,27 @@ +diff --git a/Makefile b/Makefile +index 1c11f74..9c4709b 100644 +--- a/Makefile ++++ b/Makefile +@@ -248,11 +248,11 @@ ifeq (,$(findstring $(DEFAULT_HYPERVISOR),$(KNOWN_HYPERVISORS))) + endif + + ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_QEMU)) +- DEFAULT_HYPERVISOR_CONFIG_PATH = $(CONFIG_PATH_QEMU) ++ DEFAULT_HYPERVISOR_CONFIG_FILE = $(CONFIG_FILE_QEMU) + endif + + ifeq ($(DEFAULT_HYPERVISOR),$(HYPERVISOR_FC)) +- DEFAULT_HYPERVISOR_CONFIG_PATH = $(CONFIG_PATH_FC) ++ DEFAULT_HYPERVISOR_CONFIG_FILE = $(CONFIG_FILE_FC) + endif + + CONFDIR := $(DEFAULTSDIR)/$(PROJECT_DIR) +@@ -546,7 +546,7 @@ install-bin-libexec: $(BINLIBEXECLIST) + + install-configs: $(CONFIGS) + $(QUIET_INST)$(foreach f,$(CONFIGS),$(call INSTALL_CONFIG,$f,$(dir $(CONFIG_PATH)))) +- $(QUIET_INST)ln -sf $(DEFAULT_HYPERVISOR_CONFIG_PATH) $(CONFIG_PATH) ++ $(QUIET_INST)(cd $(dir $(DESTDIR)/$(CONFIG_PATH)) && ln -sf $(DEFAULT_HYPERVISOR_CONFIG_FILE) $(CONFIG_FILE)) + + install-scripts: $(SCRIPTS) + $(QUIET_INST)$(foreach f,$(SCRIPTS),$(call INSTALL_EXEC,$f,$(SCRIPTS_DIR))) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 345e829c72..a9c7d21621 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -33,6 +33,7 @@ parts: build-attributes: [no-patchelf] override-build: | pkg_name=runtime + patch1="$(realpath ../../../snap/1162-runtime.patch)" # set GOPATH export GOPATH=$(realpath go) @@ -46,6 +47,9 @@ parts: git clone -b ${SNAPCRAFT_PROJECT_VERSION} https://github.com/kata-containers/${pkg_name} ${pkg_gopath} cd ${pkg_gopath} + #Issue: https://github.com/kata-containers/runtime/pull/1162 + patch -p1 < "${patch1}" + # build and install make \ PREFIX=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr \ From 70876b33433bf4f448e97958a50731c397262d95 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Mon, 28 Jan 2019 09:15:36 -0600 Subject: [PATCH 04/13] snap: apply patch in runtime to fix config paths This is a hotfix to fix https://github.com/kata-containers/runtime/issues/1185 Signed-off-by: Julio Montes --- snap/1185-runtime.patch | 22 ++++++++++++++++++++++ snap/snapcraft.yaml | 3 +++ 2 files changed, 25 insertions(+) create mode 100644 snap/1185-runtime.patch diff --git a/snap/1185-runtime.patch b/snap/1185-runtime.patch new file mode 100644 index 0000000000..a256ed107b --- /dev/null +++ b/snap/1185-runtime.patch @@ -0,0 +1,22 @@ +diff --git a/cli/main.go b/cli/main.go +index a01a02dc5..a021a0a77 100644 +--- a/cli/main.go ++++ b/cli/main.go +@@ -250,6 +250,8 @@ func beforeSubcommands(c *cli.Context) error { + var runtimeConfig oci.RuntimeConfig + var err error + ++ katautils.SetConfigOptions(name, defaultRuntimeConfiguration, defaultSysConfRuntimeConfiguration) ++ + handleShowConfig(c) + + if userWantsUsage(c) || (c.NArg() == 1 && (c.Args()[0] == checkCmd)) { +@@ -302,8 +304,6 @@ func beforeSubcommands(c *cli.Context) error { + ignoreLogging = true + } + +- katautils.SetConfigOptions(name, defaultRuntimeConfiguration, defaultSysConfRuntimeConfiguration) +- + configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreLogging, false) + if err != nil { + fatal(err) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index a9c7d21621..9dd170653a 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -34,6 +34,7 @@ parts: override-build: | pkg_name=runtime patch1="$(realpath ../../../snap/1162-runtime.patch)" + patch2="$(realpath ../../../snap/1185-runtime.patch)" # set GOPATH export GOPATH=$(realpath go) @@ -49,6 +50,8 @@ parts: #Issue: https://github.com/kata-containers/runtime/pull/1162 patch -p1 < "${patch1}" + #Issue: https://github.com/kata-containers/runtime/issues/1185 + patch -p1 < "${patch2}" # build and install make \ From 51f5c22409233509926aab2cb235d6d290b70824 Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Mon, 28 Jan 2019 09:16:47 -0600 Subject: [PATCH 05/13] snap: remove commands used for debugging remove set and env commands that were used for debugging Signed-off-by: Julio Montes --- snap/snapcraft.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 9dd170653a..af6e0aaae8 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -123,11 +123,9 @@ parts: - docker.io - cpio override-build: | - set -x # go was built in parts/go/build export GOROOT=$(realpath ../../go/build) export PATH="${GOROOT}/bin:${PATH}" - env if [ -n "$http_proxy" ]; then echo "Setting proxy $http_proxy" systemctl set-environment http_proxy=$http_proxy || true From 7e312f833b8f4e90f74b55eec8ca7ac8d13b6a0d Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Mon, 28 Jan 2019 09:22:15 -0600 Subject: [PATCH 06/13] snap: use new GOPATH to build image Use a new GOPATH to build image in order to avoid clashes with user's GOPATH, otherwise user's kata agent will be used causing problem if that repository is not up to date. Signed-off-by: Julio Montes --- snap/snapcraft.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index af6e0aaae8..66a4bb6892 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -123,6 +123,9 @@ parts: - docker.io - cpio override-build: | + # set GOPATH + export GOPATH=$(realpath go) + mkdir -p "${GOPATH}" # go was built in parts/go/build export GOROOT=$(realpath ../../go/build) export PATH="${GOROOT}/bin:${PATH}" @@ -138,7 +141,7 @@ parts: # use the same go version for all packages sed -i 's|^GO_VERSION=.*|GO_VERSION='$(go version | cut -d' ' -f3 | tr -d go)'|g' rootfs-builder/versions.txt - sudo -E PATH=$PATH make DISTRO=alpine AGENT_INIT=yes USE_DOCKER=1 initrd + sudo -E PATH=$PATH make AGENT_VERSION=${SNAPCRAFT_PROJECT_VERSION} DISTRO=alpine AGENT_INIT=yes USE_DOCKER=1 initrd kata_image_dir=${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers mkdir -p ${kata_image_dir} From e12442bc652aa36bde710f75d559114bf46ae82d Mon Sep 17 00:00:00 2001 From: Julio Montes Date: Wed, 23 Jan 2019 09:16:12 -0600 Subject: [PATCH 07/13] snap: release 1.5.0 release kata containers 1.5.0 Signed-off-by: Julio Montes --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 66a4bb6892..20d0998af3 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,5 +1,5 @@ name: kata-containers -version: "1.5.0-rc2" +version: "1.5.0" summary: Build lightweight VMs that seamlessly plug into the containers ecosystem description: | Kata Containers is an open source project and community working to build a From 5db1ba5710d71eae441aec203e3e8d3522ff714d Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Mon, 7 Jan 2019 22:15:39 +0000 Subject: [PATCH 08/13] kata-deploy: add support for runtime class, firecracker Simplify the yaml and combine the prior scripts. The resulting script, kata-deploy.sh, is used for install and configuration and removal for CRI-O and containerd. While this could be used standalone outside of daemonsets, today it will sleep infinity after processing the request, since it is assumed to be called by a daemon. By checking the CRI runtime within the script itself, we no longer need to support many daemonsets for deploy - just a single. Still requires a seperate cleanup daemonset (for restarting the CRI runtime), and an RBAC. Verified with CRI-O -- containerd testing WIP Throwing this up now for feedback since I do not bash good. Signed-off-by: Eric Ernst Signed-off-by: Saikrishna Edupuganti --- kata-deploy/Dockerfile | 6 +- kata-deploy/kata-cleanup.yaml | 9 +- kata-deploy/kata-deploy.yaml | 125 +----------- .../scripts/install-kata-containerd.sh | 26 --- kata-deploy/scripts/install-kata-crio.sh | 14 -- kata-deploy/scripts/kata-deploy.sh | 178 ++++++++++++++++++ kata-deploy/scripts/remove-kata-containerd.sh | 8 - kata-deploy/scripts/remove-kata-crio.sh | 4 - 8 files changed, 192 insertions(+), 178 deletions(-) delete mode 100755 kata-deploy/scripts/install-kata-containerd.sh delete mode 100755 kata-deploy/scripts/install-kata-crio.sh create mode 100755 kata-deploy/scripts/kata-deploy.sh delete mode 100755 kata-deploy/scripts/remove-kata-containerd.sh delete mode 100755 kata-deploy/scripts/remove-kata-crio.sh diff --git a/kata-deploy/Dockerfile b/kata-deploy/Dockerfile index bc1e046c26..940407a432 100644 --- a/kata-deploy/Dockerfile +++ b/kata-deploy/Dockerfile @@ -1,9 +1,9 @@ FROM centos/systemd -ARG KATA_VER=1.4.0 +ARG KATA_VER ARG ARCH=x86_64 +ARG KUBE_ARCH=amd64 ARG KATA_URL=https://github.com/kata-containers/runtime/releases/download/${KATA_VER} ARG KATA_FILE=kata-static-${KATA_VER}-${ARCH}.tar.xz -ARG KUBECTL_VER=v1.10.2 RUN \ curl -sOL ${KATA_URL}/${KATA_FILE} && \ @@ -12,7 +12,7 @@ tar xvf ${KATA_FILE} -C /opt/kata-artifacts/ && \ rm ${KATA_FILE} RUN \ -curl -s -o /bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VER}/bin/linux/amd64/kubectl && \ +curl -Lso /bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/${KUBE_ARCH}/kubectl && \ chmod +x /bin/kubectl COPY scripts /opt/kata-artifacts/scripts diff --git a/kata-deploy/kata-cleanup.yaml b/kata-deploy/kata-cleanup.yaml index 92b75d9efe..7f7308424b 100644 --- a/kata-deploy/kata-cleanup.yaml +++ b/kata-deploy/kata-cleanup.yaml @@ -20,14 +20,7 @@ spec: - name: kube-kata-cleanup image: katadocker/kata-deploy imagePullPolicy: Always - command: [ "sh", "-c" ] - args: - - kubectl label node $NODE_NAME kata-containers.io/container-runtime- kata-containers.io/kata-runtime-; - systemctl daemon-reload; - systemctl restart containerd; - systemctl restart crio; - systemctl restart kubelet; - sleep infinity; + command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh", "reset" ] env: - name: NODE_NAME valueFrom: diff --git a/kata-deploy/kata-deploy.yaml b/kata-deploy/kata-deploy.yaml index f618784fe4..a30c93a5e6 100644 --- a/kata-deploy/kata-deploy.yaml +++ b/kata-deploy/kata-deploy.yaml @@ -2,71 +2,27 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: kubelet-runtime-labeler + name: kata-deploy namespace: kube-system spec: selector: matchLabels: - name: kubelet-runtime-labeler + name: kata-deploy template: metadata: labels: - name: kubelet-runtime-labeler + name: kata-deploy spec: serviceAccountName: kata-label-node containers: - - name: kubelet-runtime-labeler-pod - image: katadocker/kata-deploy - imagePullPolicy: Always - command: [ "sh", "-c" ] - args: - - printenv NODE_NAME; - kubectl get node $NODE_NAME --show-labels; - kubectl label node $NODE_NAME kata-containers.io/container-runtime=$(kubectl describe node $NODE_NAME | awk -F'[:]' '/Container Runtime Version/ {print $2}' | tr -d ' '); - kubectl get node $NODE_NAME --show-labels; - sleep infinity; - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - privileged: false - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kubelet-cri-o-kata - namespace: kube-system -spec: - selector: - matchLabels: - name: kubelet-cri-o-kata - template: - metadata: - labels: - name: kubelet-cri-o-kata - spec: - serviceAccountName: kata-label-node - nodeSelector: - kata-containers.io/container-runtime: cri-o - containers: - name: kube-kata image: katadocker/kata-deploy imagePullPolicy: Always lifecycle: preStop: exec: - command: ["sh", "-c", "/opt/kata-artifacts/scripts/remove-kata-crio.sh && kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup"] - command: [ "sh", "-ce" ] - args: - - /opt/kata-artifacts/scripts/install-kata-crio.sh && kubectl label node $NODE_NAME kata-containers.io/kata-runtime=true; - kubectl get node $NODE_NAME --show-labels; - sleep infinity; + command: ["bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh", "cleanup"] + command: [ "bash", "-c", "/opt/kata-artifacts/scripts/kata-deploy.sh", "install" ] env: - name: NODE_NAME valueFrom: @@ -77,6 +33,8 @@ spec: volumeMounts: - name: crio-conf mountPath: /etc/crio/ + - name: containerd-conf + mountPath: /etc/containerd/ - name: kata-artifacts mountPath: /opt/kata/ - name: dbus @@ -87,6 +45,9 @@ spec: - name: crio-conf hostPath: path: /etc/crio/ + - name: containerd-conf + hostPath: + path: /etc/containerd/ - name: kata-artifacts hostPath: path: /opt/kata/ @@ -101,69 +62,3 @@ spec: rollingUpdate: maxUnavailable: 1 type: RollingUpdate ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kubelet-cri-containerd-kata - namespace: kube-system -spec: - selector: - matchLabels: - name: kubelet-cri-containerd-kata - template: - metadata: - labels: - name: kubelet-cri-containerd-kata - spec: - serviceAccountName: kata-label-node - nodeSelector: - kata-containers.io/container-runtime: containerd - containers: - - name: kube-kata - image: katadocker/kata-deploy - imagePullPolicy: Always - lifecycle: - preStop: - exec: - command: ["sh", "-c", "/opt/kata-artifacts/scripts/remove-kata-containerd.sh && kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup"] - command: [ "sh", "-c" ] - args: - - /opt/kata-artifacts/scripts/install-kata-containerd.sh && kubectl label node $NODE_NAME kata-containers.io/kata-runtime=true; - kubectl get node $NODE_NAME --show-labels; - sleep infinity; - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - privileged: false - volumeMounts: - - name: containerd-conf - mountPath: /etc/containerd/ - - name: kata-artifacts - mountPath: /opt/kata/ - - name: dbus - mountPath: /var/run/dbus - - name: systemd - mountPath: /run/systemd - volumes: - - name: containerd-conf - hostPath: - path: /etc/containerd/ - type: DirectoryOrCreate - - name: kata-artifacts - hostPath: - path: /opt/kata/ - type: DirectoryOrCreate - - name: dbus - hostPath: - path: /var/run/dbus - - name: systemd - hostPath: - path: /run/systemd - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate diff --git a/kata-deploy/scripts/install-kata-containerd.sh b/kata-deploy/scripts/install-kata-containerd.sh deleted file mode 100755 index 085b83971b..0000000000 --- a/kata-deploy/scripts/install-kata-containerd.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -echo "copying kata artifacts onto host" -cp -R /opt/kata-artifacts/opt/kata/* /opt/kata/ -chmod +x /opt/kata/bin/* - -# Configure containerd to use Kata: -echo "create containerd configuration for Kata" -mkdir -p /etc/containerd/ - -if [ -f /etc/containerd/config.toml ]; then - cp /etc/containerd/config.toml /etc/containerd/config.toml.bak -fi - -cat <&2 + exit 1 +} + +function print_usage() { + echo "Usage: $0 [install/cleanup/reset]" +} + +function get_container_runtime() { + local runtime=$(kubectl describe node $NODE_NAME) + if [ "$?" -ne 0 ]; then + die "invalid node name" + fi + echo "$runtime" | awk -F'[:]' '/Container Runtime Version/ {print $2}' | tr -d ' ' +} + +function install_artifacts() { + echo "copying kata artifacts onto host" + cp -a /opt/kata-artifacts/opt/kata/* /opt/kata/ + chmod +x /opt/kata/bin/* +} + +function configure_cri_runtime() { + case $1 in + crio) + configure_crio + ;; + containerd) + configure_containerd + ;; + esac + systemctl daemon-reload + systemctl restart $1 +} + +function configure_crio() { + # Configure crio to use Kata: + echo "Add Kata Containers as a supported runtime for CRIO:" + + # backup the CRIO.conf only if a backup doesn't already exist (don't override original) + cp -n "$crio_conf_file" "$crio_conf_file_backup" + + cat < Date: Tue, 22 Jan 2019 22:24:38 +0000 Subject: [PATCH 09/13] kata-deploy: updates for Dockerfile Additional packages are necessary and/or were removed from the base image. Signed-off-by: Eric Ernst --- kata-deploy/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kata-deploy/Dockerfile b/kata-deploy/Dockerfile index 940407a432..189d5c9f87 100644 --- a/kata-deploy/Dockerfile +++ b/kata-deploy/Dockerfile @@ -6,6 +6,8 @@ ARG KATA_URL=https://github.com/kata-containers/runtime/releases/download/${KATA ARG KATA_FILE=kata-static-${KATA_VER}-${ARCH}.tar.xz RUN \ +yum install -y epel-release && \ +yum install -y bzip2 jq && \ curl -sOL ${KATA_URL}/${KATA_FILE} && \ mkdir -p /opt/kata-artifacts && \ tar xvf ${KATA_FILE} -C /opt/kata-artifacts/ && \ From c0cdc045a58f3519f18482b197cf8c897d06fdf7 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Tue, 22 Jan 2019 22:24:38 +0000 Subject: [PATCH 10/13] kata-deploy: add script for configuring Docker Before the kata-deploy container image was intended to be used with only Kubernetes. This commit adds a script for configuring Kata to run with Docker. This assumes > release 1.5 of Kata, as Firecracker is being configured as well as QEMU based Kata. Note, in order for this to work, Docker must be configured to use a block-based storage driver. To succeed, it the following directories must be mounted: - /opt/kata - this is the location that the kata artifacts are stored - /run/systemd - for reloading the docker service - /var/run/dbus - for reloading the docker service - /etc/docker - for updating the docker configuration (daemon.json) usage: kata-deploy-kata [install | remove] Signed-off-by: Eric Ernst --- kata-deploy/Dockerfile | 3 + kata-deploy/scripts/kata-deploy-docker.sh | 111 ++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100755 kata-deploy/scripts/kata-deploy-docker.sh diff --git a/kata-deploy/Dockerfile b/kata-deploy/Dockerfile index 189d5c9f87..1795179424 100644 --- a/kata-deploy/Dockerfile +++ b/kata-deploy/Dockerfile @@ -18,3 +18,6 @@ curl -Lso /bin/kubectl https://storage.googleapis.com/kubernetes-release/release chmod +x /bin/kubectl COPY scripts /opt/kata-artifacts/scripts +RUN \ +ln -s /opt/kata-artifacts/scripts/kata-deploy-docker.sh /usr/bin/kata-deploy-docker && \ +ln -s /opt/kata-artifacts/scripts/kata-deploy.sh /usr/bin/kata-deploy diff --git a/kata-deploy/scripts/kata-deploy-docker.sh b/kata-deploy/scripts/kata-deploy-docker.sh new file mode 100755 index 0000000000..9987cdf99e --- /dev/null +++ b/kata-deploy/scripts/kata-deploy-docker.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Copyright (c) 2019 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o pipefail +set -o nounset + +conf_file="/etc/docker/daemon.json" +conf_file_backup="${conf_file}.bak" +snippet="${conf_file}.snip" +tmp_file="${conf_file}.tmp" + +# If we fail for any reason a message will be displayed +die() { + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +function print_usage() { + echo "Usage: $0 [install/remove]" +} + +function install_artifacts() { + echo "copying kata artifacts onto host" + cp -a /opt/kata-artifacts/opt/kata/* /opt/kata/ + chmod +x /opt/kata/bin/* +} + +function configure_docker() { + echo "configuring docker" + + cat < "${tmp_file}" + mv "${tmp_file}" "${conf_file}" + rm "${snippet}" + else + mv "${snippet}" "${conf_file}" + fi + + systemctl daemon-reload + systemctl reload docker +} + +function remove_artifacts() { + echo "deleting kata artifacts" + rm -rf /opt/kata/ +} + +function cleanup_runtime() { + echo "cleanup docker" + rm -f "${conf_file}" + + if [ -f "${conf_file_backup}" ]; then + cp "${conf_file_backup}" "${conf_file}" + fi + systemctl daemon-reload + systemctl reload docker +} + +function main() { + # script requires that user is root + euid=`id -u` + if [[ $euid -ne 0 ]]; then + die "This script must be run as root" + fi + + action=${1:-} + if [ -z $action ]; then + print_usage + die "invalid arguments" + fi + + case $action in + install) + install_artifacts + configure_docker + ;; + remove) + cleanup_runtime + remove_artifacts + ;; + *) + echo invalid arguments + print_usage + ;; + esac +} + + +main $@ From 53115c0de9d0c12467c2a5a53a9443ed2079c0e1 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Wed, 30 Jan 2019 10:34:01 -0800 Subject: [PATCH 11/13] kata-deploy: add docker details to readme Add details for Docker configuration to the kata-deploy README Signed-off-by: Eric Ernst --- kata-deploy/README.md | 52 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/kata-deploy/README.md b/kata-deploy/README.md index 9a181a0cb1..52df13e67b 100644 --- a/kata-deploy/README.md +++ b/kata-deploy/README.md @@ -2,7 +2,11 @@ - [kata-deploy](#kata-deploy) - * [Quick start](#quick-start-) + * [Docker quick start](#docker-quick-start-) + + [Install Kata and configure Docker](#install-kata-and-configure-docker) + + [Run a sample workload](#run-a-sample-workload-utilizing-kata-containers) + + [Remove Kata](#remove-kata) + * [Kubernetes quick start](#kubernetes-quick-start-) + [Install Kata on a running Kubernetes cluster](#install-kata-on-a-running-kubernetes-cluster) + [Run a sample workload](#run-a-sample-workload-) + [Remove Kata from the Kubernetes cluster](#remove-kata-from-the-kubernetes-cluster-) @@ -16,12 +20,54 @@ [kata-deploy](kata-deploy) provides a Dockerfile, which contains all of the binaries and artifacts required to run Kata Containers, as well as reference daemonsets, which can -be utilized to install Kata Containers on a running Kubernetes cluster. +be utilized to install Kata Containers for both Docker and on a running Kubernetes cluster. Note, installation through daemonsets successfully installs `kata-containers.io/kata-runtime` on a node only if it uses either containerd or CRI-O CRI-shims. -## Quick start: +## Docker quick start: + +The kata-deploy container image makes use of a script, `kata-deploy-docker`, for installation of +Kata artifacts and configuration of Docker to utilize the runtime. The following volumes are required to be mounted +to aid in this: +- /opt/kata: this is where all kata artifacts are installed on the system +- /var/run/dbus, /run/systemd: this is require for reloading the the Docker service +- /etc/docker: this is required for updating `daemon.json` in order to configure the kata runtimes in Docker + + +### Install Kata and configure Docker + +To install: + +``` +docker run -v /opt/kata:/opt/kata -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd -v /etc/docker:/etc/docker -it katadocker/kata-deploy kata-deploy-docker install +``` + +Once complete, `/etc/docker/daemon.json` is updated or created to include the Kata runtimes: kata-qemu and kata-fc, for utilizing +QEMU and Firecracker, respectively, for the VM isolation layer. + +### Run a sample workload utilizing Kata containers: + +Run a QEMU QEMU isolated Kata container: +``` +docker run --runtime=kata-qemu -itd alpine +``` + +Run a Firecracker isolated Kata container: +``` +docker run --runtime=kata-fc -itd alpine +``` + +### Remove Kata + +To uninstall: +``` +docker run -v /opt/kata:/opt/kata -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd -v /etc/docker:/etc/docker -it katadocker/kata-deploy kata-deploy-docker remove +``` + +After completing, the original daemon.json, if it existed, is restored and all Kata artifacts from /opt/kata are removed. + +## Kubernetes quick start ### Install Kata on a running Kubernetes cluster From 5f955968e660523003b12ff9a1d3d06baba4e7f7 Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Wed, 30 Jan 2019 11:00:10 -0800 Subject: [PATCH 12/13] kata-deploy: update documentation after 1.5 rewrite With the 1.5 release, we made several changes: -simplification of daemonsets -introduction of runtimeClass Update documentation to take this into account. Signed-off-by: Eric Ernst --- kata-deploy/README.md | 99 ++++++++----------- kata-deploy/examples/test-deploy-kata-fc.yaml | 42 ++++++++ .../examples/test-deploy-kata-qemu.yaml | 45 +++++++++ kata-deploy/examples/test-deploy-runc.yaml | 41 ++++++++ 4 files changed, 171 insertions(+), 56 deletions(-) create mode 100644 kata-deploy/examples/test-deploy-kata-fc.yaml create mode 100644 kata-deploy/examples/test-deploy-kata-qemu.yaml create mode 100644 kata-deploy/examples/test-deploy-runc.yaml diff --git a/kata-deploy/README.md b/kata-deploy/README.md index 52df13e67b..ebe2292d43 100644 --- a/kata-deploy/README.md +++ b/kata-deploy/README.md @@ -78,49 +78,44 @@ kubectl apply -f kata-deploy.yaml ### Run a sample workload -Untrusted workloads can node-select based on ```kata-containers.io/kata-runtime=true```, and are -run through ```kata-containers.io/kata-runtime``` if they are marked with the appropriate CRIO or containerd -annotation: -``` -CRIO: io.kubernetes.cri-o.TrustedSandbox: "false" -containerd: io.kubernetes.cri.untrusted-workload: "true" -``` -The following is a sample workload for running untrusted on a kata-enabled node: +Workloads which utilize Kata can node-select based on ```kata-containers.io/kata-runtime=true```, and are +run through an applicable runtime if they are marked with the appropriate runtimeClass annotation. + + +The following YAML snippet shows how to specify a workload should use Kata with QEMU: ``` -apiVersion: v1 -kind: Pod -metadata: - name: nginx - annotations: - io.kubernetes.cri-o.TrustedSandbox: "false" - io.kubernetes.cri.untrusted-workload: "true" - labels: - env: test spec: - containers: - - name: nginx - image: nginx - imagePullPolicy: IfNotPresent - nodeSelector: - kata-containers.io/kata-runtime: "true" -``` - -To run: -``` -kubectl apply -f examples/nginx-untrusted.yaml + template: + spec: + runtimeClassName: kata-qemu ``` -Now, you should see the pod start. You can verify that the pod is making use of -```kata-containers.io/kata-runtime``` by comparing the container ID observed with the following: +The following YAML snippet shows how to specify a workload should use Kata with Firecracker: ``` -/opt/kata/bin/kata-containers.io/kata-runtime list -kubectl describe pod nginx-untrusted +spec: + template: + spec: + runtimeClassName: kata-fc ``` -The following removes the test pod: + +To run an example with kata-qemu: + ``` -kubectl delete -f examples/nginx-untrusted.yaml +kubectl apply -f https://raw.githubusercontent.com/kata-containers/packaging/master/kata-deploy/examples/test-deploy-kata-qemu.yaml +``` + +To run an example with kata-fc: + +``` +kubectl apply -f https://raw.githubusercontent.com/kata-containers/packaging/master/kata-deploy/examples/test-deploy-kata-fc.yaml +``` + +The following removes the test pods: +``` +kubectl delete -f https://raw.githubusercontent.com/kata-containers/packaging/master/kata-deploy/examples/test-deploy-kata-qemu.yaml +kubectl delete -f https://raw.githubusercontent.com/kata-containers/packaging/master/kata-deploy/examples/test-deploy-kata-fc.yaml ``` ### Remove Kata from the Kubernetes cluster @@ -135,16 +130,16 @@ kubectl delete -f kata-rbac.yaml ## kata-deploy Details ### Dockerfile - -The Dockerfile used to create the container image deployed in the DaemonSet is provided here. -This image contains all the necessary artifacts for running Kata Containers. +The [Dockerfile](kata-deploy/Dockerfile) used to create the container image deployed in the DaemonSet is provided here. +This image contains all the necessary artifacts for running Kata Containers, all of which are pulled +from the [Kata Containers release page](https://github.com/kata-containers/runtime/releases). Host artifacts: -* kata-containers.io/kata-runtime: pulled from Kata GitHub releases page -* kata-proxy: pulled from Kata GitHub releases page -* kata-shim: pulled from Kata GitHub releases page -* qemu-system-x86_64: statically built and included in this repo, based on Kata's QEMU repo -* qemu/* : supporting binaries required for qemu-system-x86_64 +* kata-containers.io/kata-runtime +* kata-proxy +* kata-shim +* firecracker +* qemu-system-x86_64 and supporting binaries Virtual Machine artifacts: * kata-containers.img: pulled from Kata github releases page @@ -152,24 +147,16 @@ Virtual Machine artifacts: ### Daemonsets and RBAC: -A few daemonsets are introduced for kata-deploy, as well as an RBAC to facilitate +Two daemonsets are introduced for kata-deploy, as well as an RBAC to facilitate applying labels to the nodes. -#### runtime-labeler: +#### Kata installer: kata-deploy -This daemonset creates a label on each node in -the cluster identifying the CRI shim in use. For example, -`kata-containers.io/container-runtime=crio` or `kata-containers.io/container-runtime=containerd.` - -#### CRI-O and containerd kata installer - -Depending on the value of `kata-containers.io/container-runtime` label on the node, either the CRI-O or -containerd kata installation daemonset executes. These daemonsets install -the necessary kata binaries, configuration files, and virtual machine artifacts on +This daemonset installs the necessary kata binaries, configuration files, and virtual machine artifacts on the node. Once installed, the daemonset adds a node label `kata-containers.io/kata-runtime=true` and reconfigures -either CRI-O or containerd to make use of Kata for untrusted workloads. As a final step the daemonset -restarts either CRI-O or containerd and kubelet. Upon deletion, the daemonset removes the kata binaries -and VM artifacts and updates the node label to `kata-containers.io/kata-runtime=cleanup.` +either CRI-O or containerd to register two runtimeClasses: `kata-qemu` (for QEMU isolation) and `kata-fc` (for Firecracker isolation). +As a final step the daemonset restarts either CRI-O or containerd. Upon deletion, the daemonset removes the +Kata binaries and VM artifacts and updates the node label to `kata-containers.io/kata-runtime=cleanup.` ### Kata cleanup: This daemonset runs of the node has the label `kata-containers.io/kata-runtime=cleanup.` These daemonsets removes diff --git a/kata-deploy/examples/test-deploy-kata-fc.yaml b/kata-deploy/examples/test-deploy-kata-fc.yaml new file mode 100644 index 0000000000..498de27ae9 --- /dev/null +++ b/kata-deploy/examples/test-deploy-kata-fc.yaml @@ -0,0 +1,42 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + run: php-apache-kata-fc + name: php-apache-kata-fc +spec: + replicas: 1 + selector: + matchLabels: + run: php-apache-kata-fc + template: + metadata: + labels: + run: php-apache-kata-fc + spec: + runtimeClassName: kata-fc + containers: + - image: k8s.gcr.io/hpa-example + imagePullPolicy: Always + name: php-apache + ports: + - containerPort: 80 + protocol: TCP + resources: + requests: + cpu: 200m + restartPolicy: Always +--- +apiVersion: v1 +kind: Service +metadata: + name: php-apache-kata-fc +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + run: php-apache-kata-fc + sessionAffinity: None + type: ClusterIP diff --git a/kata-deploy/examples/test-deploy-kata-qemu.yaml b/kata-deploy/examples/test-deploy-kata-qemu.yaml new file mode 100644 index 0000000000..84fd1bfeda --- /dev/null +++ b/kata-deploy/examples/test-deploy-kata-qemu.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + run: php-apache-kata-qemu + name: php-apache-kata-qemu +spec: + replicas: 1 + selector: + matchLabels: + run: php-apache-kata-qemu + template: + metadata: + annotations: + io.kubernetes.cri-o.TrustedSandbox: "false" + io.kubernetes.cri.untrusted-workload: "true" + labels: + run: php-apache-kata-qemu + spec: + runtimeClassName: kata-qemu + containers: + - image: k8s.gcr.io/hpa-example + imagePullPolicy: Always + name: php-apache + ports: + - containerPort: 80 + protocol: TCP + resources: + requests: + cpu: 200m + restartPolicy: Always +--- +apiVersion: v1 +kind: Service +metadata: + name: php-apache-kata-qemu +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + run: php-apache-kata-qemu + sessionAffinity: None + type: ClusterIP diff --git a/kata-deploy/examples/test-deploy-runc.yaml b/kata-deploy/examples/test-deploy-runc.yaml new file mode 100644 index 0000000000..c7702bc0f5 --- /dev/null +++ b/kata-deploy/examples/test-deploy-runc.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + run: php-apache-runc + name: php-apache-runc +spec: + replicas: 1 + selector: + matchLabels: + run: php-apache-runc + template: + metadata: + labels: + run: php-apache-runc + spec: + containers: + - image: k8s.gcr.io/hpa-example + imagePullPolicy: Always + name: php-apache + ports: + - containerPort: 80 + protocol: TCP + resources: + requests: + cpu: 200m + restartPolicy: Always +--- +apiVersion: v1 +kind: Service +metadata: + name: php-apache-runc +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + run: php-apache-runc + sessionAffinity: None + type: ClusterIP From b4ba52d432a4bd024e33510280a4adecd3786a9a Mon Sep 17 00:00:00 2001 From: Eric Ernst Date: Thu, 31 Jan 2019 20:31:23 -0800 Subject: [PATCH 13/13] kata-deploy: s/kata-containers.io/katacontainers.io To be consistent with project URL, use katacontainers.io instead of kata-containers.io Signed-off-by: Eric Ernst --- kata-deploy/README.md | 18 ++++++++++-------- kata-deploy/kata-cleanup.yaml | 2 +- kata-deploy/scripts/kata-deploy.sh | 4 ++-- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/kata-deploy/README.md b/kata-deploy/README.md index ebe2292d43..f66f6b0e14 100644 --- a/kata-deploy/README.md +++ b/kata-deploy/README.md @@ -22,7 +22,7 @@ and artifacts required to run Kata Containers, as well as reference daemonsets, which can be utilized to install Kata Containers for both Docker and on a running Kubernetes cluster. -Note, installation through daemonsets successfully installs `kata-containers.io/kata-runtime` on +Note, installation through daemonsets successfully installs `katacontainers.io/kata-runtime` on a node only if it uses either containerd or CRI-O CRI-shims. ## Docker quick start: @@ -79,7 +79,7 @@ kubectl apply -f kata-deploy.yaml ### Run a sample workload -Workloads which utilize Kata can node-select based on ```kata-containers.io/kata-runtime=true```, and are +Workloads which utilize Kata can node-select based on ```katacontainers.io/kata-runtime=true```, and are run through an applicable runtime if they are marked with the appropriate runtimeClass annotation. @@ -135,7 +135,9 @@ This image contains all the necessary artifacts for running Kata Containers, all from the [Kata Containers release page](https://github.com/kata-containers/runtime/releases). Host artifacts: -* kata-containers.io/kata-runtime +* kata-runtime +* kata-fc +* kata-qemu * kata-proxy * kata-shim * firecracker @@ -153,13 +155,13 @@ applying labels to the nodes. #### Kata installer: kata-deploy This daemonset installs the necessary kata binaries, configuration files, and virtual machine artifacts on -the node. Once installed, the daemonset adds a node label `kata-containers.io/kata-runtime=true` and reconfigures +the node. Once installed, the daemonset adds a node label `katacontainers.io/kata-runtime=true` and reconfigures either CRI-O or containerd to register two runtimeClasses: `kata-qemu` (for QEMU isolation) and `kata-fc` (for Firecracker isolation). As a final step the daemonset restarts either CRI-O or containerd. Upon deletion, the daemonset removes the -Kata binaries and VM artifacts and updates the node label to `kata-containers.io/kata-runtime=cleanup.` +Kata binaries and VM artifacts and updates the node label to `katacontainers.io/kata-runtime=cleanup.` ### Kata cleanup: -This daemonset runs of the node has the label `kata-containers.io/kata-runtime=cleanup.` These daemonsets removes -the `kata-containers.io/container-runtime` and `kata-containers.io/kata-runtime` labels as well as restarts either CRI-O or containerd systemctl -daemon and kubelet. You cannot execute these resets during the preStopHook of the Kata installer daemonset, +This daemonset runs of the node has the label `katacontainers.io/kata-runtime=cleanup.` These daemonsets removes +the `katacontainers.io/kata-runtime` label as well as restarts either CRI-O or containerd systemctl +daemon. You cannot execute these resets during the preStopHook of the Kata installer daemonset, which necessitated this final cleanup daemonset. diff --git a/kata-deploy/kata-cleanup.yaml b/kata-deploy/kata-cleanup.yaml index 7f7308424b..f2c26d0243 100644 --- a/kata-deploy/kata-cleanup.yaml +++ b/kata-deploy/kata-cleanup.yaml @@ -15,7 +15,7 @@ spec: spec: serviceAccountName: kata-label-node nodeSelector: - kata-containers.io/kata-runtime: cleanup + katacontainers.io/kata-runtime: cleanup containers: - name: kube-kata-cleanup image: katadocker/kata-deploy diff --git a/kata-deploy/scripts/kata-deploy.sh b/kata-deploy/scripts/kata-deploy.sh index a91f0fd217..c073e5a4ac 100755 --- a/kata-deploy/scripts/kata-deploy.sh +++ b/kata-deploy/scripts/kata-deploy.sh @@ -120,7 +120,7 @@ function cleanup_containerd() { } function reset_runtime() { - kubectl label node $NODE_NAME kata-containers.io/container-runtime- kata-containers.io/kata-runtime- + kubectl label node $NODE_NAME katacontainers.io/kata-runtime- systemctl daemon-reload systemctl restart $1 systemctl restart kubelet @@ -158,7 +158,7 @@ function main() { cleanup) remove_artifacts cleanup_cri_runtime $runtime - kubectl label node $NODE_NAME --overwrite kata-containers.io/kata-runtime=cleanup + kubectl label node $NODE_NAME --overwrite katacontainers.io/kata-runtime=cleanup ;; reset) reset_runtime $runtime