Merge pull request #5193 from openanolis/origin/kata-deploy

kata-deploy: ship the rustified runtime binary

src/runtime-rs/Makefile

@@ -44,29 +44,43 @@ else
 endif
 
 ifeq ($(PREFIX),)
-    PREFIX := /usr
+PREFIX := /usr
+EXEC_PREFIX := $(PREFIX)/local
+##VAR BINDIR=<path> is a directory for installing executable programs
+BINDIR := $(EXEC_PREFIX)/bin
+else
+EXEC_PREFIX := $(PREFIX)
+##VAR BINDIR=<path> is a directory for installing executable programs
+# when creating the kata-deploy image, the default installation path for go runtime is $(EXEC_PREFIX)/bin, so we put it here for multiple runtime
+BINDIR := $(EXEC_PREFIX)/runtime-rs/bin/
 endif
 
-PREFIXDEPS    := $(PREFIX)
+PREFIXDEPS := $(PREFIX)
 LIBEXECDIR := $(PREFIXDEPS)/libexec
 SHAREDIR := $(PREFIX)/share
 DEFAULTSDIR := $(SHAREDIR)/defaults
 PROJECT_DIR = $(PROJECT_TAG)
 IMAGENAME = $(PROJECT_TAG).img
 TARGET = $(PROJECT_COMPONENT)
-
+SYSCONFDIR := /etc
 CONFIG_FILE = configuration.toml
 HYPERVISOR_DB = dragonball
+HYPERVISOR_ACRN = acrn
+HYPERVISOR_FC = firecracker
+HYPERVISOR_QEMU = qemu
+HYPERVISOR_CLH = cloud-hypervisor
 
 
 DEFAULT_HYPERVISOR ?= $(HYPERVISOR_DB)
 
-HYPERVISORS := $(HYPERVISOR_DB)
+# List of hypervisors this build system can generate configuration for.
+HYPERVISORS := $(HYPERVISOR_DB) $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH)
 
 DBVALIDHYPERVISORPATHS := []
 PKGDATADIR := $(PREFIXDEPS)/share/$(PROJECT_DIR)
 KERNELDIR := $(PKGDATADIR)
 IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME)
+PKGLIBEXECDIR := $(LIBEXECDIR)/$(PROJECT_DIR)
 FIRMWAREPATH :=
 FIRMWAREVOLUMEPATH :=
 
@@ -290,9 +304,7 @@ endif
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
 
 ##VAR DESTDIR=<path> is a directory prepended to each installed target file
-DESTDIR :=
-##VAR BINDIR=<path> is a directory for installing executable programs
-BINDIR := /usr/local/bin
+DESTDIR ?= /
 
 GENERATED_CODE = crates/shim/src/config.rs
 
@@ -330,8 +342,12 @@ define get_toolchain_version
 $(shell printf "%s: %s\\n" "toolchain" "$(or $(shell rustup show active-toolchain 2>/dev/null), (unknown))")
 endef
 
+# Install a configuration file
+# params:
+# $1 : file to install
+# $2 : directory path where file will be installed
 define INSTALL_FILE
-	install -D -m 644 $1 $(DESTDIR)$2/$1 || exit 1;
+	install --mode 0644 -D $1 $(DESTDIR)$2/$(notdir $1);
 endef
 
 # Returns the name of the kernel file to use based on the provided KERNELTYPE.
@@ -382,20 +398,62 @@ show-header:
 	@printf "%s - version %s (commit %s)\n\n" "$(TARGET)" "$(VERSION)" "$(COMMIT_MSG)"
 
 show-summary: show-header
-	@printf "project:\n"
+	@printf "• Project:\n"
 	@printf "  name: $(PROJECT_NAME)\n"
 	@printf "  url: $(PROJECT_URL)\n"
 	@printf "  component: $(PROJECT_COMPONENT)\n"
-	@printf "target: $(TARGET)\n"
-	@printf "architecture:\n"
-	@printf "  host: $(ARCH)\n"
-	@printf "rust:\n"
+	@printf "\n"
+	@printf "• Target: $(TARGET)\n"
+	@printf "\n"
+	@printf "• Architecture: $(ARCH)\n"
+	@printf "\n"
+	@printf "• Rust:\n"
 	@printf "  %s\n" "$(call get_command_version,cargo)"
 	@printf "  %s\n" "$(call get_command_version,rustc)"
 	@printf "  %s\n" "$(call get_command_version,rustup)"
 	@printf "  %s\n" "$(call get_toolchain_version)"
 	@printf "\n"
-
+	@printf "• Hypervisors:\n"
+	@printf "\tDefault: $(DEFAULT_HYPERVISOR)\n"
+	@printf "\tKnown: $(sort $(HYPERVISORS))\n"
+	@printf "\tAvailable for this architecture: $(sort $(KNOWN_HYPERVISORS))\n"
+	@printf "\n"
+	@printf "• Summary:\n"
+	@printf "\n"
+	@printf "\tdestination install path (DESTDIR) : %s\n" $(abspath $(DESTDIR))
+	@printf "\tbinary installation path (BINDIR) : %s\n" $(abspath $(BINDIR))
+	@printf "\tbinaries to install :\n"
+	@printf \
+          "$(foreach b,$(sort $(SHIMV2)),$(shell printf "\\t - $(shell readlink -m $(DESTDIR)/$(BINDIR)/$(b))\\\n"))"
+	@printf "\tconfigs to install (CONFIGS) :\n"
+	@printf \
+	  "$(foreach c,$(sort $(CONFIGS)),$(shell printf "\\t - $(c)\\\n"))"
+	@printf "\tinstall paths (CONFIG_PATHS) :\n"
+	@printf \
+	  "$(foreach c,$(sort $(CONFIG_PATHS)),$(shell printf "\\t - $(c)\\\n"))"
+	@printf "\talternate config paths (SYSCONFIG_PATHS) : %s\n"
+	@printf \
+	  "$(foreach c,$(sort $(SYSCONFIG_PATHS)),$(shell printf "\\t - $(c)\\\n"))"
+	@printf "\tdefault install path for $(DEFAULT_HYPERVISOR) (CONFIG_PATH) : %s\n" $(abspath $(CONFIG_PATH))
+	@printf "\tdefault alternate config path (SYSCONFIG) : %s\n" $(abspath $(SYSCONFIG))
+ifneq (,$(findstring $(HYPERVISOR_QEMU),$(KNOWN_HYPERVISORS)))
+	@printf "\t$(HYPERVISOR_QEMU) hypervisor path (QEMUPATH) : %s\n" $(abspath $(QEMUPATH))
+endif
+ifneq (,$(findstring $(HYPERVISOR_QEMU_VIRTIOFS),$(KNOWN_HYPERVISORS)))
+	@printf "\t$(HYPERVISOR_QEMU_VIRTIOFS) hypervisor path (QEMUVIRTIOFSPATH) : %s\n" $(abspath $(QEMUVIRTIOFSPATH))
+endif
+ifneq (,$(findstring $(HYPERVISOR_CLH),$(KNOWN_HYPERVISORS)))
+	@printf "\t$(HYPERVISOR_CLH) hypervisor path (CLHPATH) : %s\n" $(abspath $(CLHPATH))
+endif
+ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS)))
+	@printf "\t$(HYPERVISOR_FC) hypervisor path (FCPATH) : %s\n" $(abspath $(FCPATH))
+endif
+ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS)))
+	@printf "\t$(HYPERVISOR_ACRN) hypervisor path (ACRNPATH) : %s\n" $(abspath $(ACRNPATH))
+endif
+	@printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR))
+	@printf "\tshim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR))
+	@printf "\n"
 ## help: Show help comments that start with `##VAR` and `##TARGET`
 help: Makefile show-summary
 	@echo "========================== Help ============================="
@@ -429,10 +487,10 @@ codecov-html: check_tarpaulin
 install: install-runtime install-configs
 
 install-runtime: runtime
-	install -D $(TARGET_PATH) $(BINDIR)
+	install -D $(TARGET_PATH) $(DESTDIR)$(BINDIR)/$(notdir $(TARGET_PATH))
 
 install-configs: $(CONFIGS)
-	$(foreach f,$(CONFIGS),$(call INSTALL_CONFIG,$f,$(dir $(CONFIG_PATH)))) \
+	$(foreach f,$(CONFIGS),$(call INSTALL_FILE,$f,$(dir $(CONFIG_PATH)))) \
     sudo ln -sf $(DEFAULT_HYPERVISOR_CONFIG) $(DESTDIR)/$(CONFIG_PATH)
 
 .PHONY: \

src/runtime-rs/crates/runtimes/common/src/types/mod.rs

@@ -128,6 +128,7 @@ pub struct ContainerConfig {
     pub bundle: String,
     pub rootfs_mounts: Vec<Mount>,
     pub terminal: bool,
+    pub options: Option<Vec<u8>>,
     pub stdin: Option<String>,
     pub stdout: Option<String>,
     pub stderr: Option<String>,

src/runtime-rs/crates/runtimes/common/src/types/trans_from_shim.rs

@@ -4,19 +4,17 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use std::{
-    convert::{From, TryFrom},
-    path::PathBuf,
-};
-
-use anyhow::{Context, Result};
-use containerd_shim_protos::api;
-use kata_types::mount::Mount;
-
 use super::{
     ContainerConfig, ContainerID, ContainerProcess, ExecProcessRequest, KillRequest, Request,
     ResizePTYRequest, ShutdownRequest, UpdateRequest,
 };
+use anyhow::{Context, Result};
+use containerd_shim_protos::api;
+use kata_types::mount::Mount;
+use std::{
+    convert::{From, TryFrom},
+    path::PathBuf,
+};
 
 fn trans_from_shim_mount(from: api::Mount) -> Mount {
     let options = from.options.to_vec();
@@ -42,6 +40,11 @@ fn trans_from_shim_mount(from: api::Mount) -> Mount {
 impl TryFrom<api::CreateTaskRequest> for Request {
     type Error = anyhow::Error;
     fn try_from(from: api::CreateTaskRequest) -> Result<Self> {
+        let options = if from.has_options() {
+            Some(from.get_options().get_value().to_vec())
+        } else {
+            None
+        };
         Ok(Request::CreateContainer(ContainerConfig {
             container_id: from.id.clone(),
             bundle: from.bundle.clone(),
@@ -52,6 +55,7 @@ impl TryFrom<api::CreateTaskRequest> for Request {
                 .map(trans_from_shim_mount)
                 .collect(),
             terminal: from.terminal,
+            options,
             stdin: (!from.stdin.is_empty()).then(|| from.stdin.clone()),
             stdout: (!from.stdout.is_empty()).then(|| from.stdout.clone()),
             stderr: (!from.stderr.is_empty()).then(|| from.stderr.clone()),

src/runtime-rs/crates/runtimes/src/manager.rs

@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use std::sync::Arc;
+use std::{str::from_utf8, sync::Arc};
 
 use anyhow::{anyhow, Context, Result};
 
@@ -74,7 +74,7 @@ impl RuntimeHandlerManagerInner {
         Ok(())
     }
 
-    async fn try_init(&mut self, spec: &oci::Spec) -> Result<()> {
+    async fn try_init(&mut self, spec: &oci::Spec, options: &Option<Vec<u8>>) -> Result<()> {
         // return if runtime instance has init
         if self.runtime_instance.is_some() {
             return Ok(());
@@ -104,7 +104,7 @@ impl RuntimeHandlerManagerInner {
             None
         };
 
-        let config = load_config(spec).context("load config")?;
+        let config = load_config(spec, options).context("load config")?;
         self.init_runtime_handler(netns, Arc::new(config))
             .await
             .context("init runtime handler")?;
@@ -182,9 +182,13 @@ impl RuntimeHandlerManager {
             .ok_or_else(|| anyhow!("runtime not ready"))
     }
 
-    async fn try_init_runtime_instance(&self, spec: &oci::Spec) -> Result<()> {
+    async fn try_init_runtime_instance(
+        &self,
+        spec: &oci::Spec,
+        options: &Option<Vec<u8>>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
-        inner.try_init(spec).await
+        inner.try_init(spec, options).await
     }
 
     pub async fn handler_message(&self, req: Request) -> Result<Response> {
@@ -193,7 +197,7 @@ impl RuntimeHandlerManager {
             let bundler_path = format!("{}/{}", req.bundle, oci::OCI_SPEC_CONFIG_FILE_NAME);
             let spec = oci::Spec::load(&bundler_path).context("load spec")?;
 
-            self.try_init_runtime_instance(&spec)
+            self.try_init_runtime_instance(&spec, &req.options)
                 .await
                 .context("try init runtime instance")?;
             let instance = self
@@ -309,13 +313,21 @@ impl RuntimeHandlerManager {
 /// 2. shimv2 create task option
 /// TODO: https://github.com/kata-containers/kata-containers/issues/3961
 /// 3. environment
-fn load_config(spec: &oci::Spec) -> Result<TomlConfig> {
+fn load_config(spec: &oci::Spec, option: &Option<Vec<u8>>) -> Result<TomlConfig> {
     const KATA_CONF_FILE: &str = "KATA_CONF_FILE";
     let annotation = Annotation::new(spec.annotations.clone());
     let config_path = if let Some(path) = annotation.get_sandbox_config_path() {
         path
     } else if let Ok(path) = std::env::var(KATA_CONF_FILE) {
         path
+    } else if let Some(option) = option {
+        // get rid of the special characters in options to get the config path
+        let path = if option.len() > 2 {
+            from_utf8(&option[2..])?.to_string()
+        } else {
+            String::from("")
+        };
+        path
     } else {
         String::from("")
     };

tools/packaging/kata-deploy/README.md

@@ -81,6 +81,14 @@ which will ensure the workload is only scheduled on a node that has Kata Contain
 ```bash
   $ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml
 ```
+The following YAML snippet shows how to specify a workload should use Kata with `Dragonball`:
+
+```yaml
+spec:
+  template:
+    spec:
+      runtimeClassName: kata-dragonball
+```
 
 The following YAML snippet shows how to specify a workload should use Kata with Cloud Hypervisor:
 
@@ -108,6 +116,11 @@ spec:
     spec:
       runtimeClassName: kata-qemu
 ```
+To run an example with `kata-dragonball`:
+
+```bash
+$ kubectl apply -f https://raw.githubusercontent.com/kata-containers/kata-containers/main/tools/packaging/kata-deploy/examples/test-deploy-kata-dragonball.yaml
+```
 
 To run an example with `kata-clh`:
 
@@ -192,7 +205,7 @@ from the [Kata Containers release page](https://github.com/kata-containers/kata-
 
 Host artifacts:
 * `cloud-hypervisor`, `firecracker`, `qemu`, and supporting binaries
-* `containerd-shim-kata-v2`
+* `containerd-shim-kata-v2` (go runtime and rust runtime)
 * `kata-collect-data.sh`
 * `kata-runtime`
 

tools/packaging/kata-deploy/action/test-kata.sh

@@ -66,7 +66,7 @@ function run_test() {
     cmd="kubectl get pods | grep $busybox_pod | grep Completed"
     wait_time=120
 
-    configurations=("nginx-deployment-qemu" "nginx-deployment-clh")
+    configurations=("nginx-deployment-qemu" "nginx-deployment-clh" "nginx-deployment-dragonball")
     for deployment in "${configurations[@]}"; do
         # start the kata pod:
         kubectl apply -f "$YAMLPATH/examples/${deployment}.yaml"

tools/packaging/kata-deploy/examples/nginx-deployment-dragonball.yaml

@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment-dragonball
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      runtimeClassName: kata-dragonball
+      containers:
+      - name: nginx
+        image: nginx:1.14
+        ports:
+        - containerPort: 80

tools/packaging/kata-deploy/examples/test-deploy-kata-dragonball.yaml

@@ -0,0 +1,42 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    run: php-apache-kata-dragonball
+  name: php-apache-kata-dragonball
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: php-apache-kata-dragonball
+  template:
+    metadata:
+      labels:
+        run: php-apache-kata-dragonball
+    spec:
+      runtimeClassName: kata-dragonball
+      containers:
+      - image: k8s.gcr.io/hpa-example
+        imagePullPolicy: Always
+        name: php-apache
+        ports:
+        - containerPort: 80
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 200m
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: php-apache-kata-dragonball
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    run: php-apache-kata-dragonball
+  sessionAffinity: None
+  type: ClusterIP

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -154,7 +154,9 @@ install_virtiofsd() {
 #Install all components that are not assets
 install_shimv2() {
 	GO_VERSION="$(yq r ${versions_yaml} languages.golang.meta.newest-version)"
+	RUST_VERSION="$(yq r ${versions_yaml} languages.rust.meta.newest-version)"
 	export GO_VERSION
+	export RUST_VERSION
 	DESTDIR="${destdir}" PREFIX="${prefix}" "${shimv2_builder}"
 }
 
@@ -179,7 +181,7 @@ handle_build() {
 		install_shimv2
 		install_virtiofsd
 		;;
-
+	
 	cloud-hypervisor) install_clh ;;
 
 	firecracker) install_firecracker ;;
@@ -198,6 +200,7 @@ handle_build() {
 
 	virtiofsd) install_virtiofsd ;;
 
+	
 	*)
 		die "Invalid build target ${build_target}"
 		;;

tools/packaging/kata-deploy/runtimeclasses/kata-runtimeClasses.yaml

@@ -37,3 +37,16 @@ overhead:
 scheduling:
   nodeSelector:
     katacontainers.io/kata-runtime: "true"
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1
+metadata:
+    name: kata-dragonball
+handler: kata-dragonball
+overhead:
+    podFixed:
+        memory: "130Mi"
+        cpu: "250m"
+scheduling:
+  nodeSelector:
+    katacontainers.io/kata-runtime: "true"

tools/packaging/kata-deploy/scripts/kata-deploy.sh

@@ -17,6 +17,7 @@ shims=(
 	"fc"
 	"qemu"
 	"clh"
+	"dragonball"
 )
 
 default_shim="qemu"
@@ -57,6 +58,7 @@ function install_artifacts() {
 	echo "copying kata artifacts onto host"
 	cp -a /opt/kata-artifacts/opt/kata/* /opt/kata/
 	chmod +x /opt/kata/bin/*
+	chmod +x /opt/kata/runtime-rs/bin/*
 }
 
 function configure_cri_runtime() {
@@ -98,7 +100,11 @@ function configure_different_shims_base() {
 			fi
 		fi
 
-		ln -sf /opt/kata/bin/containerd-shim-kata-v2 "${shim_file}"
+		if [[ "${shim}" == "dragonball" ]]; then
+			ln -sf /opt/kata/runtime-rs/bin/containerd-shim-kata-v2 "${shim_file}"
+		else
+			ln -sf /opt/kata/bin/containerd-shim-kata-v2 "${shim_file}"
+		fi
 		chmod +x "$shim_file"
 
 		if [ "${shim}" == "${default_shim}" ]; then

tools/packaging/static-build/shim-v2/Dockerfile

@@ -13,10 +13,13 @@ RUN apt-get update && \
         gcc \
         git \
         make \
+        musl-tools \
         sudo && \
     apt-get clean && rm -rf /var/lib/apt/lists/
 
-COPY install_go.sh /usr/bin/install_go.sh
+COPY install_go_rust.sh /usr/bin/install_go_rust.sh
 ARG GO_VERSION
-RUN install_go.sh "${GO_VERSION}"
+ARG RUST_VERSION
+RUN install_go_rust.sh "${GO_VERSION}" "${RUST_VERSION}"
 ENV PATH=/usr/local/go/bin:${PATH}
+ENV PATH=/root/.cargo/bin/:${PATH}

tools/packaging/static-build/shim-v2/build.sh

@@ -14,18 +14,29 @@ readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh
 
 
 GO_VERSION=${GO_VERSION}
+RUST_VERSION=${RUST_VERSION}
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
 container_image="shim-v2-builder"
 
-sudo docker build  --build-arg GO_VERSION="${GO_VERSION}" -t "${container_image}" "${script_dir}"
+sudo docker build  --build-arg GO_VERSION="${GO_VERSION}"  --build-arg RUST_VERSION="${RUST_VERSION}" -t "${container_image}" "${script_dir}"
 
 arch=$(uname -m)
 if [ ${arch} = "ppc64le" ]; then
 	arch="ppc64"
 fi
 
+sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
+	-w "${repo_root_dir}/src/runtime-rs" \
+	"${container_image}" \
+	bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch}"
+
+sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
+	-w "${repo_root_dir}/src/runtime-rs" \
+	"${container_image}" \
+	bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX="${PREFIX}" DESTDIR="${DESTDIR}" install"
+	
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${repo_root_dir}/src/runtime" \
 	"${container_image}" \

tools/packaging/static-build/shim-v2/install_go_rust.sh

@@ -50,6 +50,13 @@ EOF
 
 trap finish EXIT
 
+rust_version=${2:-}
+ARCH=${ARCH:-$(uname -m)}
+LIBC=${LIBC:-musl}
+curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSLf | sh -s -- -y --default-toolchain ${rust_version} -t ${ARCH}-unknown-linux-${LIBC}
+source /root/.cargo/env
+rustup target add x86_64-unknown-linux-musl
+
 pushd "${tmp_dir}"
 
 while getopts "d:fh" opt