Merge pull request #7961 from ChengyuZhu6/update_nydus

Bump nydus versions and update nydus tests

tests/common.bash

@@ -414,6 +414,28 @@ function install_cri_tools() {
 	rm -f "${tarball_name}"
 }
 
+function install_nydus() {
+	version="${1}"
+
+	project="dragonflyoss/image-service"
+	tarball_name="nydus-static-${version}-linux-$(${repo_root_dir}/tests/kata-arch.sh -g).tgz"
+
+	download_github_project_tarball "${project}" "${version}" "${tarball_name}"
+	sudo tar xfz "${tarball_name}" -C /usr/local/bin --strip-components=1
+	rm -f "${tarball_name}"
+}
+
+function install_nydus_snapshotter() {
+	version="${1}"
+
+	project="containerd/nydus-snapshotter"
+	tarball_name="nydus-snapshotter-${version}-$(${repo_root_dir}/tests/kata-arch.sh).tgz"
+
+	download_github_project_tarball "${project}" "${version}" "${tarball_name}"
+	sudo tar xfz "${tarball_name}" -C /usr/local/bin --strip-components=1
+	rm -f "${tarball_name}"
+}
+
 # Convert architecture to the name used by golang
 function arch_to_golang() {
 	local arch="$(uname -m)"

tests/integration/nydus/gha-run.sh

@@ -16,20 +16,48 @@ source "${nydus_dir}/../../common.bash"
 function install_dependencies() {
 	info "Installing the dependencies needed for running the nydus tests"
 
-	return 0
+	# Dependency list of projects that we can rely on the system packages
+	# - jq
+	declare -a system_deps=(
+		jq
+	)
+	
+	sudo apt-get update
+	sudo apt-get -y install "${system_deps[@]}"
+
+	ensure_yq
+
+	# Dependency list of projects that we can install them
+	# directly from their releases on GitHub:
+	# - containerd
+	#   - cri-container-cni release tarball already includes CNI plugins
+	# - cri-tools
+	# - nydus
+	# - nydus-snapshotter
+	declare -a github_deps
+	github_deps[0]="cri_containerd:$(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}")"
+	github_deps[1]="cri_tools:$(get_from_kata_deps "externals.critools.latest")"
+	github_deps[2]="nydus:$(get_from_kata_deps "externals.nydus.version")"
+	github_deps[3]="nydus_snapshotter:$(get_from_kata_deps "externals.nydus-snapshotter.version")"
+	
+	for github_dep in "${github_deps[@]}"; do
+	        IFS=":" read -r -a dep <<< "${github_dep}"
+	        install_${dep[0]} "${dep[1]}"
+	done
 }
 
 function run() {
 	info "Running nydus tests using ${KATA_HYPERVISOR} hypervisor"
 
-	return 0
+	enabling_hypervisor
+	bash -c "${nydus_dir}/nydus_tests.sh"
 }
 
 function main() {
 	action="${1:-}"
 	case "${action}" in
 		install-dependencies) install_dependencies ;;
-		install-kata) return 0 ;;
+		install-kata) install_kata ;;
 		run) run ;;
 		*) >&2 die "Invalid argument" ;;
 	esac

tests/integration/nydus/nydus-container.yaml

@@ -1,5 +1,7 @@
 metadata:
   name: nydus-container
+  namespace: default
+  uid: nydus-containerd-uid
 image:
   image: ghcr.io/dragonflyoss/image-service/alpine:nydus-latest
 command:

tests/integration/nydus/nydus-sandbox.yaml

@@ -2,4 +2,5 @@ metadata:
   attempt: 1
   name: nydus-sandbox
   namespace: default
+  uid: nydus-sandbox-uid
 log_directory: /tmp

tests/integration/nydus/nydus_tests.sh

@@ -12,8 +12,7 @@ set -o pipefail
 set -o errtrace
 
 dir_path=$(dirname "$0")
-source "${dir_path}/../../lib/common.bash"
+source "${dir_path}/../../common.bash"
-source "${dir_path}/../../.ci/lib.sh"
 source "/etc/os-release" || source "/usr/lib/os-release"
 KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
 
@@ -30,56 +29,20 @@ containerd_config_backup="/tmp/containerd.config.toml"
 # test image for container
 IMAGE="${IMAGE:-ghcr.io/dragonflyoss/image-service/alpine:nydus-latest}"
 
-if [ "$KATA_HYPERVISOR" != "qemu" ] && [ "$KATA_HYPERVISOR" != "cloud-hypervisor" ] && [ "$KATA_HYPERVISOR" != "dragonball" ]; then
+if [ "$KATA_HYPERVISOR" != "qemu" ] && [ "$KATA_HYPERVISOR" != "clh" ]; then
-	echo "Skip nydus test for $KATA_HYPERVISOR, it only works for QEMU/CLH/DB now."
+	echo "Skip nydus test for $KATA_HYPERVISOR, it only works for QEMU/CLH now."
 	exit 0
 fi
 
-arch="$(uname -m)"
-if [ "$arch" != "x86_64" ]; then
-	echo "Skip nydus test for $arch, it only works for x86_64 now. See https://github.com/kata-containers/tests/issues/4445"
-	exit 0
-fi
-
-function install_from_tarball() {
-	local package_name="$1"
-	local binary_name="$2"
-	[ -n "$package_name" ] || die "need package_name"
-	[ -n "$binary_name" ] || die "need package release binary_name"
-
-	local url=$(get_version "externals.${package_name}.url")
-	local version=$(get_version "externals.${package_name}.version")
-	local tarball_url="${url}/releases/download/${version}/${binary_name}-${version}-$arch.tgz"
-	if [ "${package_name}" == "nydus" ]; then
-		local goarch="$(${dir_path}/../../.ci/kata-arch.sh --golang)"
-		tarball_url="${url}/releases/download/${version}/${binary_name}-${version}-linux-$goarch.tgz"
-	fi
-	echo "Download tarball from ${tarball_url}"
-	curl -Ls "$tarball_url" | sudo tar xfz - -C /usr/local/bin --strip-components=1
-}
-
 function setup_nydus() {
-	# install nydus
-	install_from_tarball "nydus" "nydus-static"
-
-	# install nydus-snapshotter
-	install_from_tarball "nydus-snapshotter" "nydus-snapshotter"
-
 	# Config nydus snapshotter
 	sudo -E cp "$dir_path/nydusd-config.json" /etc/
+	sudo -E cp "$dir_path/snapshotter-config.toml" /etc/
 
 	# start nydus-snapshotter
-	nohup /usr/local/bin/containerd-nydus-grpc \
+	sudo nohup /usr/local/bin/containerd-nydus-grpc \
-		--config-path /etc/nydusd-config.json \
+		--config /etc/snapshotter-config.toml \
-		--shared-daemon \
+		--nydusd-config /etc/nydusd-config.json &
-		--log-level debug \
-		--root /var/lib/containerd/io.containerd.snapshotter.v1.nydus \
-		--cache-dir /var/lib/nydus/cache \
-		--nydusd-path /usr/local/bin/nydusd \
-		--nydusimg-path /usr/local/bin/nydus-image \
-		--disable-cache-manager true \
-		--enable-nydus-overlayfs true \
-		--log-to-stdout >/dev/null 2>&1 &
 }
 
 function config_kata() {
@@ -136,40 +99,51 @@ function config_containerd() {
          [plugins.cri.containerd.runtimes.runc.options]
            BinaryName = "${runc_path}"
            Root = ""
-      [plugins.cri.containerd.runtimes.kata]
+      [plugins.cri.containerd.runtimes.kata-${KATA_HYPERVISOR}]
-         runtime_type = "io.containerd.kata.v2"
+         runtime_type = "io.containerd.kata-${KATA_HYPERVISOR}.v2"
          privileged_without_host_devices = true
 EOF
 }
 
+function check_nydus_snapshotter_exist() {
+	echo "check_nydus_snapshotter_exist"
+	bin="containerd-nydus-grpc"
+	if pgrep -f "$bin" >/dev/null; then
+		echo "nydus-snapshotter is running"
+	else
+		die "nydus-snapshotter is not running"
+	fi
+}
+
 function setup() {
 	setup_nydus
 	config_kata
 	config_containerd
 	restart_containerd_service
 	check_processes
+	check_nydus_snapshotter_exist
 	extract_kata_env
 }
 
 function run_test() {
-	sudo -E crictl pull "${IMAGE}"
+	sudo -E crictl --timeout=20s pull "${IMAGE}"
-	pod=$(sudo -E crictl runp -r kata $dir_path/nydus-sandbox.yaml)
+	pod=$(sudo -E crictl --timeout=20s runp -r kata-${KATA_HYPERVISOR} $dir_path/nydus-sandbox.yaml)
 	echo "Pod $pod created"
-	cnt=$(sudo -E crictl create $pod $dir_path/nydus-container.yaml $dir_path/nydus-sandbox.yaml)
+	cnt=$(sudo -E crictl --timeout=20s create $pod $dir_path/nydus-container.yaml $dir_path/nydus-sandbox.yaml)
 	echo "Container $cnt created"
-	sudo -E crictl start $cnt
+	sudo -E crictl --timeout=20s start $cnt
 	echo "Container $cnt started"
 
 	# ensure container is running
-	state=$(sudo -E crictl inspect $cnt | jq .status.state | tr -d '"')
+	state=$(sudo -E crictl --timeout=20s inspect $cnt | jq .status.state | tr -d '"')
 	[ $state == "CONTAINER_RUNNING" ] || die "Container is not running($state)"
 	# run a command in container
-	crictl exec $cnt ls
+	sudo -E crictl --timeout=20s exec $cnt ls
 
 	# cleanup containers
-	sudo -E crictl stop $cnt
+	sudo -E crictl --timeout=20s stop $cnt
-	sudo -E crictl stopp $pod
+	sudo -E crictl --timeout=20s stopp $pod
-	sudo -E crictl rmp $pod
+	sudo -E crictl --timeout=20s rmp $pod
 }
 
 function teardown() {
@@ -177,11 +151,11 @@ function teardown() {
 
 	# kill nydus-snapshotter
 	bin=containerd-nydus-grpc
-	kill -9 $(pidof $bin) || true
+	sudo -E kill -9 $(pidof $bin) || true
 	[ "$(pidof $bin)" == "" ] || die "$bin is running"
 
 	bin=nydusd
-	kill -9 $(pidof $bin) || true
+	sudo -E kill -9 $(pidof $bin) || true
 	[ "$(pidof $bin)" == "" ] || die "$bin is running"
 
 	# restore kata configuratiom.toml if needed

tests/integration/nydus/nydusd-config.json

@@ -3,17 +3,13 @@
     "backend": {
       "type": "registry",
       "config": {
-        "scheme": "https",
         "timeout": 5,
         "connect_timeout": 5,
         "retry_limit": 2
       }
     },
     "cache": {
-      "type": "blobcache",
+      "type": "blobcache"
-      "config": {
-        "work_dir": "/var/lib/nydus/cache"
-      }
     }
   },
   "mode": "direct",
@@ -22,6 +18,8 @@
   "enable_xattr": true,
   "fs_prefetch": {
     "enable": true,
-    "threads_count": 2
+    "threads_count": 8,
+    "merging_size": 1048576,
+    "prefetch_all": true
   }
-}
+}

tests/integration/nydus/snapshotter-config.toml

@@ -0,0 +1,128 @@
+version = 1
+# Snapshotter's own home directory where it stores and creates necessary resources
+root = "/var/lib/containerd-nydus"
+# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket
+address = "/run/containerd-nydus/containerd-nydus-grpc.sock"
+daemon_mode = "dedicated"
+# Whether snapshotter should try to clean up resources when it is closed
+cleanup_on_close = false
+
+[system]
+# Snapshotter's debug and trace HTTP server interface
+enable = true
+# Unix domain socket path where system controller is listening on
+address = "/run/containerd-nydus/system.sock"
+
+[system.debug]
+# Snapshotter can profile the CPU utilization of each nydusd daemon when it is being started.
+# This option specifies the profile duration when nydusd is downloading and uncomproessing data.
+daemon_cpu_profile_duration_secs = 5
+# Enable by assigning an address, empty indicates pprof server is disabled
+pprof_address = ""
+
+[daemon]
+# Specify a configuration file for nydusd
+nydusd_config = "/etc/nydusd-config.json"
+nydusd_path = "/usr/local/bin/nydusd"
+nydusimage_path = "/usr/local/bin/nydus-image"
+# fusedev or fscache
+fs_driver = "fusedev"
+# How to process when daemon dies: "none", "restart" or "failover"
+recover_policy = "restart"
+# Nydusd worker thread number to handle FUSE or fscache requests, [0-1024].
+# Setting to 0 will use the default configuration of nydusd.
+threads_number = 4
+# Log rotation size for nydusd, in unit MB(megabytes)
+log_rotation_size = 100
+
+
+[cgroup]
+# Whether to use separate cgroup for nydusd.
+enable = true
+# The memory limit for nydusd cgroup, which contains all nydusd processes.
+# Percentage is supported as well, please ensure it is end with "%".
+# The default unit is bytes. Acceptable values include "209715200", "200MiB", "200Mi" and "10%".
+memory_limit = ""
+
+[log]
+# Print logs to stdout rather than logging files
+log_to_stdout = false
+# Snapshotter's log level
+level = "info"
+log_rotation_compress = true
+log_rotation_local_time = true
+# Max number of days to retain logs
+log_rotation_max_age = 7
+log_rotation_max_backups = 5
+# In unit MB(megabytes)
+log_rotation_max_size = 100
+
+[metrics]
+# Enable by assigning an address, empty indicates metrics server is disabled
+address = ":9110"
+
+[remote]
+convert_vpc_registry = false
+
+[remote.mirrors_config]
+# Snapshotter will overwrite daemon's mirrors configuration
+# if the values loaded from this driectory are not null before starting a daemon.
+# Set to "" or an empty directory to disable it.
+#dir = "/etc/nydus/certs.d"
+
+[remote.auth]
+# Fetch the private registry auth by listening to K8s API server
+enable_kubeconfig_keychain = false
+# synchronize `kubernetes.io/dockerconfigjson` secret from kubernetes API server with specified kubeconfig (default `$KUBECONFIG` or `~/.kube/config`)
+kubeconfig_path = ""
+# Fetch the private registry auth as CRI image service proxy
+enable_cri_keychain = false
+# the target image service when using image proxy
+#image_service_address = "/run/containerd/containerd.sock"
+
+[snapshot]
+# Let containerd use nydus-overlayfs mount helper
+enable_nydus_overlayfs = true
+# Insert Kata Virtual Volume option to `Mount.Options`
+enable_kata_volume = false
+# Whether to remove resources when a snapshot is removed
+sync_remove = false
+
+[cache_manager]
+disable = false
+gc_period = "24h"
+# Directory to host cached files
+cache_dir = ""
+
+[image]
+public_key_file = ""
+validate_signature = false
+
+# The configuraions for features that are not production ready
+[experimental]
+# Whether to enable stargz support
+enable_stargz = false
+# Whether to enable referrers support
+# The option enables trying to fetch the Nydus image associated with the OCI image and run it.
+# Also see https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers
+enable_referrer_detect = false
+[experimental.tarfs]
+# Whether to enable nydus tarfs mode. Tarfs is supported by:
+# - The EROFS filesystem driver since Linux 6.4
+# - Nydus Image Service release v2.3
+enable_tarfs = false
+# Mount rafs on host by loopdev and EROFS
+mount_tarfs_on_host = false
+# Only enable nydus tarfs mode for images with `tarfs hint` label when true
+tarfs_hint = false
+# Maximum of concurrence to converting OCIv1 images to tarfs, 0 means default
+max_concurrent_proc = 0
+# Mode to export tarfs images:
+# - "none" or "": do not export tarfs
+# - "layer_verity_only": only generate disk verity information for a layer blob
+# - "image_verity_only": only generate disk verity information for all blobs of an image
+# - "layer_block": generate a raw block disk image with tarfs for a layer
+# - "image_block": generate a raw block disk image with tarfs for an image
+# - "layer_block_with_verity": generate a raw block disk image with tarfs for a layer with dm-verity info
+# - "image_block_with_verity": generate a raw block disk image with tarfs for an image with dm-verity info
+export_mode = ""

tools/packaging/static-build/nydus/build.sh

@@ -12,7 +12,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 source "${script_dir}/../../scripts/lib.sh"
 
-ARCH=${ARCH:-$(arch_to_golang "$(uname -m)")}
+arch="$(uname -m)"
 
 nydus_url="${nydus_url:-}"
 nydus_version="${nydus_version:-}"
@@ -25,7 +25,7 @@ info "Get nydus information from runtime versions.yaml"
 
 nydus_tarball_url="${nydus_url}/releases/download"
 
-file_name="nydus-static-${nydus_version}-linux-${ARCH}.tgz"
+file_name="nydus-static-${nydus_version}-linux-$(arch_to_golang $arch).tgz"
 download_url="${nydus_tarball_url}/${nydus_version}/${file_name}"
 
 info "Download nydus version: ${nydus_version} from ${download_url}"

versions.yaml

@@ -277,12 +277,12 @@ externals:
   nydus:
     description: "Nydus image acceleration service"
     url: "https://github.com/dragonflyoss/image-service"
-    version: "v2.2.1"
+    version: "v2.2.3"
 
   nydus-snapshotter:
     description: "Snapshotter for Nydus image acceleration service"
     url: "https://github.com/containerd/nydus-snapshotter"
-    version: "v0.3.3"
+    version: "v0.12.0"
 
   open-policy-agent:
     description: "Open Policy Agent"