Merge pull request #7676 from ChengyuZhu6/pull_image_in_guest

CC | image pulling in the guest without forked containerd

src/agent/src/image_rpc.rs

@@ -81,14 +81,14 @@ impl ImageService {
     }
 
     // pause image is packaged in rootfs for CC
-    fn unpack_pause_image(cid: &str) -> Result<()> {
+    fn unpack_pause_image(cid: &str, target_subpath: &str) -> Result<String> {
         let cc_pause_bundle = Path::new(KATA_CC_PAUSE_BUNDLE);
         if !cc_pause_bundle.exists() {
             return Err(anyhow!("Pause image not present in rootfs"));
         }
 
         info!(sl(), "use guest pause image cid {:?}", cid);
-        let pause_bundle = Path::new(CONTAINER_BASE).join(cid);
+        let pause_bundle = Path::new(CONTAINER_BASE).join(cid).join(target_subpath);
         let pause_rootfs = pause_bundle.join("rootfs");
         let pause_config = pause_bundle.join(CONFIG_JSON);
         let pause_binary = pause_rootfs.join("pause");
@@ -103,7 +103,7 @@ impl ImageService {
             fs::copy(cc_pause_bundle.join("rootfs").join("pause"), pause_binary)?;
         }
 
-        Ok(())
+        Ok(pause_rootfs.display().to_string())
     }
 
     /// Determines the container id (cid) to use for a given request.
@@ -127,25 +127,20 @@ impl ImageService {
         Ok(cid)
     }
 
-    async fn pull_image(&self, req: &image::PullImageRequest) -> Result<String> {
+    /// Set proxy environment from AGENT_CONFIG
+    fn set_proxy_env_vars() {
         let https_proxy = &AGENT_CONFIG.https_proxy;
         if !https_proxy.is_empty() {
             env::set_var("HTTPS_PROXY", https_proxy);
         }
-
         let no_proxy = &AGENT_CONFIG.no_proxy;
         if !no_proxy.is_empty() {
             env::set_var("NO_PROXY", no_proxy);
         }
+    }
 
-        let cid = self.cid_from_request(req)?;
-        let image = req.image();
-        if cid.starts_with("pause") {
-            Self::unpack_pause_image(&cid)?;
-            self.add_image(String::from(image), cid).await;
-            return Ok(image.to_owned());
-        }
-
+    /// init atestation agent and read config from AGENT_CONFIG
+    async fn get_security_config(&self) -> Result<String> {
         let aa_kbc_params = &AGENT_CONFIG.aa_kbc_params;
         // If the attestation-agent is being used, then enable the authenticated credentials support
         info!(
@@ -163,22 +158,24 @@ impl ImageService {
             "enable_signature_verification set to: {}", enable_signature_verification
         );
         self.image_client.lock().await.config.security_validate = *enable_signature_verification;
+        Ok(decrypt_config)
+    }
 
-        let source_creds = (!req.source_creds().is_empty()).then(|| req.source_creds());
-
-        let bundle_path = Path::new(CONTAINER_BASE).join(&cid);
-        fs::create_dir_all(&bundle_path)?;
-
-        info!(sl(), "pull image {:?}, bundle path {:?}", cid, bundle_path);
-        // Image layers will store at KATA_CC_IMAGE_WORK_DIR, generated bundles
-        // with rootfs and config.json will store under CONTAINER_BASE/cid.
+    /// Call image-rs to pull and unpack image.
+    async fn common_image_pull(
+        &self,
+        image: &str,
+        bundle_path: &Path,
+        decrypt_config: &str,
+        source_creds: Option<&str>,
+        cid: &str,
+    ) -> Result<()> {
         let res = self
             .image_client
             .lock()
             .await
-            .pull_image(image, &bundle_path, &source_creds, &Some(&decrypt_config))
+            .pull_image(image, bundle_path, &source_creds, &Some(decrypt_config))
             .await;
-
         match res {
             Ok(image) => {
                 info!(
@@ -197,8 +194,64 @@ impl ImageService {
                 return Err(e);
             }
         };
+        self.add_image(String::from(image), String::from(cid)).await;
+        Ok(())
+    }
 
-        self.add_image(String::from(image), cid).await;
+    /// Pull image when creating container and return the bundle path with rootfs.
+    pub async fn pull_image_for_container(
+        &self,
+        image: &str,
+        cid: &str,
+        image_metadata: &HashMap<String, String>,
+    ) -> Result<String> {
+        info!(sl(), "image metadata: {:?}", image_metadata);
+        Self::set_proxy_env_vars();
+        if image_metadata["io.kubernetes.cri.container-type"] == "sandbox" {
+            let mount_path = Self::unpack_pause_image(cid, "pause")?;
+            self.add_image(String::from(image), String::from(cid)).await;
+            return Ok(mount_path);
+        }
+        let bundle_path = Path::new(CONTAINER_BASE).join(cid).join("images");
+        fs::create_dir_all(&bundle_path)?;
+        info!(sl(), "pull image {:?}, bundle path {:?}", cid, bundle_path);
+
+        let decrypt_config = self.get_security_config().await?;
+
+        let source_creds = None; // You need to determine how to obtain this.
+
+        self.common_image_pull(image, &bundle_path, &decrypt_config, source_creds, cid)
+            .await?;
+        Ok(format! {"{}/rootfs",bundle_path.display()})
+    }
+
+    /// Pull image when recieving the PullImageRequest and return the image digest.
+    async fn pull_image(&self, req: &image::PullImageRequest) -> Result<String> {
+        Self::set_proxy_env_vars();
+        let cid = self.cid_from_request(req)?;
+        let image = req.image();
+        if cid.starts_with("pause") {
+            Self::unpack_pause_image(&cid, "")?;
+            self.add_image(String::from(image), cid).await;
+            return Ok(image.to_owned());
+        }
+
+        // Image layers will store at KATA_CC_IMAGE_WORK_DIR, generated bundles
+        // with rootfs and config.json will store under CONTAINER_BASE/cid.
+        let bundle_path = Path::new(CONTAINER_BASE).join(&cid);
+        fs::create_dir_all(&bundle_path)?;
+
+        let decrypt_config = self.get_security_config().await?;
+        let source_creds = (!req.source_creds().is_empty()).then(|| req.source_creds());
+
+        self.common_image_pull(
+            image,
+            &bundle_path,
+            &decrypt_config,
+            source_creds,
+            cid.clone().as_str(),
+        )
+        .await?;
         Ok(image.to_owned())
     }
 

src/agent/src/storage/image_pull_handler.rs

@@ -0,0 +1,102 @@
+// Copyright (c) 2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use anyhow::{anyhow, Result};
+use kata_types::mount::KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL;
+use kata_types::mount::{ImagePullVolume, StorageDevice};
+use protocols::agent::Storage;
+use std::sync::Arc;
+use tracing::instrument;
+
+use crate::image_rpc;
+use crate::storage::{StorageContext, StorageHandler};
+
+use super::{common_storage_handler, new_device};
+
+#[derive(Debug)]
+pub struct ImagePullHandler {}
+
+impl ImagePullHandler {
+    fn get_image_info(storage: &Storage) -> Result<ImagePullVolume> {
+        for option in storage.driver_options.iter() {
+            if let Some((key, value)) = option.split_once('=') {
+                if key == KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL {
+                    let imagepull_volume: ImagePullVolume = serde_json::from_str(value)?;
+                    return Ok(imagepull_volume);
+                }
+            }
+        }
+        Err(anyhow!("missing Image information for ImagePull volume"))
+    }
+}
+
+#[async_trait::async_trait]
+impl StorageHandler for ImagePullHandler {
+    #[instrument]
+    async fn create_device(
+        &self,
+        mut storage: Storage,
+        ctx: &mut StorageContext,
+    ) -> Result<Arc<dyn StorageDevice>> {
+        //Currently the image metadata is not used to pulling image in the guest.
+        let image_pull_volume = Self::get_image_info(&storage)?;
+        debug!(ctx.logger, "image_pull_volume = {:?}", image_pull_volume);
+        let image_name = storage.source();
+        debug!(ctx.logger, "image_name = {:?}", image_name);
+
+        let cid = ctx
+            .cid
+            .clone()
+            .ok_or_else(|| anyhow!("failed to get container id"))?;
+        let image_service = image_rpc::ImageService::singleton().await?;
+        let bundle_path = image_service
+            .pull_image_for_container(image_name, &cid, &image_pull_volume.metadata)
+            .await?;
+
+        storage.source = bundle_path;
+        storage.options = vec!["bind".to_string(), "ro".to_string()];
+
+        common_storage_handler(ctx.logger, &storage)?;
+
+        new_device(storage.mount_point)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use kata_types::mount::{ImagePullVolume, KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL};
+    use protocols::agent::Storage;
+
+    use crate::storage::image_pull_handler::ImagePullHandler;
+
+    #[test]
+    fn test_get_image_info() {
+        let mut res = HashMap::new();
+        res.insert("key1".to_string(), "value1".to_string());
+        res.insert("key2".to_string(), "value2".to_string());
+
+        let image_pull = ImagePullVolume {
+            metadata: res.clone(),
+        };
+
+        let image_pull_str = serde_json::to_string(&image_pull);
+        assert!(image_pull_str.is_ok());
+
+        let storage = Storage {
+            driver: KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL.to_string(),
+            driver_options: vec![format!("image_guest_pull={}", image_pull_str.ok().unwrap())],
+            ..Default::default()
+        };
+
+        match ImagePullHandler::get_image_info(&storage) {
+            Ok(image_info) => {
+                assert_eq!(image_info.metadata, res);
+            }
+            Err(e) => panic!("err = {}", e),
+        }
+    }
+}

src/agent/src/storage/mod.rs

@@ -12,7 +12,10 @@ use std::sync::Arc;
 
 use anyhow::{anyhow, Context, Result};
 use kata_sys_util::mount::{create_mount_destination, parse_mount_options};
-use kata_types::mount::{StorageDevice, StorageHandlerManager, KATA_SHAREDFS_GUEST_PREMOUNT_TAG};
+use kata_types::mount::{
+    StorageDevice, StorageHandlerManager, KATA_SHAREDFS_GUEST_PREMOUNT_TAG,
+    KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL,
+};
 use kata_types::volume::KATA_VOLUME_TYPE_DMVERITY;
 use nix::unistd::{Gid, Uid};
 use protocols::agent::Storage;
@@ -26,7 +29,9 @@ use self::block_handler::{PmemHandler, ScsiHandler, VirtioBlkMmioHandler, Virtio
 use self::dm_verity::DmVerityHandler;
 use self::ephemeral_handler::EphemeralHandler;
 use self::fs_handler::{OverlayfsHandler, Virtio9pHandler, VirtioFsHandler};
+use self::image_pull_handler::ImagePullHandler;
 use self::local_handler::LocalHandler;
+
 use crate::device::{
     DRIVER_9P_TYPE, DRIVER_BLK_MMIO_TYPE, DRIVER_BLK_PCI_TYPE, DRIVER_EPHEMERAL_TYPE,
     DRIVER_LOCAL_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE,
@@ -42,6 +47,7 @@ mod block_handler;
 mod dm_verity;
 mod ephemeral_handler;
 mod fs_handler;
+mod image_pull_handler;
 mod local_handler;
 
 const RW_MASK: u32 = 0o660;
@@ -149,6 +155,7 @@ lazy_static! {
         manager.add_handler(DRIVER_VIRTIOFS_TYPE, Arc::new(VirtioFsHandler{})).unwrap();
         manager.add_handler(DRIVER_WATCHABLE_BIND_TYPE, Arc::new(BindWatcherHandler{})).unwrap();
         manager.add_handler(KATA_VOLUME_TYPE_DMVERITY, Arc::new(DmVerityHandler{})).unwrap();
+        manager.add_handler(KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL, Arc::new(ImagePullHandler{})).unwrap();
         manager
     };
 }

src/runtime/virtcontainers/container.go

@@ -821,9 +821,6 @@ func (c *Container) createVirtualVolumeDevices() ([]config.DeviceInfo, error) {
 					return nil, err
 				}
 				deviceInfos = append(deviceInfos, *di)
-			} else if virtVolume.VolumeType == types.KataVirtualVolumeImageGuestPullType {
-				///TODO implement the logic with pulling image in the guest.
-				continue
 			}
 		}
 	}

src/runtime/virtcontainers/fs_share_linux.go

@@ -472,6 +472,11 @@ func handleVirtualVolume(c *Container) ([]*grpc.Storage, string, error) {
 						break
 					}
 				}
+			} else if virtVolume.VolumeType == types.KataVirtualVolumeImageGuestPullType {
+				vol, err = handleVirtualVolumeStorageObject(c, "", virtVolume)
+				if err != nil {
+					return nil, "", err
+				}
 			}
 			if vol != nil {
 				volumes = append(volumes, vol)
@@ -521,7 +526,7 @@ func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container)
 
 	// In the confidential computing, there is no Image information on the host,
 	// so there is no Rootfs.Target.
-	if f.sandbox.config.ServiceOffload && c.rootFs.Target == "" {
+	if f.sandbox.config.ServiceOffload && c.rootFs.Target == "" && !HasOptionPrefix(c.rootFs.Options, VirtualVolumePrefix) {
 		return &SharedFile{
 			containerStorages: nil,
 			guestPath:         rootfsGuestPath,

src/runtime/virtcontainers/kata_agent.go

@@ -1575,6 +1575,34 @@ func handleDmVerityBlockVolume(driverType, source string, verityInfo *types.DmVe
 	return vol, nil
 }
 
+func handleImageGuestPullBlockVolume(c *Container, virtualVolumeInfo *types.KataVirtualVolume, vol *grpc.Storage) (*grpc.Storage, error) {
+	container_annotations := c.GetAnnotations()
+	container_type := container_annotations["io.kubernetes.cri.container-type"]
+	if virtualVolumeInfo.Source == "" {
+		var image_ref string
+		if container_type == "sandbox" {
+			image_ref = "pause"
+		} else {
+			image_ref = container_annotations["io.kubernetes.cri.image-name"]
+			if image_ref == "" {
+				return nil, fmt.Errorf("Failed to get image name from annotations")
+			}
+		}
+		virtualVolumeInfo.Source = image_ref
+		virtualVolumeInfo.ImagePull.Metadata = container_annotations
+	}
+
+	no, err := json.Marshal(virtualVolumeInfo.ImagePull)
+	if err != nil {
+		return nil, err
+	}
+	vol.Driver = types.KataVirtualVolumeImageGuestPullType
+	vol.DriverOptions = append(vol.DriverOptions, types.KataVirtualVolumeImageGuestPullType+"="+string(no))
+	vol.Source = virtualVolumeInfo.Source
+	vol.Fstype = typeOverlayFS
+	return vol, nil
+}
+
 func handleBlockVolume(c *Container, device api.Device) (*grpc.Storage, error) {
 	vol := &grpc.Storage{}
 
@@ -1631,8 +1659,13 @@ func handleVirtualVolumeStorageObject(c *Container, blockDeviceId string, virtVo
 			}
 		}
 	} else if virtVolume.VolumeType == types.KataVirtualVolumeImageGuestPullType {
-		///TODO implement the logic with pulling image in the guest.
-		return nil, nil
+		var err error
+		vol = &grpc.Storage{}
+		vol, err = handleImageGuestPullBlockVolume(c, virtVolume, vol)
+		vol.MountPoint = filepath.Join("/run/kata-containers/", c.id, c.rootfsSuffix)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	return vol, nil

tools/osbuilder/scripts/lib.sh

@@ -236,6 +236,9 @@ generate_dockerfile()
 	readonly install_rust="
 ENV http_proxy=${http_proxy:-}
 ENV https_proxy=${http_proxy:-}
+ENV RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-}"
+ENV RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER:-}"
+
 RUN curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSLf | \
     sh -s -- -y --default-toolchain ${RUST_VERSION} -t ${rustarch}-unknown-linux-${LIBC}
 RUN . /root/.cargo/env; cargo install cargo-when

tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh

@@ -18,6 +18,8 @@ uid=$(id -u ${USER})
 gid=$(id -g ${USER})
 http_proxy="${http_proxy:-}"
 https_proxy="${https_proxy:-}"
+RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-}"
+RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER:-}"
 
 ARCH=${ARCH:-$(uname -m)}
 CROSS_BUILD=
@@ -99,6 +101,8 @@ docker run \
 	--env CROSS_BUILD="${CROSS_BUILD}" \
 	--env TARGET_ARCH="${TARGET_ARCH}" \
 	--env ARCH="${ARCH}" \
+	--env RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT}" \
+	--env RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER}" \
 	--rm \
 	-w ${script_dir} \
 	build-kata-deploy "${kata_deploy_create}" $@

tools/packaging/static-build/nydus/build.sh

@@ -12,7 +12,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 source "${script_dir}/../../scripts/lib.sh"
 
-ARCH=${ARCH:-$(arch_to_golang "$(uname -m)")}
+arch="$(uname -m)"
 
 nydus_url="${nydus_url:-}"
 nydus_version="${nydus_version:-}"
@@ -25,7 +25,7 @@ info "Get nydus information from runtime versions.yaml"
 
 nydus_tarball_url="${nydus_url}/releases/download"
 
-file_name="nydus-static-${nydus_version}-linux-${ARCH}.tgz"
+file_name="nydus-static-${nydus_version}-linux-$(arch_to_golang $arch).tgz"
 download_url="${nydus_tarball_url}/${nydus_version}/${file_name}"
 
 info "Download nydus version: ${nydus_version} from ${download_url}"

tools/packaging/static-build/shim-v2/build.sh

@@ -20,6 +20,8 @@ VMM_CONFIGS="qemu fc"
 GO_VERSION=${GO_VERSION}
 RUST_VERSION=${RUST_VERSION}
 CC=""
+RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-}"
+RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER:-}"
 
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
@@ -53,6 +55,8 @@ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	--env CROSS_BUILD=${CROSS_BUILD} \
 	--env ARCH=${ARCH} \
 	--env CC="${CC}" \
+        --env RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT}" \
+        --env RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER}" \
 	-w "${repo_root_dir}/src/runtime-rs" \
 	"${container_image}" \
 	bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch}"
@@ -61,6 +65,8 @@ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	--env CROSS_BUILD=${CROSS_BUILD} \
         --env ARCH=${ARCH} \
         --env CC="${CC}" \
+        --env RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT}" \
+        --env RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER}" \
 	-w "${repo_root_dir}/src/runtime-rs" \
 	"${container_image}" \
 	bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX="${PREFIX}" DESTDIR="${DESTDIR}" install"

tools/packaging/static-build/shim-v2/install_go_rust.sh

@@ -9,6 +9,9 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-}"
+RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER:-}"
+
 tmp_dir=$(mktemp -d -t install-go-tmp.XXXXXXXXXX)
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 script_name="$(basename "${BASH_SOURCE[0]}")"
@@ -82,6 +85,14 @@ case "${ARCH}" in
 		;;
 esac
 
+if [ -n "${RUSTUP_UPDATE_ROOT}" ]; then
+	export RUSTUP_UPDATE_ROOT=${RUSTUP_UPDATE_ROOT}
+fi
+
+if [ -n "${RUSTUP_DIST_SERVER}" ]; then
+	export RUSTUP_DIST_SERVER=${RUSTUP_DIST_SERVER}
+fi
+
 curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSLf | sh -s -- -y --default-toolchain ${rust_version} -t ${ARCH}-unknown-linux-${LIBC}
 source /root/.cargo/env
 rustup target add ${ARCH}-unknown-linux-${LIBC}