diff --git a/src/agent/src/image_rpc.rs b/src/agent/src/image_rpc.rs index 1915a9f4fa..94d45cb2fd 100644 --- a/src/agent/src/image_rpc.rs +++ b/src/agent/src/image_rpc.rs @@ -81,14 +81,14 @@ impl ImageService { } // pause image is packaged in rootfs for CC - fn unpack_pause_image(cid: &str) -> Result<()> { + fn unpack_pause_image(cid: &str, target_subpath: &str) -> Result { let cc_pause_bundle = Path::new(KATA_CC_PAUSE_BUNDLE); if !cc_pause_bundle.exists() { return Err(anyhow!("Pause image not present in rootfs")); } info!(sl(), "use guest pause image cid {:?}", cid); - let pause_bundle = Path::new(CONTAINER_BASE).join(cid); + let pause_bundle = Path::new(CONTAINER_BASE).join(cid).join(target_subpath); let pause_rootfs = pause_bundle.join("rootfs"); let pause_config = pause_bundle.join(CONFIG_JSON); let pause_binary = pause_rootfs.join("pause"); @@ -103,7 +103,7 @@ impl ImageService { fs::copy(cc_pause_bundle.join("rootfs").join("pause"), pause_binary)?; } - Ok(()) + Ok(pause_rootfs.display().to_string()) } /// Determines the container id (cid) to use for a given request. @@ -127,25 +127,20 @@ impl ImageService { Ok(cid) } - async fn pull_image(&self, req: &image::PullImageRequest) -> Result { + /// Set proxy environment from AGENT_CONFIG + fn set_proxy_env_vars() { let https_proxy = &AGENT_CONFIG.https_proxy; if !https_proxy.is_empty() { env::set_var("HTTPS_PROXY", https_proxy); } - let no_proxy = &AGENT_CONFIG.no_proxy; if !no_proxy.is_empty() { env::set_var("NO_PROXY", no_proxy); } + } - let cid = self.cid_from_request(req)?; - let image = req.image(); - if cid.starts_with("pause") { - Self::unpack_pause_image(&cid)?; - self.add_image(String::from(image), cid).await; - return Ok(image.to_owned()); - } - + /// init atestation agent and read config from AGENT_CONFIG + async fn get_security_config(&self) -> Result { let aa_kbc_params = &AGENT_CONFIG.aa_kbc_params; // If the attestation-agent is being used, then enable the authenticated credentials support info!( @@ -163,22 +158,24 @@ impl ImageService { "enable_signature_verification set to: {}", enable_signature_verification ); self.image_client.lock().await.config.security_validate = *enable_signature_verification; + Ok(decrypt_config) + } - let source_creds = (!req.source_creds().is_empty()).then(|| req.source_creds()); - - let bundle_path = Path::new(CONTAINER_BASE).join(&cid); - fs::create_dir_all(&bundle_path)?; - - info!(sl(), "pull image {:?}, bundle path {:?}", cid, bundle_path); - // Image layers will store at KATA_CC_IMAGE_WORK_DIR, generated bundles - // with rootfs and config.json will store under CONTAINER_BASE/cid. + /// Call image-rs to pull and unpack image. + async fn common_image_pull( + &self, + image: &str, + bundle_path: &Path, + decrypt_config: &str, + source_creds: Option<&str>, + cid: &str, + ) -> Result<()> { let res = self .image_client .lock() .await - .pull_image(image, &bundle_path, &source_creds, &Some(&decrypt_config)) + .pull_image(image, bundle_path, &source_creds, &Some(decrypt_config)) .await; - match res { Ok(image) => { info!( @@ -197,8 +194,64 @@ impl ImageService { return Err(e); } }; + self.add_image(String::from(image), String::from(cid)).await; + Ok(()) + } - self.add_image(String::from(image), cid).await; + /// Pull image when creating container and return the bundle path with rootfs. + pub async fn pull_image_for_container( + &self, + image: &str, + cid: &str, + image_metadata: &HashMap, + ) -> Result { + info!(sl(), "image metadata: {:?}", image_metadata); + Self::set_proxy_env_vars(); + if image_metadata["io.kubernetes.cri.container-type"] == "sandbox" { + let mount_path = Self::unpack_pause_image(cid, "pause")?; + self.add_image(String::from(image), String::from(cid)).await; + return Ok(mount_path); + } + let bundle_path = Path::new(CONTAINER_BASE).join(cid).join("images"); + fs::create_dir_all(&bundle_path)?; + info!(sl(), "pull image {:?}, bundle path {:?}", cid, bundle_path); + + let decrypt_config = self.get_security_config().await?; + + let source_creds = None; // You need to determine how to obtain this. + + self.common_image_pull(image, &bundle_path, &decrypt_config, source_creds, cid) + .await?; + Ok(format! {"{}/rootfs",bundle_path.display()}) + } + + /// Pull image when recieving the PullImageRequest and return the image digest. + async fn pull_image(&self, req: &image::PullImageRequest) -> Result { + Self::set_proxy_env_vars(); + let cid = self.cid_from_request(req)?; + let image = req.image(); + if cid.starts_with("pause") { + Self::unpack_pause_image(&cid, "")?; + self.add_image(String::from(image), cid).await; + return Ok(image.to_owned()); + } + + // Image layers will store at KATA_CC_IMAGE_WORK_DIR, generated bundles + // with rootfs and config.json will store under CONTAINER_BASE/cid. + let bundle_path = Path::new(CONTAINER_BASE).join(&cid); + fs::create_dir_all(&bundle_path)?; + + let decrypt_config = self.get_security_config().await?; + let source_creds = (!req.source_creds().is_empty()).then(|| req.source_creds()); + + self.common_image_pull( + image, + &bundle_path, + &decrypt_config, + source_creds, + cid.clone().as_str(), + ) + .await?; Ok(image.to_owned()) } diff --git a/src/agent/src/storage/image_pull_handler.rs b/src/agent/src/storage/image_pull_handler.rs new file mode 100644 index 0000000000..3ddfcbe87a --- /dev/null +++ b/src/agent/src/storage/image_pull_handler.rs @@ -0,0 +1,102 @@ +// Copyright (c) 2023 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::{anyhow, Result}; +use kata_types::mount::KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL; +use kata_types::mount::{ImagePullVolume, StorageDevice}; +use protocols::agent::Storage; +use std::sync::Arc; +use tracing::instrument; + +use crate::image_rpc; +use crate::storage::{StorageContext, StorageHandler}; + +use super::{common_storage_handler, new_device}; + +#[derive(Debug)] +pub struct ImagePullHandler {} + +impl ImagePullHandler { + fn get_image_info(storage: &Storage) -> Result { + for option in storage.driver_options.iter() { + if let Some((key, value)) = option.split_once('=') { + if key == KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL { + let imagepull_volume: ImagePullVolume = serde_json::from_str(value)?; + return Ok(imagepull_volume); + } + } + } + Err(anyhow!("missing Image information for ImagePull volume")) + } +} + +#[async_trait::async_trait] +impl StorageHandler for ImagePullHandler { + #[instrument] + async fn create_device( + &self, + mut storage: Storage, + ctx: &mut StorageContext, + ) -> Result> { + //Currently the image metadata is not used to pulling image in the guest. + let image_pull_volume = Self::get_image_info(&storage)?; + debug!(ctx.logger, "image_pull_volume = {:?}", image_pull_volume); + let image_name = storage.source(); + debug!(ctx.logger, "image_name = {:?}", image_name); + + let cid = ctx + .cid + .clone() + .ok_or_else(|| anyhow!("failed to get container id"))?; + let image_service = image_rpc::ImageService::singleton().await?; + let bundle_path = image_service + .pull_image_for_container(image_name, &cid, &image_pull_volume.metadata) + .await?; + + storage.source = bundle_path; + storage.options = vec!["bind".to_string(), "ro".to_string()]; + + common_storage_handler(ctx.logger, &storage)?; + + new_device(storage.mount_point) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use kata_types::mount::{ImagePullVolume, KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL}; + use protocols::agent::Storage; + + use crate::storage::image_pull_handler::ImagePullHandler; + + #[test] + fn test_get_image_info() { + let mut res = HashMap::new(); + res.insert("key1".to_string(), "value1".to_string()); + res.insert("key2".to_string(), "value2".to_string()); + + let image_pull = ImagePullVolume { + metadata: res.clone(), + }; + + let image_pull_str = serde_json::to_string(&image_pull); + assert!(image_pull_str.is_ok()); + + let storage = Storage { + driver: KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL.to_string(), + driver_options: vec![format!("image_guest_pull={}", image_pull_str.ok().unwrap())], + ..Default::default() + }; + + match ImagePullHandler::get_image_info(&storage) { + Ok(image_info) => { + assert_eq!(image_info.metadata, res); + } + Err(e) => panic!("err = {}", e), + } + } +} diff --git a/src/agent/src/storage/mod.rs b/src/agent/src/storage/mod.rs index a8a3b3717a..1f3d86c875 100644 --- a/src/agent/src/storage/mod.rs +++ b/src/agent/src/storage/mod.rs @@ -12,7 +12,10 @@ use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use kata_sys_util::mount::{create_mount_destination, parse_mount_options}; -use kata_types::mount::{StorageDevice, StorageHandlerManager, KATA_SHAREDFS_GUEST_PREMOUNT_TAG}; +use kata_types::mount::{ + StorageDevice, StorageHandlerManager, KATA_SHAREDFS_GUEST_PREMOUNT_TAG, + KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL, +}; use kata_types::volume::KATA_VOLUME_TYPE_DMVERITY; use nix::unistd::{Gid, Uid}; use protocols::agent::Storage; @@ -26,7 +29,9 @@ use self::block_handler::{PmemHandler, ScsiHandler, VirtioBlkMmioHandler, Virtio use self::dm_verity::DmVerityHandler; use self::ephemeral_handler::EphemeralHandler; use self::fs_handler::{OverlayfsHandler, Virtio9pHandler, VirtioFsHandler}; +use self::image_pull_handler::ImagePullHandler; use self::local_handler::LocalHandler; + use crate::device::{ DRIVER_9P_TYPE, DRIVER_BLK_MMIO_TYPE, DRIVER_BLK_PCI_TYPE, DRIVER_EPHEMERAL_TYPE, DRIVER_LOCAL_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_OVERLAYFS_TYPE, DRIVER_SCSI_TYPE, @@ -42,6 +47,7 @@ mod block_handler; mod dm_verity; mod ephemeral_handler; mod fs_handler; +mod image_pull_handler; mod local_handler; const RW_MASK: u32 = 0o660; @@ -149,6 +155,7 @@ lazy_static! { manager.add_handler(DRIVER_VIRTIOFS_TYPE, Arc::new(VirtioFsHandler{})).unwrap(); manager.add_handler(DRIVER_WATCHABLE_BIND_TYPE, Arc::new(BindWatcherHandler{})).unwrap(); manager.add_handler(KATA_VOLUME_TYPE_DMVERITY, Arc::new(DmVerityHandler{})).unwrap(); + manager.add_handler(KATA_VIRTUAL_VOLUME_IMAGE_GUEST_PULL, Arc::new(ImagePullHandler{})).unwrap(); manager }; } diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 314c2b1d73..ae07671d20 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -821,9 +821,6 @@ func (c *Container) createVirtualVolumeDevices() ([]config.DeviceInfo, error) { return nil, err } deviceInfos = append(deviceInfos, *di) - } else if virtVolume.VolumeType == types.KataVirtualVolumeImageGuestPullType { - ///TODO implement the logic with pulling image in the guest. - continue } } } diff --git a/src/runtime/virtcontainers/fs_share_linux.go b/src/runtime/virtcontainers/fs_share_linux.go index 06c21c3afa..566f3bdd2d 100644 --- a/src/runtime/virtcontainers/fs_share_linux.go +++ b/src/runtime/virtcontainers/fs_share_linux.go @@ -472,6 +472,11 @@ func handleVirtualVolume(c *Container) ([]*grpc.Storage, string, error) { break } } + } else if virtVolume.VolumeType == types.KataVirtualVolumeImageGuestPullType { + vol, err = handleVirtualVolumeStorageObject(c, "", virtVolume) + if err != nil { + return nil, "", err + } } if vol != nil { volumes = append(volumes, vol) @@ -521,7 +526,7 @@ func (f *FilesystemShare) ShareRootFilesystem(ctx context.Context, c *Container) // In the confidential computing, there is no Image information on the host, // so there is no Rootfs.Target. - if f.sandbox.config.ServiceOffload && c.rootFs.Target == "" { + if f.sandbox.config.ServiceOffload && c.rootFs.Target == "" && !HasOptionPrefix(c.rootFs.Options, VirtualVolumePrefix) { return &SharedFile{ containerStorages: nil, guestPath: rootfsGuestPath, diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 21b99d3579..bcf47c65a3 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -1575,6 +1575,34 @@ func handleDmVerityBlockVolume(driverType, source string, verityInfo *types.DmVe return vol, nil } +func handleImageGuestPullBlockVolume(c *Container, virtualVolumeInfo *types.KataVirtualVolume, vol *grpc.Storage) (*grpc.Storage, error) { + container_annotations := c.GetAnnotations() + container_type := container_annotations["io.kubernetes.cri.container-type"] + if virtualVolumeInfo.Source == "" { + var image_ref string + if container_type == "sandbox" { + image_ref = "pause" + } else { + image_ref = container_annotations["io.kubernetes.cri.image-name"] + if image_ref == "" { + return nil, fmt.Errorf("Failed to get image name from annotations") + } + } + virtualVolumeInfo.Source = image_ref + virtualVolumeInfo.ImagePull.Metadata = container_annotations + } + + no, err := json.Marshal(virtualVolumeInfo.ImagePull) + if err != nil { + return nil, err + } + vol.Driver = types.KataVirtualVolumeImageGuestPullType + vol.DriverOptions = append(vol.DriverOptions, types.KataVirtualVolumeImageGuestPullType+"="+string(no)) + vol.Source = virtualVolumeInfo.Source + vol.Fstype = typeOverlayFS + return vol, nil +} + func handleBlockVolume(c *Container, device api.Device) (*grpc.Storage, error) { vol := &grpc.Storage{} @@ -1631,8 +1659,13 @@ func handleVirtualVolumeStorageObject(c *Container, blockDeviceId string, virtVo } } } else if virtVolume.VolumeType == types.KataVirtualVolumeImageGuestPullType { - ///TODO implement the logic with pulling image in the guest. - return nil, nil + var err error + vol = &grpc.Storage{} + vol, err = handleImageGuestPullBlockVolume(c, virtVolume, vol) + vol.MountPoint = filepath.Join("/run/kata-containers/", c.id, c.rootfsSuffix) + if err != nil { + return nil, err + } } return vol, nil diff --git a/tools/osbuilder/scripts/lib.sh b/tools/osbuilder/scripts/lib.sh index 633bc115c5..ffc39a7683 100644 --- a/tools/osbuilder/scripts/lib.sh +++ b/tools/osbuilder/scripts/lib.sh @@ -236,6 +236,9 @@ generate_dockerfile() readonly install_rust=" ENV http_proxy=${http_proxy:-} ENV https_proxy=${http_proxy:-} +ENV RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-}" +ENV RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER:-}" + RUN curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSLf | \ sh -s -- -y --default-toolchain ${RUST_VERSION} -t ${rustarch}-unknown-linux-${LIBC} RUN . /root/.cargo/env; cargo install cargo-when diff --git a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh index c45bfb085c..2ed32654a6 100755 --- a/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh +++ b/tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh @@ -18,6 +18,8 @@ uid=$(id -u ${USER}) gid=$(id -g ${USER}) http_proxy="${http_proxy:-}" https_proxy="${https_proxy:-}" +RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-}" +RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER:-}" ARCH=${ARCH:-$(uname -m)} CROSS_BUILD= @@ -99,6 +101,8 @@ docker run \ --env CROSS_BUILD="${CROSS_BUILD}" \ --env TARGET_ARCH="${TARGET_ARCH}" \ --env ARCH="${ARCH}" \ + --env RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT}" \ + --env RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER}" \ --rm \ -w ${script_dir} \ build-kata-deploy "${kata_deploy_create}" $@ diff --git a/tools/packaging/static-build/nydus/build.sh b/tools/packaging/static-build/nydus/build.sh index 97686642ab..908f46af55 100755 --- a/tools/packaging/static-build/nydus/build.sh +++ b/tools/packaging/static-build/nydus/build.sh @@ -12,7 +12,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${script_dir}/../../scripts/lib.sh" -ARCH=${ARCH:-$(arch_to_golang "$(uname -m)")} +arch="$(uname -m)" nydus_url="${nydus_url:-}" nydus_version="${nydus_version:-}" @@ -25,7 +25,7 @@ info "Get nydus information from runtime versions.yaml" nydus_tarball_url="${nydus_url}/releases/download" -file_name="nydus-static-${nydus_version}-linux-${ARCH}.tgz" +file_name="nydus-static-${nydus_version}-linux-$(arch_to_golang $arch).tgz" download_url="${nydus_tarball_url}/${nydus_version}/${file_name}" info "Download nydus version: ${nydus_version} from ${download_url}" diff --git a/tools/packaging/static-build/shim-v2/build.sh b/tools/packaging/static-build/shim-v2/build.sh index eba47d7ad0..39e85b9e4f 100755 --- a/tools/packaging/static-build/shim-v2/build.sh +++ b/tools/packaging/static-build/shim-v2/build.sh @@ -20,6 +20,8 @@ VMM_CONFIGS="qemu fc" GO_VERSION=${GO_VERSION} RUST_VERSION=${RUST_VERSION} CC="" +RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-}" +RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER:-}" DESTDIR=${DESTDIR:-${PWD}} PREFIX=${PREFIX:-/opt/kata} @@ -53,6 +55,8 @@ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ --env CROSS_BUILD=${CROSS_BUILD} \ --env ARCH=${ARCH} \ --env CC="${CC}" \ + --env RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT}" \ + --env RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER}" \ -w "${repo_root_dir}/src/runtime-rs" \ "${container_image}" \ bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX=${PREFIX} QEMUCMD=qemu-system-${arch}" @@ -61,6 +65,8 @@ sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \ --env CROSS_BUILD=${CROSS_BUILD} \ --env ARCH=${ARCH} \ --env CC="${CC}" \ + --env RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT}" \ + --env RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER}" \ -w "${repo_root_dir}/src/runtime-rs" \ "${container_image}" \ bash -c "git config --global --add safe.directory ${repo_root_dir} && make PREFIX="${PREFIX}" DESTDIR="${DESTDIR}" install" diff --git a/tools/packaging/static-build/shim-v2/install_go_rust.sh b/tools/packaging/static-build/shim-v2/install_go_rust.sh index 078f49d37d..4a1e21e22f 100755 --- a/tools/packaging/static-build/shim-v2/install_go_rust.sh +++ b/tools/packaging/static-build/shim-v2/install_go_rust.sh @@ -9,6 +9,9 @@ set -o errexit set -o nounset set -o pipefail +RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-}" +RUSTUP_DIST_SERVER="${RUSTUP_DIST_SERVER:-}" + tmp_dir=$(mktemp -d -t install-go-tmp.XXXXXXXXXX) script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" script_name="$(basename "${BASH_SOURCE[0]}")" @@ -82,6 +85,14 @@ case "${ARCH}" in ;; esac +if [ -n "${RUSTUP_UPDATE_ROOT}" ]; then + export RUSTUP_UPDATE_ROOT=${RUSTUP_UPDATE_ROOT} +fi + +if [ -n "${RUSTUP_DIST_SERVER}" ]; then + export RUSTUP_DIST_SERVER=${RUSTUP_DIST_SERVER} +fi + curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSLf | sh -s -- -y --default-toolchain ${rust_version} -t ${ARCH}-unknown-linux-${LIBC} source /root/.cargo/env rustup target add ${ARCH}-unknown-linux-${LIBC}