github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-06-26 07:22:20 +00:00
Code Issues Projects Releases Wiki Activity

genpolicy: add bind mounts for image volumes

Browse Source 
Add bind mounts for volumes defined by docker container images, unless
those mounts have been defined in the input K8s YAML file too.

For example, quay.io/opstree/redis defines two mounts:
/data
/node-conf
Before these changes, if these mounts were not defined in the YAML file
too, the auto-generated policy did not allow this container image to
start.

Signed-off-by: Dan Mihai <dmihai@microsoft.com>
This commit is contained in:
Dan Mihai 2024-07-11 20:08:54 +00:00
parent b203f715e5
commit c22ac4f72c
17 changed files with 280 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/tools/genpolicy/genpolicy-settings.json
View File

@ -189,6 +189,18 @@
"rprivate",
"ro"
]
},
"image_volume": {
"mount_type": "bind",
"mount_source": "$(sfprefix)",
"driver": "local",
"source": "local",
"fstype": "bind",
"options": [
"rbind",
"rprivate",
"rw"
]
}
},
"mount_destinations": [

16
src/tools/genpolicy/src/cronjob.rs
View File

@ -94,15 +94,13 @@ impl yaml::K8sResource for CronJob {
container: &pod::Container,
settings: &settings::Settings,
) {
if let Some(volumes) = &self.spec.jobTemplate.spec.template.spec.volumes {
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
volumes,
);
}
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
&self.spec.jobTemplate.spec.template.spec.volumes,
);
}
fn generate_policy(&self, agent_policy: &policy::AgentPolicy) -> String {

16
src/tools/genpolicy/src/daemon_set.rs
View File

@ -96,15 +96,13 @@ impl yaml::K8sResource for DaemonSet {
container: &pod::Container,
settings: &settings::Settings,
) {
if let Some(volumes) = &self.spec.template.spec.volumes {
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
volumes,
)
}
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
&self.spec.template.spec.volumes,
);
}
fn generate_policy(&self, agent_policy: &policy::AgentPolicy) -> String {

16
src/tools/genpolicy/src/deployment.rs
View File

@ -94,15 +94,13 @@ impl yaml::K8sResource for Deployment {
container: &pod::Container,
settings: &settings::Settings,
) {
if let Some(volumes) = &self.spec.template.spec.volumes {
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
volumes,
);
}
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
&self.spec.template.spec.volumes,
);
}
fn generate_policy(&self, agent_policy: &policy::AgentPolicy) -> String {

16
src/tools/genpolicy/src/job.rs
View File

@ -68,15 +68,13 @@ impl yaml::K8sResource for Job {
container: &pod::Container,
settings: &settings::Settings,
) {
if let Some(volumes) = &self.spec.template.spec.volumes {
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
volumes,
);
}
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
&self.spec.template.spec.volumes,
);
}
fn generate_policy(&self, agent_policy: &policy::AgentPolicy) -> String {

60
src/tools/genpolicy/src/mount_and_storage.rs
View File

@ -106,6 +106,11 @@ pub fn get_mount_and_storage(
yaml_volume: &volume::Volume,
yaml_mount: &pod::VolumeMount,
) {
debug!(
"get_mount_and_storage: adding mount and storage for: {:?}",
&yaml_volume
);
if let Some(emptyDir) = &yaml_volume.emptyDir {
let settings_volumes = &settings.volumes;
let mut volume: Option<&settings::EmptyDirVolume> = None;
@ -351,3 +356,58 @@ fn get_downward_api_mount(yaml_mount: &pod::VolumeMount, p_mounts: &mut Vec<poli
});
}
}
pub fn get_image_mount_and_storage(
settings: &settings::Settings,
p_mounts: &mut Vec<policy::KataMount>,
storages: &mut Vec<agent::Storage>,
destination: &str,
) {
// https://github.com/kubernetes/examples/blob/master/cassandra/image/Dockerfile
// has a volume mount starting with two '/' characters:
//
// CASSANDRA_DATA=/cassandra_data
// VOLUME ["/$CASSANDRA_DATA"]
let mut destination_string = destination.to_string();
while destination_string.contains("//") {
destination_string = destination_string.replace("//", "/");
}
debug!("get_image_mount_and_storage: image dest = {destination}, dest = {destination_string}");
for mount in &mut *p_mounts {
if mount.destination == destination_string {
debug!(
"get_image_mount_and_storage: mount {destination_string} already defined by YAML"
);
return;
}
}
let settings_image = &settings.volumes.image_volume;
debug!(
"get_image_mount_and_storage: settings for container image volumes: {:?}",
settings_image
);
storages.push(agent::Storage {
driver: settings_image.driver.clone(),
driver_options: Vec::new(),
source: settings_image.source.clone(),
fstype: settings_image.fstype.clone(),
options: settings_image.options.clone(),
mount_point: destination_string.clone(),
fs_group: protobuf::MessageField::none(),
special_fields: ::protobuf::SpecialFields::new(),
});
let file_name = Path::new(&destination_string).file_name().unwrap();
let name = OsString::from(file_name).into_string().unwrap();
let source = format!("{}{name}$", &settings_image.mount_source);
p_mounts.push(policy::KataMount {
destination: destination_string,
type_: settings_image.fstype.clone(),
source,
options: settings_image.options.clone(),
});
}

16
src/tools/genpolicy/src/pod.rs
View File

@ -842,15 +842,13 @@ impl yaml::K8sResource for Pod {
container: &Container,
settings: &settings::Settings,
) {
if let Some(volumes) = &self.spec.volumes {
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
volumes,
);
}
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
&self.spec.volumes,
);
}
fn generate_policy(&self, agent_policy: &policy::AgentPolicy) -> String {

29
src/tools/genpolicy/src/registry.rs
View File

@ -23,12 +23,15 @@ use oci_distribution::{
};
use serde::{Deserialize, Serialize};
use sha2::{digest::typenum::Unsigned, digest::OutputSizeUser, Sha256};
use std::{fs::OpenOptions, io, io::BufWriter, io::Seek, io::Write, path::Path};
use std::{
collections::BTreeMap, fs::OpenOptions, io, io::BufWriter, io::Seek, io::Write, path::Path,
};
use tokio::io::AsyncWriteExt;
/// Container image properties obtained from an OCI repository.
#[derive(Clone, Debug, Default)]
pub struct Container {
pub image: String,
pub config_layer: DockerConfigLayer,
pub image_layers: Vec<ImageLayer>,
}
@ -37,19 +40,20 @@ pub struct Container {
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct DockerConfigLayer {
architecture: String,
config: DockerImageConfig,
pub config: DockerImageConfig,
pub rootfs: DockerRootfs,
}
/// Image config properties.
/// See: https://docs.docker.com/reference/dockerfile/.
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
struct DockerImageConfig {
pub struct DockerImageConfig {
User: Option<String>,
Tty: Option<bool>,
Env: Option<Vec<String>>,
Cmd: Option<Vec<String>>,
WorkingDir: Option<String>,
Entrypoint: Option<Vec<String>>,
pub Volumes: Option<BTreeMap<String, DockerVolumeHostDirectory>>,
}
/// Container rootfs information.
@ -66,11 +70,21 @@ pub struct ImageLayer {
pub verity_hash: String,
}
/// See https://docs.docker.com/reference/dockerfile/#volume.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct DockerVolumeHostDirectory {
// This struct is empty because, according to the documentation:
// "The VOLUME instruction does not support specifying a host-dir
// parameter. You must specify the mountpoint when you create or
// run the container."
}
impl Container {
pub async fn new(config: &Config, image: &str) -> Result<Self> {
info!("============================================");
info!("Pulling manifest and config for {:?}", image);
let reference: Reference = image.to_string().parse().unwrap();
info!("Pulling manifest and config for {image}");
let image_string = image.to_string();
let reference: Reference = image_string.parse().unwrap();
let auth = build_auth(&reference);
let mut client = Client::new(ClientConfig {
@ -96,6 +110,8 @@ impl Container {
let config_layer: DockerConfigLayer =
serde_json::from_str(&config_layer_str).unwrap();
debug!("config_layer: {:?}", &config_layer);
let image_layers = get_image_layers(
config.layers_cache_file_path.clone(),
&mut client,
@ -107,6 +123,7 @@ impl Container {
.unwrap();
Ok(Container {
image: image_string,
config_layer,
image_layers,
})

4
src/tools/genpolicy/src/registry_containerd.rs
View File

@ -46,7 +46,8 @@ impl Container {
let ctrd_client = containerd_client::Client::from(containerd_channel.clone());
let k8_cri_image_client = ImageServiceClient::new(containerd_channel);
let image_ref: Reference = image.to_string().parse().unwrap();
let image_str = image.to_string();
let image_ref: Reference = image_str.parse().unwrap();
info!("Pulling image: {:?}", image_ref);
@ -67,6 +68,7 @@ impl Container {
.await?;
Ok(Container {
image: image_str,
config_layer,
image_layers,
})

16
src/tools/genpolicy/src/replica_set.rs
View File

@ -66,15 +66,13 @@ impl yaml::K8sResource for ReplicaSet {
container: &pod::Container,
settings: &settings::Settings,
) {
if let Some(volumes) = &self.spec.template.spec.volumes {
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
volumes,
);
}
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
&self.spec.template.spec.volumes,
);
}
fn generate_policy(&self, agent_policy: &policy::AgentPolicy) -> String {

16
src/tools/genpolicy/src/replication_controller.rs
View File

@ -68,15 +68,13 @@ impl yaml::K8sResource for ReplicationController {
container: &pod::Container,
settings: &settings::Settings,
) {
if let Some(volumes) = &self.spec.template.spec.volumes {
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
volumes,
);
}
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
&self.spec.template.spec.volumes,
);
}
fn generate_policy(&self, agent_policy: &policy::AgentPolicy) -> String {

12
src/tools/genpolicy/src/settings.rs
View File

@ -34,6 +34,7 @@ pub struct Volumes {
pub emptyDir_memory: EmptyDirVolume,
pub configMap: ConfigMapVolume,
pub confidential_configMap: ConfigMapVolume,
pub image_volume: ImageVolume,
}
/// EmptyDir volume settings loaded from genpolicy-settings.json.
@ -59,6 +60,17 @@ pub struct ConfigMapVolume {
pub options: Vec<String>,
}
/// Container image volume settings loaded from genpolicy-settings.json.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ImageVolume {
pub mount_type: String,
pub mount_source: String,
pub driver: String,
pub source: String,
pub fstype: String,
pub options: Vec<String>,
}
/// Data corresponding to the kata runtime config file data, loaded from
/// genpolicy-settings.json.
#[derive(Clone, Debug, Serialize, Deserialize)]

18
src/tools/genpolicy/src/stateful_set.rs
View File

@ -116,16 +116,6 @@ impl yaml::K8sResource for StatefulSet {
container: &pod::Container,
settings: &settings::Settings,
) {
if let Some(volumes) = &self.spec.template.spec.volumes {
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
volumes,
);
}
// Example:
//
// containers:
@ -150,6 +140,14 @@ impl yaml::K8sResource for StatefulSet {
StatefulSet::get_mounts_and_storages(policy_mounts, volume_mounts, claims);
}
}
yaml::get_container_mounts_and_storages(
policy_mounts,
storages,
container,
settings,
&self.spec.template.spec.volumes,
);
}
fn generate_policy(&self, agent_policy: &policy::AgentPolicy) -> String {

41
src/tools/genpolicy/src/yaml.rs
View File

@ -278,23 +278,40 @@ pub fn get_container_mounts_and_storages(
storages: &mut Vec<agent::Storage>,
container: &pod::Container,
settings: &settings::Settings,
volumes: &Vec<volume::Volume>,
volumes_option: &Option<Vec<volume::Volume>>,
) {
if let Some(volume_mounts) = &container.volumeMounts {
for volume in volumes {
for volume_mount in volume_mounts {
if volume_mount.name.eq(&volume.name) {
mount_and_storage::get_mount_and_storage(
settings,
policy_mounts,
storages,
volume,
volume_mount,
);
if let Some(volumes) = volumes_option {
if let Some(volume_mounts) = &container.volumeMounts {
for volume in volumes {
for volume_mount in volume_mounts {
if volume_mount.name.eq(&volume.name) {
mount_and_storage::get_mount_and_storage(
settings,
policy_mounts,
storages,
volume,
volume_mount,
);
}
}
}
}
}
// Add storage and mount for each volume defined in the docker container image
// configuration layer.
if let Some(volumes) = &container.registry.config_layer.config.Volumes {
for volume in volumes {
debug!("get_container_mounts_and_storages: {:?}", &volume);
mount_and_storage::get_image_mount_and_storage(
settings,
policy_mounts,
storages,
volume.0,
);
}
}
}
/// Add the "io.katacontainers.config.agent.policy" annotation into

47
tests/integration/kubernetes/k8s-policy-deployment.bats Normal file
View File

@ -0,0 +1,47 @@
#!/usr/bin/env bats
#
# Copyright (c) 2024 Microsoft.
#
# SPDX-License-Identifier: Apache-2.0
#
load "${BATS_TEST_DIRNAME}/../../common.bash"
load "${BATS_TEST_DIRNAME}/tests_common.sh"
setup() {
auto_generate_policy_enabled || skip "Auto-generated policy tests are disabled."
get_pod_config_dir
deployment_name="policy-redis-deployment"
deployment_yaml="${pod_config_dir}/k8s-policy-deployment.yaml"
# Add an appropriate policy to the correct YAML file.
policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
add_requests_to_policy_settings "${policy_settings_dir}" "ReadStreamRequest"
auto_generate_policy "${policy_settings_dir}" "${deployment_yaml}"
}
@test "Successful deployment with auto-generated policy and container image volumes" {
# Initiate deployment
kubectl apply -f "${deployment_yaml}"
# Wait for the deployment to be created
cmd="kubectl rollout status --timeout=1s deployment/${deployment_name} | grep 'successfully rolled out'"
info "Waiting for: ${cmd}"
waitForProcess "${wait_time}" "${sleep_time}" "${cmd}"
}
teardown() {
auto_generate_policy_enabled || skip "Auto-generated policy tests are disabled."
# Debugging information
info "Deployment ${deployment_name}:"
kubectl describe deployment "${deployment_name}"
kubectl rollout status deployment/${deployment_name}
# Clean-up
kubectl delete deployment "${deployment_name}"
delete_tmp_policy_settings_dir "${policy_settings_dir}"
}

1
tests/integration/kubernetes/run_kubernetes_tests.sh
View File

@ -57,6 +57,7 @@ else
"k8s-pid-ns.bats" \
"k8s-pod-quota.bats" \
"k8s-policy-hard-coded.bats" \
"k8s-policy-deployment.bats" \
"k8s-policy-job.bats" \
"k8s-policy-pod.bats" \
"k8s-policy-pvc.bats" \

36
tests/integration/kubernetes/runtimeclass_workloads/k8s-policy-deployment.yaml Normal file
View File

@ -0,0 +1,36 @@
#
# Copyright (c) 2024 Microsoft
#
# SPDX-License-Identifier: Apache-2.0
#
apiVersion: apps/v1
kind: Deployment
metadata:
name: policy-redis-deployment
labels:
app: policyredis
spec:
selector:
matchLabels:
app: policyredis
role: master
tier: backend
replicas: 1
template:
metadata:
labels:
app: policyredis
role: master
tier: backend
spec:
terminationGracePeriodSeconds: 0
runtimeClassName: kata
containers:
- name: master
image: quay.io/opstree/redis
resources:
requests:
cpu: 100m
memory: 100Mi
ports:
- containerPort: 6379