Compare commits

...

10 Commits

Author SHA1 Message Date
Fabiano Fidêncio
81e03aee20 tests: Fix kubelet data dir for k0s in trusted ephemeral storage test
k0s uses /var/lib/k0s/kubelet instead of /var/lib/kubelet as its
kubelet data directory. Introduce get_kubelet_data_dir() in
tests_common.sh and use it in k8s-trusted-ephemeral-data-storage.bats
instead of hardcoding /var/lib/kubelet.

Signed-off-by: Fabiano Fidêncio <ffidencio@nvidia.com>
2026-03-05 19:24:41 -06:00
Aurélien Bombo
8b176e8a52 build: Add artifacts to .gitignore
This adds various files that are generated during development.

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
2026-03-05 19:24:41 -06:00
Aurélien Bombo
94fff98ad1 tests: Improve logging for some tests
Use modern test semantics to ease debugging.

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
2026-03-05 19:24:41 -06:00
Aurélien Bombo
d15016454a tests: Introduce new env variables to ease development
It can be useful to set these variables during local testing:

 * AZ_REGION: Region for the cluster.
 * AZ_NODEPOOL_TAGS: Node pool tags for the cluster.
 * GENPOLICY_BINARY: Path to the genpolicy binary.
 * GENPOLICY_SETTINGS_DIR: Directory holding the genpolicy settings.

I've also made it so that tests_common.sh modifies the duplicated
genpolicy-settings.json (used for testing) instead of the original git-tracked
one.

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
2026-03-05 19:24:41 -06:00
Aurélien Bombo
59719f64d2 ci: Stop deploying the CSI driver
The design moved away from CSI driver so stop deploying that.

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
2026-03-05 19:24:41 -06:00
Aurélien Bombo
02aa27c7b5 tests: Add test for trusted ephemeral data storage
This tests the feature on CoCo machines. We use virtio-blk as the basic
virtio-scsi flow is already tested in k8s-empty-dirs.bats.

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
2026-03-05 19:24:41 -06:00
Aurélien Bombo
b655494dab genpolicy: Support trusted ephemeral data storage
* Introduces a new cluster_config setting encrypted_emptydir defaulting to true.
 * Adapts genpolicy for encrypted emptyDirs.

Crucially, the rules.rego change checks that the mount and the storage are
well-formed together:

 * i_storage.source matches a known regex.
 * i_storage.mount_point == $(spath)/BASE64(i_storage.source)
 * i_storage.mount_point == p_storage.mount_point
 * i_storage.mount_point == i_mount.source

Note that policy enforcement is necessary to prevent rogue device injection.
E.g. the agent could not blindly encrypt all block devices as some use cases
only need dm-verity.

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
2026-03-05 19:24:34 -06:00
Aurélien Bombo
6cc2365a9f agent: Support trusted ephemeral data storage
Handles block-based emptyDirs plugged via virtio-blk and virtio-scsi by
encrypting and formatting them.

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
2026-03-05 09:32:11 +01:00
Aurélien Bombo
e863558285 runtime: Support trusted ephemeral data storage
* Introduces the `emptydir_mode` config flag to allow instructing the runtime
   to create a block device for emptyDir volumes.
 * The block device is created in the original emptyDir folder on the host
   so that Kubelet can monitors its disk usage and evict the pod if it exceeds
   its sizeLimit. This matches runc and virtio-fs.
 * The block device's disk image file is sparse to minimize host disk
   footprint.

Fixes: #10560

Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
2026-03-05 09:32:11 +01:00
Alex Lyn
b45fd3bbc2 runtime: Support plugging a disk as an image file
Some VMMs support plugging a disk as an image file instead of a block device,
so we adapt the runtime to support that.

Signed-off-by: Alex Lyn <alex.lyn@antgroup.com>
Signed-off-by: Aurélien Bombo <abombo@microsoft.com>
Co-authored-by: Aurélien Bombo <abombo@microsoft.com>
2026-03-05 09:32:11 +01:00
59 changed files with 1670 additions and 769 deletions

View File

@@ -216,61 +216,6 @@ jobs:
platforms: linux/amd64, linux/s390x
file: tests/integration/kubernetes/runtimeclass_workloads/confidential/unencrypted/Dockerfile
publish-csi-driver-amd64:
name: publish-csi-driver-amd64
needs: build-kata-static-tarball-amd64
permissions:
contents: read
packages: write
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.commit-hash }}
fetch-depth: 0
persist-credentials: false
- name: Rebase atop of the latest target branch
run: |
./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
env:
TARGET_BRANCH: ${{ inputs.target-branch }}
- name: get-kata-tools-tarball
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: kata-tools-static-tarball-amd64-${{ inputs.tag }}
path: kata-tools-artifacts
- name: Install kata-tools
run: bash tests/integration/kubernetes/gha-run.sh install-kata-tools kata-tools-artifacts
- name: Copy binary into Docker context
run: |
# Copy to the location where the Dockerfile expects the binary.
mkdir -p src/tools/csi-kata-directvolume/bin/
cp /opt/kata/bin/csi-kata-directvolume src/tools/csi-kata-directvolume/bin/directvolplugin
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
- name: Login to Kata Containers ghcr.io
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Docker build and push
uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0
with:
tags: ghcr.io/kata-containers/csi-kata-directvolume:${{ inputs.pr-number }}
push: true
context: src/tools/csi-kata-directvolume/
platforms: linux/amd64
file: src/tools/csi-kata-directvolume/Dockerfile
run-kata-monitor-tests:
if: ${{ inputs.skip-test != 'yes' }}
needs: build-kata-static-tarball-amd64
@@ -349,7 +294,6 @@ jobs:
needs:
- publish-kata-deploy-payload-amd64
- build-and-publish-tee-confidential-unencrypted-image
- publish-csi-driver-amd64
uses: ./.github/workflows/run-kata-coco-tests.yaml
permissions:
contents: read

7
.gitignore vendored
View File

@@ -20,3 +20,10 @@ tools/packaging/static-build/agent/install_libseccomp.sh
.direnv
**/.DS_Store
site/
opt/
tools/packaging/kernel/configs/**/.config
root_hash.txt
# Generated by genpolicy and stored in the working directory, so only
# the basename is ignored.
layers-cache.json

7
src/agent/Cargo.lock generated
View File

@@ -979,6 +979,12 @@ dependencies = [
"parking_lot_core",
]
[[package]]
name = "data-encoding"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476"
[[package]]
name = "deranged"
version = "0.5.5"
@@ -3428,6 +3434,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843c3d97f07e3b5ac0955d53ad0af4c91fe4a4f8525843ece5bf014f27829b73"
dependencies = [
"anyhow",
"data-encoding",
"lazy_static",
"rand",
"regex",

View File

@@ -18,6 +18,8 @@ serde_json.workspace = true
# Agent Policy
regorus = { version = "0.2.8", default-features = false, features = [
"arc",
"base64",
"base64url",
"regex",
"std",
] }

View File

@@ -2308,9 +2308,6 @@ fn is_sealed_secret_path(source_path: &str) -> bool {
}
async fn cdh_handler_trusted_storage(oci: &mut Spec) -> Result<()> {
if !confidential_data_hub::is_cdh_client_initialized() {
return Ok(());
}
let linux = oci
.linux()
.as_ref()
@@ -2320,26 +2317,8 @@ async fn cdh_handler_trusted_storage(oci: &mut Spec) -> Result<()> {
for specdev in devices.iter() {
if specdev.path().as_path().to_str() == Some(TRUSTED_IMAGE_STORAGE_DEVICE) {
let dev_major_minor = format!("{}:{}", specdev.major(), specdev.minor());
let secure_storage_integrity = AGENT_CONFIG.secure_storage_integrity.to_string();
info!(
sl(),
"trusted_store device major:min {}, enable data integrity {}",
dev_major_minor,
secure_storage_integrity
);
let options = std::collections::HashMap::from([
("deviceId".to_string(), dev_major_minor),
("encryptType".to_string(), "LUKS".to_string()),
("dataIntegrity".to_string(), secure_storage_integrity),
]);
confidential_data_hub::secure_mount(
"BlockDevice",
&options,
vec![],
KATA_IMAGE_WORK_DIR,
)
.await?;
cdh_secure_mount("BlockDevice", &dev_major_minor, "LUKS", KATA_IMAGE_WORK_DIR)
.await?;
break;
}
}
@@ -2347,6 +2326,38 @@ async fn cdh_handler_trusted_storage(oci: &mut Spec) -> Result<()> {
Ok(())
}
pub(crate) async fn cdh_secure_mount(
device_type: &str,
device_id: &str,
encrypt_type: &str,
mount_point: &str,
) -> Result<()> {
if !confidential_data_hub::is_cdh_client_initialized() {
return Ok(());
}
let integrity = AGENT_CONFIG.secure_storage_integrity.to_string();
info!(
sl(),
"cdh_secure_mount: device_type {}, device_id {}, encrypt_type {}, integrity {}",
device_type,
device_id,
encrypt_type,
integrity
);
let options = std::collections::HashMap::from([
("deviceId".to_string(), device_id.to_string()),
("encryptType".to_string(), encrypt_type.to_string()),
("dataIntegrity".to_string(), integrity),
]);
confidential_data_hub::secure_mount(device_type, &options, vec![], mount_point).await?;
Ok(())
}
async fn cdh_handler_sealed_secrets(oci: &mut Spec) -> Result<()> {
if !confidential_data_hub::is_cdh_client_initialized() {
return Ok(());

View File

@@ -65,6 +65,12 @@ type UeventWatcher = (Box<dyn UeventMatcher>, oneshot::Sender<Uevent>);
pub struct StorageState {
count: Arc<AtomicU32>,
device: Arc<dyn StorageDevice>,
/// Whether the storage is shared across multiple containers (e.g.
/// block-based emptyDirs). Shared storages should not be cleaned up
/// when a container exits; cleanup happens only when the sandbox is
/// destroyed.
shared: bool,
}
impl Debug for StorageState {
@@ -74,17 +80,11 @@ impl Debug for StorageState {
}
impl StorageState {
fn new() -> Self {
fn new(shared: bool) -> Self {
StorageState {
count: Arc::new(AtomicU32::new(1)),
device: Arc::new(StorageDeviceGeneric::default()),
}
}
pub fn from_device(device: Arc<dyn StorageDevice>) -> Self {
Self {
count: Arc::new(AtomicU32::new(1)),
device,
shared,
}
}
@@ -92,6 +92,10 @@ impl StorageState {
self.device.path()
}
pub fn is_shared(&self) -> bool {
self.shared
}
pub async fn ref_count(&self) -> u32 {
self.count.load(Ordering::Relaxed)
}
@@ -171,8 +175,10 @@ impl Sandbox {
/// Add a new storage object or increase reference count of existing one.
/// The caller may detect new storage object by checking `StorageState.refcount == 1`.
/// The `shared` flag indicates if this storage is shared across multiple containers;
/// if true, cleanup will be skipped when containers exit.
#[instrument]
pub async fn add_sandbox_storage(&mut self, path: &str) -> StorageState {
pub async fn add_sandbox_storage(&mut self, path: &str, shared: bool) -> StorageState {
match self.storages.entry(path.to_string()) {
Entry::Occupied(e) => {
let state = e.get().clone();
@@ -180,7 +186,7 @@ impl Sandbox {
state
}
Entry::Vacant(e) => {
let state = StorageState::new();
let state = StorageState::new(shared);
e.insert(state.clone());
state
}
@@ -188,22 +194,32 @@ impl Sandbox {
}
/// Update the storage device associated with a path.
/// Preserves the existing shared flag and reference count.
pub fn update_sandbox_storage(
&mut self,
path: &str,
device: Arc<dyn StorageDevice>,
) -> std::result::Result<Arc<dyn StorageDevice>, Arc<dyn StorageDevice>> {
if !self.storages.contains_key(path) {
return Err(device);
match self.storages.get(path) {
None => Err(device),
Some(existing) => {
let state = StorageState {
device,
..existing.clone()
};
// Safe to unwrap() because we have just ensured existence of entry via get().
let state = self.storages.insert(path.to_string(), state).unwrap();
Ok(state.device)
}
}
let state = StorageState::from_device(device);
// Safe to unwrap() because we have just ensured existence of entry.
let state = self.storages.insert(path.to_string(), state).unwrap();
Ok(state.device)
}
/// Decrease reference count and destroy the storage object if reference count reaches zero.
///
/// For shared storages (e.g., emptyDir volumes), cleanup is skipped even when refcount
/// reaches zero. The storage entry is kept in the map so subsequent containers can reuse
/// the already-mounted storage. Actual cleanup happens when the sandbox is destroyed.
///
/// Returns `Ok(true)` if the reference count has reached zero and the storage object has been
/// removed.
#[instrument]
@@ -212,6 +228,10 @@ impl Sandbox {
None => Err(anyhow!("Sandbox storage with path {} not found", path)),
Some(state) => {
if state.dec_and_test_ref_count().await {
if state.is_shared() {
state.count.store(1, Ordering::Release);
return Ok(false);
}
if let Some(storage) = self.storages.remove(path) {
storage.device.cleanup()?;
}
@@ -720,7 +740,7 @@ mod tests {
let tmpdir_path = tmpdir.path().to_str().unwrap();
// Add a new sandbox storage
let new_storage = s.add_sandbox_storage(tmpdir_path).await;
let new_storage = s.add_sandbox_storage(tmpdir_path, false).await;
// Check the reference counter
let ref_count = new_storage.ref_count().await;
@@ -730,7 +750,7 @@ mod tests {
);
// Use the existing sandbox storage
let new_storage = s.add_sandbox_storage(tmpdir_path).await;
let new_storage = s.add_sandbox_storage(tmpdir_path, false).await;
// Since we are using existing storage, the reference counter
// should be 2 by now.
@@ -771,7 +791,7 @@ mod tests {
assert!(bind_mount(srcdir_path, destdir_path, &logger).is_ok());
s.add_sandbox_storage(destdir_path).await;
s.add_sandbox_storage(destdir_path, false).await;
let storage = StorageDeviceGeneric::new(destdir_path.to_string());
assert!(s
.update_sandbox_storage(destdir_path, Arc::new(storage))
@@ -789,7 +809,7 @@ mod tests {
let other_dir_path = other_dir.path().to_str().unwrap();
other_dir_str = other_dir_path.to_string();
s.add_sandbox_storage(other_dir_path).await;
s.add_sandbox_storage(other_dir_path, false).await;
let storage = StorageDeviceGeneric::new(other_dir_path.to_string());
assert!(s
.update_sandbox_storage(other_dir_path, Arc::new(storage))
@@ -808,9 +828,9 @@ mod tests {
let storage_path = "/tmp/testEphe";
// Add a new sandbox storage
s.add_sandbox_storage(storage_path).await;
s.add_sandbox_storage(storage_path, false).await;
// Use the existing sandbox storage
let state = s.add_sandbox_storage(storage_path).await;
let state = s.add_sandbox_storage(storage_path, false).await;
assert!(
state.ref_count().await > 1,
"Expects false as the storage is not new."

View File

@@ -6,7 +6,7 @@
use crate::linux_abi::pcipath_from_dev_tree_path;
use std::fs;
use std::os::unix::fs::PermissionsExt;
use std::os::unix::fs::{MetadataExt, PermissionsExt};
use std::path::Path;
use std::sync::Arc;
@@ -17,6 +17,7 @@ use kata_types::device::{
DRIVER_BLK_MMIO_TYPE, DRIVER_BLK_PCI_TYPE, DRIVER_NVDIMM_TYPE, DRIVER_SCSI_TYPE,
};
use kata_types::mount::StorageDevice;
use nix::sys::stat::{major, minor};
use protocols::agent::Storage;
use tracing::instrument;
@@ -29,10 +30,44 @@ use crate::device::block_device_handler::{
};
use crate::device::nvdimm_device_handler::wait_for_pmem_device;
use crate::device::scsi_device_handler::get_scsi_device_name;
use crate::storage::{common_storage_handler, new_device, StorageContext, StorageHandler};
use crate::storage::{
common_storage_handler, new_device, set_ownership, StorageContext, StorageHandler,
};
use slog::Logger;
#[cfg(target_arch = "s390x")]
use std::str::FromStr;
fn get_device_number(dev_path: &str, metadata: Option<&fs::Metadata>) -> Result<String> {
let dev_id = match metadata {
Some(m) => m.rdev(),
None => {
let m =
fs::metadata(dev_path).context(format!("get metadata on file {:?}", dev_path))?;
m.rdev()
}
};
Ok(format!("{}:{}", major(dev_id), minor(dev_id)))
}
async fn handle_block_storage(
logger: &Logger,
storage: &Storage,
dev_num: &str,
) -> Result<Arc<dyn StorageDevice>> {
let has_ephemeral_encryption = storage
.driver_options
.contains(&"encryption_key=ephemeral".to_string());
if has_ephemeral_encryption {
crate::rpc::cdh_secure_mount("BlockDevice", dev_num, "LUKS", &storage.mount_point).await?;
set_ownership(logger, storage)?;
new_device(storage.mount_point.clone())
} else {
let path = common_storage_handler(logger, storage)?;
new_device(path)
}
}
#[derive(Debug)]
pub struct VirtioBlkMmioHandler {}
@@ -75,6 +110,8 @@ impl StorageHandler for VirtioBlkPciHandler {
mut storage: Storage,
ctx: &mut StorageContext,
) -> Result<Arc<dyn StorageDevice>> {
let dev_num: String;
// If hot-plugged, get the device node path based on the PCI path
// otherwise use the virt path provided in Storage Source
if storage.source.starts_with("/dev") {
@@ -84,15 +121,16 @@ impl StorageHandler for VirtioBlkPciHandler {
if mode & libc::S_IFBLK == 0 {
return Err(anyhow!("Invalid device {}", &storage.source));
}
dev_num = get_device_number(&storage.source, Some(&metadata))?;
} else {
let (root_complex, pcipath) = pcipath_from_dev_tree_path(&storage.source)?;
let dev_path =
get_virtio_blk_pci_device_name(ctx.sandbox, root_complex, &pcipath).await?;
storage.source = dev_path;
dev_num = get_device_number(&storage.source, None)?;
}
let path = common_storage_handler(ctx.logger, &storage)?;
new_device(path)
handle_block_storage(ctx.logger, &storage, &dev_num).await
}
}
@@ -151,10 +189,10 @@ impl StorageHandler for ScsiHandler {
) -> Result<Arc<dyn StorageDevice>> {
// Retrieve the device path from SCSI address.
let dev_path = get_scsi_device_name(ctx.sandbox, &storage.source).await?;
storage.source = dev_path;
storage.source = dev_path.clone();
let path = common_storage_handler(ctx.logger, &storage)?;
new_device(path)
let dev_num = get_device_number(&dev_path, None)?;
handle_block_storage(ctx.logger, &storage, &dev_num).await
}
}

View File

@@ -172,7 +172,11 @@ pub async fn add_storages(
for storage in storages {
let path = storage.mount_point.clone();
let state = sandbox.lock().await.add_sandbox_storage(&path).await;
let state = sandbox
.lock()
.await
.add_sandbox_storage(&path, storage.shared)
.await;
if state.ref_count().await > 1 {
if let Some(path) = state.path() {
if !path.is_empty() {

View File

@@ -520,6 +520,11 @@ message Storage {
// FSGroup consists of the group ID and group ownership change policy
// that the mounted volume must have its group ID changed to when specified.
FSGroup fs_group = 7;
// Shared indicates this storage is shared across multiple containers
// (e.g., block-based emptyDirs). When true, the agent should not clean up
// the storage when a container using it exits, as other containers
// may still need it. Cleanup will happen when the sandbox is destroyed.
bool shared = 8;
}
// Device represents only the devices that could have been defined through the

View File

@@ -225,6 +225,8 @@ DEFENABLEANNOTATIONS := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_pa
DEFENABLEANNOTATIONS_COCO := [\"enable_iommu\", \"virtio_fs_extra_args\", \"kernel_params\", \"kernel_verity_params\", \"default_vcpus\", \"default_memory\", \"cc_init_data\"]
DEFDISABLEGUESTSECCOMP := true
DEFDISABLEGUESTEMPTYDIR := false
DEFEMPTYDIRMODE := shared-fs
DEFEMPTYDIRMODE_COCO := block-encrypted
#Default experimental features enabled
DEFAULTEXPFEATURES := []
@@ -750,6 +752,8 @@ USER_VARS += DEFNETWORKMODEL_FC
USER_VARS += DEFNETWORKMODEL_QEMU
USER_VARS += DEFNETWORKMODEL_STRATOVIRT
USER_VARS += DEFDISABLEGUESTEMPTYDIR
USER_VARS += DEFEMPTYDIRMODE
USER_VARS += DEFEMPTYDIRMODE_COCO
USER_VARS += DEFDISABLEGUESTSECCOMP
USER_VARS += DEFDISABLESELINUX
USER_VARS += DEFDISABLEGUESTSELINUX

View File

@@ -463,6 +463,18 @@ vfio_mode = "@DEFVFIOMODE@"
# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
# Specifies how Kubernetes emptyDir volumes are handled.
# Options:
#
# - shared-fs (default)
# Shares the emptyDir folder with the guest using the method given
# by the `shared_fs` setting.
#
# - block-encrypted
# Plugs a block device to be encrypted in the guest.
#
emptydir_mode = "@DEFEMPTYDIRMODE@"
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.

View File

@@ -354,6 +354,18 @@ static_sandbox_resource_mgmt = @DEFSTATICRESOURCEMGMT_FC@
# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
# Specifies how Kubernetes emptyDir volumes are handled.
# Options:
#
# - shared-fs (default)
# Shares the emptyDir folder with the guest using the method given
# by the `shared_fs` setting.
#
# - block-encrypted
# Plugs a block device to be encrypted in the guest.
#
emptydir_mode = "@DEFEMPTYDIRMODE@"
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.

View File

@@ -638,6 +638,18 @@ vfio_mode = "@DEFVFIOMODE@"
# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
# Specifies how Kubernetes emptyDir volumes are handled.
# Options:
#
# - shared-fs (default)
# Shares the emptyDir folder with the guest using the method given
# by the `shared_fs` setting.
#
# - block-encrypted
# Plugs a block device to be encrypted in the guest.
#
emptydir_mode = "@DEFEMPTYDIRMODE@"
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.

View File

@@ -701,6 +701,18 @@ vfio_mode = "@DEFVFIOMODE@"
# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
# Specifies how Kubernetes emptyDir volumes are handled.
# Options:
#
# - shared-fs (default)
# Shares the emptyDir folder with the guest using the method given
# by the `shared_fs` setting.
#
# - block-encrypted
# Plugs a block device to be encrypted in the guest.
#
emptydir_mode = "@DEFEMPTYDIRMODE_COCO@"
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.

View File

@@ -717,6 +717,18 @@ vfio_mode = "@DEFVFIOMODE@"
# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
# Specifies how Kubernetes emptyDir volumes are handled.
# Options:
#
# - shared-fs (default)
# Shares the emptyDir folder with the guest using the method given
# by the `shared_fs` setting.
#
# - block-encrypted
# Plugs a block device to be encrypted in the guest.
#
emptydir_mode = "@DEFEMPTYDIRMODE@"
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.

View File

@@ -694,6 +694,18 @@ vfio_mode = "@DEFVFIOMODE@"
# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
# Specifies how Kubernetes emptyDir volumes are handled.
# Options:
#
# - shared-fs (default)
# Shares the emptyDir folder with the guest using the method given
# by the `shared_fs` setting.
#
# - block-encrypted
# Plugs a block device to be encrypted in the guest.
#
emptydir_mode = "@DEFEMPTYDIRMODE@"
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.

View File

@@ -696,6 +696,18 @@ vfio_mode = "@DEFVFIOMODE@"
# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
# Specifies how Kubernetes emptyDir volumes are handled.
# Options:
#
# - shared-fs (default)
# Shares the emptyDir folder with the guest using the method given
# by the `shared_fs` setting.
#
# - block-encrypted
# Plugs a block device to be encrypted in the guest.
#
emptydir_mode = "@DEFEMPTYDIRMODE@"
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.

View File

@@ -679,6 +679,18 @@ vfio_mode = "@DEFVFIOMODE_SE@"
# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
# Specifies how Kubernetes emptyDir volumes are handled.
# Options:
#
# - shared-fs (default)
# Shares the emptyDir folder with the guest using the method given
# by the `shared_fs` setting.
#
# - block-encrypted
# Plugs a block device to be encrypted in the guest.
#
emptydir_mode = "@DEFEMPTYDIRMODE@"
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.

View File

@@ -704,6 +704,18 @@ vfio_mode = "@DEFVFIOMODE@"
# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
# Specifies how Kubernetes emptyDir volumes are handled.
# Options:
#
# - shared-fs (default)
# Shares the emptyDir folder with the guest using the method given
# by the `shared_fs` setting.
#
# - block-encrypted
# Plugs a block device to be encrypted in the guest.
#
emptydir_mode = "@DEFEMPTYDIRMODE_COCO@"
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.

View File

@@ -686,6 +686,18 @@ vfio_mode = "@DEFVFIOMODE@"
# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
# Specifies how Kubernetes emptyDir volumes are handled.
# Options:
#
# - shared-fs (default)
# Shares the emptyDir folder with the guest using the method given
# by the `shared_fs` setting.
#
# - block-encrypted
# Plugs a block device to be encrypted in the guest.
#
emptydir_mode = "@DEFEMPTYDIRMODE_COCO@"
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.

View File

@@ -695,6 +695,18 @@ vfio_mode = "@DEFVFIOMODE@"
# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
# Specifies how Kubernetes emptyDir volumes are handled.
# Options:
#
# - shared-fs (default)
# Shares the emptyDir folder with the guest using the method given
# by the `shared_fs` setting.
#
# - block-encrypted
# Plugs a block device to be encrypted in the guest.
#
emptydir_mode = "@DEFEMPTYDIRMODE@"
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.

View File

@@ -262,6 +262,18 @@ vfio_mode = "@DEFVFIOMODE@"
# Note: remote hypervisor has no sharing of emptydir mounts from host to guest
disable_guest_empty_dir = false
# Specifies how Kubernetes emptyDir volumes are handled.
# Options:
#
# - shared-fs (default)
# Shares the emptyDir folder with the guest using the method given
# by the `shared_fs` setting.
#
# - block-encrypted
# Plugs a block device to be encrypted in the guest.
#
emptydir_mode = "@DEFEMPTYDIRMODE@"
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.

View File

@@ -397,6 +397,18 @@ static_sandbox_resource_mgmt = @DEFSTATICRESOURCEMGMT_STRATOVIRT@
# be created on the host and shared via virtio-fs. This is potentially slower, but allows sharing of files from host to guest.
disable_guest_empty_dir = @DEFDISABLEGUESTEMPTYDIR@
# Specifies how Kubernetes emptyDir volumes are handled.
# Options:
#
# - shared-fs (default)
# Shares the emptyDir folder with the guest using the method given
# by the `shared_fs` setting.
#
# - block-encrypted
# Plugs a block device to be encrypted in the guest.
#
emptydir_mode = "@DEFEMPTYDIRMODE@"
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.

View File

@@ -17,6 +17,7 @@ import (
const (
mountInfoFileName = "mountInfo.json"
EncryptionKeyMetadataKey = "encryptionKey"
FSGroupMetadataKey = "fsGroup"
FSGroupChangePolicyMetadataKey = "fsGroupChangePolicy"
)
@@ -77,6 +78,14 @@ func Add(volumePath string, mountInfo string) error {
return os.WriteFile(filepath.Join(volumeDir, mountInfoFileName), []byte(mountInfo), 0600)
}
func AddMountInfo(volumePath string, mountInfo MountInfo) error {
s, err := json.Marshal(&mountInfo)
if err != nil {
return err
}
return Add(volumePath, string(s))
}
// Remove deletes the direct volume path including all the files inside it.
func Remove(volumePath string) error {
return os.RemoveAll(filepath.Join(kataDirectVolumeRootPath, b64.URLEncoding.EncodeToString([]byte(volumePath))))
@@ -99,7 +108,18 @@ func VolumeMountInfo(volumePath string) (*MountInfo, error) {
return &mountInfo, nil
}
// RecordSandboxID associates a sandbox id with a direct volume.
// IsVolumeMounted returns whether the direct volume mount is present.
func IsVolumeMounted(volumePath string) (bool, error) {
if _, err := VolumeMountInfo(volumePath); err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return true, nil
}
// RecordSandboxId associates a sandbox id with a direct volume.
func RecordSandboxID(sandboxID string, volumePath string) error {
encodedPath := b64.URLEncoding.EncodeToString([]byte(volumePath))
mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, encodedPath, mountInfoFileName)

View File

@@ -197,6 +197,7 @@ type runtime struct {
StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"`
EnablePprof bool `toml:"enable_pprof"`
DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"`
EmptyDirMode string `toml:"emptydir_mode"`
CreateContainerTimeout uint64 `toml:"create_container_timeout"`
DanConf string `toml:"dan_conf"`
ForceGuestPull bool `toml:"experimental_force_guest_pull"`
@@ -204,6 +205,22 @@ type runtime struct {
KubeletRootDir string `toml:"kubelet_root_dir"`
}
// emptyDirMode returns a valid emptydir_mode value, defaulting to shared-fs
// if the TOML field is unset.
func (r runtime) emptyDirMode() (string, error) {
if r.EmptyDirMode == "" {
return vc.EmptyDirModeSharedFs, nil
}
switch r.EmptyDirMode {
case vc.EmptyDirModeSharedFs, vc.EmptyDirModeVirtioBlkEncrypted:
return r.EmptyDirMode, nil
default:
return "", fmt.Errorf("invalid emptydir_mode=%q, allowed values: %q, %q",
r.EmptyDirMode, vc.EmptyDirModeSharedFs, vc.EmptyDirModeVirtioBlkEncrypted)
}
}
type agent struct {
KernelModules []string `toml:"kernel_modules"`
Debug bool `toml:"enable_debug"`
@@ -1390,6 +1407,16 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
return nil
}
func updateRuntimeConfigRuntime(configPath string, tomlConf tomlConfig, config *oci.RuntimeConfig) error {
emptyDirMode, err := tomlConf.Runtime.emptyDirMode()
if err != nil {
return fmt.Errorf("%v: %v", configPath, err)
}
config.EmptyDirMode = emptyDirMode
return nil
}
// SetKernelParams adds the user-specified kernel parameters (from the
// configuration file) to the defaults so that the former take priority.
func SetKernelParams(runtimeConfig *oci.RuntimeConfig) error {
@@ -1454,6 +1481,10 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run
return err
}
if err := updateRuntimeConfigRuntime(configPath, tomlConf, config); err != nil {
return err
}
fConfig, err := newFactoryConfig(tomlConf.Factory)
if err != nil {
return fmt.Errorf("%v: %v", configPath, err)

View File

@@ -218,6 +218,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (testConfig testRuntime
JaegerPassword: jaegerPassword,
FactoryConfig: factoryConfig,
EmptyDirMode: vc.EmptyDirModeSharedFs,
}
err = SetKernelParams(&runtimeConfig)
@@ -599,6 +600,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
AgentConfig: expectedAgentConfig,
FactoryConfig: expectedFactoryConfig,
EmptyDirMode: vc.EmptyDirModeSharedFs,
}
err = SetKernelParams(&expectedConfig)
if err != nil {
@@ -1609,6 +1611,39 @@ func TestCheckNetNsConfig(t *testing.T) {
assert.Error(err)
}
func TestCheckEmptyDirMode(t *testing.T) {
assert := assert.New(t)
// Valid values
r := runtime{EmptyDirMode: vc.EmptyDirModeSharedFs}
mode, err := r.emptyDirMode()
assert.NoError(err)
assert.Equal(vc.EmptyDirModeSharedFs, mode)
r = runtime{EmptyDirMode: vc.EmptyDirModeVirtioBlkEncrypted}
mode, err = r.emptyDirMode()
assert.NoError(err)
assert.Equal(vc.EmptyDirModeVirtioBlkEncrypted, mode)
r = runtime{}
mode, err = r.emptyDirMode()
assert.NoError(err)
assert.Equal(vc.EmptyDirModeSharedFs, mode)
// Invalid values
r = runtime{EmptyDirMode: "invalid"}
_, err = r.emptyDirMode()
assert.Error(err)
r = runtime{EmptyDirMode: "shared_fs"}
_, err = r.emptyDirMode()
assert.Error(err)
r = runtime{EmptyDirMode: "block_encrypted"}
_, err = r.emptyDirMode()
assert.Error(err)
}
func TestCheckFactoryConfig(t *testing.T) {
assert := assert.New(t)

View File

@@ -98,12 +98,12 @@ func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeCon
// For the given pod ephemeral volume is created only once
// backed by tmpfs inside the VM. For successive containers
// of the same pod the already existing volume is reused.
func SetEphemeralStorageType(ociSpec specs.Spec, disableGuestEmptyDir bool) specs.Spec {
func SetEphemeralStorageType(ociSpec specs.Spec, disableGuestEmptyDir bool, emptyDirMode string) specs.Spec {
for idx, mnt := range ociSpec.Mounts {
if vc.IsEphemeralStorage(mnt.Source) {
ociSpec.Mounts[idx].Type = vc.KataEphemeralDevType
}
if vc.Isk8sHostEmptyDir(mnt.Source) && !disableGuestEmptyDir {
if vc.Isk8sHostEmptyDir(mnt.Source) && !disableGuestEmptyDir && emptyDirMode != vc.EmptyDirModeVirtioBlkEncrypted {
ociSpec.Mounts[idx].Type = vc.KataLocalDevType
}
}
@@ -243,7 +243,8 @@ func CreateContainer(ctx context.Context, sandbox vc.VCSandbox, ociSpec specs.Sp
// The value of this annotation is sent to the sandbox using init data.
delete(ociSpec.Annotations, vcAnnotations.Initdata)
ociSpec = SetEphemeralStorageType(ociSpec, disableGuestEmptyDir)
emptyDirMode := sandbox.Status().EmptyDirMode
ociSpec = SetEphemeralStorageType(ociSpec, disableGuestEmptyDir, emptyDirMode)
contConfig, err := oci.ContainerConfig(ociSpec, bundlePath, containerID, disableOutput)
if err != nil {

View File

@@ -141,7 +141,7 @@ func TestSetEphemeralStorageType(t *testing.T) {
ociMounts = append(ociMounts, mount)
ociSpec.Mounts = ociMounts
ociSpec = SetEphemeralStorageType(ociSpec, false)
ociSpec = SetEphemeralStorageType(ociSpec, false, vc.EmptyDirModeSharedFs)
mountType := ociSpec.Mounts[0].Type
assert.Equal(mountType, "ephemeral",

View File

@@ -165,6 +165,10 @@ type RuntimeConfig struct {
// Determines if Kata creates emptyDir on the guest
DisableGuestEmptyDir bool
// EmptyDirMode specifies how Kubernetes emptyDir volumes are handled.
// Valid values are "shared-fs" (default) or "block-encrypted".
EmptyDirMode string
// CreateContainer timeout which, if provided, indicates the createcontainer request timeout
// needed for the workload ( Mostly used for pulling images in the guest )
CreateContainerTimeout uint64
@@ -1211,6 +1215,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st
DisableGuestSeccomp: runtime.DisableGuestSeccomp,
EmptyDirMode: runtime.EmptyDirMode,
EnableVCPUsPinning: runtime.EnableVCPUsPinning,
GuestSeLinuxLabel: runtime.GuestSeLinuxLabel,

View File

@@ -304,6 +304,20 @@ type ContainerDevice struct {
// GID is group ID in the container namespace
GID uint32
// Shared indicates whether the device is shared across containers.
Shared bool
}
// EphemeralDisk holds information about an ephemeral disk created for
// block-based emptyDir volumes.
type EphemeralDisk struct {
// DiskPath is the path to the disk image file.
DiskPath string
// SourcePath is the emptyDir source path, ie. the folder created by
// Kubelet on the host.
SourcePath string
}
// RootFs describes the container's rootfs.
@@ -604,13 +618,16 @@ func filterDevices(c *Container, devices []ContainerDevice) (ret []ContainerDevi
// device ID for the particular mount. This'll occur when the mountpoint source
// is a block device.
func (c *Container) createBlockDevices(ctx context.Context) error {
if !c.checkBlockDeviceSupport(ctx) {
c.Logger().Warn("Block device not supported")
return nil
}
// iterate all mounts and create block device if it's block based.
for i := range c.mounts {
// If block devices are disabled, we selectively only hotplug if
// the mount is an encrypted block-based emptyDir, to avoid
// cases that could regress 20ca4d2.
if !c.checkBlockDeviceSupport(ctx) && (c.sandbox.config.EmptyDirMode != EmptyDirModeVirtioBlkEncrypted || !Isk8sHostEmptyDir(c.mounts[i].Source)) {
c.Logger().Warn("Block device not supported")
continue
}
if len(c.mounts[i].BlockDeviceID) > 0 {
// Non-empty m.BlockDeviceID indicates there's already one device
// associated with the mount,so no need to create a new device for it
@@ -638,6 +655,12 @@ func (c *Container) createBlockDevices(ctx context.Context) error {
c.Logger().WithError(err).Error("error writing sandbox info")
}
// When using direct volume assignment, we assume the source file is a disk if it's a regular file.
fileInfo, err := os.Stat(mntInfo.Device)
if err == nil && fileInfo.Mode().IsRegular() {
isBlockFile = true
}
readonly := false
for _, flag := range mntInfo.Options {
if flag == "ro" {
@@ -653,6 +676,8 @@ func (c *Container) createBlockDevices(ctx context.Context) error {
for key, value := range mntInfo.Metadata {
switch key {
case volume.EncryptionKeyMetadataKey:
c.mounts[i].EncryptionKey = value
case volume.FSGroupMetadataKey:
gid, err := strconv.Atoi(value)
if err != nil {
@@ -760,6 +785,10 @@ func newContainer(ctx context.Context, sandbox *Sandbox, contConfig *ContainerCo
return nil, err
}
if err := c.createEphemeralDisks(); err != nil {
return nil, err
}
// If mounts are block devices, add to devmanager
if err := c.createMounts(ctx); err != nil {
return nil, err
@@ -825,6 +854,109 @@ func (c *Container) createVirtualVolumeDevices() ([]config.DeviceInfo, error) {
return deviceInfos, nil
}
// getFilesystemCapacity return the total size in bytes of the filesystem
// under path.
func getFilesystemCapacity(path string) (uint64, error) {
var stat unix.Statfs_t
if err := unix.Statfs(path, &stat); err != nil {
return 0, err
}
return stat.Blocks * uint64(stat.Bsize), nil
}
func (c *Container) createEphemeralDisks() error {
if c.sandbox.config.EmptyDirMode != EmptyDirModeVirtioBlkEncrypted {
return nil
}
for i := range c.mounts {
if !Isk8sHostEmptyDir(c.mounts[i].Source) {
continue
}
// Mark the mount as shared so the block device isn't removed when a container stops.
c.mounts[i].Shared = true
if mounted, err := volume.IsVolumeMounted(c.mounts[i].Source); err != nil {
return err
} else if mounted {
continue
}
diskPath, err := c.setupEphemeralDisk(c.mounts[i].Source)
if err != nil {
return err
}
c.sandbox.ephemeralDisks = append(c.sandbox.ephemeralDisks, EphemeralDisk{
DiskPath: diskPath,
SourcePath: c.mounts[i].Source,
})
}
return nil
}
// setupEphemeralDisk creates and configures an ephemeral disk image
// inside the given emptyDir. It returns the path to the created disk
// image. The fd is always closed and the disk image is removed if any
// step after creation fails.
func (c *Container) setupEphemeralDisk(emptyDirPath string) (diskPath string, err error) {
// Create the disk file in the same folder as the original
// emptyDir mount so that Kubelet can enforce the sizeLimit.
diskPath = filepath.Join(emptyDirPath, "disk.img")
f, err := os.Create(diskPath)
if err != nil {
c.Logger().WithError(err).Errorf("failed to create disk file at %s", diskPath)
return
}
defer f.Close()
defer func() {
if err != nil {
if removeErr := os.Remove(diskPath); removeErr != nil {
c.Logger().WithError(removeErr).Errorf("failed to clean up disk file %s after error", diskPath)
}
}
}()
emptyDirFsCapacity, err := getFilesystemCapacity(emptyDirPath)
if err != nil {
c.Logger().WithError(err).Errorf("failed to get filesystem capacity for mount %s", emptyDirPath)
return
}
if err = f.Truncate(int64(emptyDirFsCapacity)); err != nil {
c.Logger().WithError(err).Errorf("failed to truncate disk file")
return
}
var sourceStat unix.Stat_t
if err = unix.Stat(emptyDirPath, &sourceStat); err != nil {
c.Logger().WithError(err).Errorf("failed to stat mount source: %s", emptyDirPath)
return
}
metadata := map[string]string{
volume.EncryptionKeyMetadataKey: "ephemeral",
}
if sourceStat.Gid != 0 {
metadata[volume.FSGroupMetadataKey] = strconv.FormatUint(uint64(sourceStat.Gid), 10)
}
if err = volume.AddMountInfo(emptyDirPath, volume.MountInfo{
VolumeType: "blk",
Device: diskPath,
FsType: "ext4",
Metadata: metadata,
}); err != nil {
c.Logger().WithError(err).Errorf("failed to assign direct volume for mount %s", emptyDirPath)
return
}
return
}
func (c *Container) createMounts(ctx context.Context) error {
// Create block devices for newly created container
return c.createBlockDevices(ctx)
@@ -1729,6 +1861,13 @@ func (c *Container) attachDevices(ctx context.Context) error {
func (c *Container) detachDevices(ctx context.Context) error {
for _, dev := range c.devices {
// Skip detaching shared devices - they are shared across
// containers (e.g., block-based emptyDirs) and will be cleaned
// up when the sandbox is deleted.
if dev.Shared {
continue
}
err := c.sandbox.devManager.DetachDevice(ctx, dev.ID, c.sandbox)
if err != nil && err != deviceManager.ErrDeviceNotAttached {
return err

View File

@@ -66,6 +66,16 @@ const (
// containers.
KataLocalDevType = "local"
// EmptyDirModeSharedFs is the emptydir_mode value for sharing emptyDir via shared filesystem.
EmptyDirModeSharedFs = "shared-fs"
// EmptyDirModeVirtioBlkEncrypted is the emptydir_mode value for encrypted virtio-blk emptyDir.
EmptyDirModeVirtioBlkEncrypted = "block-encrypted"
// encryptionKeyDriverOption is the driver option used to specify
// an encryption key for a Storage struct.
encryptionKeyDriverOption = "encryption_key"
// Allocating an FSGroup that owns the pod's volumes
fsGid = "fsgid"
@@ -1805,7 +1815,7 @@ func (k *kataAgent) handleDeviceBlockVolume(c *Container, m Mount, device api.De
if *m.FSGroup > 0 && *m.FSGroup <= math.MaxUint32 {
safeFsgroup = uint32(*m.FSGroup)
} else {
return nil, fmt.Errorf("m.FSGroup value was out of range: %d", m.FSGroup)
return nil, fmt.Errorf("m.FSGroup value was out of range: %d", *m.FSGroup)
}
vol.FsGroup = &grpc.FSGroup{
@@ -1814,6 +1824,13 @@ func (k *kataAgent) handleDeviceBlockVolume(c *Container, m Mount, device api.De
}
}
if m.EncryptionKey != "" {
option := fmt.Sprintf("%s=%s", encryptionKeyDriverOption, m.EncryptionKey)
vol.DriverOptions = append(vol.DriverOptions, option)
}
vol.Shared = m.Shared
return vol, nil
}
@@ -1883,7 +1900,11 @@ func (k *kataAgent) handleBlkOCIMounts(c *Container, spec *specs.Spec) ([]*grpc.
// Add the block device to the list of container devices, to make sure the
// device is detached with detachDevices() for a container.
c.devices = append(c.devices, ContainerDevice{ID: id, ContainerPath: m.Destination})
c.devices = append(c.devices, ContainerDevice{
ID: id,
ContainerPath: m.Destination,
Shared: m.Shared,
})
// Create Storage structure
vol, err := k.createBlkStorageObject(c, m)

View File

@@ -272,6 +272,14 @@ type Mount struct {
// FSGroupChangePolicy specifies the policy that will be used when applying
// group id ownership change for a volume.
FSGroupChangePolicy volume.FSGroupChangePolicy
// EncryptionKey is the encryption key used for the volume.
// Currently, valid values are "" for no encryption or "ephemeral"
// to instruct the agent to generate a one-time key.
EncryptionKey string
// Shared indicates whether the mount is shared across containers.
Shared bool
}
func isSymlink(path string) bool {

View File

@@ -90,6 +90,7 @@ func (s *Sandbox) dumpDevices(ss *persistapi.SandboxState, cs map[string]persist
FileMode: dev.FileMode,
UID: dev.UID,
GID: dev.GID,
Shared: dev.Shared,
})
}
@@ -337,6 +338,7 @@ func (c *Container) loadContDevices(cs persistapi.ContainerState) {
FileMode: dev.FileMode,
UID: dev.UID,
GID: dev.GID,
Shared: dev.Shared,
})
}
}

View File

@@ -31,6 +31,9 @@ type DeviceMap struct {
// GID is group ID in the container namespace
GID uint32
// Shared indicates whether the device is shared across containers
Shared bool
}
// Mount describes a container mount.

View File

@@ -3438,6 +3438,11 @@ type Storage struct {
// FSGroup consists of the group ID and group ownership change policy
// that the mounted volume must have its group ID changed to when specified.
FsGroup *FSGroup `protobuf:"bytes,7,opt,name=fs_group,json=fsGroup,proto3" json:"fs_group,omitempty"`
// Shared indicates this storage is shared across multiple containers
// (e.g., block-based emptyDirs). When true, the agent should not clean up
// the storage when a container using it exits, as other containers
// may still need it. Cleanup will happen when the sandbox is destroyed.
Shared bool `protobuf:"varint,8,opt,name=shared,proto3" json:"shared,omitempty"`
}
func (x *Storage) Reset() {
@@ -3521,6 +3526,13 @@ func (x *Storage) GetFsGroup() *FSGroup {
return nil
}
func (x *Storage) GetShared() bool {
if x != nil {
return x.Shared
}
return false
}
// Device represents only the devices that could have been defined through the
// Linux Device list of the OCI specification.
type Device struct {
@@ -4838,7 +4850,7 @@ var file_agent_proto_rawDesc = []byte{
0x61, 0x74, 0x68, 0x12, 0x17, 0x0a, 0x07, 0x64, 0x73, 0x74, 0x5f, 0x63, 0x74, 0x72, 0x18, 0x04,
0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x73, 0x74, 0x43, 0x74, 0x72, 0x12, 0x19, 0x0a, 0x08,
0x64, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07,
0x64, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x22, 0xdd, 0x01, 0x0a, 0x07, 0x53, 0x74, 0x6f, 0x72,
0x64, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x22, 0xf5, 0x01, 0x0a, 0x07, 0x53, 0x74, 0x6f, 0x72,
0x61, 0x67, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x72, 0x69, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x72, 0x69, 0x76, 0x65, 0x72, 0x12, 0x25, 0x0a, 0x0e, 0x64,
0x72, 0x69, 0x76, 0x65, 0x72, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20,
@@ -4852,312 +4864,314 @@ var file_agent_proto_rawDesc = []byte{
0x09, 0x52, 0x0a, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x28, 0x0a,
0x08, 0x66, 0x73, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x46, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x07,
0x66, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x86, 0x01, 0x0a, 0x06, 0x44, 0x65, 0x76, 0x69,
0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02,
0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x76, 0x6d, 0x5f, 0x70, 0x61, 0x74,
0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x6d, 0x50, 0x61, 0x74, 0x68, 0x12,
0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x70, 0x61, 0x74,
0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e,
0x65, 0x72, 0x50, 0x61, 0x74, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e,
0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73,
0x22, 0x58, 0x0a, 0x0a, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x55, 0x73, 0x65, 0x72, 0x12, 0x10,
0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64,
0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x67,
0x69, 0x64, 0x12, 0x26, 0x0a, 0x0e, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c,
0x47, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x64, 0x64, 0x69,
0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x47, 0x69, 0x64, 0x73, 0x22, 0xca, 0x01, 0x0a, 0x0f, 0x43,
0x6f, 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12,
0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61,
0x74, 0x68, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18,
0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12,
0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01,
0x28, 0x0d, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x19, 0x0a, 0x08,
0x64, 0x69, 0x72, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07,
0x64, 0x69, 0x72, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x05,
0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64,
0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x67, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f,
0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66,
0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x08, 0x20, 0x01, 0x28,
0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4f, 0x4f,
0x4d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x2d, 0x0a,
0x08, 0x4f, 0x4f, 0x4d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e,
0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0x2a, 0x0a, 0x0e,
0x41, 0x64, 0x64, 0x53, 0x77, 0x61, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18,
0x0a, 0x07, 0x50, 0x43, 0x49, 0x50, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0d, 0x52,
0x07, 0x50, 0x43, 0x49, 0x50, 0x61, 0x74, 0x68, 0x22, 0x28, 0x0a, 0x12, 0x41, 0x64, 0x64, 0x53,
0x77, 0x61, 0x70, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12,
0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61,
0x74, 0x68, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x23, 0x0a, 0x07, 0x4d, 0x65, 0x74, 0x72, 0x69,
0x63, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x22, 0x40, 0x0a, 0x12,
0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x67, 0x75, 0x65,
0x73, 0x74, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x76,
0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x47, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x22, 0x55,
0x0a, 0x13, 0x52, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f,
0x67, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x0f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x47, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x74,
0x68, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52,
0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x2a, 0x0a, 0x10, 0x53, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69,
0x63, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6f, 0x6c,
0x69, 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63,
0x79, 0x22, 0xfc, 0x03, 0x0a, 0x13, 0x4d, 0x65, 0x6d, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4d, 0x65,
0x6d, 0x63, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x08, 0x64, 0x69, 0x73,
0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x08, 0x64,
0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x88, 0x01, 0x01, 0x12, 0x17, 0x0a, 0x04, 0x73, 0x77,
0x61, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x48, 0x01, 0x52, 0x04, 0x73, 0x77, 0x61, 0x70,
0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x73, 0x77, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x65, 0x73,
0x73, 0x5f, 0x6d, 0x61, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x02, 0x52, 0x0d, 0x73,
0x77, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x4d, 0x61, 0x78, 0x88, 0x01, 0x01, 0x12,
0x24, 0x0a, 0x0b, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x73, 0x65, 0x63, 0x73, 0x18, 0x04,
0x20, 0x01, 0x28, 0x04, 0x48, 0x03, 0x52, 0x0a, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x53, 0x65,
0x63, 0x73, 0x88, 0x01, 0x01, 0x12, 0x3c, 0x0a, 0x18, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f,
0x70, 0x73, 0x69, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69,
0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x04, 0x52, 0x15, 0x70, 0x65, 0x72, 0x69, 0x6f,
0x64, 0x50, 0x73, 0x69, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x6d, 0x69, 0x74,
0x88, 0x01, 0x01, 0x12, 0x40, 0x0a, 0x1a, 0x65, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f,
0x70, 0x73, 0x69, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69,
0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x05, 0x52, 0x17, 0x65, 0x76, 0x69, 0x63, 0x74,
0x69, 0x6f, 0x6e, 0x50, 0x73, 0x69, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x6d,
0x69, 0x74, 0x88, 0x01, 0x01, 0x12, 0x43, 0x0a, 0x1c, 0x65, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f,
0x6e, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x63, 0x6f, 0x75, 0x6e,
0x74, 0x5f, 0x6d, 0x69, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x48, 0x06, 0x52, 0x18, 0x65,
0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x75, 0x6e, 0x41, 0x67, 0x69, 0x6e, 0x67, 0x43,
0x6f, 0x75, 0x6e, 0x74, 0x4d, 0x69, 0x6e, 0x88, 0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64,
0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x73, 0x77, 0x61, 0x70,
0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x73, 0x77, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x5f,
0x6d, 0x61, 0x78, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x73,
0x65, 0x63, 0x73, 0x42, 0x1b, 0x0a, 0x19, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x70,
0x73, 0x69, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74,
0x42, 0x1d, 0x0a, 0x1b, 0x5f, 0x65, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x73,
0x66, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65,
0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x22,
0x86, 0x01, 0x0a, 0x06, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79,
0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x17,
0x0a, 0x07, 0x76, 0x6d, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
0x06, 0x76, 0x6d, 0x50, 0x61, 0x74, 0x68, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61,
0x69, 0x6e, 0x65, 0x72, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x50, 0x61, 0x74, 0x68, 0x12, 0x18,
0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52,
0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x58, 0x0a, 0x0a, 0x53, 0x74, 0x72, 0x69,
0x6e, 0x67, 0x55, 0x73, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x67, 0x69, 0x64, 0x12, 0x26, 0x0a, 0x0e, 0x61, 0x64,
0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x47, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03,
0x28, 0x09, 0x52, 0x0e, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x47, 0x69,
0x64, 0x73, 0x22, 0xca, 0x01, 0x0a, 0x0f, 0x43, 0x6f, 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01,
0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69,
0x6c, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x66,
0x69, 0x6c, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f,
0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65,
0x4d, 0x6f, 0x64, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x64, 0x69, 0x72, 0x5f, 0x6d, 0x6f, 0x64, 0x65,
0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x64, 0x69, 0x72, 0x4d, 0x6f, 0x64, 0x65, 0x12,
0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x75, 0x69,
0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03,
0x67, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x07, 0x20,
0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x64,
0x61, 0x74, 0x61, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22,
0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4f, 0x4f, 0x4d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x2d, 0x0a, 0x08, 0x4f, 0x4f, 0x4d, 0x45, 0x76, 0x65, 0x6e,
0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69,
0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e,
0x65, 0x72, 0x49, 0x64, 0x22, 0x2a, 0x0a, 0x0e, 0x41, 0x64, 0x64, 0x53, 0x77, 0x61, 0x70, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x43, 0x49, 0x50, 0x61, 0x74,
0x68, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x07, 0x50, 0x43, 0x49, 0x50, 0x61, 0x74, 0x68,
0x22, 0x28, 0x0a, 0x12, 0x41, 0x64, 0x64, 0x53, 0x77, 0x61, 0x70, 0x50, 0x61, 0x74, 0x68, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01,
0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x13, 0x0a, 0x11, 0x47, 0x65,
0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22,
0x23, 0x0a, 0x07, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65,
0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x74,
0x72, 0x69, 0x63, 0x73, 0x22, 0x40, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x74,
0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x76, 0x6f,
0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x67, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x47, 0x75, 0x65,
0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x22, 0x55, 0x0a, 0x13, 0x52, 0x65, 0x73, 0x69, 0x7a, 0x65,
0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a,
0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x67, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x70, 0x61,
0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65,
0x47, 0x75, 0x65, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a,
0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x2a, 0x0a,
0x10, 0x53, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x22, 0xfc, 0x03, 0x0a, 0x13, 0x4d, 0x65,
0x6d, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x6d, 0x63, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69,
0x67, 0x12, 0x1f, 0x0a, 0x08, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20,
0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x08, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x88,
0x01, 0x01, 0x12, 0x17, 0x0a, 0x04, 0x73, 0x77, 0x61, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08,
0x48, 0x01, 0x52, 0x04, 0x73, 0x77, 0x61, 0x70, 0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x73,
0x77, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x5f, 0x6d, 0x61, 0x78, 0x18, 0x03, 0x20,
0x01, 0x28, 0x0d, 0x48, 0x02, 0x52, 0x0d, 0x73, 0x77, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x65, 0x73,
0x73, 0x4d, 0x61, 0x78, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x0b, 0x70, 0x65, 0x72, 0x69, 0x6f,
0x64, 0x5f, 0x73, 0x65, 0x63, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x48, 0x03, 0x52, 0x0a,
0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x53, 0x65, 0x63, 0x73, 0x88, 0x01, 0x01, 0x12, 0x3c, 0x0a,
0x18, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x70, 0x73, 0x69, 0x5f, 0x70, 0x65, 0x72, 0x63,
0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x48,
0x04, 0x52, 0x15, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x50, 0x73, 0x69, 0x50, 0x65, 0x72, 0x63,
0x65, 0x6e, 0x74, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x88, 0x01, 0x01, 0x12, 0x40, 0x0a, 0x1a, 0x65,
0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x73, 0x69, 0x5f, 0x70, 0x65, 0x72, 0x63,
0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x48,
0x05, 0x52, 0x17, 0x65, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x73, 0x69, 0x50, 0x65,
0x72, 0x63, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x88, 0x01, 0x01, 0x12, 0x43, 0x0a,
0x1c, 0x65, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x61, 0x67,
0x69, 0x6e, 0x67, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6d, 0x69, 0x6e, 0x18, 0x07, 0x20,
0x01, 0x28, 0x04, 0x48, 0x06, 0x52, 0x18, 0x65, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52,
0x75, 0x6e, 0x41, 0x67, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x4d, 0x69, 0x6e, 0x88,
0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x42,
0x07, 0x0a, 0x05, 0x5f, 0x73, 0x77, 0x61, 0x70, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x73, 0x77, 0x61,
0x70, 0x70, 0x69, 0x6e, 0x65, 0x73, 0x73, 0x5f, 0x6d, 0x61, 0x78, 0x42, 0x0e, 0x0a, 0x0c, 0x5f,
0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x73, 0x65, 0x63, 0x73, 0x42, 0x1b, 0x0a, 0x19, 0x5f,
0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x70, 0x73, 0x69, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65,
0x6e, 0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x42, 0x1d, 0x0a, 0x1b, 0x5f, 0x65, 0x76, 0x69,
0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x73, 0x69, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e,
0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x42, 0x1f, 0x0a, 0x1d, 0x5f, 0x65, 0x76, 0x69, 0x63,
0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x63,
0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6d, 0x69, 0x6e, 0x22, 0xc6, 0x04, 0x0a, 0x15, 0x4d, 0x65, 0x6d,
0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x66,
0x69, 0x67, 0x12, 0x1f, 0x0a, 0x08, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01,
0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x08, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64,
0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x0b, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x73, 0x65,
0x63, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x48, 0x01, 0x52, 0x0a, 0x70, 0x65, 0x72, 0x69,
0x6f, 0x64, 0x53, 0x65, 0x63, 0x73, 0x88, 0x01, 0x01, 0x12, 0x3c, 0x0a, 0x18, 0x70, 0x65, 0x72,
0x69, 0x6f, 0x64, 0x5f, 0x70, 0x73, 0x69, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f,
0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x02, 0x52, 0x15, 0x70,
0x65, 0x72, 0x69, 0x6f, 0x64, 0x50, 0x73, 0x69, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x4c,
0x69, 0x6d, 0x69, 0x74, 0x88, 0x01, 0x01, 0x12, 0x3e, 0x0a, 0x19, 0x63, 0x6f, 0x6d, 0x70, 0x61,
0x63, 0x74, 0x5f, 0x70, 0x73, 0x69, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x6c,
0x69, 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x03, 0x52, 0x16, 0x63, 0x6f,
0x6d, 0x70, 0x61, 0x63, 0x74, 0x50, 0x73, 0x69, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x4c,
0x69, 0x6d, 0x69, 0x74, 0x88, 0x01, 0x01, 0x12, 0x2b, 0x0a, 0x0f, 0x63, 0x6f, 0x6d, 0x70, 0x61,
0x63, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x5f, 0x6d, 0x61, 0x78, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03,
0x48, 0x04, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x53, 0x65, 0x63, 0x4d, 0x61,
0x78, 0x88, 0x01, 0x01, 0x12, 0x28, 0x0a, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f,
0x6f, 0x72, 0x64, 0x65, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x05, 0x52, 0x0c, 0x63,
0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x88, 0x01, 0x01, 0x12, 0x30,
0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68,
0x6f, 0x6c, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x48, 0x06, 0x52, 0x10, 0x63, 0x6f, 0x6d,
0x70, 0x61, 0x63, 0x74, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x88, 0x01, 0x01,
0x12, 0x33, 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x66, 0x6f, 0x72, 0x63,
0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x48, 0x07, 0x52,
0x11, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x46, 0x6f, 0x72, 0x63, 0x65, 0x54, 0x69, 0x6d,
0x65, 0x73, 0x88, 0x01, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c,
0x65, 0x64, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x73, 0x65,
0x63, 0x73, 0x42, 0x1b, 0x0a, 0x19, 0x5f, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x70, 0x73,
0x69, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x42,
0x1f, 0x0a, 0x1d, 0x5f, 0x65, 0x76, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x75, 0x6e,
0x5f, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x6d, 0x69, 0x6e,
0x22, 0xc6, 0x04, 0x0a, 0x15, 0x4d, 0x65, 0x6d, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6d,
0x70, 0x61, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x08, 0x64, 0x69,
0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x08,
0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x0b, 0x70,
0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x73, 0x65, 0x63, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04,
0x48, 0x01, 0x52, 0x0a, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x53, 0x65, 0x63, 0x73, 0x88, 0x01,
0x01, 0x12, 0x3c, 0x0a, 0x18, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x70, 0x73, 0x69, 0x5f,
0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20,
0x01, 0x28, 0x0d, 0x48, 0x02, 0x52, 0x15, 0x70, 0x65, 0x72, 0x69, 0x6f, 0x64, 0x50, 0x73, 0x69,
0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x88, 0x01, 0x01, 0x12,
0x3e, 0x0a, 0x19, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x70, 0x73, 0x69, 0x5f, 0x70,
0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x04, 0x20, 0x01,
0x28, 0x0d, 0x48, 0x03, 0x52, 0x16, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x50, 0x73, 0x69,
0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x88, 0x01, 0x01, 0x12,
0x2b, 0x0a, 0x0f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x5f, 0x6d,
0x61, 0x78, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x48, 0x04, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70,
0x61, 0x63, 0x74, 0x53, 0x65, 0x63, 0x4d, 0x61, 0x78, 0x88, 0x01, 0x01, 0x12, 0x28, 0x0a, 0x0d,
0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x18, 0x06, 0x20,
0x01, 0x28, 0x0d, 0x48, 0x05, 0x52, 0x0c, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x4f, 0x72,
0x64, 0x65, 0x72, 0x88, 0x01, 0x01, 0x12, 0x30, 0x0a, 0x11, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63,
0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28,
0x04, 0x48, 0x06, 0x52, 0x10, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x54, 0x68, 0x72, 0x65,
0x73, 0x68, 0x6f, 0x6c, 0x64, 0x88, 0x01, 0x01, 0x12, 0x33, 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x70,
0x61, 0x63, 0x74, 0x5f, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x18,
0x08, 0x20, 0x01, 0x28, 0x04, 0x48, 0x07, 0x52, 0x11, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74,
0x46, 0x6f, 0x72, 0x63, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x88, 0x01, 0x01, 0x42, 0x0b, 0x0a,
0x09, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x70,
0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x73, 0x65, 0x63, 0x73, 0x42, 0x1b, 0x0a, 0x19, 0x5f, 0x70,
0x65, 0x72, 0x69, 0x6f, 0x64, 0x5f, 0x70, 0x73, 0x69, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e,
0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x42, 0x1c, 0x0a, 0x1a, 0x5f, 0x63, 0x6f, 0x6d, 0x70,
0x61, 0x63, 0x74, 0x5f, 0x70, 0x73, 0x69, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f,
0x6c, 0x69, 0x6d, 0x69, 0x74, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63,
0x74, 0x5f, 0x73, 0x65, 0x63, 0x5f, 0x6d, 0x61, 0x78, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x63, 0x6f,
0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x42, 0x14, 0x0a, 0x12, 0x5f,
0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c,
0x64, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x66, 0x6f,
0x72, 0x63, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x32, 0x94, 0x16, 0x0a, 0x0c, 0x41, 0x67,
0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x47, 0x0a, 0x0f, 0x43, 0x72,
0x65, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1c, 0x2e,
0x67, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61,
0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f,
0x1c, 0x0a, 0x1a, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x70, 0x73, 0x69, 0x5f,
0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x42, 0x12, 0x0a,
0x10, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x5f, 0x6d, 0x61,
0x78, 0x42, 0x10, 0x0a, 0x0e, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x6f, 0x72,
0x64, 0x65, 0x72, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f,
0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x42, 0x16, 0x0a, 0x14, 0x5f, 0x63, 0x6f,
0x6d, 0x70, 0x61, 0x63, 0x74, 0x5f, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65,
0x73, 0x32, 0x94, 0x16, 0x0a, 0x0c, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x12, 0x47, 0x0a, 0x0f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74,
0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x72, 0x65,
0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x45, 0x0a, 0x0e, 0x53,
0x74, 0x61, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1b, 0x2e,
0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69,
0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f,
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70,
0x74, 0x79, 0x12, 0x47, 0x0a, 0x0f, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74,
0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x6d,
0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3f, 0x0a, 0x0b, 0x45,
0x78, 0x65, 0x63, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x18, 0x2e, 0x67, 0x72, 0x70,
0x63, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x0d,
0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x2e,
0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x63, 0x65,
0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74,
0x79, 0x12, 0x42, 0x0a, 0x0b, 0x57, 0x61, 0x69, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73,
0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x50, 0x72, 0x6f, 0x63,
0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x67, 0x72, 0x70,
0x63, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a, 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43,
0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53,
0x0a, 0x15, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61,
0x6c, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x55,
0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x4d, 0x6f,
0x75, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d,
0x70, 0x74, 0x79, 0x12, 0x45, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x74,
0x70, 0x74, 0x79, 0x12, 0x4b, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x74,
0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61,
0x72, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x47, 0x0a, 0x0f, 0x52, 0x65,
0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1c, 0x2e,
0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61,
0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d,
0x70, 0x74, 0x79, 0x12, 0x3f, 0x0a, 0x0b, 0x45, 0x78, 0x65, 0x63, 0x50, 0x72, 0x6f, 0x63, 0x65,
0x73, 0x73, 0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x50, 0x72,
0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67,
0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45,
0x6d, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x0d, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x50, 0x72,
0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x69, 0x67,
0x6e, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x42, 0x0a, 0x0b, 0x57, 0x61, 0x69,
0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
0x57, 0x61, 0x69, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x50, 0x72,
0x6f, 0x63, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x47, 0x0a,
0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
0x12, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f,
0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16,
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53, 0x0a, 0x15, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65,
0x45, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x12,
0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x45, 0x70, 0x68,
0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75,
0x74, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x43,
0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x12, 0x45, 0x0a, 0x0e, 0x50, 0x61, 0x75, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e,
0x65, 0x72, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x61, 0x75, 0x73, 0x65, 0x43,
0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x47, 0x0a, 0x0f, 0x52, 0x65, 0x73, 0x75, 0x6d,
0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1c, 0x2e, 0x67, 0x72, 0x70,
0x63, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65,
0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
0x12, 0x65, 0x0a, 0x1e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x74, 0x61, 0x6c, 0x65, 0x56,
0x69, 0x72, 0x74, 0x69, 0x6f, 0x66, 0x73, 0x53, 0x68, 0x61, 0x72, 0x65, 0x4d, 0x6f, 0x75, 0x6e,
0x74, 0x73, 0x12, 0x2b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65,
0x53, 0x74, 0x61, 0x6c, 0x65, 0x56, 0x69, 0x72, 0x74, 0x69, 0x6f, 0x66, 0x73, 0x53, 0x68, 0x61,
0x72, 0x65, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x41, 0x0a, 0x0a, 0x57, 0x72, 0x69, 0x74, 0x65,
0x53, 0x74, 0x64, 0x69, 0x6e, 0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x57, 0x72, 0x69,
0x74, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x53, 0x74, 0x72, 0x65,
0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x0a, 0x52, 0x65,
0x61, 0x64, 0x53, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x12, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
0x52, 0x65, 0x61, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x53, 0x74, 0x72,
0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x0a, 0x52,
0x65, 0x61, 0x64, 0x53, 0x74, 0x64, 0x65, 0x72, 0x72, 0x12, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63,
0x2e, 0x52, 0x65, 0x61, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x53, 0x74,
0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x0a,
0x43, 0x6c, 0x6f, 0x73, 0x65, 0x53, 0x74, 0x64, 0x69, 0x6e, 0x12, 0x17, 0x2e, 0x67, 0x72, 0x70,
0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x53, 0x74, 0x64, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x4b, 0x0a, 0x0e, 0x53,
0x74, 0x61, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1b, 0x2e,
0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69,
0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70,
0x63, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0e, 0x50, 0x61, 0x75, 0x73,
0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70,
0x63, 0x2e, 0x50, 0x61, 0x75, 0x73, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12,
0x47, 0x0a, 0x0f, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e,
0x65, 0x72, 0x12, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65,
0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x65, 0x0a, 0x1e, 0x52, 0x65, 0x6d, 0x6f,
0x76, 0x65, 0x53, 0x74, 0x61, 0x6c, 0x65, 0x56, 0x69, 0x72, 0x74, 0x69, 0x6f, 0x66, 0x73, 0x53,
0x68, 0x61, 0x72, 0x65, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x12, 0x2b, 0x2e, 0x67, 0x72, 0x70,
0x63, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x74, 0x61, 0x6c, 0x65, 0x56, 0x69, 0x72,
0x74, 0x69, 0x6f, 0x66, 0x73, 0x53, 0x68, 0x61, 0x72, 0x65, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x73,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12,
0x41, 0x0a, 0x0a, 0x57, 0x72, 0x69, 0x74, 0x65, 0x53, 0x74, 0x64, 0x69, 0x6e, 0x12, 0x18, 0x2e,
0x67, 0x72, 0x70, 0x63, 0x2e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x57,
0x72, 0x69, 0x74, 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x12, 0x3f, 0x0a, 0x0a, 0x52, 0x65, 0x61, 0x64, 0x53, 0x74, 0x64, 0x6f, 0x75, 0x74,
0x12, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x53, 0x74, 0x72, 0x65,
0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63,
0x2e, 0x52, 0x65, 0x61, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x0a, 0x52, 0x65, 0x61, 0x64, 0x53, 0x74, 0x64, 0x65, 0x72,
0x72, 0x12, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x53, 0x74, 0x72,
0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x67, 0x72, 0x70,
0x63, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x0a, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x53, 0x74, 0x64,
0x69, 0x6e, 0x12, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x53,
0x74, 0x64, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d,
0x70, 0x74, 0x79, 0x12, 0x41, 0x0a, 0x0c, 0x54, 0x74, 0x79, 0x57, 0x69, 0x6e, 0x52, 0x65, 0x73,
0x69, 0x7a, 0x65, 0x12, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x54, 0x74, 0x79, 0x57, 0x69,
0x6e, 0x52, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16,
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x41, 0x0a, 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65,
0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63,
0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e,
0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x37, 0x0a, 0x0c, 0x55, 0x70, 0x64,
0x61, 0x74, 0x65, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63,
0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x6f, 0x75, 0x74,
0x65, 0x73, 0x12, 0x3f, 0x0a, 0x0e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66,
0x61, 0x63, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x69, 0x73, 0x74,
0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x10, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61,
0x63, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65,
0x73, 0x12, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75,
0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x67, 0x72, 0x70,
0x63, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x47, 0x0a, 0x0f, 0x41, 0x64, 0x64, 0x41,
0x52, 0x50, 0x4e, 0x65, 0x69, 0x67, 0x68, 0x62, 0x6f, 0x72, 0x73, 0x12, 0x1c, 0x2e, 0x67, 0x72,
0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x41, 0x52, 0x50, 0x4e, 0x65, 0x69, 0x67, 0x68, 0x62, 0x6f,
0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74,
0x79, 0x12, 0x42, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x49, 0x50, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73,
0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x50, 0x54, 0x61, 0x62,
0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x67, 0x72, 0x70,
0x63, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x50, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x49, 0x50, 0x54, 0x61,
0x62, 0x6c, 0x65, 0x73, 0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x65, 0x74, 0x49,
0x50, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19,
0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x65, 0x74, 0x49, 0x50, 0x54, 0x61, 0x62, 0x6c, 0x65,
0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x0a, 0x47, 0x65, 0x74,
0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47,
0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12,
0x45, 0x0a, 0x10, 0x4d, 0x65, 0x6d, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x6d, 0x63, 0x67,
0x53, 0x65, 0x74, 0x12, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x6d, 0x41, 0x67,
0x65, 0x6e, 0x74, 0x4d, 0x65, 0x6d, 0x63, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x16,
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x49, 0x0a, 0x12, 0x4d, 0x65, 0x6d, 0x41, 0x67, 0x65,
0x6e, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x53, 0x65, 0x74, 0x12, 0x1b, 0x2e, 0x67,
0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x6d, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6d, 0x70,
0x61, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74,
0x79, 0x12, 0x43, 0x0a, 0x0d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x61, 0x6e, 0x64, 0x62,
0x6f, 0x78, 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65,
0x53, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16,
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x45, 0x0a, 0x0e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f,
0x79, 0x53, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x53, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x41, 0x0a,
0x0c, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x43, 0x50, 0x55, 0x4d, 0x65, 0x6d, 0x12, 0x19, 0x2e,
0x67, 0x72, 0x70, 0x63, 0x2e, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x43, 0x50, 0x55, 0x4d, 0x65,
0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
0x12, 0x47, 0x0a, 0x0f, 0x52, 0x65, 0x73, 0x65, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x64, 0x6f, 0x6d,
0x44, 0x65, 0x76, 0x12, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x65,
0x64, 0x52, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x44, 0x65, 0x76, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x41, 0x0a, 0x0c, 0x54,
0x74, 0x79, 0x57, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x19, 0x2e, 0x67, 0x72,
0x70, 0x63, 0x2e, 0x54, 0x74, 0x79, 0x57, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x41,
0x0a, 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63,
0x65, 0x12, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49,
0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x10, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63,
0x65, 0x12, 0x37, 0x0a, 0x0c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x6f, 0x75, 0x74, 0x65,
0x73, 0x12, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52,
0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x67,
0x72, 0x70, 0x63, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x3f, 0x0a, 0x0e, 0x4c, 0x69,
0x73, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x67,
0x72, 0x70, 0x63, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63,
0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x67, 0x72, 0x70, 0x63,
0x2e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x0a, 0x4c,
0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63,
0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x0c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73,
0x12, 0x47, 0x0a, 0x0f, 0x41, 0x64, 0x64, 0x41, 0x52, 0x50, 0x4e, 0x65, 0x69, 0x67, 0x68, 0x62,
0x6f, 0x72, 0x73, 0x12, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x41, 0x52,
0x50, 0x4e, 0x65, 0x69, 0x67, 0x68, 0x62, 0x6f, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x48, 0x0a, 0x0f, 0x47, 0x65, 0x74,
0x47, 0x75, 0x65, 0x73, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x19, 0x2e, 0x67,
0x72, 0x70, 0x63, 0x2e, 0x47, 0x75, 0x65, 0x73, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47,
0x75, 0x65, 0x73, 0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x11, 0x4d, 0x65, 0x6d, 0x48, 0x6f, 0x74, 0x70, 0x6c, 0x75,
0x67, 0x42, 0x79, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x12, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
0x4d, 0x65, 0x6d, 0x48, 0x6f, 0x74, 0x70, 0x6c, 0x75, 0x67, 0x42, 0x79, 0x50, 0x72, 0x6f, 0x62,
0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x42, 0x0a, 0x0b, 0x47, 0x65, 0x74,
0x49, 0x50, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
0x47, 0x65, 0x74, 0x49, 0x50, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x50, 0x54,
0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a,
0x0b, 0x53, 0x65, 0x74, 0x49, 0x50, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x18, 0x2e, 0x67,
0x72, 0x70, 0x63, 0x2e, 0x53, 0x65, 0x74, 0x49, 0x50, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x65,
0x74, 0x49, 0x50, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x12, 0x34, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12,
0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63,
0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x45, 0x0a, 0x10, 0x4d, 0x65, 0x6d, 0x41, 0x67,
0x65, 0x6e, 0x74, 0x4d, 0x65, 0x6d, 0x63, 0x67, 0x53, 0x65, 0x74, 0x12, 0x19, 0x2e, 0x67, 0x72,
0x70, 0x63, 0x2e, 0x4d, 0x65, 0x6d, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x6d, 0x63, 0x67,
0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x49,
0x0a, 0x12, 0x4d, 0x65, 0x6d, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x63,
0x74, 0x53, 0x65, 0x74, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x6d, 0x41,
0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69,
0x67, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x0d, 0x43, 0x72, 0x65,
0x61, 0x74, 0x65, 0x53, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70,
0x63, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x45,
0x0a, 0x0e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x53, 0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78,
0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x73, 0x74, 0x72, 0x6f, 0x79, 0x53,
0x61, 0x6e, 0x64, 0x62, 0x6f, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e,
0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x41, 0x0a, 0x0c, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x43,
0x50, 0x55, 0x4d, 0x65, 0x6d, 0x12, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4f, 0x6e, 0x6c,
0x69, 0x6e, 0x65, 0x43, 0x50, 0x55, 0x4d, 0x65, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x47, 0x0a, 0x0f, 0x52, 0x65, 0x73, 0x65,
0x65, 0x64, 0x52, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x44, 0x65, 0x76, 0x12, 0x1c, 0x2e, 0x67, 0x72,
0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x44,
0x65, 0x76, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74,
0x79, 0x12, 0x48, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x47, 0x75, 0x65, 0x73, 0x74, 0x44, 0x65, 0x74,
0x61, 0x69, 0x6c, 0x73, 0x12, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x75, 0x65, 0x73,
0x74, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x75, 0x65, 0x73, 0x74, 0x44, 0x65, 0x74, 0x61,
0x69, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x11, 0x4d,
0x65, 0x6d, 0x48, 0x6f, 0x74, 0x70, 0x6c, 0x75, 0x67, 0x42, 0x79, 0x50, 0x72, 0x6f, 0x62, 0x65,
0x12, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x65, 0x6d, 0x48, 0x6f, 0x74, 0x70, 0x6c,
0x75, 0x67, 0x42, 0x79, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x49, 0x0a, 0x10, 0x53, 0x65, 0x74, 0x47,
0x75, 0x65, 0x73, 0x74, 0x44, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x67,
0x72, 0x70, 0x63, 0x2e, 0x53, 0x65, 0x74, 0x47, 0x75, 0x65, 0x73, 0x74, 0x44, 0x61, 0x74, 0x65,
0x54, 0x69, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d,
0x70, 0x74, 0x79, 0x12, 0x39, 0x0a, 0x08, 0x43, 0x6f, 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12,
0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6f, 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x37,
0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4f, 0x4f, 0x4d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x18, 0x2e,
0x67, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x4f, 0x4d, 0x45, 0x76, 0x65, 0x6e, 0x74,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4f,
0x4f, 0x4d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x07, 0x41, 0x64, 0x64, 0x53, 0x77,
0x61, 0x70, 0x12, 0x14, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x53, 0x77, 0x61,
0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
0x12, 0x49, 0x0a, 0x10, 0x53, 0x65, 0x74, 0x47, 0x75, 0x65, 0x73, 0x74, 0x44, 0x61, 0x74, 0x65,
0x54, 0x69, 0x6d, 0x65, 0x12, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x65, 0x74, 0x47,
0x75, 0x65, 0x73, 0x74, 0x44, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75,
0x12, 0x3f, 0x0a, 0x0b, 0x41, 0x64, 0x64, 0x53, 0x77, 0x61, 0x70, 0x50, 0x61, 0x74, 0x68, 0x12,
0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x53, 0x77, 0x61, 0x70, 0x50, 0x61,
0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74,
0x79, 0x12, 0x45, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x74,
0x61, 0x74, 0x73, 0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d,
0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e,
0x67, 0x72, 0x70, 0x63, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x69,
0x7a, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
0x52, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x39, 0x0a, 0x08, 0x43,
0x6f, 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x43,
0x6f, 0x70, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16,
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x37, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4f, 0x4f, 0x4d,
0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74,
0x4f, 0x4f, 0x4d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4f, 0x4f, 0x4d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12,
0x37, 0x0a, 0x07, 0x41, 0x64, 0x64, 0x53, 0x77, 0x61, 0x70, 0x12, 0x14, 0x2e, 0x67, 0x72, 0x70,
0x63, 0x2e, 0x41, 0x64, 0x64, 0x53, 0x77, 0x61, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3b, 0x0a, 0x09, 0x53,
0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e,
0x53, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3f, 0x0a, 0x0b, 0x41, 0x64, 0x64, 0x53,
0x77, 0x61, 0x70, 0x50, 0x61, 0x74, 0x68, 0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x41,
0x64, 0x64, 0x53, 0x77, 0x61, 0x70, 0x50, 0x61, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x45, 0x0a, 0x0e, 0x47, 0x65, 0x74,
0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x18, 0x2e, 0x67, 0x72,
0x70, 0x63, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x56, 0x6f, 0x6c,
0x75, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x12, 0x41, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65,
0x12, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x69, 0x7a, 0x65, 0x56, 0x6f,
0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d,
0x70, 0x74, 0x79, 0x12, 0x3b, 0x0a, 0x09, 0x53, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79,
0x12, 0x16, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x65, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63,
0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
0x42, 0x60, 0x5a, 0x5e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b,
0x61, 0x74, 0x61, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x2f, 0x6b,
0x61, 0x74, 0x61, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x2f, 0x73,
0x72, 0x63, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x69, 0x72, 0x74, 0x63,
0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x67,
0x65, 0x6e, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x2f, 0x67, 0x72,
0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x60, 0x5a, 0x5e, 0x67, 0x69, 0x74, 0x68,
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x61, 0x74, 0x61, 0x2d, 0x63, 0x6f, 0x6e, 0x74,
0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x2f, 0x6b, 0x61, 0x74, 0x61, 0x2d, 0x63, 0x6f, 0x6e, 0x74,
0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69,
0x6d, 0x65, 0x2f, 0x76, 0x69, 0x72, 0x74, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
}
var (

View File

@@ -34,6 +34,7 @@ import (
"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
deviceManager "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager"
volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol"
exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
@@ -108,6 +109,7 @@ type SandboxStatus struct {
ContainersStatus []ContainerStatus
State types.SandboxState
HypervisorConfig HypervisorConfig
EmptyDirMode string
}
// SandboxStats describes a sandbox's stats
@@ -180,6 +182,10 @@ type SandboxConfig struct {
// DisableGuestSeccomp disable seccomp within the guest
DisableGuestSeccomp bool
// EmptyDirMode specifies how Kubernetes emptyDir volumes are handled.
// Valid values are "shared-fs" (default) or "block-encrypted".
EmptyDirMode string
// EnableVCPUsPinning controls whether each vCPU thread should be scheduled to a fixed CPU
EnableVCPUsPinning bool
@@ -226,8 +232,9 @@ type Sandbox struct {
store persistapi.PersistDriver
fsShare FilesystemSharer
swapDevices []*config.BlockDrive
volumes []types.Volume
swapDevices []*config.BlockDrive
volumes []types.Volume
ephemeralDisks []EphemeralDisk
monitor *monitor
config *SandboxConfig
@@ -381,6 +388,7 @@ func (s *Sandbox) Status() SandboxStatus {
HypervisorConfig: s.config.HypervisorConfig,
ContainersStatus: contStatusList,
Annotations: s.config.Annotations,
EmptyDirMode: s.config.EmptyDirMode,
}
}
@@ -998,9 +1006,31 @@ func (s *Sandbox) Delete(ctx context.Context) error {
s.Logger().WithError(err).Error("failed to cleanup share files")
}
if err := s.cleanupEphemeralDisks(); err != nil {
s.Logger().WithError(err).Error("failed to cleanup ephemeral disks")
}
return s.store.Destroy(s.id)
}
// cleanupEphemeralDisks removes ephemeral disk images and their mount info.
func (s *Sandbox) cleanupEphemeralDisks() error {
if s.config.EmptyDirMode != EmptyDirModeVirtioBlkEncrypted {
return nil
}
for _, disk := range s.ephemeralDisks {
if err := os.Remove(disk.DiskPath); err != nil && !os.IsNotExist(err) {
s.Logger().WithError(err).Errorf("Failed to remove disk file: %s", disk.DiskPath)
}
if err := volume.Remove(disk.SourcePath); err != nil && !os.IsNotExist(err) {
s.Logger().WithError(err).Errorf("Failed to remove volume: %s", disk.SourcePath)
}
}
return nil
}
func (s *Sandbox) createNetwork(ctx context.Context) error {
if s.config.NetworkConfig.DisableNewNetwork ||
s.config.NetworkConfig.NetworkID == "" {

View File

@@ -1,2 +1 @@
src/version.rs
tests/**/layers-cache.json

View File

@@ -644,6 +644,12 @@ dependencies = [
"syn 2.0.104",
]
[[package]]
name = "data-encoding"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476"
[[package]]
name = "derive-new"
version = "0.5.9"
@@ -2420,6 +2426,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843c3d97f07e3b5ac0955d53ad0af4c91fe4a4f8525843ece5bf014f27829b73"
dependencies = [
"anyhow",
"data-encoding",
"lazy_static",
"rand",
"regex",

View File

@@ -4,6 +4,11 @@
"path": "/cluster_config/guest_pull",
"value": false
},
{
"op": "replace",
"path": "/cluster_config/encrypted_emptydir",
"value": false
},
{
"op": "replace",
"path": "/cluster_config/pause_container_id_policy",

View File

@@ -4,6 +4,11 @@
"path": "/cluster_config/guest_pull",
"value": false
},
{
"op": "replace",
"path": "/cluster_config/encrypted_emptydir",
"value": false
},
{
"op": "replace",
"path": "/cluster_config/pause_container_id_policy",

View File

@@ -4,6 +4,11 @@
"path": "/cluster_config/guest_pull",
"value": false
},
{
"op": "replace",
"path": "/cluster_config/encrypted_emptydir",
"value": false
},
{
"op": "replace",
"path": "/common/root_path",

View File

@@ -0,0 +1,7 @@
[
{
"op": "replace",
"path": "/cluster_config/encrypted_emptydir",
"value": false
}
]

View File

@@ -157,26 +157,44 @@
"mount_source": "^$(cpath)/$(sandbox-id)/rootfs/local/",
"mount_point": "^$(cpath)/$(sandbox-id)/rootfs/local/",
"driver": "local",
"driver_options": [],
"source": "local",
"fstype": "local",
"options": [
"mode=0777"
]
],
"shared": false
},
"emptyDir_encrypted": {
"mount_source": "",
"mount_type": "bind",
"driver": "",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "",
"mount_point": "^$(spath)/$(b64_device_id)$",
"fstype": "ext4",
"options": [],
"shared": true
},
"emptyDir_memory": {
"mount_type": "bind",
"mount_source": "^/run/kata-containers/sandbox/ephemeral/",
"mount_point": "^/run/kata-containers/sandbox/ephemeral/",
"driver": "ephemeral",
"driver_options": [],
"source": "tmpfs",
"fstype": "tmpfs",
"options": []
"options": [],
"shared": false
},
"configMap": {
"mount_type": "bind",
"mount_source": "$(sfprefix)",
"mount_point": "$(sfprefix)",
"driver": "local",
"driver_options": [],
"fstype": "bind",
"options": [
"rbind",
@@ -241,6 +259,7 @@
},
"common": {
"cpath": "/run/kata-containers/shared/containers(?:/passthrough)?",
"spath": "/run/kata-containers/sandbox/storage",
"root_path": "/run/kata-containers/$(bundle-id)/rootfs",
"sfprefix": "^$(cpath)/(watchable/)?$(bundle-id)-[a-z0-9]{16}-",
"ip_p": "[0-9]{1,5}",
@@ -315,7 +334,8 @@
"cluster_config": {
"pause_container_image": "mcr.microsoft.com/oss/kubernetes/pause:3.6",
"guest_pull": true,
"pause_container_id_policy": "v1"
"pause_container_id_policy": "v1",
"encrypted_emptydir": true
},
"request_defaults": {
"CreateContainerRequest": {

View File

@@ -801,10 +801,12 @@ allow_by_bundle_or_sandbox_id(p_oci, i_oci, p_storages, i_storages) if {
# Match each input mount with a Policy mount.
# Reject possible attempts to match multiple input mounts with a single Policy mount.
p_matches := { p_index | some i_index; p_index = allow_mount(p_oci, input.OCI.Mounts[i_index], bundle_id, sandbox_id) }
print("allow_by_bundle_or_sandbox_id: p_oci.Mounts =", p_oci.Mounts)
print("allow_by_bundle_or_sandbox_id: i_oci.Mounts =", i_oci.Mounts)
p_matches := { p_index | some i_index; p_index = allow_mount(p_oci, i_oci.Mounts[i_index], i_storages, bundle_id, sandbox_id) }
print("allow_by_bundle_or_sandbox_id: p_matches =", p_matches)
count(p_matches) == count(input.OCI.Mounts)
count(p_matches) == count(i_oci.Mounts)
allow_storages(p_storages, i_storages, bundle_id, sandbox_id)
@@ -1098,30 +1100,45 @@ is_ip_other_byte(component) if {
# device mounts
# allow_mount returns the policy index (p_index) if a given input mount matches a policy mount.
allow_mount(p_oci, i_mount, bundle_id, sandbox_id):= p_index if {
print("-------- allow_mount: i_mount =", i_mount)
allow_mount(p_oci, i_mount, i_storages, bundle_id, sandbox_id):= p_index if {
print("-------- allow_mount 1: i_mount =", i_mount)
some p_index, p_mount in p_oci.Mounts
print("allow_mount: p_index =", p_index, "p_mount =", p_mount)
check_mount(p_mount, i_mount, bundle_id, sandbox_id)
some i_storage in i_storages
print("allow_mount: true, p_index =", p_index)
print("allow_mount 1: p_index =", p_index, "p_mount =", p_mount)
check_mount(p_mount, i_mount, i_storage, bundle_id, sandbox_id)
print("allow_mount 1: true, p_index =", p_index)
}
# This rule is identical to the first one, minus the i_storage requirement:
# - Rule 1 accepts mounts that rule 2 rejects (e.g. when i_storage.mount_point == i_mount.source).
# - Rule 2 does NOT accept all the mounts that rule 1 rejects (because it's missing i_storage).
allow_mount(p_oci, i_mount, i_storages, bundle_id, sandbox_id):= p_index if {
print("allow_mount 2: i_mount =", i_mount)
some p_index, p_mount in p_oci.Mounts
print("allow_mount 2: p_index =", p_index, "p_mount =", p_mount)
check_mount(p_mount, i_mount, null, bundle_id, sandbox_id)
print("allow_mount 2: true, p_index =", p_index)
}
check_mount(p_mount, i_mount, bundle_id, sandbox_id) if {
check_mount(p_mount, i_mount, i_storage, bundle_id, sandbox_id) if {
p_mount == i_mount
print("check_mount 1: true")
}
check_mount(p_mount, i_mount, bundle_id, sandbox_id) if {
check_mount(p_mount, i_mount, i_storage, bundle_id, sandbox_id) if {
p_mount.destination == i_mount.destination
p_mount.type_ == i_mount.type_
p_mount.options == i_mount.options
mount_source_allows(p_mount, i_mount, bundle_id, sandbox_id)
mount_source_allows(p_mount, i_mount, i_storage, bundle_id, sandbox_id)
print("check_mount 2: true")
}
check_mount(p_mount, i_mount, bundle_id, sandbox_id) if {
check_mount(p_mount, i_mount, i_storage, bundle_id, sandbox_id) if {
# This check passes if the policy container has RW, the input container has
# RO and the volume type is sysfs, working around different handling of
# privileged containers after containerd 2.0.4.
@@ -1136,7 +1153,9 @@ check_mount(p_mount, i_mount, bundle_id, sandbox_id) if {
print("check_mount 3: true")
}
mount_source_allows(p_mount, i_mount, bundle_id, sandbox_id) if {
mount_source_allows(p_mount, i_mount, i_storage, bundle_id, sandbox_id) if {
p_mount.source != ""
regex1 := p_mount.source
print("mount_source_allows 1: regex1 =", regex1)
@@ -1152,7 +1171,9 @@ mount_source_allows(p_mount, i_mount, bundle_id, sandbox_id) if {
print("mount_source_allows 1: true")
}
mount_source_allows(p_mount, i_mount, bundle_id, sandbox_id) if {
mount_source_allows(p_mount, i_mount, i_storage, bundle_id, sandbox_id) if {
p_mount.source != ""
regex1 := p_mount.source
print("mount_source_allows 2: regex1 =", regex1)
@@ -1168,6 +1189,10 @@ mount_source_allows(p_mount, i_mount, bundle_id, sandbox_id) if {
print("mount_source_allows 2: true")
}
mount_source_allows(p_mount, i_mount, i_storage, bundle_id, sandbox_id) if {
i_storage.mount_point == i_mount.source
print("mount_source_allows 3: true")
}
######################################################################
# Create container Storages
@@ -1196,14 +1221,10 @@ allow_storage(p_storages, i_storage, bundle_id, sandbox_id) if {
print("allow_storage: p_storage =", p_storage)
print("allow_storage: i_storage =", i_storage)
p_storage.driver == i_storage.driver
p_storage.driver_options == i_storage.driver_options
p_storage.fs_group == i_storage.fs_group
p_storage.fstype == i_storage.fstype
p_storage.driver == i_storage.driver
allow_storage_source(p_storage, i_storage, bundle_id)
allow_storage_options(p_storage, i_storage)
allow_mount_point(p_storage, i_storage, bundle_id, sandbox_id)
allow_storage_base(p_storage, i_storage, bundle_id, sandbox_id)
print("allow_storage: true")
}
@@ -1212,10 +1233,58 @@ allow_storage(p_storages, i_storage, bundle_id, sandbox_id) if {
print("allow_storage with image_guest_pull: start")
i_storage.fstype == "overlay"
i_storage.fs_group == null
i_storage.shared == false
count(i_storage.options) == 0
# TODO: Check Mount Point, Source, Driver Options, etc.
print("allow_storage with image_guest_pull: true")
}
allow_storage(p_storages, i_storage, bundle_id, sandbox_id) if {
print("allow_storage with scsi: start")
i_storage.driver == "scsi"
regex.match("^[0-9]+:[0-9]+$", i_storage.source)
allow_block_storage(p_storages, i_storage, bundle_id, sandbox_id)
print("allow_storage with scsi: true")
}
allow_storage(p_storages, i_storage, bundle_id, sandbox_id) if {
print("allow_storage with blk: start")
i_storage.driver == "blk"
regex.match("^[0-9]{2}/[0-9]{2}$", i_storage.source)
allow_block_storage(p_storages, i_storage, bundle_id, sandbox_id)
print("allow_storage with blk: true")
}
# Validates all storage fields except driver and source.
allow_storage_base(p_storage, i_storage, bundle_id, sandbox_id) if {
# Not logging as this is reused multiple times.
p_storage.driver_options == i_storage.driver_options
p_storage.fs_group == i_storage.fs_group
p_storage.fstype == i_storage.fstype
p_storage.shared == i_storage.shared
allow_mount_point(p_storage, i_storage, bundle_id, sandbox_id)
allow_storage_options(p_storage, i_storage)
}
allow_block_storage(p_storages, i_storage, bundle_id, sandbox_id) if {
print("allow_block_storage: start")
some p_storage in p_storages
# This matches the `volumes.emptyDir_encrypted` setting.
# `i_storage.driver` is validated by the caller.
p_storage.driver == ""
allow_storage_base(p_storage, i_storage, bundle_id, sandbox_id)
print("allow_block_storage: true")
}
allow_storage_source(p_storage, i_storage, bundle_id) if {
print("allow_storage_source 1: start")
@@ -1249,7 +1318,6 @@ allow_storage_source(p_storage, i_storage, bundle_id) if {
allow_storage_options(p_storage, i_storage) if {
print("allow_storage_options 1: start")
p_storage.driver != "blk"
p_storage.driver != "overlayfs"
p_storage.options == i_storage.options
@@ -1257,6 +1325,8 @@ allow_storage_options(p_storage, i_storage) if {
}
allow_mount_point(p_storage, i_storage, bundle_id, sandbox_id) if {
print("allow_mount_point 1: start")
p_storage.fstype == "local"
mount1 := p_storage.mount_point
@@ -1273,6 +1343,8 @@ allow_mount_point(p_storage, i_storage, bundle_id, sandbox_id) if {
print("allow_mount_point 1: true")
}
allow_mount_point(p_storage, i_storage, bundle_id, sandbox_id) if {
print("allow_mount_point 2: start")
p_storage.fstype == "bind"
mount1 := p_storage.mount_point
@@ -1289,6 +1361,8 @@ allow_mount_point(p_storage, i_storage, bundle_id, sandbox_id) if {
print("allow_mount_point 2: true")
}
allow_mount_point(p_storage, i_storage, bundle_id, sandbox_id) if {
print("allow_mount_point 3: start")
p_storage.fstype == "tmpfs"
mount1 := p_storage.mount_point
@@ -1298,6 +1372,39 @@ allow_mount_point(p_storage, i_storage, bundle_id, sandbox_id) if {
print("allow_mount_point 3: true")
}
allow_mount_point(p_storage, i_storage, bundle_id, sandbox_id) if {
print("allow_mount_point 4: start")
i_storage.driver == "blk"
allow_mount_point_by_device_id(p_storage, i_storage)
print("allow_mount_point 4: true")
}
allow_mount_point(p_storage, i_storage, bundle_id, sandbox_id) if {
print("allow_mount_point 5: start")
i_storage.driver == "scsi"
allow_mount_point_by_device_id(p_storage, i_storage)
print("allow_mount_point 5: true")
}
allow_mount_point_by_device_id(p_storage, i_storage) if {
print("allow_mount_point_by_device_id: start")
mount1 := p_storage.mount_point
print("allow_mount_point_by_device_id: mount1 =", mount1)
mount2 := replace(mount1, "$(spath)", policy_data.common.spath)
print("allow_mount_point_by_device_id: mount2 =", mount2)
mount3 := replace(mount2, "$(b64_device_id)", base64url.encode(i_storage.source))
print("allow_mount_point_by_device_id: mount3 =", mount3)
regex.match(mount3, i_storage.mount_point)
print("allow_mount_point_by_device_id: true")
}
# ExecProcessRequest.process.Capabilities
allow_exec_caps(i_caps) if {

View File

@@ -114,24 +114,18 @@ pub fn get_mount_and_storage(
if let Some(emptyDir) = &yaml_volume.emptyDir {
let settings_volumes = &settings.volumes;
let mut volume: Option<&settings::EmptyDirVolume> = None;
if let Some(medium) = &emptyDir.medium {
if medium == "Memory" {
volume = Some(&settings_volumes.emptyDir_memory);
}
}
if volume.is_none() {
volume = Some(&settings_volumes.emptyDir);
}
let volume = match emptyDir.medium.as_deref() {
Some("Memory") => &settings_volumes.emptyDir_memory,
_ if settings.cluster_config.encrypted_emptydir => &settings_volumes.emptyDir_encrypted,
_ => &settings_volumes.emptyDir,
};
get_empty_dir_mount_and_storage(
settings,
p_mounts,
storages,
yaml_mount,
volume.unwrap(),
volume,
pod_security_context,
);
} else if yaml_volume.persistentVolumeClaim.is_some() || yaml_volume.azureFile.is_some() {
@@ -170,12 +164,17 @@ fn get_empty_dir_mount_and_storage(
}
storages.push(agent::Storage {
driver: settings_empty_dir.driver.clone(),
driver_options: Vec::new(),
driver_options: settings_empty_dir.driver_options.clone(),
source: settings_empty_dir.source.clone(),
fstype: settings_empty_dir.fstype.clone(),
options,
mount_point: format!("{}{}$", &settings_empty_dir.mount_point, &yaml_mount.name),
mount_point: if settings_empty_dir.mount_point.ends_with('$') {
settings_empty_dir.mount_point.clone()
} else {
format!("{}{}$", &settings_empty_dir.mount_point, &yaml_mount.name)
},
fs_group: protobuf::MessageField::none(),
shared: settings_empty_dir.shared,
special_fields: ::protobuf::SpecialFields::new(),
});
}
@@ -184,6 +183,8 @@ fn get_empty_dir_mount_and_storage(
let file_name = Path::new(&yaml_mount.mountPath).file_name().unwrap();
let name = OsString::from(file_name).into_string().unwrap();
format!("{}{name}$", &settings.volumes.configMap.mount_source)
} else if settings_empty_dir.mount_source.is_empty() {
String::new()
} else {
format!("{}{}$", &settings_empty_dir.mount_source, &yaml_mount.name)
};
@@ -298,6 +299,7 @@ fn get_config_map_mount_and_storage(
options: settings_config_map.options.clone(),
mount_point: format!("{}{mount_path_str}$", &settings_config_map.mount_point),
fs_group: protobuf::MessageField::none(),
shared: false,
special_fields: ::protobuf::SpecialFields::new(),
});
}

View File

@@ -420,6 +420,9 @@ pub struct CommonData {
/// Regex prefix for shared file paths - e.g., "^$(cpath)/$(bundle-id)-[a-z0-9]{16}-".
pub sfprefix: String,
/// Path to the shared sandbox storage - e.g., "/run/kata-containers/sandbox/storage".
pub spath: String,
/// Regex for an IPv4 address.
pub ipv4_a: String,
@@ -463,6 +466,10 @@ pub struct ClusterConfig {
/// - When changing the GID via runAsUser or runAsGroup, the new GID value *gets added
/// as the only value* in AdditionalGids.
pub pause_container_id_policy: String,
/// Whether emptyDirs are encrypted with modified metadata in the
/// mount and a storage object for the block device.
pub encrypted_emptydir: bool,
}
/// Describes patterns for supported VFIO devices.

View File

@@ -34,6 +34,7 @@ pub struct Settings {
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Volumes {
pub emptyDir: EmptyDirVolume,
pub emptyDir_encrypted: EmptyDirVolume,
pub emptyDir_memory: EmptyDirVolume,
pub configMap: ConfigMapVolume,
pub image_volume: ImageVolume,
@@ -46,9 +47,11 @@ pub struct EmptyDirVolume {
pub mount_source: String,
pub mount_point: String,
pub driver: String,
pub driver_options: Vec<String>,
pub fstype: String,
pub options: Vec<String>,
pub source: String,
pub shared: bool,
}
/// ConfigMap volume settings loaded from genpolicy-settings.json.

View File

@@ -163,8 +163,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/ba77faf442028eb9f3d849acb8ff55f3bc7e35d579dddcfec43fe75de2e7b2da/rootfs/local/tmp",
"type_": "local"
"source": "/run/kata-containers/sandbox/storage/MDAvMDA=",
"type_": "bind"
},
{
"destination": "/etc/hosts",
@@ -337,19 +337,22 @@
"fstype": "overlay",
"mount_point": "/run/kata-containers/f86ec3cfa9c5e5a732744dbe844033126f0fdd84408df8809d19274b448e255d/rootfs",
"options": [],
"source": "registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db"
"source": "registry.k8s.io/pause@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db",
"shared": false
},
{
"driver": "local",
"driver_options": [],
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/ba77faf442028eb9f3d849acb8ff55f3bc7e35d579dddcfec43fe75de2e7b2da/rootfs/local/tmp",
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDAvMDA=",
"options": [
"mode=0777",
"fsgid=1000"
],
"source": "local"
"source": "00/00",
"shared": true
}
],
"string_user": null

View File

@@ -67,8 +67,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"type_": "local"
"source": "",
"type_": "bind"
},
{
"destination": "/mnt/test2",
@@ -77,8 +77,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"type_": "local"
"source": "",
"type_": "bind"
},
{
"destination": "/mnt/test3",
@@ -87,8 +87,8 @@
"rprivate",
"ro"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"type_": "local"
"source": "",
"type_": "bind"
}
],
"Process": {
@@ -166,48 +166,52 @@
},
"storages": [
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDAvMDA=",
"options": [],
"source": "00/00",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDEvMDE=",
"options": [],
"source": "01/01",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDIvMDI=",
"options": [],
"source": "02/02",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume2",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDMvMDM=",
"options": [],
"source": "03/03",
"shared": true
}
]
}
@@ -280,8 +284,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume-fake$",
"type_": "local"
"source": "",
"type_": "bind"
}
],
"Process": {
@@ -359,48 +363,52 @@
},
"storages": [
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/invalid-mount",
"options": [],
"source": "invalid-source",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/invalid-mount",
"options": [],
"source": "invalid-source",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/invalid-mount",
"options": [],
"source": "invalid-source",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume2",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/invalid-mount",
"options": [],
"source": "invalid-source",
"shared": true
}
]
}
@@ -473,8 +481,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"type_": "local"
"source": "",
"type_": "bind"
},
{
"destination": "/mnt/test4",
@@ -483,8 +491,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume2",
"type_": "local"
"source": "",
"type_": "bind"
}
],
"Process": {
@@ -562,48 +570,52 @@
},
"storages": [
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDAvMDA=",
"options": [],
"source": "00/00",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDEvMDE=",
"options": [],
"source": "01/01",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDIvMDI=",
"options": [],
"source": "02/02",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume2",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDMvMDM=",
"options": [],
"source": "03/03",
"shared": true
}
]
}
@@ -671,7 +683,8 @@
"Mounts": [
{
"destination": "/mnt/test",
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume"
"source": "",
"type_": "bind"
}
],
"Process": {
@@ -749,48 +762,52 @@
},
"storages": [
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDAvMDA=",
"options": [],
"source": "00/00",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDEvMDE=",
"options": [],
"source": "01/01",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDIvMDI=",
"options": [],
"source": "02/02",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume2",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDMvMDM=",
"options": [],
"source": "03/03",
"shared": true
}
]
}
@@ -863,8 +880,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"type_": "local"
"source": "",
"type_": "bind"
},
{
"destination": "/mnt/test",
@@ -873,8 +890,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"type_": "local"
"source": "",
"type_": "bind"
}
],
"Process": {
@@ -1020,8 +1037,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"type_": "local"
"source": "",
"type_": "bind"
},
{
"destination": "/mnt/test",
@@ -1030,8 +1047,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume2",
"type_": "local"
"source": "",
"type_": "bind"
}
],
"Process": {
@@ -1177,8 +1194,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"type_": "local"
"source": "",
"type_": "bind"
},
{
"destination": "/mnt/test2",
@@ -1187,8 +1204,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"type_": "local"
"source": "",
"type_": "bind"
},
{
"destination": "/mnt/test3",
@@ -1197,8 +1214,8 @@
"rprivate",
"ro"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"type_": "local"
"source": "",
"type_": "bind"
},
{
"destination": "/mnt/test4",
@@ -1207,8 +1224,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume2",
"type_": "local"
"source": "",
"type_": "bind"
},
{
"destination": "/mnt/test5",
@@ -1217,8 +1234,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume2",
"type_": "local"
"source": "",
"type_": "bind"
}
],
"Process": {
@@ -1296,48 +1313,52 @@
},
"storages": [
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDAvMDA=",
"options": [],
"source": "00/00",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDEvMDE=",
"options": [],
"source": "01/01",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDIvMDI=",
"options": [],
"source": "02/02",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume2",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDMvMDM=",
"options": [],
"source": "03/03",
"shared": true
}
]
}
@@ -1410,8 +1431,8 @@
"rshared",
"ro"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"type_": "local"
"source": "",
"type_": "bind"
}
],
"Process": {
@@ -1489,48 +1510,60 @@
},
"storages": [
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDAvMDA=",
"options": [
"invalid_option=true"
],
"source": "00/00",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDEvMDE=",
"options": [
"invalid_option=true"
],
"source": "01/01",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDIvMDI=",
"options": [
"invalid_option=true"
],
"source": "02/02",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume2",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "ext4",
"mount_point": "/run/kata-containers/sandbox/storage/MDMvMDM=",
"options": [
"invalid_option=true"
],
"source": "03/03",
"shared": true
}
]
}
@@ -1603,8 +1636,8 @@
"rprivate",
"rw"
],
"source": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"type_": "someweirdfstype"
"source": "",
"type_": "bind"
}
],
"Process": {
@@ -1682,48 +1715,52 @@
},
"storages": [
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "invalid_fstype",
"mount_point": "/run/kata-containers/sandbox/storage/MDAvMDA=",
"options": [],
"source": "00/00",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "invalid_fstype",
"mount_point": "/run/kata-containers/sandbox/storage/MDEvMDE=",
"options": [],
"source": "01/01",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "invalid_fstype",
"mount_point": "/run/kata-containers/sandbox/storage/MDIvMDI=",
"options": [],
"source": "02/02",
"shared": true
},
{
"driver": "local",
"driver_options": [],
"fs_group": null,
"fstype": "local",
"mount_point": "/run/kata-containers/shared/containers/0000000000000000000000000000000000000000000000000000000000000000/rootfs/local/test-volume2",
"options": [
"mode=0777"
"driver": "blk",
"driver_options": [
"encryption_key=ephemeral"
],
"source": "local"
"fs_group": null,
"fstype": "invalid_fstype",
"mount_point": "/run/kata-containers/sandbox/storage/MDMvMDM=",
"options": [],
"source": "03/03",
"shared": true
}
]
}

View File

@@ -9,6 +9,8 @@ source "${tests_dir}/common.bash"
kubernetes_dir="${tests_dir}/integration/kubernetes"
helm_chart_dir="${repo_root_dir}/tools/packaging/kata-deploy/helm-chart/kata-deploy"
AZ_REGION="${AZ_REGION:-eastus}"
AZ_NODEPOOL_TAGS="${AZ_NODEPOOL_TAGS:-}"
GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-oci-distribution}"
GH_PR_NUMBER="${GH_PR_NUMBER:-}"
HELM_DEFAULT_INSTALLATION="${HELM_DEFAULT_INSTALLATION:-false}"
@@ -138,7 +140,7 @@ function create_cluster() {
"GENPOLICY_PULL_METHOD=${GENPOLICY_PULL_METHOD:0:1}")
az group create \
-l eastus \
-l "${AZ_REGION}" \
-n "${rg}"
# Required by e.g. AKS App Routing for KBS installation.
@@ -154,6 +156,7 @@ function create_cluster() {
--generate-ssh-keys
--tags "${tags[@]}")
[[ "${KATA_HOST_OS}" = "cbl-mariner" ]] && aks_create+=( --os-sku AzureLinux --workload-runtime KataVmIsolation)
[[ -n "${AZ_NODEPOOL_TAGS}" ]] && aks_create+=(--nodepool-tags "${AZ_NODEPOOL_TAGS}")
"${aks_create[@]}"
}
@@ -610,6 +613,7 @@ function deploy_k8s() {
function set_test_cluster_namespace() {
# Delete any spurious tests namespace that was left behind
echo "Deleting test namespace ${TEST_CLUSTER_NAMESPACE}"
kubectl delete namespace "${TEST_CLUSTER_NAMESPACE}" &> /dev/null || true
# Create a new namespace for the tests and switch to it
@@ -622,6 +626,7 @@ function set_default_cluster_namespace() {
}
function delete_test_cluster_namespace() {
echo "Deleting test namespace ${TEST_CLUSTER_NAMESPACE}"
kubectl delete namespace "${TEST_CLUSTER_NAMESPACE}"
set_default_cluster_namespace
}

View File

@@ -587,7 +587,7 @@ function main() {
install-bats) install_bats ;;
install-kata-tools) install_kata_tools "${2:-}" ;;
install-kbs-client) install_kbs_client ;;
get-cluster-credentials) get_cluster_credentials "" ;;
get-cluster-credentials) get_cluster_credentials ;;
deploy-csi-driver) return 0 ;;
deploy-kata) deploy_kata ;;
deploy-kata-aks) deploy_kata "aks" ;;

View File

@@ -91,11 +91,6 @@ setup() {
}
teardown() {
# Debugging information
kubectl describe "pod/$pod_name"
kubectl delete pod "$pod_name"
[ ! -f "$pod_logs_file" ] || rm -f "$pod_logs_file"
delete_tmp_policy_settings_dir "${policy_settings_dir}"

View File

@@ -70,10 +70,6 @@ setup() {
teardown() {
[[ "${KATA_HYPERVISOR}" == qemu-se* ]] && \
skip "See: https://github.com/kata-containers/kata-containers/issues/10002"
# Debugging information
kubectl describe "pod/$pod_name" || true
kubectl delete pod "$pod_name" || true
delete_tmp_policy_settings_dir "${policy_settings_dir}"
teardown_common "${node}" "${node_start_time:-}"
}

View File

@@ -0,0 +1,92 @@
#!/usr/bin/env bats
# Copyright (c) 2025 Microsoft Corporation
# SPDX-License-Identifier: Apache-2.0
load "${BATS_TEST_DIRNAME}/lib.sh"
load "${BATS_TEST_DIRNAME}/../../common.bash"
load "${BATS_TEST_DIRNAME}/confidential_common.sh"
load "${BATS_TEST_DIRNAME}/tests_common.sh"
setup() {
is_confidential_runtime_class || skip "Only supported for CoCo"
[[ "$(uname -m)" == "s390x" ]] && skip "Not supported on s390x"
[[ "${KATA_HYPERVISOR}" == *-runtime-rs ]] && skip "Not supported with runtime-rs"
setup_common
get_pod_config_dir
pod_name="trusted-ephemeral-data-storage"
mountpoint="/mnt/temp-encrypted"
host_df="$(exec_host "${node}" df -P -B1 "$(get_kubelet_data_dir)" | tail -n +2)"
host_capacity_bytes="$(echo "${host_df}" | awk '{print $3}')"
yaml_file="${pod_config_dir}/pod-trusted-ephemeral-data-storage.yaml"
policy_settings_dir="$(create_tmp_policy_settings_dir "${pod_config_dir}")"
# The policy would only block container creation, so allow these
# requests to make writing tests easier.
allow_requests "${policy_settings_dir}" "ExecProcessRequest" "ReadStreamRequest"
auto_generate_policy "${policy_settings_dir}" "${yaml_file}"
}
@test "Trusted ephemeral data storage" {
kubectl apply -f "${yaml_file}"
kubectl wait --for=condition=Ready --timeout="${timeout}" pod "${pod_name}"
# With long device names, df adds line breaks by default, so we pass -P to prevent that.
df="$(kubectl exec "${pod_name}" -- df -PT -B1 "${mountpoint}" | tail -n +2)"
info "df output:"
info "${df}"
dm_device="$(echo "${df}" | awk '{print $1}')"
fs_type="$(echo "${df}" | awk '{print $2}')"
guest_capacity_bytes="$(echo "${df}" | awk '{print $3}')"
guest_available_bytes="$(echo "${df}" | awk '{print $5}')"
# The output of the cryptsetup command will contain something like this:
#
# /dev/mapper/encrypted_disk_N6PxO is active and is in use.
# type: LUKS2
# cipher: aes-xts-plain64
# keysize: 768 bits
# key location: keyring
# integrity: hmac(sha256)
# integrity keysize: 256 bits
# device: /dev/vda
# sector size: 4096
# offset: 0 sectors
# size: 2031880 sectors
# mode: read/write
crypt_status="$(kubectl exec "${pod_name}" -- cryptsetup status "${dm_device}")"
info "cryptsetup status output:"
info "${crypt_status}"
# Check filesystem type and capacity.
[[ "${fs_type}" == "ext4" ]]
# Allow up to 7% LUKS metadata overhead.
(( guest_capacity_bytes >= host_capacity_bytes * 93 / 100 ))
# Allow up to 15% LUKS + ext4 metadata overhead.
(( guest_available_bytes >= host_capacity_bytes * 85 / 100 ))
# Check encryption settings.
grep -q "${dm_device} is active and is in use" <<< "${crypt_status}"
grep -Eq "type: +LUKS2" <<< "${crypt_status}"
grep -Eq "cipher: +aes-xts-plain64" <<< "${crypt_status}"
grep -Eq "integrity: +hmac\(sha256\)" <<< "${crypt_status}"
# Check I/O.
kubectl exec "${pod_name}" -- sh -c "echo foo > '${mountpoint}/foo.txt'"
[[ "$(kubectl exec "${pod_name}" -- cat "${mountpoint}/foo.txt")" == "foo" ]]
}
teardown() {
is_confidential_runtime_class || skip "Only supported for CoCo"
[[ "$(uname -m)" == "s390x" ]] && skip "Not supported on s390x"
[[ "${KATA_HYPERVISOR}" == *-runtime-rs ]] && skip "Not supported with runtime-rs"
confidential_teardown_common "${node}" "${node_start_time:-}"
}

View File

@@ -103,6 +103,43 @@ k8s_create_pod() {
fi
}
# Creates a debugger pod if one doesn't already exist.
#
# Parameters:
# $1 - the node name
#
create_debugger_pod() {
local node="$1"
local pod_name="custom-node-debugger-$(echo -n "$node" | sha1sum | cut -c1-7)"
# Check if there is an existing node debugger pod and reuse it
# Otherwise, create a new one
if ! kubectl get pod -n kube-system "${pod_name}" > /dev/null 2>&1; then
POD_NAME="${pod_name}" NODE_NAME="${node}" envsubst < runtimeclass_workloads/custom-node-debugger.yaml | \
kubectl apply -n kube-system -f - > /dev/null
# Wait for the newly created pod to be ready
kubectl wait pod -n kube-system --timeout="30s" --for=condition=ready "${pod_name}" > /dev/null
fi
echo "${pod_name}"
}
# Copies a file into the host filesystem.
#
# Parameters:
# $1 - source file path on the client
# $2 - node
# $3 - destination path on the node
#
copy_file_to_host() {
local source="$1"
local node="$2"
local destination="$3"
debugger_pod="$(create_debugger_pod "${node}")"
kubectl cp -n kube-system "${source}" "${debugger_pod}:/host/${destination}"
}
# Runs a command in the host filesystem.
#
# Parameters:
@@ -114,25 +151,9 @@ exec_host() {
if ! kubectl get node "${node}" > /dev/null 2>&1; then
die "A given node ${node} is not valid"
fi
# `kubectl debug` always returns 0, so we hack it to return the right exit code.
local command="${@:2}"
# Make 7 character hash from the node name
local pod_name="custom-node-debugger-$(echo -n "$node" | sha1sum | cut -c1-7)"
# Run a debug pod
# Check if there is an existing node debugger pod and reuse it
# Otherwise, create a new one
if ! kubectl get pod -n kube-system "${pod_name}" > /dev/null 2>&1; then
POD_NAME="${pod_name}" NODE_NAME="${node}" envsubst < runtimeclass_workloads/custom-node-debugger.yaml | \
kubectl apply -n kube-system -f - > /dev/null
# Wait for the newly created pod to be ready
kubectl wait pod -n kube-system --timeout="30s" --for=condition=ready "${pod_name}" > /dev/null
# Manually check the exit status of the previous command to handle errors explicitly
# since `set -e` is not enabled, allowing subsequent commands to run if needed.
if [ $? -ne 0 ]; then
return $?
fi
fi
local command="${@:2}"
local pod_name="$(create_debugger_pod "${node}")"
# Execute the command and capture the output
# We're trailing the `\r` here due to: https://github.com/kata-containers/kata-containers/issues/8051

View File

@@ -93,6 +93,7 @@ else
"k8s-sysctls.bats" \
"k8s-security-context.bats" \
"k8s-shared-volume.bats" \
"k8s-trusted-ephemeral-data-storage.bats" \
"k8s-volume.bats" \
"k8s-nginx-connectivity.bats" \
)

View File

@@ -0,0 +1,9 @@
# Copyright (c) 2026 Microsoft Corporation
# SPDX-License-Identifier: Apache-2.0
#
# This is used by k8s-trusted-ephemeral-data-storage.bats.
FROM alpine:3.22.1
RUN apk add --no-cache cryptsetup
CMD ["/bin/sh"]

View File

@@ -0,0 +1,38 @@
---
kind: Pod
apiVersion: v1
metadata:
name: trusted-ephemeral-data-storage
spec:
# Explicit user/group/supplementary groups to support nydus guest-pull.
# See issue https://github.com/kata-containers/kata-containers/issues/11162 and
# other references to this issue in the genpolicy source folder.
securityContext:
runAsUser: 0
runAsGroup: 0
supplementalGroups: [1, 2, 3, 4, 6, 10, 11, 20, 26, 27]
runtimeClassName: kata
restartPolicy: Never
containers:
- image: quay.io/kata-containers/alpine:3.22.1-cryptsetup
name: busybox
command: ["sleep", "infinity"]
securityContext:
capabilities:
# Only needed to use cryptsetup in the test code (to access /dev/mapper/control).
# NOT needed for the actual encrypted emptyDir functionality.
add: ["SYS_ADMIN"]
volumeMounts:
- name: temp-encrypted
mountPath: /mnt/temp-encrypted
- name: guest-dev
mountPath: /dev
volumes:
- name: temp-encrypted
emptyDir:
sizeLimit: 10G
- name: guest-dev
# hostPath mounts guest devices for /dev, see
# https://github.com/kata-containers/kata-containers/blob/main/docs/Limitations.md#kubernetes-hostpath-volumes
hostPath:
path: /dev

View File

@@ -37,6 +37,8 @@ K8S_TEST_DIR="${kubernetes_dir:-"${BATS_TEST_DIRNAME}"}"
AUTO_GENERATE_POLICY="${AUTO_GENERATE_POLICY:-}"
GENPOLICY_PULL_METHOD="${GENPOLICY_PULL_METHOD:-}"
GENPOLICY_BINARY="${GENPOLICY_BINARY:-"/opt/kata/bin/genpolicy"}"
GENPOLICY_SETTINGS_DIR="${GENPOLICY_SETTINGS_DIR:-"/opt/kata/share/defaults/kata-containers"}"
KATA_HYPERVISOR="${KATA_HYPERVISOR:-}"
KATA_HOST_OS="${KATA_HOST_OS:-}"
RUNS_ON_AKS="${RUNS_ON_AKS:-false}"
@@ -113,6 +115,18 @@ is_k3s_or_rke2() {
esac
}
# Return the kubelet data directory, which varies by Kubernetes distribution.
get_kubelet_data_dir() {
case "${KUBERNETES:-}" in
k0s) echo "/var/lib/k0s/kubelet" ;;
*) echo "/var/lib/kubelet" ;;
esac
}
is_runtime_rs() {
[[ "${KATA_HYPERVISOR}" == *-runtime-rs ]]
}
# Copy the right combination of drop-ins from drop-in-examples/ into
# genpolicy-settings.d/. Drop-ins are layered: 10-* for platform base,
# 20-* for OCI version and other overlays.
@@ -144,6 +158,11 @@ install_genpolicy_drop_ins() {
if [[ "${PULL_TYPE:-}" == "experimental-force-guest-pull" ]]; then
cp "${examples_dir}/20-experimental-force-guest-pull-drop-in.json" "${settings_d}/"
fi
# 20-* runtime-rs overlay (disable encrypted emptyDir, not supported yet)
if is_runtime_rs; then
cp "${examples_dir}/20-runtime-rs-drop-in.json" "${settings_d}/"
fi
}
# If auto-generated policy testing is enabled, make a copy of the genpolicy settings
@@ -155,8 +174,8 @@ create_common_genpolicy_settings() {
auto_generate_policy_enabled || return 0
cp "${default_genpolicy_settings_dir}/genpolicy-settings.json" "${genpolicy_settings_dir}"
cp "${default_genpolicy_settings_dir}/rules.rego" "${genpolicy_settings_dir}"
cp "${GENPOLICY_SETTINGS_DIR}/genpolicy-settings.json" "${genpolicy_settings_dir}"
cp "${GENPOLICY_SETTINGS_DIR}/rules.rego" "${genpolicy_settings_dir}"
mkdir -p "${genpolicy_settings_dir}/genpolicy-settings.d"
install_genpolicy_drop_ins \
@@ -213,7 +232,7 @@ auto_generate_policy_no_added_flags() {
declare -r additional_flags="${4:-""}"
auto_generate_policy_enabled || return 0
local genpolicy_command="RUST_LOG=info /opt/kata/bin/genpolicy -u -y ${yaml_file}"
local genpolicy_command="RUST_LOG=info ${GENPOLICY_BINARY} -u -y ${yaml_file}"
genpolicy_command+=" -p ${settings_dir}/rules.rego"
genpolicy_command+=" -j ${settings_dir}"
@@ -282,6 +301,21 @@ add_requests_to_policy_settings() {
done
}
# Change Rego rules to allow one or more ttrpc requests from the Host to the Guest.
allow_requests() {
declare -r settings_dir="$1"
shift
declare -r requests=("$@")
auto_generate_policy_enabled || return 0
for request in "${requests[@]}"
do
info "${settings_dir}/rules.rego: allowing ${request}"
sed -i "s/^default \(${request}\).\+/default \1 := true/" "${settings_dir}"/rules.rego
done
}
# Change genpolicy settings to allow executing on the Guest VM the commands
# used by "kubectl cp" from the Host to the Guest.
add_copy_from_host_to_policy_settings() {
@@ -449,12 +483,7 @@ teardown_common() {
node_end_time=$(measure_node_time "${node}")
echo "Journal LOG starts at ${node_start_time:-}, ends at ${node_end_time:-}"
# Print the node journal since the test start time if a bats test is not completed
if [[ -n "${node_start_time}" && -z "${BATS_TEST_COMPLETED}" ]]; then
echo "DEBUG: system logs of node '${node}' since test start time (${node_start_time})"
exec_host "${node}" journalctl -x -t "kata" --since '"'"${node_start_time}"'"' || true
fi
print_node_journal_since_test_start "${node}" "${node_start_time}" "${BATS_TEST_COMPLETED:-}"
}
measure_node_time() {
@@ -573,6 +602,6 @@ print_node_journal_since_test_start() {
if [[ -n "${node_start_time:-}" && -z "${BATS_TEST_COMPLETED:-}" ]]; then
echo "DEBUG: system logs of node '${node}' since test start time (${node_start_time})"
exec_host "${node}" journalctl -x -t "kata" --since '"'"${node_start_time}"'"' || true
exec_host "${node}" journalctl -t "kata" --since '"'"${node_start_time}"'"' -o cat || true
fi
}