Merge pull request #3406 from fengwang666/direct-blk-assignment

Implement direct-assigned volume
This commit is contained in:
Eric Ernst 2022-03-04 11:58:37 -08:00 committed by GitHub
commit 1e301482e7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 3373 additions and 353 deletions

71
src/agent/Cargo.lock generated
View File

@ -214,6 +214,12 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "core-foundation-sys"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]] [[package]]
name = "crc32fast" name = "crc32fast"
version = "1.3.0" version = "1.3.0"
@ -233,6 +239,30 @@ dependencies = [
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]]
name = "crossbeam-deque"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97242a70df9b89a65d0b6df3c4bf5b9ce03c5b7309019777fbde37e7537f8762"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils",
"lazy_static",
"memoffset",
"scopeguard",
]
[[package]] [[package]]
name = "crossbeam-utils" name = "crossbeam-utils"
version = "0.8.5" version = "0.8.5"
@ -565,6 +595,7 @@ dependencies = [
"slog", "slog",
"slog-scope", "slog-scope",
"slog-stdlog", "slog-stdlog",
"sysinfo",
"tempfile", "tempfile",
"thiserror", "thiserror",
"tokio", "tokio",
@ -1238,6 +1269,31 @@ dependencies = [
"rand_core", "rand_core",
] ]
[[package]]
name = "rayon"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90"
dependencies = [
"autocfg",
"crossbeam-deque",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"lazy_static",
"num_cpus",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.2.10" version = "0.2.10"
@ -1518,6 +1574,21 @@ dependencies = [
"unicode-xid", "unicode-xid",
] ]
[[package]]
name = "sysinfo"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e757000a4bed2b1be9be65a3f418b9696adf30bb419214c73997422de73a591"
dependencies = [
"cfg-if 1.0.0",
"core-foundation-sys",
"libc",
"ntapi",
"once_cell",
"rayon",
"winapi",
]
[[package]] [[package]]
name = "take_mut" name = "take_mut"
version = "0.2.2" version = "0.2.2"

View File

@ -20,6 +20,7 @@ scopeguard = "1.0.0"
thiserror = "1.0.26" thiserror = "1.0.26"
regex = "1.5.4" regex = "1.5.4"
serial_test = "0.5.1" serial_test = "0.5.1"
sysinfo = "0.23.0"
# Async helpers # Async helpers
async-trait = "0.1.42" async-trait = "0.1.42"

View File

@ -25,6 +25,7 @@ allowed = [
"ReadStreamRequest", "ReadStreamRequest",
"RemoveContainerRequest", "RemoveContainerRequest",
"ReseedRandomDevRequest", "ReseedRandomDevRequest",
"ResizeVolumeRequest",
"ResumeContainerRequest", "ResumeContainerRequest",
"SetGuestDateTimeRequest", "SetGuestDateTimeRequest",
"SignalProcessRequest", "SignalProcessRequest",
@ -34,6 +35,7 @@ allowed = [
"UpdateContainerRequest", "UpdateContainerRequest",
"UpdateInterfaceRequest", "UpdateInterfaceRequest",
"UpdateRoutesRequest", "UpdateRoutesRequest",
"VolumeStatsRequest",
"WaitProcessRequest", "WaitProcessRequest",
"WriteStreamRequest" "WriteStreamRequest"
] ]

View File

@ -193,13 +193,6 @@ async fn ephemeral_storage_handler(
storage: &Storage, storage: &Storage,
sandbox: Arc<Mutex<Sandbox>>, sandbox: Arc<Mutex<Sandbox>>,
) -> Result<String> { ) -> Result<String> {
let mut sb = sandbox.lock().await;
let new_storage = sb.set_sandbox_storage(&storage.mount_point);
if !new_storage {
return Ok("".to_string());
}
// hugetlbfs // hugetlbfs
if storage.fstype == FS_TYPE_HUGETLB { if storage.fstype == FS_TYPE_HUGETLB {
return handle_hugetlbfs_storage(logger, storage).await; return handle_hugetlbfs_storage(logger, storage).await;
@ -255,13 +248,6 @@ async fn local_storage_handler(
storage: &Storage, storage: &Storage,
sandbox: Arc<Mutex<Sandbox>>, sandbox: Arc<Mutex<Sandbox>>,
) -> Result<String> { ) -> Result<String> {
let mut sb = sandbox.lock().await;
let new_storage = sb.set_sandbox_storage(&storage.mount_point);
if !new_storage {
return Ok("".to_string());
}
fs::create_dir_all(&storage.mount_point).context(format!( fs::create_dir_all(&storage.mount_point).context(format!(
"failed to create dir all {:?}", "failed to create dir all {:?}",
&storage.mount_point &storage.mount_point
@ -401,7 +387,7 @@ fn get_pagesize_and_size_from_option(options: &[String]) -> Result<(u64, u64)> {
async fn virtiommio_blk_storage_handler( async fn virtiommio_blk_storage_handler(
logger: &Logger, logger: &Logger,
storage: &Storage, storage: &Storage,
_sandbox: Arc<Mutex<Sandbox>>, sandbox: Arc<Mutex<Sandbox>>,
) -> Result<String> { ) -> Result<String> {
//The source path is VmPath //The source path is VmPath
common_storage_handler(logger, storage) common_storage_handler(logger, storage)
@ -641,6 +627,14 @@ pub async fn add_storages(
"subsystem" => "storage", "subsystem" => "storage",
"storage-type" => handler_name.to_owned())); "storage-type" => handler_name.to_owned()));
{
let mut sb = sandbox.lock().await;
let new_storage = sb.set_sandbox_storage(&storage.mount_point);
if !new_storage {
continue;
}
}
let res = match handler_name.as_str() { let res = match handler_name.as_str() {
DRIVER_BLK_TYPE => virtio_blk_storage_handler(&logger, &storage, sandbox.clone()).await, DRIVER_BLK_TYPE => virtio_blk_storage_handler(&logger, &storage, sandbox.clone()).await,
DRIVER_BLK_CCW_TYPE => { DRIVER_BLK_CCW_TYPE => {

View File

@ -23,9 +23,10 @@ use oci::{LinuxNamespace, Root, Spec};
use protobuf::{Message, RepeatedField, SingularPtrField}; use protobuf::{Message, RepeatedField, SingularPtrField};
use protocols::agent::{ use protocols::agent::{
AddSwapRequest, AgentDetails, CopyFileRequest, GuestDetailsResponse, Interfaces, Metrics, AddSwapRequest, AgentDetails, CopyFileRequest, GuestDetailsResponse, Interfaces, Metrics,
OOMEvent, ReadStreamResponse, Routes, StatsContainerResponse, WaitProcessResponse, OOMEvent, ReadStreamResponse, Routes, StatsContainerResponse, VolumeStatsRequest,
WriteStreamResponse, WaitProcessResponse, WriteStreamResponse,
}; };
use protocols::csi::{VolumeCondition, VolumeStatsResponse, VolumeUsage, VolumeUsage_Unit};
use protocols::empty::Empty; use protocols::empty::Empty;
use protocols::health::{ use protocols::health::{
HealthCheckResponse, HealthCheckResponse_ServingStatus, VersionCheckResponse, HealthCheckResponse, HealthCheckResponse_ServingStatus, VersionCheckResponse,
@ -43,12 +44,14 @@ use nix::sys::stat;
use nix::unistd::{self, Pid}; use nix::unistd::{self, Pid};
use rustjail::process::ProcessOperations; use rustjail::process::ProcessOperations;
use sysinfo::{DiskExt, System, SystemExt};
use crate::device::{ use crate::device::{
add_devices, get_virtio_blk_pci_device_name, update_device_cgroup, update_env_pci, add_devices, get_virtio_blk_pci_device_name, update_device_cgroup, update_env_pci,
}; };
use crate::linux_abi::*; use crate::linux_abi::*;
use crate::metrics::get_metrics; use crate::metrics::get_metrics;
use crate::mount::{add_storages, baremount, remove_mounts, STORAGE_HANDLER_LIST}; use crate::mount::{add_storages, baremount, STORAGE_HANDLER_LIST};
use crate::namespace::{NSTYPEIPC, NSTYPEPID, NSTYPEUTS}; use crate::namespace::{NSTYPEIPC, NSTYPEPID, NSTYPEUTS};
use crate::network::setup_guest_dns; use crate::network::setup_guest_dns;
use crate::pci; use crate::pci;
@ -68,6 +71,7 @@ use tracing::instrument;
use libc::{self, c_char, c_ushort, pid_t, winsize, TIOCSWINSZ}; use libc::{self, c_char, c_ushort, pid_t, winsize, TIOCSWINSZ};
use std::convert::TryFrom; use std::convert::TryFrom;
use std::fs; use std::fs;
use std::os::unix::fs::MetadataExt;
use std::os::unix::prelude::PermissionsExt; use std::os::unix::prelude::PermissionsExt;
use std::process::{Command, Stdio}; use std::process::{Command, Stdio};
use std::time::Duration; use std::time::Duration;
@ -283,8 +287,6 @@ impl AgentService {
// Find the sandbox storage used by this container // Find the sandbox storage used by this container
let mounts = sandbox.container_mounts.get(&cid); let mounts = sandbox.container_mounts.get(&cid);
if let Some(mounts) = mounts { if let Some(mounts) = mounts {
remove_mounts(mounts)?;
for m in mounts.iter() { for m in mounts.iter() {
if sandbox.storages.get(m).is_some() { if sandbox.storages.get(m).is_some() {
cmounts.push(m.to_string()); cmounts.push(m.to_string());
@ -1254,6 +1256,47 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
Err(ttrpc_error!(ttrpc::Code::INTERNAL, "")) Err(ttrpc_error!(ttrpc::Code::INTERNAL, ""))
} }
async fn get_volume_stats(
&self,
ctx: &TtrpcContext,
req: VolumeStatsRequest,
) -> ttrpc::Result<VolumeStatsResponse> {
trace_rpc_call!(ctx, "get_volume_stats", req);
is_allowed!(req);
info!(sl!(), "get volume stats!");
let mut resp = VolumeStatsResponse::new();
let mut condition = VolumeCondition::new();
match File::open(&req.volume_guest_path) {
Ok(_) => {
condition.abnormal = false;
condition.message = String::from("OK");
}
Err(e) => {
info!(sl!(), "failed to open the volume");
return Err(ttrpc_error!(ttrpc::Code::INTERNAL, e));
}
};
let mut usage_vec = Vec::new();
// to get volume capacity stats
get_volume_capacity_stats(&req.volume_guest_path)
.map(|u| usage_vec.push(u))
.map_err(|e| ttrpc_error!(ttrpc::Code::INTERNAL, e))?;
// to get volume inode stats
get_volume_inode_stats(&req.volume_guest_path)
.map(|u| usage_vec.push(u))
.map_err(|e| ttrpc_error!(ttrpc::Code::INTERNAL, e))?;
resp.usage = RepeatedField::from_vec(usage_vec);
resp.volume_condition = SingularPtrField::some(condition);
Ok(resp)
}
async fn add_swap( async fn add_swap(
&self, &self,
ctx: &TtrpcContext, ctx: &TtrpcContext,
@ -1341,6 +1384,48 @@ fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> {
Ok((size, plug)) Ok((size, plug))
} }
fn get_volume_capacity_stats(path: &str) -> Result<VolumeUsage> {
let mut usage = VolumeUsage::new();
let s = System::new();
for disk in s.disks() {
if let Some(v) = disk.name().to_str() {
if v.to_string().eq(path) {
usage.available = disk.available_space();
usage.total = disk.total_space();
usage.used = usage.total - usage.available;
usage.unit = VolumeUsage_Unit::BYTES; // bytes
break;
}
} else {
return Err(anyhow!(nix::Error::EINVAL));
}
}
Ok(usage)
}
fn get_volume_inode_stats(path: &str) -> Result<VolumeUsage> {
let mut usage = VolumeUsage::new();
let s = System::new();
for disk in s.disks() {
if let Some(v) = disk.name().to_str() {
if v.to_string().eq(path) {
let meta = fs::metadata(disk.mount_point())?;
let inode = meta.ino();
usage.used = inode;
usage.unit = VolumeUsage_Unit::INODES;
break;
}
} else {
return Err(anyhow!(nix::Error::EINVAL));
}
}
Ok(usage)
}
pub fn have_seccomp() -> bool { pub fn have_seccomp() -> bool {
if cfg!(feature = "seccomp") { if cfg!(feature = "seccomp") {
return true; return true;

View File

@ -95,6 +95,7 @@ fn real_main() -> Result<(), std::io::Error> {
let protos = vec![ let protos = vec![
"protos/agent.proto", "protos/agent.proto",
"protos/csi.proto",
"protos/google/protobuf/empty.proto", "protos/google/protobuf/empty.proto",
"protos/health.proto", "protos/health.proto",
"protos/oci.proto", "protos/oci.proto",

View File

@ -51,6 +51,8 @@ generate_go_sources() {
--gogottrpc_out=plugins=ttrpc+fieldpath,\ --gogottrpc_out=plugins=ttrpc+fieldpath,\
import_path=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\ import_path=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
\ \
Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/csi.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
\
Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/types.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols,\ Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/types.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols,\
\ \
Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/oci.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\ Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/oci.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\
@ -69,7 +71,7 @@ if [ "$(basename $(pwd))" != "agent" ]; then
fi fi
# Protocol buffer files required to generate golang/rust bindings. # Protocol buffer files required to generate golang/rust bindings.
proto_files_list=(agent.proto health.proto oci.proto types.proto) proto_files_list=(agent.proto csi.proto health.proto oci.proto types.proto)
if [ "$1" = "" ]; then if [ "$1" = "" ]; then
show_usage "${proto_files_list[@]}" show_usage "${proto_files_list[@]}"

View File

@ -12,6 +12,7 @@ option go_package = "github.com/kata-containers/kata-containers/src/runtime/virt
package grpc; package grpc;
import "oci.proto"; import "oci.proto";
import "csi.proto";
import "types.proto"; import "types.proto";
import "google/protobuf/empty.proto"; import "google/protobuf/empty.proto";
@ -65,6 +66,8 @@ service AgentService {
rpc CopyFile(CopyFileRequest) returns (google.protobuf.Empty); rpc CopyFile(CopyFileRequest) returns (google.protobuf.Empty);
rpc GetOOMEvent(GetOOMEventRequest) returns (OOMEvent); rpc GetOOMEvent(GetOOMEventRequest) returns (OOMEvent);
rpc AddSwap(AddSwapRequest) returns (google.protobuf.Empty); rpc AddSwap(AddSwapRequest) returns (google.protobuf.Empty);
rpc GetVolumeStats(VolumeStatsRequest) returns (VolumeStatsResponse);
rpc ResizeVolume(ResizeVolumeRequest) returns (google.protobuf.Empty);
} }
message CreateContainerRequest { message CreateContainerRequest {
@ -505,3 +508,14 @@ message GetMetricsRequest {}
message Metrics { message Metrics {
string metrics = 1; string metrics = 1;
} }
message VolumeStatsRequest {
// The volume path on the guest outside the container
string volume_guest_path = 1;
}
message ResizeVolumeRequest {
// Full VM guest path of the volume (outside the container)
string volume_guest_path = 1;
uint64 size = 2;
}

View File

@ -0,0 +1,60 @@
// Copyright (c) 2022 Databricks Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
syntax = "proto3";
option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc";
package grpc;
import "gogo/protobuf/gogoproto/gogo.proto";
option (gogoproto.equal_all) = true;
option (gogoproto.populate_all) = true;
option (gogoproto.testgen_all) = true;
option (gogoproto.benchgen_all) = true;
// This should be kept in sync with CSI NodeGetVolumeStatsResponse (https://github.com/container-storage-interface/spec/blob/v1.5.0/csi.proto)
message VolumeStatsResponse {
// This field is OPTIONAL.
repeated VolumeUsage usage = 1;
// Information about the current condition of the volume.
// This field is OPTIONAL.
// This field MUST be specified if the VOLUME_CONDITION node
// capability is supported.
VolumeCondition volume_condition = 2;
}
message VolumeUsage {
enum Unit {
UNKNOWN = 0;
BYTES = 1;
INODES = 2;
}
// The available capacity in specified Unit. This field is OPTIONAL.
// The value of this field MUST NOT be negative.
uint64 available = 1;
// The total capacity in specified Unit. This field is REQUIRED.
// The value of this field MUST NOT be negative.
uint64 total = 2;
// The used capacity in specified Unit. This field is OPTIONAL.
// The value of this field MUST NOT be negative.
uint64 used = 3;
// Units by which values are measured. This field is REQUIRED.
Unit unit = 4;
}
// VolumeCondition represents the current condition of a volume.
message VolumeCondition {
// Normal volumes are available for use and operating optimally.
// An abnormal volume does not meet these criteria.
// This field is REQUIRED.
bool abnormal = 1;
// The message describing the condition of the volume.
// This field is REQUIRED.
string message = 2;
}

View File

@ -6,11 +6,9 @@
// //
syntax = "proto3"; syntax = "proto3";
option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"; option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc";
package grpc; package grpc;
import "gogo/protobuf/gogoproto/gogo.proto"; import "gogo/protobuf/gogoproto/gogo.proto";
option (gogoproto.equal_all) = true; option (gogoproto.equal_all) = true;

View File

@ -7,6 +7,7 @@
pub mod agent; pub mod agent;
pub mod agent_ttrpc; pub mod agent_ttrpc;
pub mod csi;
pub mod empty; pub mod empty;
pub mod health; pub mod health;
pub mod health_ttrpc; pub mod health_ttrpc;

View File

@ -19,8 +19,8 @@ import (
"time" "time"
"github.com/containerd/console" "github.com/containerd/console"
kataMonitor "github.com/kata-containers/kata-containers/src/runtime/pkg/kata-monitor"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
"github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient"
clientUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/client" clientUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/client"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/urfave/cli" "github.com/urfave/cli"
@ -154,7 +154,7 @@ func (s *iostream) Read(data []byte) (n int, err error) {
} }
func getConn(sandboxID string, port uint64) (net.Conn, error) { func getConn(sandboxID string, port uint64) (net.Conn, error) {
client, err := kataMonitor.BuildShimClient(sandboxID, defaultTimeout) client, err := shimclient.BuildShimClient(sandboxID, defaultTimeout)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -0,0 +1,145 @@
// Copyright (c) 2022 Databricks Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
package main
import (
"encoding/json"
"net/url"
containerdshim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2"
"github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
"github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient"
"github.com/urfave/cli"
)
var volumeSubCmds = []cli.Command{
addCommand,
removeCommand,
statsCommand,
resizeCommand,
}
var (
mountInfo string
volumePath string
size uint64
)
var kataVolumeCommand = cli.Command{
Name: "direct-volume",
Usage: "directly assign a volume to Kata Containers to manage",
Subcommands: volumeSubCmds,
Action: func(context *cli.Context) {
cli.ShowSubcommandHelp(context)
},
}
var addCommand = cli.Command{
Name: "add",
Usage: "add a direct assigned block volume device to the Kata Containers runtime",
Flags: []cli.Flag{
cli.StringFlag{
Name: "volume-path",
Usage: "the target volume path the volume is published to",
Destination: &volumePath,
},
cli.StringFlag{
Name: "mount-info",
Usage: "the mount info for the Kata Containers runtime to manage the volume",
Destination: &mountInfo,
},
},
Action: func(c *cli.Context) error {
return volume.Add(volumePath, mountInfo)
},
}
var removeCommand = cli.Command{
Name: "remove",
Usage: "remove a direct assigned block volume device from the Kata Containers runtime",
Flags: []cli.Flag{
cli.StringFlag{
Name: "volume-path",
Usage: "the target volume path the volume is published to",
Destination: &volumePath,
},
},
Action: func(c *cli.Context) error {
return volume.Remove(volumePath)
},
}
var statsCommand = cli.Command{
Name: "stats",
Usage: "get the filesystem stat of a direct assigned volume",
Flags: []cli.Flag{
cli.StringFlag{
Name: "volume-path",
Usage: "the target volume path the volume is published to",
Destination: &volumePath,
},
},
Action: func(c *cli.Context) (string, error) {
stats, err := Stats(volumePath)
if err != nil {
return "", err
}
return string(stats), nil
},
}
var resizeCommand = cli.Command{
Name: "resize",
Usage: "resize a direct assigned block volume",
Flags: []cli.Flag{
cli.StringFlag{
Name: "volume-path",
Usage: "the target volume path the volume is published to",
Destination: &volumePath,
},
cli.Uint64Flag{
Name: "size",
Usage: "the new size of the volume",
Destination: &size,
},
},
Action: func(c *cli.Context) error {
return Resize(volumePath, size)
},
}
// Stats retrieves the filesystem stats of the direct volume inside the guest.
func Stats(volumePath string) ([]byte, error) {
sandboxId, err := volume.GetSandboxIdForVolume(volumePath)
if err != nil {
return nil, err
}
urlSafeDevicePath := url.PathEscape(volumePath)
body, err := shimclient.DoGet(sandboxId, defaultTimeout, containerdshim.DirectVolumeStatUrl+"/"+urlSafeDevicePath)
if err != nil {
return nil, err
}
return body, nil
}
// Resize resizes a direct volume inside the guest.
func Resize(volumePath string, size uint64) error {
sandboxId, err := volume.GetSandboxIdForVolume(volumePath)
if err != nil {
return err
}
resizeReq := containerdshim.ResizeRequest{
VolumePath: volumePath,
Size: size,
}
encoded, err := json.Marshal(resizeReq)
if err != nil {
return err
}
return shimclient.DoPost(sandboxId, defaultTimeout, containerdshim.DirectVolumeResizeUrl, encoded)
}

View File

@ -124,6 +124,7 @@ var runtimeCommands = []cli.Command{
kataExecCLICommand, kataExecCLICommand,
kataMetricsCLICommand, kataMetricsCLICommand,
factoryCLICommand, factoryCLICommand,
kataVolumeCommand,
} }
// runtimeBeforeSubcommands is the function to run before command-line // runtimeBeforeSubcommands is the function to run before command-line

View File

@ -7,26 +7,33 @@ package containerdshim
import ( import (
"context" "context"
"encoding/json"
"expvar" "expvar"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"net/http/pprof" "net/http/pprof"
"net/url"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
"google.golang.org/grpc/codes"
cdshim "github.com/containerd/containerd/runtime/v2/shim" cdshim "github.com/containerd/containerd/runtime/v2/shim"
mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
"github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/runtime-spec/specs-go"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go" dto "github.com/prometheus/client_model/go"
"github.com/prometheus/common/expfmt" "github.com/prometheus/common/expfmt"
)
"google.golang.org/grpc/codes" const (
DirectVolumeStatUrl = "/direct-volume/stats"
mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" DirectVolumeResizeUrl = "/direct-volume/resize"
) )
var ( var (
@ -34,6 +41,11 @@ var (
shimMgtLog = shimLog.WithField("subsystem", "shim-management") shimMgtLog = shimLog.WithField("subsystem", "shim-management")
) )
type ResizeRequest struct {
VolumePath string
Size uint64
}
// agentURL returns URL for agent // agentURL returns URL for agent
func (s *service) agentURL(w http.ResponseWriter, r *http.Request) { func (s *service) agentURL(w http.ResponseWriter, r *http.Request) {
url, err := s.sandbox.GetAgentURL() url, err := s.sandbox.GetAgentURL()
@ -126,6 +138,52 @@ func decodeAgentMetrics(body string) []*dto.MetricFamily {
return list return list
} }
func (s *service) serveVolumeStats(w http.ResponseWriter, r *http.Request) {
volumePath, err := url.PathUnescape(strings.TrimPrefix(r.URL.Path, DirectVolumeStatUrl))
if err != nil {
shimMgtLog.WithError(err).Error("failed to unescape the volume stat url path")
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
buf, err := s.sandbox.GuestVolumeStats(context.Background(), volumePath)
if err != nil {
shimMgtLog.WithError(err).WithField("volume-path", volumePath).Error("failed to get volume stats")
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
w.Write(buf)
}
func (s *service) serveVolumeResize(w http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
shimMgtLog.WithError(err).Error("failed to read request body")
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
var resizeReq ResizeRequest
err = json.Unmarshal(body, &resizeReq)
if err != nil {
shimMgtLog.WithError(err).Error("failed to unmarshal the http request body")
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
err = s.sandbox.ResizeGuestVolume(context.Background(), resizeReq.VolumePath, resizeReq.Size)
if err != nil {
shimMgtLog.WithError(err).WithField("volume-path", resizeReq.VolumePath).Error("failed to resize the volume")
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
w.Write([]byte(""))
}
func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec) { func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec) {
// metrics socket will under sandbox's bundle path // metrics socket will under sandbox's bundle path
metricsAddress := SocketAddress(s.id) metricsAddress := SocketAddress(s.id)
@ -148,6 +206,8 @@ func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec
m := http.NewServeMux() m := http.NewServeMux()
m.Handle("/metrics", http.HandlerFunc(s.serveMetrics)) m.Handle("/metrics", http.HandlerFunc(s.serveMetrics))
m.Handle("/agent-url", http.HandlerFunc(s.agentURL)) m.Handle("/agent-url", http.HandlerFunc(s.agentURL))
m.Handle(DirectVolumeStatUrl, http.HandlerFunc(s.serveVolumeStats))
m.Handle(DirectVolumeResizeUrl, http.HandlerFunc(s.serveVolumeResize))
s.mountPprofHandle(m, ociSpec) s.mountPprofHandle(m, ociSpec)
// register shim metrics // register shim metrics

View File

@ -0,0 +1,108 @@
// Copyright (c) 2022 Databricks Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
package volume
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
)
const (
mountInfoFileName = "mountInfo.json"
)
var kataDirectVolumeRootPath = "/run/kata-containers/shared/direct-volumes"
// MountInfo contains the information needed by Kata to consume a host block device and mount it as a filesystem inside the guest VM.
type MountInfo struct {
// The type of the volume (ie. block)
VolumeType string `json:"volume-type"`
// The device backing the volume.
Device string `json:"device"`
// The filesystem type to be mounted on the volume.
FsType string `json:"fstype"`
// Additional metadata to pass to the agent regarding this volume.
Metadata map[string]string `json:"metadata,omitempty"`
// Additional mount options.
Options []string `json:"options,omitempty"`
}
// Add writes the mount info of a direct volume into a filesystem path known to Kata Container.
func Add(volumePath string, mountInfo string) error {
volumeDir := filepath.Join(kataDirectVolumeRootPath, volumePath)
stat, err := os.Stat(volumeDir)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if stat != nil && !stat.IsDir() {
return fmt.Errorf("%s should be a directory", volumeDir)
}
if errors.Is(err, os.ErrNotExist) {
if err := os.MkdirAll(volumeDir, 0700); err != nil {
return err
}
}
var deserialized MountInfo
if err := json.Unmarshal([]byte(mountInfo), &deserialized); err != nil {
return err
}
return ioutil.WriteFile(filepath.Join(volumeDir, mountInfoFileName), []byte(mountInfo), 0600)
}
// Remove deletes the direct volume path including all the files inside it.
func Remove(volumePath string) error {
// Find the base of the volume path to delete the whole volume path
base := strings.SplitN(volumePath, string(os.PathSeparator), 2)[0]
return os.RemoveAll(filepath.Join(kataDirectVolumeRootPath, base))
}
// VolumeMountInfo retrieves the mount info of a direct volume.
func VolumeMountInfo(volumePath string) (*MountInfo, error) {
mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, volumePath, mountInfoFileName)
if _, err := os.Stat(mountInfoFilePath); err != nil {
return nil, err
}
buf, err := ioutil.ReadFile(mountInfoFilePath)
if err != nil {
return nil, err
}
var mountInfo MountInfo
if err := json.Unmarshal(buf, &mountInfo); err != nil {
return nil, err
}
return &mountInfo, nil
}
// RecordSandboxId associates a sandbox id with a direct volume.
func RecordSandboxId(sandboxId string, volumePath string) error {
mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, volumePath, mountInfoFileName)
if _, err := os.Stat(mountInfoFilePath); err != nil {
return err
}
return ioutil.WriteFile(filepath.Join(kataDirectVolumeRootPath, volumePath, sandboxId), []byte(""), 0600)
}
func GetSandboxIdForVolume(volumePath string) (string, error) {
files, err := ioutil.ReadDir(filepath.Join(kataDirectVolumeRootPath, volumePath))
if err != nil {
return "", err
}
// Find the id of the first sandbox.
// We expect a direct-assigned volume is associated with only a sandbox at a time.
for _, file := range files {
if file.Name() != mountInfoFileName {
return file.Name(), nil
}
}
return "", fmt.Errorf("no sandbox found for %s", volumePath)
}

View File

@ -0,0 +1,94 @@
// Copyright (c) 2022 Databricks Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
package volume
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"testing"
"github.com/kata-containers/kata-containers/src/runtime/pkg/uuid"
"github.com/stretchr/testify/assert"
)
func TestAdd(t *testing.T) {
var err error
kataDirectVolumeRootPath, err = os.MkdirTemp(os.TempDir(), "add-test")
assert.Nil(t, err)
defer os.RemoveAll(kataDirectVolumeRootPath)
var volumePath = "/a/b/c"
var basePath = "a"
actual := MountInfo{
VolumeType: "block",
Device: "/dev/sda",
FsType: "ext4",
Options: []string{"journal_dev", "noload"},
}
buf, err := json.Marshal(actual)
assert.Nil(t, err)
// Add the mount info
assert.Nil(t, Add(volumePath, string(buf)))
// Validate the mount info
expected, err := VolumeMountInfo(volumePath)
assert.Nil(t, err)
assert.Equal(t, expected.Device, actual.Device)
assert.Equal(t, expected.FsType, actual.FsType)
assert.Equal(t, expected.Options, actual.Options)
// Remove the file
err = Remove(volumePath)
assert.Nil(t, err)
_, err = os.Stat(filepath.Join(kataDirectVolumeRootPath, basePath))
assert.True(t, errors.Is(err, os.ErrNotExist))
// Test invalid mount info json
assert.Error(t, Add(volumePath, "{invalid json}"))
}
func TestRecordSandboxId(t *testing.T) {
var err error
kataDirectVolumeRootPath, err = os.MkdirTemp(os.TempDir(), "recordSanboxId-test")
assert.Nil(t, err)
defer os.RemoveAll(kataDirectVolumeRootPath)
var volumePath = "/a/b/c"
mntInfo := MountInfo{
VolumeType: "block",
Device: "/dev/sda",
FsType: "ext4",
Options: []string{"journal_dev", "noload"},
}
buf, err := json.Marshal(mntInfo)
assert.Nil(t, err)
// Add the mount info
assert.Nil(t, Add(volumePath, string(buf)))
sandboxId := uuid.Generate().String()
err = RecordSandboxId(sandboxId, volumePath)
assert.Nil(t, err)
id, err := GetSandboxIdForVolume(volumePath)
assert.Nil(t, err)
assert.Equal(t, sandboxId, id)
}
func TestRecordSandboxIdNoMountInfoFile(t *testing.T) {
var err error
kataDirectVolumeRootPath, err = os.MkdirTemp(os.TempDir(), "recordSanboxId-test")
assert.Nil(t, err)
defer os.RemoveAll(kataDirectVolumeRootPath)
var volumePath = "/a/b/c"
sandboxId := uuid.Generate().String()
err = RecordSandboxId(sandboxId, volumePath)
assert.Error(t, err)
assert.True(t, errors.Is(err, os.ErrNotExist))
}

View File

@ -16,7 +16,7 @@ import (
"time" "time"
mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
"github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/expfmt" "github.com/prometheus/common/expfmt"
@ -224,7 +224,7 @@ func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error {
} }
func getParsedMetrics(sandboxID string, sandboxMetadata sandboxCRIMetadata) ([]*dto.MetricFamily, error) { func getParsedMetrics(sandboxID string, sandboxMetadata sandboxCRIMetadata) ([]*dto.MetricFamily, error) {
body, err := doGet(sandboxID, defaultTimeout, "metrics") body, err := shimclient.DoGet(sandboxID, defaultTimeout, "metrics")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -234,7 +234,7 @@ func getParsedMetrics(sandboxID string, sandboxMetadata sandboxCRIMetadata) ([]*
// GetSandboxMetrics will get sandbox's metrics from shim // GetSandboxMetrics will get sandbox's metrics from shim
func GetSandboxMetrics(sandboxID string) (string, error) { func GetSandboxMetrics(sandboxID string) (string, error) {
body, err := doGet(sandboxID, defaultTimeout, "metrics") body, err := shimclient.DoGet(sandboxID, defaultTimeout, "metrics")
if err != nil { if err != nil {
return "", err return "", err
} }

View File

@ -14,6 +14,8 @@ import (
"sync" "sync"
"time" "time"
"github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient"
"github.com/fsnotify/fsnotify" "github.com/fsnotify/fsnotify"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
@ -180,7 +182,7 @@ func (km *KataMonitor) GetAgentURL(w http.ResponseWriter, r *http.Request) {
return return
} }
data, err := doGet(sandboxID, defaultTimeout, "agent-url") data, err := shimclient.DoGet(sandboxID, defaultTimeout, "agent-url")
if err != nil { if err != nil {
commonServeError(w, http.StatusBadRequest, err) commonServeError(w, http.StatusBadRequest, err)
return return

View File

@ -7,13 +7,9 @@ package katamonitor
import ( import (
"fmt" "fmt"
"io"
"net"
"net/http" "net/http"
"time" "time"
cdshim "github.com/containerd/containerd/runtime/v2/shim"
shim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2" shim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2"
) )
@ -40,51 +36,3 @@ func getSandboxIDFromReq(r *http.Request) (string, error) {
func getSandboxFS() string { func getSandboxFS() string {
return shim.GetSandboxesStoragePath() return shim.GetSandboxesStoragePath()
} }
// BuildShimClient builds and returns an http client for communicating with the provided sandbox
func BuildShimClient(sandboxID string, timeout time.Duration) (*http.Client, error) {
return buildUnixSocketClient(shim.SocketAddress(sandboxID), timeout)
}
// buildUnixSocketClient build http client for Unix socket
func buildUnixSocketClient(socketAddr string, timeout time.Duration) (*http.Client, error) {
transport := &http.Transport{
DisableKeepAlives: true,
Dial: func(proto, addr string) (conn net.Conn, err error) {
return cdshim.AnonDialer(socketAddr, timeout)
},
}
client := &http.Client{
Transport: transport,
}
if timeout > 0 {
client.Timeout = timeout
}
return client, nil
}
func doGet(sandboxID string, timeoutInSeconds time.Duration, urlPath string) ([]byte, error) {
client, err := BuildShimClient(sandboxID, timeoutInSeconds)
if err != nil {
return nil, err
}
resp, err := client.Get(fmt.Sprintf("http://shim/%s", urlPath))
if err != nil {
return nil, err
}
defer func() {
resp.Body.Close()
}()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return body, nil
}

View File

@ -0,0 +1,79 @@
// Copyright (c) 2022 Databricks Inc.
//
// SPDX-License-Identifier: Apache-2.0
//
package shimclient
import (
"bytes"
"fmt"
"io"
"net"
"net/http"
"time"
cdshim "github.com/containerd/containerd/runtime/v2/shim"
shim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2"
)
// BuildShimClient builds and returns an http client for communicating with the provided sandbox
func BuildShimClient(sandboxID string, timeout time.Duration) (*http.Client, error) {
return buildUnixSocketClient(shim.SocketAddress(sandboxID), timeout)
}
// buildUnixSocketClient build http client for Unix socket
func buildUnixSocketClient(socketAddr string, timeout time.Duration) (*http.Client, error) {
transport := &http.Transport{
DisableKeepAlives: true,
Dial: func(proto, addr string) (conn net.Conn, err error) {
return cdshim.AnonDialer(socketAddr, timeout)
},
}
client := &http.Client{
Transport: transport,
}
if timeout > 0 {
client.Timeout = timeout
}
return client, nil
}
func DoGet(sandboxID string, timeoutInSeconds time.Duration, urlPath string) ([]byte, error) {
client, err := BuildShimClient(sandboxID, timeoutInSeconds)
if err != nil {
return nil, err
}
resp, err := client.Get(fmt.Sprintf("http://shim/%s", urlPath))
if err != nil {
return nil, err
}
defer func() {
resp.Body.Close()
}()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return body, nil
}
func DoPost(sandboxID string, timeoutInSeconds time.Duration, urlPath string, payload []byte) error {
client, err := BuildShimClient(sandboxID, timeoutInSeconds)
if err != nil {
return err
}
resp, err := client.Post(fmt.Sprintf("http://shim/%s", urlPath), "application/json", bytes.NewBuffer(payload))
defer func() {
resp.Body.Close()
}()
return err
}

View File

@ -190,4 +190,10 @@ type agent interface {
// getAgentMetrics get metrics of agent and guest through agent // getAgentMetrics get metrics of agent and guest through agent
getAgentMetrics(context.Context, *grpc.GetMetricsRequest) (*grpc.Metrics, error) getAgentMetrics(context.Context, *grpc.GetMetricsRequest) (*grpc.Metrics, error)
//getGuestVolumeStats get the filesystem stats of a volume specified by the volume mount path on the guest.
getGuestVolumeStats(ctx context.Context, volumeGuestPath string) ([]byte, error)
// resizeGuestVolume resizes a volume specified by the volume mount path on the guest.
resizeGuestVolume(ctx context.Context, volumeGuestPath string, size uint64) error
} }

View File

@ -15,6 +15,7 @@ import (
"syscall" "syscall"
"time" "time"
volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
@ -598,22 +599,50 @@ func (c *Container) createBlockDevices(ctx context.Context) error {
} }
// iterate all mounts and create block device if it's block based. // iterate all mounts and create block device if it's block based.
for i, m := range c.mounts { for i := range c.mounts {
if len(m.BlockDeviceID) > 0 { if len(c.mounts[i].BlockDeviceID) > 0 {
// Non-empty m.BlockDeviceID indicates there's already one device // Non-empty m.BlockDeviceID indicates there's already one device
// associated with the mount,so no need to create a new device for it // associated with the mount,so no need to create a new device for it
// and we only create block device for bind mount // and we only create block device for bind mount
continue continue
} }
if m.Type != "bind" { if c.mounts[i].Type != "bind" {
// We only handle for bind-mounts // We only handle for bind-mounts
continue continue
} }
// Handle directly assigned volume. Update the mount info based on the mount info json.
mntInfo, e := volume.VolumeMountInfo(c.mounts[i].Source)
if e != nil && !os.IsNotExist(e) {
c.Logger().WithError(e).WithField("mount-source", c.mounts[i].Source).
Error("failed to parse the mount info file for a direct assigned volume")
continue
}
if mntInfo != nil {
// Write out sandbox info file on the mount source to allow CSI to communicate with the runtime
if err := volume.RecordSandboxId(c.sandboxID, c.mounts[i].Source); err != nil {
c.Logger().WithError(err).Error("error writing sandbox info")
}
readonly := false
for _, flag := range mntInfo.Options {
if flag == "ro" {
readonly = true
break
}
}
c.mounts[i].Source = mntInfo.Device
c.mounts[i].Type = mntInfo.FsType
c.mounts[i].Options = mntInfo.Options
c.mounts[i].ReadOnly = readonly
}
var stat unix.Stat_t var stat unix.Stat_t
if err := unix.Stat(m.Source, &stat); err != nil { if err := unix.Stat(c.mounts[i].Source, &stat); err != nil {
return fmt.Errorf("stat %q failed: %v", m.Source, err) return fmt.Errorf("stat %q failed: %v", c.mounts[i].Source, err)
} }
var di *config.DeviceInfo var di *config.DeviceInfo
@ -623,17 +652,17 @@ func (c *Container) createBlockDevices(ctx context.Context) error {
// instead of passing this as a shared mount. // instead of passing this as a shared mount.
if stat.Mode&unix.S_IFBLK == unix.S_IFBLK { if stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
di = &config.DeviceInfo{ di = &config.DeviceInfo{
HostPath: m.Source, HostPath: c.mounts[i].Source,
ContainerPath: m.Destination, ContainerPath: c.mounts[i].Destination,
DevType: "b", DevType: "b",
Major: int64(unix.Major(uint64(stat.Rdev))), Major: int64(unix.Major(uint64(stat.Rdev))),
Minor: int64(unix.Minor(uint64(stat.Rdev))), Minor: int64(unix.Minor(uint64(stat.Rdev))),
ReadOnly: m.ReadOnly, ReadOnly: c.mounts[i].ReadOnly,
} }
// Check whether source can be used as a pmem device // Check whether source can be used as a pmem device
} else if di, err = config.PmemDeviceInfo(m.Source, m.Destination); err != nil { } else if di, err = config.PmemDeviceInfo(c.mounts[i].Source, c.mounts[i].Destination); err != nil {
c.Logger().WithError(err). c.Logger().WithError(err).
WithField("mount-source", m.Source). WithField("mount-source", c.mounts[i].Source).
Debug("no loop device") Debug("no loop device")
} }
@ -642,7 +671,7 @@ func (c *Container) createBlockDevices(ctx context.Context) error {
if err != nil { if err != nil {
// Do not return an error, try to create // Do not return an error, try to create
// devices for other mounts // devices for other mounts
c.Logger().WithError(err).WithField("mount-source", m.Source). c.Logger().WithError(err).WithField("mount-source", c.mounts[i].Source).
Error("device manager failed to create new device") Error("device manager failed to create new device")
continue continue

View File

@ -76,6 +76,9 @@ type VCSandbox interface {
UpdateRuntimeMetrics() error UpdateRuntimeMetrics() error
GetAgentMetrics(ctx context.Context) (string, error) GetAgentMetrics(ctx context.Context) (string, error)
GetAgentURL() (string, error) GetAgentURL() (string, error)
GuestVolumeStats(ctx context.Context, volumePath string) ([]byte, error)
ResizeGuestVolume(ctx context.Context, volumePath string, size uint64) error
} }
// VCContainer is the Container interface // VCContainer is the Container interface

View File

@ -6,6 +6,7 @@
package virtcontainers package virtcontainers
import ( import (
b64 "encoding/base64"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
@ -138,6 +139,8 @@ const (
grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" grpcGetOOMEventRequest = "grpc.GetOOMEventRequest"
grpcGetMetricsRequest = "grpc.GetMetricsRequest" grpcGetMetricsRequest = "grpc.GetMetricsRequest"
grpcAddSwapRequest = "grpc.AddSwapRequest" grpcAddSwapRequest = "grpc.AddSwapRequest"
grpcVolumeStatsRequest = "grpc.VolumeStatsRequest"
grpcResizeVolumeRequest = "grpc.ResizeVolumeRequest"
) )
// newKataAgent returns an agent from an agent type. // newKataAgent returns an agent from an agent type.
@ -882,35 +885,6 @@ func (k *kataAgent) removeIgnoredOCIMount(spec *specs.Spec, ignoredMounts map[st
return nil return nil
} }
func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages []*grpc.Storage) error {
ociMounts := spec.Mounts
var index int
var m specs.Mount
for i, v := range volumeStorages {
for index, m = range ociMounts {
if m.Destination != v.MountPoint {
continue
}
// Create a temporary location to mount the Storage. Mounting to the correct location
// will be handled by the OCI mount structure.
filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination))
path := filepath.Join(kataGuestSandboxStorageDir(), filename)
k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path)
ociMounts[index].Source = path
volumeStorages[i].MountPoint = path
break
}
if index == len(ociMounts) {
return fmt.Errorf("OCI mount not found for block volume %s", v.MountPoint)
}
}
return nil
}
func (k *kataAgent) constrainGRPCSpec(grpcSpec *grpc.Spec, passSeccomp bool, stripVfio bool) { func (k *kataAgent) constrainGRPCSpec(grpcSpec *grpc.Spec, passSeccomp bool, stripVfio bool) {
// Disable Hooks since they have been handled on the host and there is // Disable Hooks since they have been handled on the host and there is
// no reason to send them to the agent. It would make no sense to try // no reason to send them to the agent. It would make no sense to try
@ -1247,19 +1221,13 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co
// Append container devices for block devices passed with --device. // Append container devices for block devices passed with --device.
ctrDevices = k.appendDevices(ctrDevices, c) ctrDevices = k.appendDevices(ctrDevices, c)
// Handle all the volumes that are block device files. // Block based volumes will require some adjustments in the OCI spec, and creation of
// Note this call modifies the list of container devices to make sure // storage objects to pass to the agent.
// all hotplugged devices are unplugged, so this needs be done volumeStorages, err := k.handleBlkOCIMounts(c, ociSpec)
// after devices passed with --device are handled.
volumeStorages, err := k.handleBlockVolumes(c)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil {
return nil, err
}
ctrStorages = append(ctrStorages, volumeStorages...) ctrStorages = append(ctrStorages, volumeStorages...)
grpcSpec, err := grpc.OCItoGRPC(ociSpec) grpcSpec, err := grpc.OCItoGRPC(ociSpec)
@ -1522,16 +1490,46 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, m Mount, device api.D
vol.Options = []string{"bind"} vol.Options = []string{"bind"}
vol.MountPoint = m.Destination vol.MountPoint = m.Destination
// Assign the type from the mount, if it's specified (e.g. direct assigned volume)
if m.Type != "" {
vol.Fstype = m.Type
vol.Options = m.Options
}
return vol, nil return vol, nil
} }
// handleBlockVolumes handles volumes that are block devices files func (k *kataAgent) createBlkStorageObject(c *Container, m Mount) (*grpc.Storage, error) {
// by passing the block devices as Storage to the agent. var vol *grpc.Storage
func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
id := m.BlockDeviceID
device := c.sandbox.devManager.GetDeviceByID(id)
if device == nil {
k.Logger().WithField("device", id).Error("failed to find device by id")
return nil, fmt.Errorf("Failed to find device by id (id=%s)", id)
}
var err error
switch device.DeviceType() {
case config.DeviceBlock:
vol, err = k.handleDeviceBlockVolume(c, m, device)
case config.VhostUserBlk:
vol, err = k.handleVhostUserBlkVolume(c, m, device)
default:
return nil, fmt.Errorf("Unknown device type")
}
return vol, err
}
// handleBlkOCIMounts will create a unique destination mountpoint in the guest for each volume in the
// given container and will update the OCI spec to utilize this mount point as the new source for the
// container volume. The container mount structure is updated to store the guest destination mountpoint.
func (k *kataAgent) handleBlkOCIMounts(c *Container, spec *specs.Spec) ([]*grpc.Storage, error) {
var volumeStorages []*grpc.Storage var volumeStorages []*grpc.Storage
for _, m := range c.mounts { for i, m := range c.mounts {
id := m.BlockDeviceID id := m.BlockDeviceID
if len(id) == 0 { if len(id) == 0 {
@ -1542,29 +1540,39 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
// device is detached with detachDevices() for a container. // device is detached with detachDevices() for a container.
c.devices = append(c.devices, ContainerDevice{ID: id, ContainerPath: m.Destination}) c.devices = append(c.devices, ContainerDevice{ID: id, ContainerPath: m.Destination})
var vol *grpc.Storage // Create Storage structure
vol, err := k.createBlkStorageObject(c, m)
device := c.sandbox.devManager.GetDeviceByID(id)
if device == nil {
k.Logger().WithField("device", id).Error("failed to find device by id")
return nil, fmt.Errorf("Failed to find device by id (id=%s)", id)
}
var err error
switch device.DeviceType() {
case config.DeviceBlock:
vol, err = k.handleDeviceBlockVolume(c, m, device)
case config.VhostUserBlk:
vol, err = k.handleVhostUserBlkVolume(c, m, device)
default:
k.Logger().Error("Unknown device type")
continue
}
if vol == nil || err != nil { if vol == nil || err != nil {
return nil, err return nil, err
} }
// Each device will be mounted at a unique location within the VM only once. Mounting
// to the container specific location is handled within the OCI spec. Let's ensure that
// the storage mount point is unique for each device. This is then utilized as the source
// in the OCI spec. If multiple containers mount the same block device, it's refcounted inside
// the guest by Kata agent.
filename := b64.StdEncoding.EncodeToString([]byte(vol.Source))
// Make the base64 encoding path safe.
filename = strings.ReplaceAll(filename, "/", "_")
path := filepath.Join(kataGuestSandboxStorageDir(), filename)
// Update applicable OCI mount source
for idx, ociMount := range spec.Mounts {
if ociMount.Destination != vol.MountPoint {
continue
}
k.Logger().WithFields(logrus.Fields{
"original-source": ociMount.Source,
"new-source": path,
}).Debug("Replacing OCI mount source")
spec.Mounts[idx].Source = path
break
}
// Update storage mountpoint, and save guest device mount path to container mount struct:
vol.MountPoint = path
c.mounts[i].GuestDeviceMount = path
volumeStorages = append(volumeStorages, vol) volumeStorages = append(volumeStorages, vol)
} }
@ -1950,6 +1958,12 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
k.reqHandlers[grpcAddSwapRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { k.reqHandlers[grpcAddSwapRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
return k.client.AgentServiceClient.AddSwap(ctx, req.(*grpc.AddSwapRequest)) return k.client.AgentServiceClient.AddSwap(ctx, req.(*grpc.AddSwapRequest))
} }
k.reqHandlers[grpcVolumeStatsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
return k.client.AgentServiceClient.GetVolumeStats(ctx, req.(*grpc.VolumeStatsRequest))
}
k.reqHandlers[grpcResizeVolumeRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
return k.client.AgentServiceClient.ResizeVolume(ctx, req.(*grpc.ResizeVolumeRequest))
}
} }
func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx context.Context, cancel context.CancelFunc) { func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx context.Context, cancel context.CancelFunc) {
@ -2167,3 +2181,22 @@ func (k *kataAgent) getAgentMetrics(ctx context.Context, req *grpc.GetMetricsReq
return resp.(*grpc.Metrics), nil return resp.(*grpc.Metrics), nil
} }
func (k *kataAgent) getGuestVolumeStats(ctx context.Context, volumeGuestPath string) ([]byte, error) {
result, err := k.sendReq(ctx, &grpc.VolumeStatsRequest{VolumeGuestPath: volumeGuestPath})
if err != nil {
return nil, err
}
buf, err := json.Marshal(result.(*grpc.VolumeStatsResponse))
if err != nil {
return nil, err
}
return buf, nil
}
func (k *kataAgent) resizeGuestVolume(ctx context.Context, volumeGuestPath string, size uint64) error {
_, err := k.sendReq(ctx, &grpc.ResizeVolumeRequest{VolumeGuestPath: volumeGuestPath, Size_: size})
return err
}

View File

@ -405,24 +405,28 @@ func TestHandleBlockVolume(t *testing.T) {
containers[c.id].sandbox = &sandbox containers[c.id].sandbox = &sandbox
containers[c.id].mounts = mounts containers[c.id].mounts = mounts
volumeStorages, err := k.handleBlockVolumes(c) vStorage, err := k.createBlkStorageObject(c, vMount)
assert.Nil(t, err, "Error while handling block volumes")
bStorage, err := k.createBlkStorageObject(c, bMount)
assert.Nil(t, err, "Error while handling block volumes")
dStorage, err := k.createBlkStorageObject(c, dMount)
assert.Nil(t, err, "Error while handling block volumes") assert.Nil(t, err, "Error while handling block volumes")
vStorage := &pb.Storage{ vStorageExpected := &pb.Storage{
MountPoint: vDestination, MountPoint: vDestination,
Fstype: "bind", Fstype: "bind",
Options: []string{"bind"}, Options: []string{"bind"},
Driver: kataBlkDevType, Driver: kataBlkDevType,
Source: vPCIPath.String(), Source: vPCIPath.String(),
} }
bStorage := &pb.Storage{ bStorageExpected := &pb.Storage{
MountPoint: bDestination, MountPoint: bDestination,
Fstype: "bind", Fstype: "bind",
Options: []string{"bind"}, Options: []string{"bind"},
Driver: kataBlkDevType, Driver: kataBlkDevType,
Source: bPCIPath.String(), Source: bPCIPath.String(),
} }
dStorage := &pb.Storage{ dStorageExpected := &pb.Storage{
MountPoint: dDestination, MountPoint: dDestination,
Fstype: "ext4", Fstype: "ext4",
Options: []string{"ro"}, Options: []string{"ro"},
@ -430,9 +434,9 @@ func TestHandleBlockVolume(t *testing.T) {
Source: dPCIPath.String(), Source: dPCIPath.String(),
} }
assert.Equal(t, vStorage, volumeStorages[0], "Error while handle VhostUserBlk type block volume") assert.Equal(t, vStorage, vStorageExpected, "Error while handle VhostUserBlk type block volume")
assert.Equal(t, bStorage, volumeStorages[1], "Error while handle BlockDevice type block volume") assert.Equal(t, bStorage, bStorageExpected, "Error while handle BlockDevice type block volume")
assert.Equal(t, dStorage, volumeStorages[2], "Error while handle direct BlockDevice type block volume") assert.Equal(t, dStorage, dStorageExpected, "Error while handle direct BlockDevice type block volume")
} }
func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) { func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) {

View File

@ -242,3 +242,11 @@ func (n *mockAgent) getOOMEvent(ctx context.Context) (string, error) {
func (n *mockAgent) getAgentMetrics(ctx context.Context, req *grpc.GetMetricsRequest) (*grpc.Metrics, error) { func (n *mockAgent) getAgentMetrics(ctx context.Context, req *grpc.GetMetricsRequest) (*grpc.Metrics, error) {
return nil, nil return nil, nil
} }
func (n *mockAgent) getGuestVolumeStats(ctx context.Context, volumeGuestPath string) ([]byte, error) {
return nil, nil
}
func (n *mockAgent) resizeGuestVolume(ctx context.Context, volumeGuestPath string, size uint64) error {
return nil
}

View File

@ -172,7 +172,7 @@ func getDeviceForPath(path string) (device, error) {
}, nil }, nil
} }
// We get the mount point by recursively peforming stat on the path // We get the mount point by recursively performing stat on the path
// The point where the device changes indicates the mountpoint // The point where the device changes indicates the mountpoint
for { for {
if mountPoint == "/" { if mountPoint == "/" {
@ -326,7 +326,9 @@ func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string
// Mount describes a container mount. // Mount describes a container mount.
type Mount struct { type Mount struct {
Source string // Source is the source of the mount.
Source string
// Destination is the destination of the mount (within the container).
Destination string Destination string
// Type specifies the type of filesystem to mount. // Type specifies the type of filesystem to mount.
@ -335,6 +337,11 @@ type Mount struct {
// HostPath used to store host side bind mount path // HostPath used to store host side bind mount path
HostPath string HostPath string
// GuestDeviceMount represents the path within the VM that the device
// is mounted. Only relevant for block devices. This is tracked in the event
// runtime wants to query the agent for mount stats.
GuestDeviceMount string
// BlockDeviceID represents block device that is attached to the // BlockDeviceID represents block device that is attached to the
// VM in case this mount is a block device file or a directory // VM in case this mount is a block device file or a directory
// backed by a block device. // backed by a block device.

View File

@ -2415,6 +2415,87 @@ func (m *Metrics) XXX_DiscardUnknown() {
var xxx_messageInfo_Metrics proto.InternalMessageInfo var xxx_messageInfo_Metrics proto.InternalMessageInfo
type VolumeStatsRequest struct {
// The volume path on the guest outside the container
VolumeGuestPath string `protobuf:"bytes,1,opt,name=volume_guest_path,json=volumeGuestPath,proto3" json:"volume_guest_path,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *VolumeStatsRequest) Reset() { *m = VolumeStatsRequest{} }
func (*VolumeStatsRequest) ProtoMessage() {}
func (*VolumeStatsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_712ce9a559fda969, []int{56}
}
func (m *VolumeStatsRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *VolumeStatsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_VolumeStatsRequest.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *VolumeStatsRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_VolumeStatsRequest.Merge(m, src)
}
func (m *VolumeStatsRequest) XXX_Size() int {
return m.Size()
}
func (m *VolumeStatsRequest) XXX_DiscardUnknown() {
xxx_messageInfo_VolumeStatsRequest.DiscardUnknown(m)
}
var xxx_messageInfo_VolumeStatsRequest proto.InternalMessageInfo
type ResizeVolumeRequest struct {
// Full VM guest path of the volume (outside the container)
VolumeGuestPath string `protobuf:"bytes,1,opt,name=volume_guest_path,json=volumeGuestPath,proto3" json:"volume_guest_path,omitempty"`
Size_ uint64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *ResizeVolumeRequest) Reset() { *m = ResizeVolumeRequest{} }
func (*ResizeVolumeRequest) ProtoMessage() {}
func (*ResizeVolumeRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_712ce9a559fda969, []int{57}
}
func (m *ResizeVolumeRequest) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *ResizeVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_ResizeVolumeRequest.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalToSizedBuffer(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (m *ResizeVolumeRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_ResizeVolumeRequest.Merge(m, src)
}
func (m *ResizeVolumeRequest) XXX_Size() int {
return m.Size()
}
func (m *ResizeVolumeRequest) XXX_DiscardUnknown() {
xxx_messageInfo_ResizeVolumeRequest.DiscardUnknown(m)
}
var xxx_messageInfo_ResizeVolumeRequest proto.InternalMessageInfo
func init() { func init() {
proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest") proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest")
proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest") proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest")
@ -2474,6 +2555,8 @@ func init() {
proto.RegisterType((*AddSwapRequest)(nil), "grpc.AddSwapRequest") proto.RegisterType((*AddSwapRequest)(nil), "grpc.AddSwapRequest")
proto.RegisterType((*GetMetricsRequest)(nil), "grpc.GetMetricsRequest") proto.RegisterType((*GetMetricsRequest)(nil), "grpc.GetMetricsRequest")
proto.RegisterType((*Metrics)(nil), "grpc.Metrics") proto.RegisterType((*Metrics)(nil), "grpc.Metrics")
proto.RegisterType((*VolumeStatsRequest)(nil), "grpc.VolumeStatsRequest")
proto.RegisterType((*ResizeVolumeRequest)(nil), "grpc.ResizeVolumeRequest")
} }
func init() { func init() {
@ -2481,194 +2564,198 @@ func init() {
} }
var fileDescriptor_712ce9a559fda969 = []byte{ var fileDescriptor_712ce9a559fda969 = []byte{
// 2978 bytes of a gzipped FileDescriptorProto // 3055 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0x4b, 0x6f, 0x23, 0xc7, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x1a, 0xcb, 0x72, 0x24, 0x47,
0xd1, 0xe6, 0x43, 0x22, 0x59, 0x7c, 0x89, 0x23, 0xad, 0x96, 0x4b, 0xdb, 0xfa, 0xd6, 0xb3, 0xf6, 0xd1, 0xf3, 0x90, 0x66, 0x26, 0xe7, 0xa5, 0x69, 0x69, 0xb5, 0xb3, 0x63, 0x5b, 0xac, 0x7b, 0xed,
0x7a, 0x6d, 0x7f, 0xa6, 0xec, 0xb5, 0xf1, 0xad, 0x1f, 0xf0, 0xb7, 0x58, 0x69, 0x65, 0x49, 0xb6, 0xf5, 0xda, 0xc6, 0x92, 0xbd, 0x76, 0xb0, 0x7e, 0x84, 0x59, 0x24, 0xad, 0x2c, 0xc9, 0xb6, 0xbc,
0xe5, 0x65, 0x46, 0x16, 0x1c, 0x24, 0x48, 0x06, 0xc3, 0x99, 0x16, 0xd9, 0x16, 0x67, 0x7a, 0xdc, 0x43, 0xcb, 0xc2, 0x04, 0x04, 0x74, 0xb4, 0xba, 0x6b, 0x47, 0x65, 0x4d, 0x77, 0xb5, 0xab, 0xab,
0xd3, 0xa3, 0x15, 0x1d, 0x20, 0xc8, 0x29, 0xb9, 0xe5, 0x98, 0x5b, 0xfe, 0x40, 0x90, 0x5b, 0x80, 0xb5, 0x92, 0x89, 0x20, 0x38, 0xc1, 0x8d, 0x23, 0x37, 0x7e, 0x80, 0xe0, 0xc6, 0x91, 0x0b, 0x07,
0x5c, 0x72, 0xcd, 0xc1, 0xc8, 0x29, 0xc7, 0x9c, 0x82, 0x78, 0x7f, 0x42, 0x7e, 0x41, 0xd0, 0xaf, 0x0e, 0x0e, 0x4e, 0x1c, 0x39, 0x11, 0x78, 0x3f, 0x81, 0x2f, 0x20, 0xea, 0xd5, 0x5d, 0x3d, 0x0f,
0x79, 0xf0, 0x21, 0x27, 0x82, 0x80, 0x5c, 0x88, 0xae, 0xea, 0xea, 0x7a, 0x75, 0x57, 0x75, 0x55, 0x19, 0x14, 0x1b, 0xc1, 0x65, 0xa2, 0x33, 0x2b, 0x2b, 0x5f, 0x55, 0x99, 0x95, 0x59, 0x35, 0x30,
0x0f, 0x61, 0x30, 0xc2, 0x6c, 0x1c, 0x0f, 0xfb, 0x2e, 0xf1, 0xb7, 0xcf, 0x1c, 0xe6, 0xbc, 0xe9, 0x1c, 0x61, 0x76, 0x92, 0x1e, 0xaf, 0xfb, 0x24, 0xdc, 0x38, 0xf5, 0x98, 0xf7, 0xba, 0x4f, 0x22,
0x92, 0x80, 0x39, 0x38, 0x40, 0x34, 0x9a, 0x83, 0x23, 0xea, 0x6e, 0x4f, 0xf0, 0x30, 0xda, 0x0e, 0xe6, 0xe1, 0x08, 0xd1, 0x64, 0x0a, 0x4e, 0xa8, 0xbf, 0x31, 0xc6, 0xc7, 0xc9, 0x46, 0x4c, 0x09,
0x29, 0x61, 0xc4, 0x25, 0x13, 0x35, 0x8a, 0xb6, 0x9d, 0x11, 0x0a, 0x58, 0x5f, 0x00, 0x46, 0x79, 0x23, 0x3e, 0x19, 0xab, 0xaf, 0x64, 0xc3, 0x1b, 0xa1, 0x88, 0xad, 0x0b, 0xc0, 0xaa, 0x8e, 0x68,
0x44, 0x43, 0xb7, 0x57, 0x23, 0x2e, 0x96, 0x88, 0x5e, 0x9d, 0x4d, 0x43, 0x14, 0x29, 0xe0, 0xf9, 0xec, 0x0f, 0x1a, 0xc4, 0xc7, 0x12, 0x31, 0x68, 0xf8, 0x89, 0xfe, 0x6c, 0xb2, 0x8b, 0x18, 0x25,
0x11, 0x21, 0xa3, 0x09, 0x92, 0x0b, 0x87, 0xf1, 0xe9, 0x36, 0xf2, 0x43, 0x36, 0x95, 0x93, 0xe6, 0x0a, 0x78, 0x76, 0x44, 0xc8, 0x68, 0x8c, 0x24, 0x8f, 0xe3, 0xf4, 0xd1, 0x06, 0x0a, 0x63, 0x76,
0x6f, 0x8b, 0xb0, 0xb9, 0x4b, 0x91, 0xc3, 0xd0, 0xae, 0x96, 0x6a, 0xa1, 0xaf, 0x63, 0x14, 0x31, 0x21, 0x07, 0xed, 0xdf, 0x97, 0x61, 0x75, 0x9b, 0x22, 0x8f, 0xa1, 0x6d, 0xad, 0x80, 0x83, 0xbe,
0xe3, 0x25, 0x68, 0x24, 0x9a, 0xd8, 0xd8, 0xeb, 0x16, 0x6e, 0x17, 0xee, 0xd5, 0xac, 0x7a, 0x82, 0x4c, 0x51, 0xc2, 0xac, 0x17, 0xa0, 0x95, 0x29, 0xe5, 0xe2, 0xa0, 0x5f, 0xba, 0x59, 0xba, 0xd3,
0x3b, 0xf4, 0x8c, 0x9b, 0x50, 0x41, 0x17, 0xc8, 0xe5, 0xb3, 0x45, 0x31, 0xbb, 0xca, 0xc1, 0x43, 0x70, 0x9a, 0x19, 0x6e, 0x3f, 0xb0, 0xae, 0x43, 0x0d, 0x9d, 0x23, 0x9f, 0x8f, 0x96, 0xc5, 0xe8,
0xcf, 0x78, 0x1b, 0xea, 0x11, 0xa3, 0x38, 0x18, 0xd9, 0x71, 0x84, 0x68, 0xb7, 0x74, 0xbb, 0x70, 0x22, 0x07, 0xf7, 0x03, 0xeb, 0x4d, 0x68, 0x26, 0x8c, 0xe2, 0x68, 0xe4, 0xa6, 0x09, 0xa2, 0xfd,
0xaf, 0x7e, 0x7f, 0xad, 0xcf, 0xf5, 0xec, 0x1f, 0x8b, 0x89, 0x93, 0x08, 0x51, 0x0b, 0xa2, 0x64, 0xca, 0xcd, 0xd2, 0x9d, 0xe6, 0xdd, 0xa5, 0x75, 0xae, 0xf2, 0xfa, 0xa1, 0x18, 0x38, 0x4a, 0x10,
0x6c, 0xdc, 0x85, 0x8a, 0x87, 0xce, 0xb1, 0x8b, 0xa2, 0x6e, 0xf9, 0x76, 0xe9, 0x5e, 0xfd, 0x7e, 0x75, 0x20, 0xc9, 0xbe, 0xad, 0xdb, 0x50, 0x0b, 0xd0, 0x19, 0xf6, 0x51, 0xd2, 0xaf, 0xde, 0xac,
0x43, 0x92, 0x3f, 0x16, 0x48, 0x4b, 0x4f, 0x1a, 0xaf, 0x41, 0x35, 0x62, 0x84, 0x3a, 0x23, 0x14, 0xdc, 0x69, 0xde, 0x6d, 0x49, 0xf2, 0x07, 0x02, 0xe9, 0xe8, 0x41, 0xeb, 0x15, 0xa8, 0x27, 0x8c,
0x75, 0x57, 0x04, 0x61, 0x53, 0xf3, 0x15, 0x58, 0x2b, 0x99, 0x36, 0x5e, 0x80, 0xd2, 0x93, 0xdd, 0x50, 0x6f, 0x84, 0x92, 0xfe, 0x82, 0x20, 0x6c, 0x6b, 0xbe, 0x02, 0xeb, 0x64, 0xc3, 0xd6, 0x73,
0xc3, 0xee, 0xaa, 0x90, 0x0e, 0x8a, 0x2a, 0x44, 0xae, 0xc5, 0xd1, 0xc6, 0x1d, 0x68, 0x46, 0x4e, 0x50, 0x79, 0xb8, 0xbd, 0xdf, 0x5f, 0x14, 0xd2, 0x41, 0x51, 0xc5, 0xc8, 0x77, 0x38, 0xda, 0xba,
0xe0, 0x0d, 0xc9, 0x85, 0x1d, 0x62, 0x2f, 0x88, 0xba, 0x95, 0xdb, 0x85, 0x7b, 0x55, 0xab, 0xa1, 0x05, 0xed, 0xc4, 0x8b, 0x82, 0x63, 0x72, 0xee, 0xc6, 0x38, 0x88, 0x92, 0x7e, 0xed, 0x66, 0xe9,
0x90, 0x03, 0x8e, 0x33, 0x3f, 0x80, 0x1b, 0xc7, 0xcc, 0xa1, 0xec, 0x0a, 0xde, 0x31, 0x4f, 0x60, 0x4e, 0xdd, 0x69, 0x29, 0xe4, 0x90, 0xe3, 0xec, 0xf7, 0xe0, 0xda, 0x21, 0xf3, 0x28, 0xbb, 0x82,
0xd3, 0x42, 0x3e, 0x39, 0xbf, 0x92, 0x6b, 0xbb, 0x50, 0x61, 0xd8, 0x47, 0x24, 0x66, 0xc2, 0xb5, 0x77, 0xec, 0x23, 0x58, 0x75, 0x50, 0x48, 0xce, 0xae, 0xe4, 0xda, 0x3e, 0xd4, 0x18, 0x0e, 0x11,
0x4d, 0x4b, 0x83, 0xe6, 0xef, 0x0b, 0x60, 0xec, 0x5d, 0x20, 0x77, 0x40, 0x89, 0x8b, 0xa2, 0xe8, 0x49, 0x99, 0x70, 0x6d, 0xdb, 0xd1, 0xa0, 0xfd, 0xc7, 0x12, 0x58, 0x3b, 0xe7, 0xc8, 0x1f, 0x52,
0xbf, 0xb4, 0x5d, 0xaf, 0x42, 0x25, 0x94, 0x0a, 0x74, 0xcb, 0x82, 0x5c, 0xed, 0x82, 0xd6, 0x4a, 0xe2, 0xa3, 0x24, 0xf9, 0x3f, 0x2d, 0xd7, 0xcb, 0x50, 0x8b, 0xa5, 0x02, 0xfd, 0xaa, 0x20, 0x57,
0xcf, 0x9a, 0x5f, 0xc1, 0xc6, 0x31, 0x1e, 0x05, 0xce, 0xe4, 0x1a, 0xf5, 0xdd, 0x84, 0xd5, 0x48, 0xab, 0xa0, 0xb5, 0xd2, 0xa3, 0xf6, 0x17, 0xb0, 0x72, 0x88, 0x47, 0x91, 0x37, 0x7e, 0x8a, 0xfa,
0xf0, 0x14, 0xaa, 0x36, 0x2d, 0x05, 0x99, 0x03, 0x30, 0xbe, 0x74, 0x30, 0xbb, 0x3e, 0x49, 0xe6, 0xae, 0xc2, 0x62, 0x22, 0x78, 0x0a, 0x55, 0xdb, 0x8e, 0x82, 0xec, 0x21, 0x58, 0x9f, 0x7b, 0x98,
0x9b, 0xb0, 0x9e, 0xe3, 0x18, 0x85, 0x24, 0x88, 0x90, 0x50, 0x80, 0x39, 0x2c, 0x8e, 0x04, 0xb3, 0x3d, 0x3d, 0x49, 0xf6, 0xeb, 0xb0, 0x5c, 0xe0, 0x98, 0xc4, 0x24, 0x4a, 0x90, 0x50, 0x80, 0x79,
0x15, 0x4b, 0x41, 0x26, 0x81, 0xcd, 0x93, 0xd0, 0xbb, 0x62, 0x34, 0xdd, 0x87, 0x1a, 0x45, 0x11, 0x2c, 0x4d, 0x04, 0xb3, 0x05, 0x47, 0x41, 0x36, 0x81, 0xd5, 0xa3, 0x38, 0xb8, 0x62, 0x34, 0xdd,
0x89, 0x29, 0x8f, 0x81, 0xa2, 0x70, 0xea, 0x86, 0x74, 0xea, 0x67, 0x38, 0x88, 0x2f, 0x2c, 0x3d, 0x85, 0x06, 0x45, 0x09, 0x49, 0x29, 0x8f, 0x81, 0xb2, 0x70, 0xea, 0x8a, 0x74, 0xea, 0x27, 0x38,
0x67, 0xa5, 0x64, 0xea, 0x7c, 0xb2, 0xe8, 0x2a, 0xe7, 0xf3, 0x03, 0xb8, 0x31, 0x70, 0xe2, 0xe8, 0x4a, 0xcf, 0x1d, 0x3d, 0xe6, 0xe4, 0x64, 0x6a, 0x7f, 0xb2, 0xe4, 0x2a, 0xfb, 0xf3, 0x3d, 0xb8,
0x2a, 0xba, 0x9a, 0x1f, 0xf2, 0xb3, 0x1d, 0xc5, 0xfe, 0x95, 0x16, 0xff, 0xae, 0x00, 0xd5, 0xdd, 0x36, 0xf4, 0xd2, 0xe4, 0x2a, 0xba, 0xda, 0xef, 0xf3, 0xbd, 0x9d, 0xa4, 0xe1, 0x95, 0x26, 0xff,
0x30, 0x3e, 0x89, 0x9c, 0x11, 0x32, 0xfe, 0x07, 0xea, 0x8c, 0x30, 0x67, 0x62, 0xc7, 0x1c, 0x14, 0xa1, 0x04, 0xf5, 0xed, 0x38, 0x3d, 0x4a, 0xbc, 0x11, 0xb2, 0xbe, 0x03, 0x4d, 0x46, 0x98, 0x37,
0xe4, 0x65, 0x0b, 0x04, 0x4a, 0x12, 0xbc, 0x04, 0x8d, 0x10, 0x51, 0x37, 0x8c, 0x15, 0x45, 0xf1, 0x76, 0x53, 0x0e, 0x0a, 0xf2, 0xaa, 0x03, 0x02, 0x25, 0x09, 0x5e, 0x80, 0x56, 0x8c, 0xa8, 0x1f,
0x76, 0xe9, 0x5e, 0xd9, 0xaa, 0x4b, 0x9c, 0x24, 0xe9, 0xc3, 0xba, 0x98, 0xb3, 0x71, 0x60, 0x9f, 0xa7, 0x8a, 0xa2, 0x7c, 0xb3, 0x72, 0xa7, 0xea, 0x34, 0x25, 0x4e, 0x92, 0xac, 0xc3, 0xb2, 0x18,
0x21, 0x1a, 0xa0, 0x89, 0x4f, 0x3c, 0x24, 0x0e, 0x47, 0xd9, 0xea, 0x88, 0xa9, 0xc3, 0xe0, 0xd3, 0x73, 0x71, 0xe4, 0x9e, 0x22, 0x1a, 0xa1, 0x71, 0x48, 0x02, 0x24, 0x36, 0x47, 0xd5, 0xe9, 0x89,
0x64, 0xc2, 0x78, 0x1d, 0x3a, 0x09, 0x3d, 0x3f, 0xf1, 0x82, 0xba, 0x2c, 0xa8, 0xdb, 0x8a, 0xfa, 0xa1, 0xfd, 0xe8, 0xe3, 0x6c, 0xc0, 0x7a, 0x15, 0x7a, 0x19, 0x3d, 0xdf, 0xf1, 0x82, 0xba, 0x2a,
0x44, 0xa1, 0xcd, 0x9f, 0x43, 0xeb, 0x8b, 0x31, 0x25, 0x8c, 0x4d, 0x70, 0x30, 0x7a, 0xec, 0x30, 0xa8, 0xbb, 0x8a, 0xfa, 0x48, 0xa1, 0xed, 0x5f, 0x42, 0xe7, 0xb3, 0x13, 0x4a, 0x18, 0x1b, 0xe3,
0x87, 0x87, 0x66, 0x88, 0x28, 0x26, 0x5e, 0xa4, 0xb4, 0xd5, 0xa0, 0xf1, 0x06, 0x74, 0x98, 0xa4, 0x68, 0xf4, 0xc0, 0x63, 0x1e, 0x0f, 0xcd, 0x18, 0x51, 0x4c, 0x82, 0x44, 0x69, 0xab, 0x41, 0xeb,
0x45, 0x9e, 0xad, 0x69, 0x8a, 0x82, 0x66, 0x2d, 0x99, 0x18, 0x28, 0xe2, 0x57, 0xa0, 0x95, 0x12, 0x35, 0xe8, 0x31, 0x49, 0x8b, 0x02, 0x57, 0xd3, 0x94, 0x05, 0xcd, 0x52, 0x36, 0x30, 0x54, 0xc4,
0xf3, 0xe0, 0x56, 0xfa, 0x36, 0x13, 0xec, 0x17, 0xd8, 0x47, 0xe6, 0xb9, 0xf0, 0x95, 0xd8, 0x64, 0x2f, 0x41, 0x27, 0x27, 0xe6, 0xc1, 0xad, 0xf4, 0x6d, 0x67, 0xd8, 0xcf, 0x70, 0x88, 0xec, 0x33,
0xe3, 0x0d, 0xa8, 0xa5, 0x7e, 0x28, 0x88, 0x13, 0xd2, 0x92, 0x27, 0x44, 0xbb, 0xd3, 0xaa, 0x26, 0xe1, 0x2b, 0xb1, 0xc8, 0xd6, 0x6b, 0xd0, 0xc8, 0xfd, 0x50, 0x12, 0x3b, 0xa4, 0x23, 0x77, 0x88,
0x4e, 0xf9, 0x08, 0xda, 0x2c, 0x51, 0xdc, 0xf6, 0x1c, 0xe6, 0xe4, 0x0f, 0x55, 0xde, 0x2a, 0xab, 0x76, 0xa7, 0x53, 0xcf, 0x9c, 0xf2, 0x01, 0x74, 0x59, 0xa6, 0xb8, 0x1b, 0x78, 0xcc, 0x2b, 0x6e,
0xc5, 0x72, 0xb0, 0xf9, 0x21, 0xd4, 0x06, 0xd8, 0x8b, 0xa4, 0xe0, 0x2e, 0x54, 0xdc, 0x98, 0x52, 0xaa, 0xa2, 0x55, 0x4e, 0x87, 0x15, 0x60, 0xfb, 0x7d, 0x68, 0x0c, 0x71, 0x90, 0x48, 0xc1, 0x7d,
0x14, 0x30, 0x6d, 0xb2, 0x02, 0x8d, 0x0d, 0x58, 0x99, 0x60, 0x1f, 0x33, 0x65, 0xa6, 0x04, 0x4c, 0xa8, 0xf9, 0x29, 0xa5, 0x28, 0x62, 0xda, 0x64, 0x05, 0x5a, 0x2b, 0xb0, 0x30, 0xc6, 0x21, 0x66,
0x02, 0x70, 0x84, 0x7c, 0x42, 0xa7, 0xc2, 0x61, 0x1b, 0xb0, 0x92, 0xdd, 0x5c, 0x09, 0x18, 0xcf, 0xca, 0x4c, 0x09, 0xd8, 0x04, 0xe0, 0x00, 0x85, 0x84, 0x5e, 0x08, 0x87, 0xad, 0xc0, 0x82, 0xb9,
0x43, 0xcd, 0x77, 0x2e, 0x92, 0x4d, 0xe5, 0x33, 0x55, 0xdf, 0xb9, 0x90, 0xca, 0x77, 0xa1, 0x72, 0xb8, 0x12, 0xb0, 0x9e, 0x85, 0x46, 0xe8, 0x9d, 0x67, 0x8b, 0xca, 0x47, 0xea, 0xa1, 0x77, 0x2e,
0xea, 0xe0, 0x89, 0x1b, 0x30, 0xe5, 0x15, 0x0d, 0xa6, 0x02, 0xcb, 0x59, 0x81, 0x7f, 0x2e, 0x42, 0x95, 0xef, 0x43, 0xed, 0x91, 0x87, 0xc7, 0x7e, 0xc4, 0x94, 0x57, 0x34, 0x98, 0x0b, 0xac, 0x9a,
0x5d, 0x4a, 0x94, 0x0a, 0x6f, 0xc0, 0x8a, 0xeb, 0xb8, 0xe3, 0x44, 0xa4, 0x00, 0x8c, 0xbb, 0x5a, 0x02, 0xff, 0x5a, 0x86, 0xa6, 0x94, 0x28, 0x15, 0x5e, 0x81, 0x05, 0xdf, 0xf3, 0x4f, 0x32, 0x91,
0x91, 0x62, 0x36, 0xc3, 0xa5, 0x9a, 0x6a, 0xd5, 0xb6, 0x01, 0xa2, 0xa7, 0x4e, 0xa8, 0x74, 0x2b, 0x02, 0xb0, 0x6e, 0x6b, 0x45, 0xca, 0x66, 0x86, 0xcb, 0x35, 0xd5, 0xaa, 0x6d, 0x00, 0x24, 0x8f,
0x2d, 0x21, 0xae, 0x71, 0x1a, 0xa9, 0xee, 0x3b, 0xd0, 0x90, 0xe7, 0x4e, 0x2d, 0x29, 0x2f, 0x59, 0xbd, 0x58, 0xe9, 0x56, 0x99, 0x43, 0xdc, 0xe0, 0x34, 0x52, 0xdd, 0xb7, 0xa0, 0x25, 0xf7, 0x9d,
0x52, 0x97, 0x54, 0x72, 0xd1, 0x1d, 0x68, 0xc6, 0x11, 0xb2, 0xc7, 0x18, 0x51, 0x87, 0xba, 0xe3, 0x9a, 0x52, 0x9d, 0x33, 0xa5, 0x29, 0xa9, 0xe4, 0xa4, 0x5b, 0xd0, 0x4e, 0x13, 0xe4, 0x9e, 0x60,
0x69, 0x77, 0x45, 0x5e, 0x40, 0x71, 0x84, 0x0e, 0x34, 0xce, 0xb8, 0x0f, 0x2b, 0x3c, 0xb7, 0x44, 0x44, 0x3d, 0xea, 0x9f, 0x5c, 0xf4, 0x17, 0xe4, 0x01, 0x94, 0x26, 0x68, 0x4f, 0xe3, 0xac, 0xbb,
0xdd, 0x55, 0x71, 0xd7, 0xbd, 0x90, 0x65, 0x29, 0x4c, 0xed, 0x8b, 0xdf, 0xbd, 0x80, 0xd1, 0xa9, 0xb0, 0xc0, 0x73, 0x4b, 0xd2, 0x5f, 0x14, 0x67, 0xdd, 0x73, 0x26, 0x4b, 0x61, 0xea, 0xba, 0xf8,
0x25, 0x49, 0x7b, 0xef, 0x01, 0xa4, 0x48, 0x63, 0x0d, 0x4a, 0x67, 0x68, 0xaa, 0xe2, 0x90, 0x0f, 0xdd, 0x89, 0x18, 0xbd, 0x70, 0x24, 0xe9, 0xe0, 0x1d, 0x80, 0x1c, 0x69, 0x2d, 0x41, 0xe5, 0x14,
0xb9, 0x73, 0xce, 0x9d, 0x49, 0xac, 0xbd, 0x2e, 0x81, 0x0f, 0x8a, 0xef, 0x15, 0x4c, 0x17, 0xda, 0x5d, 0xa8, 0x38, 0xe4, 0x9f, 0xdc, 0x39, 0x67, 0xde, 0x38, 0xd5, 0x5e, 0x97, 0xc0, 0x7b, 0xe5,
0x3b, 0x93, 0x33, 0x4c, 0x32, 0xcb, 0x37, 0x60, 0xc5, 0x77, 0xbe, 0x22, 0x54, 0x7b, 0x52, 0x00, 0x77, 0x4a, 0xb6, 0x0f, 0xdd, 0xad, 0xf1, 0x29, 0x26, 0xc6, 0xf4, 0x15, 0x58, 0x08, 0xbd, 0x2f,
0x02, 0x8b, 0x03, 0x42, 0x35, 0x0b, 0x01, 0x18, 0x2d, 0x28, 0x92, 0x50, 0xf8, 0xab, 0x66, 0x15, 0x08, 0xd5, 0x9e, 0x14, 0x80, 0xc0, 0xe2, 0x88, 0x50, 0xcd, 0x42, 0x00, 0x56, 0x07, 0xca, 0x24,
0x49, 0x98, 0x0a, 0x2a, 0x67, 0x04, 0x99, 0x7f, 0x2f, 0x03, 0xa4, 0x52, 0x0c, 0x0b, 0x7a, 0x98, 0x16, 0xfe, 0x6a, 0x38, 0x65, 0x12, 0xe7, 0x82, 0xaa, 0x86, 0x20, 0xfb, 0x9f, 0x55, 0x80, 0x5c,
0xd8, 0x11, 0xa2, 0xfc, 0x7e, 0xb7, 0x87, 0x53, 0x86, 0x22, 0x9b, 0x22, 0x37, 0xa6, 0x11, 0x3e, 0x8a, 0xe5, 0xc0, 0x00, 0x13, 0x37, 0x41, 0x94, 0x9f, 0xef, 0xee, 0xf1, 0x05, 0x43, 0x89, 0x4b,
0xe7, 0xfb, 0xc7, 0xcd, 0xbe, 0x21, 0xcd, 0x9e, 0xd1, 0xcd, 0xba, 0x89, 0xc9, 0xb1, 0x5c, 0xb7, 0x91, 0x9f, 0xd2, 0x04, 0x9f, 0xf1, 0xf5, 0xe3, 0x66, 0x5f, 0x93, 0x66, 0x4f, 0xe8, 0xe6, 0x5c,
0xc3, 0x97, 0x59, 0x7a, 0x95, 0x71, 0x08, 0x37, 0x52, 0x9e, 0x5e, 0x86, 0x5d, 0xf1, 0x32, 0x76, 0xc7, 0xe4, 0x50, 0xce, 0xdb, 0xe2, 0xd3, 0x1c, 0x3d, 0xcb, 0xda, 0x87, 0x6b, 0x39, 0xcf, 0xc0,
0xeb, 0x09, 0x3b, 0x2f, 0x65, 0xb5, 0x07, 0xeb, 0x98, 0xd8, 0x5f, 0xc7, 0x28, 0xce, 0x31, 0x2a, 0x60, 0x57, 0xbe, 0x8c, 0xdd, 0x72, 0xc6, 0x2e, 0xc8, 0x59, 0xed, 0xc0, 0x32, 0x26, 0xee, 0x97,
0x5d, 0xc6, 0xa8, 0x83, 0xc9, 0x0f, 0xc4, 0x82, 0x94, 0xcd, 0x00, 0x6e, 0x65, 0xac, 0xe4, 0xe1, 0x29, 0x4a, 0x0b, 0x8c, 0x2a, 0x97, 0x31, 0xea, 0x61, 0xf2, 0x43, 0x31, 0x21, 0x67, 0x33, 0x84,
0x9e, 0x61, 0x56, 0xbe, 0x8c, 0xd9, 0x66, 0xa2, 0x15, 0xcf, 0x07, 0x29, 0xc7, 0x4f, 0x60, 0x13, 0x1b, 0x86, 0x95, 0x3c, 0xdc, 0x0d, 0x66, 0xd5, 0xcb, 0x98, 0xad, 0x66, 0x5a, 0xf1, 0x7c, 0x90,
0x13, 0xfb, 0xa9, 0x83, 0xd9, 0x2c, 0xbb, 0x95, 0xef, 0x31, 0x92, 0xdf, 0x68, 0x79, 0x5e, 0xd2, 0x73, 0xfc, 0x08, 0x56, 0x31, 0x71, 0x1f, 0x7b, 0x98, 0x4d, 0xb2, 0x5b, 0xf8, 0x16, 0x23, 0xf9,
0x48, 0x1f, 0xd1, 0x51, 0xce, 0xc8, 0xd5, 0xef, 0x31, 0xf2, 0x48, 0x2c, 0x48, 0xd9, 0x3c, 0x82, 0x89, 0x56, 0xe4, 0x25, 0x8d, 0x0c, 0x11, 0x1d, 0x15, 0x8c, 0x5c, 0xfc, 0x16, 0x23, 0x0f, 0xc4,
0x0e, 0x26, 0xb3, 0xda, 0x54, 0x2e, 0x63, 0xd2, 0xc6, 0x24, 0xaf, 0xc9, 0x0e, 0x74, 0x22, 0xe4, 0x84, 0x9c, 0xcd, 0x26, 0xf4, 0x30, 0x99, 0xd4, 0xa6, 0x76, 0x19, 0x93, 0x2e, 0x26, 0x45, 0x4d,
0x32, 0x42, 0xb3, 0x87, 0xa0, 0x7a, 0x19, 0x8b, 0x35, 0x45, 0x9f, 0xf0, 0x30, 0x7f, 0x0c, 0x8d, 0xb6, 0xa0, 0x97, 0x20, 0x9f, 0x11, 0x6a, 0x6e, 0x82, 0xfa, 0x65, 0x2c, 0x96, 0x14, 0x7d, 0xc6,
0x83, 0x78, 0x84, 0xd8, 0x64, 0x98, 0x24, 0x83, 0x6b, 0xcb, 0x3f, 0xe6, 0x3f, 0x8b, 0x50, 0xdf, 0xc3, 0xfe, 0x29, 0xb4, 0xf6, 0xd2, 0x11, 0x62, 0xe3, 0xe3, 0x2c, 0x19, 0x3c, 0xb5, 0xfc, 0x63,
0x1d, 0x51, 0x12, 0x87, 0xb9, 0x9c, 0x2c, 0x83, 0x74, 0x36, 0x27, 0x0b, 0x12, 0x91, 0x93, 0x25, 0xff, 0xbb, 0x0c, 0xcd, 0xed, 0x11, 0x25, 0x69, 0x5c, 0xc8, 0xc9, 0x32, 0x48, 0x27, 0x73, 0xb2,
0xf1, 0xbb, 0xd0, 0xf0, 0x45, 0xe8, 0x2a, 0x7a, 0x99, 0x87, 0x3a, 0x73, 0x41, 0x6d, 0xd5, 0xfd, 0x20, 0x11, 0x39, 0x59, 0x12, 0xbf, 0x0d, 0xad, 0x50, 0x84, 0xae, 0xa2, 0x97, 0x79, 0xa8, 0x37,
0x4c, 0x32, 0xeb, 0x03, 0x84, 0xd8, 0x8b, 0xd4, 0x1a, 0x99, 0x8e, 0xda, 0xaa, 0xdc, 0xd2, 0x29, 0x15, 0xd4, 0x4e, 0x33, 0x34, 0x92, 0xd9, 0x3a, 0x40, 0x8c, 0x83, 0x44, 0xcd, 0x91, 0xe9, 0xa8,
0xda, 0xaa, 0x85, 0x49, 0xb6, 0x7e, 0x1b, 0xea, 0x43, 0xee, 0x24, 0xb5, 0x20, 0x97, 0x8c, 0x52, 0xab, 0xca, 0x2d, 0x9d, 0xa2, 0x9d, 0x46, 0x9c, 0x65, 0xeb, 0x37, 0xa1, 0x79, 0xcc, 0x9d, 0xa4,
0xef, 0x59, 0x30, 0x4c, 0x83, 0xf0, 0x00, 0x9a, 0x63, 0xe9, 0x32, 0xb5, 0x48, 0x9e, 0xa1, 0x3b, 0x26, 0x14, 0x92, 0x51, 0xee, 0x3d, 0x07, 0x8e, 0xf3, 0x20, 0xdc, 0x83, 0xf6, 0x89, 0x74, 0x99,
0xca, 0x92, 0xd4, 0xde, 0x7e, 0xd6, 0xb3, 0x72, 0x03, 0x1a, 0xe3, 0x0c, 0xaa, 0x77, 0x0c, 0x9d, 0x9a, 0x24, 0xf7, 0xd0, 0x2d, 0x65, 0x49, 0x6e, 0xef, 0xba, 0xe9, 0x59, 0xb9, 0x00, 0xad, 0x13,
0x39, 0x92, 0x05, 0x39, 0xe8, 0x5e, 0x36, 0x07, 0xd5, 0xef, 0x1b, 0x52, 0x50, 0x76, 0x65, 0x36, 0x03, 0x35, 0x38, 0x84, 0xde, 0x14, 0xc9, 0x8c, 0x1c, 0x74, 0xc7, 0xcc, 0x41, 0xcd, 0xbb, 0x96,
0x2f, 0xfd, 0xba, 0x08, 0x8d, 0xcf, 0x11, 0x7b, 0x4a, 0xe8, 0x99, 0xd4, 0xd7, 0x80, 0x72, 0xe0, 0x14, 0x64, 0xce, 0x34, 0xf3, 0xd2, 0x6f, 0xcb, 0xd0, 0xfa, 0x14, 0xb1, 0xc7, 0x84, 0x9e, 0x4a,
0xf8, 0x48, 0x71, 0x14, 0x63, 0xe3, 0x16, 0x54, 0xe9, 0x85, 0x4c, 0x20, 0x6a, 0x3f, 0x2b, 0xf4, 0x7d, 0x2d, 0xa8, 0x46, 0x5e, 0x88, 0x14, 0x47, 0xf1, 0x6d, 0xdd, 0x80, 0x3a, 0x3d, 0x97, 0x09,
0x42, 0x24, 0x06, 0xe3, 0x45, 0x00, 0x7a, 0x61, 0x87, 0x8e, 0x7b, 0x86, 0x94, 0x07, 0xcb, 0x56, 0x44, 0xad, 0x67, 0x8d, 0x9e, 0x8b, 0xc4, 0x60, 0x3d, 0x0f, 0x40, 0xcf, 0xdd, 0xd8, 0xf3, 0x4f,
0x8d, 0x5e, 0x0c, 0x24, 0x82, 0x1f, 0x05, 0x7a, 0x61, 0x23, 0x4a, 0x09, 0x8d, 0x54, 0xae, 0xaa, 0x91, 0xf2, 0x60, 0xd5, 0x69, 0xd0, 0xf3, 0xa1, 0x44, 0xf0, 0xad, 0x40, 0xcf, 0x5d, 0x44, 0x29,
0xd2, 0x8b, 0x3d, 0x01, 0xab, 0xb5, 0x1e, 0x25, 0x61, 0x88, 0x3c, 0x91, 0xa3, 0xc5, 0xda, 0xc7, 0xa1, 0x89, 0xca, 0x55, 0x75, 0x7a, 0xbe, 0x23, 0x60, 0x35, 0x37, 0xa0, 0x24, 0x8e, 0x51, 0x20,
0x12, 0xc1, 0xa5, 0x32, 0x2d, 0x75, 0x55, 0x4a, 0x65, 0xa9, 0x54, 0x96, 0x4a, 0xad, 0xc8, 0x95, 0x72, 0xb4, 0x98, 0xfb, 0x40, 0x22, 0xb8, 0x54, 0xa6, 0xa5, 0x2e, 0x4a, 0xa9, 0x2c, 0x97, 0xca,
0x2c, 0x2b, 0x95, 0x25, 0x52, 0xab, 0x52, 0x2a, 0xcb, 0x48, 0x65, 0xa9, 0xd4, 0x9a, 0x5e, 0xab, 0x72, 0xa9, 0x35, 0x39, 0x93, 0x99, 0x52, 0x59, 0x26, 0xb5, 0x2e, 0xa5, 0x32, 0x43, 0x2a, 0xcb,
0xa4, 0x9a, 0xbf, 0x2a, 0xc0, 0xe6, 0x6c, 0xe1, 0xa7, 0x6a, 0xd3, 0x77, 0xa1, 0xe1, 0x8a, 0xfd, 0xa5, 0x36, 0xf4, 0x5c, 0x25, 0xd5, 0xfe, 0x4d, 0x09, 0x56, 0x27, 0x0b, 0x3f, 0x55, 0x9b, 0xbe,
0xca, 0x9d, 0xc9, 0xce, 0xdc, 0x4e, 0x5a, 0x75, 0x37, 0x73, 0x8c, 0x1f, 0x40, 0x33, 0x90, 0x0e, 0x0d, 0x2d, 0x5f, 0xac, 0x57, 0x61, 0x4f, 0xf6, 0xa6, 0x56, 0xd2, 0x69, 0xfa, 0xc6, 0x36, 0xbe,
0x4e, 0x8e, 0x66, 0x29, 0xdd, 0x97, 0xac, 0xef, 0xad, 0x46, 0x90, 0x81, 0x4c, 0x0f, 0x8c, 0x2f, 0x07, 0xed, 0x48, 0x3a, 0x38, 0xdb, 0x9a, 0x95, 0x7c, 0x5d, 0x4c, 0xdf, 0x3b, 0xad, 0xc8, 0x80,
0x29, 0x66, 0xe8, 0x98, 0x51, 0xe4, 0xf8, 0xd7, 0x51, 0xdd, 0x1b, 0x50, 0x16, 0xd5, 0x0a, 0xdf, 0xec, 0x00, 0xac, 0xcf, 0x29, 0x66, 0xe8, 0x90, 0x51, 0xe4, 0x85, 0x4f, 0xa3, 0xba, 0xb7, 0xa0,
0xa6, 0x86, 0x25, 0xc6, 0xe6, 0xab, 0xb0, 0x9e, 0x93, 0xa2, 0x6c, 0x5d, 0x83, 0xd2, 0x04, 0x05, 0x2a, 0xaa, 0x15, 0xbe, 0x4c, 0x2d, 0x47, 0x7c, 0xdb, 0x2f, 0xc3, 0x72, 0x41, 0x8a, 0xb2, 0x75,
0x82, 0x7b, 0xd3, 0xe2, 0x43, 0xd3, 0x81, 0x8e, 0x85, 0x1c, 0xef, 0xfa, 0xb4, 0x51, 0x22, 0x4a, 0x09, 0x2a, 0x63, 0x14, 0x09, 0xee, 0x6d, 0x87, 0x7f, 0xda, 0x1e, 0xf4, 0x1c, 0xe4, 0x05, 0x4f,
0xa9, 0x88, 0x7b, 0x60, 0x64, 0x45, 0x28, 0x55, 0xb4, 0xd6, 0x85, 0x8c, 0xd6, 0x4f, 0xa0, 0xb3, 0x4f, 0x1b, 0x25, 0xa2, 0x92, 0x8b, 0xb8, 0x03, 0x96, 0x29, 0x42, 0xa9, 0xa2, 0xb5, 0x2e, 0x19,
0x3b, 0x21, 0x11, 0x3a, 0x66, 0x1e, 0x0e, 0xae, 0xa3, 0x1d, 0xf9, 0x19, 0xac, 0x7f, 0xc1, 0xa6, 0x5a, 0x3f, 0x84, 0xde, 0xf6, 0x98, 0x24, 0xe8, 0x90, 0x05, 0x38, 0x7a, 0x1a, 0xed, 0xc8, 0x2f,
0x5f, 0x72, 0x66, 0x11, 0xfe, 0x06, 0x5d, 0x93, 0x7d, 0x94, 0x3c, 0xd5, 0xf6, 0x51, 0xf2, 0x94, 0x60, 0xf9, 0x33, 0x76, 0xf1, 0x39, 0x67, 0x96, 0xe0, 0xaf, 0xd0, 0x53, 0xb2, 0x8f, 0x92, 0xc7,
0x37, 0x37, 0x2e, 0x99, 0xc4, 0x7e, 0x20, 0x42, 0xa1, 0x69, 0x29, 0xc8, 0xdc, 0x81, 0x86, 0xac, 0xda, 0x3e, 0x4a, 0x1e, 0xf3, 0xe6, 0xc6, 0x27, 0xe3, 0x34, 0x8c, 0x44, 0x28, 0xb4, 0x1d, 0x05,
0xa1, 0x8f, 0x88, 0x17, 0x4f, 0xd0, 0xc2, 0x18, 0xdc, 0x02, 0x08, 0x1d, 0xea, 0xf8, 0x88, 0x21, 0xd9, 0x5b, 0xd0, 0x92, 0x35, 0xf4, 0x01, 0x09, 0xd2, 0x31, 0x9a, 0x19, 0x83, 0x6b, 0x00, 0xb1,
0x2a, 0xcf, 0x50, 0xcd, 0xca, 0x60, 0xcc, 0xdf, 0x14, 0x61, 0x43, 0xbe, 0x37, 0x1c, 0xcb, 0x36, 0x47, 0xbd, 0x10, 0x31, 0x44, 0xe5, 0x1e, 0x6a, 0x38, 0x06, 0xc6, 0xfe, 0x5d, 0x19, 0x56, 0xe4,
0x5b, 0x9b, 0xd0, 0x83, 0xea, 0x98, 0x44, 0x2c, 0xc3, 0x30, 0x81, 0xb9, 0x8a, 0xbc, 0x3f, 0x97, 0x7d, 0xc3, 0xa1, 0x6c, 0xb3, 0xb5, 0x09, 0x03, 0xa8, 0x9f, 0x90, 0x84, 0x19, 0x0c, 0x33, 0x98,
0xdc, 0xf8, 0x30, 0xf7, 0x08, 0x50, 0xba, 0xfc, 0x11, 0x60, 0xae, 0xcd, 0x2f, 0xcf, 0xb7, 0xf9, 0xab, 0xc8, 0xfb, 0x73, 0xc9, 0x8d, 0x7f, 0x16, 0x2e, 0x01, 0x2a, 0x97, 0x5f, 0x02, 0x4c, 0xb5,
0x3c, 0xda, 0x34, 0x11, 0x96, 0x31, 0x5e, 0xb3, 0x6a, 0x0a, 0x73, 0xe8, 0x19, 0x77, 0xa1, 0x3d, 0xf9, 0xd5, 0xe9, 0x36, 0x9f, 0x47, 0x9b, 0x26, 0xc2, 0x32, 0xc6, 0x1b, 0x4e, 0x43, 0x61, 0xf6,
0xe2, 0x5a, 0xda, 0x63, 0x42, 0xce, 0xec, 0xd0, 0x61, 0x63, 0x11, 0xea, 0x35, 0xab, 0x29, 0xd0, 0x03, 0xeb, 0x36, 0x74, 0x47, 0x5c, 0x4b, 0xf7, 0x84, 0x90, 0x53, 0x37, 0xf6, 0xd8, 0x89, 0x08,
0x07, 0x84, 0x9c, 0x0d, 0x1c, 0x36, 0x36, 0xde, 0x87, 0x96, 0x2a, 0x03, 0x7d, 0xe1, 0xa2, 0x48, 0xf5, 0x86, 0xd3, 0x16, 0xe8, 0x3d, 0x42, 0x4e, 0x87, 0x1e, 0x3b, 0xb1, 0xde, 0x85, 0x8e, 0x2a,
0x5d, 0x7e, 0x2a, 0x8a, 0xb2, 0xde, 0xb3, 0x9a, 0x67, 0x19, 0x28, 0x32, 0x6f, 0xc2, 0x8d, 0xc7, 0x03, 0x43, 0xe1, 0xa2, 0x44, 0x1d, 0x7e, 0x2a, 0x8a, 0x4c, 0xef, 0x39, 0xed, 0x53, 0x03, 0x4a,
0x28, 0x62, 0x94, 0x4c, 0xf3, 0x8e, 0x31, 0xff, 0x1f, 0xe0, 0x30, 0x60, 0x88, 0x9e, 0x3a, 0x2e, 0xec, 0xeb, 0x70, 0xed, 0x01, 0x4a, 0x18, 0x25, 0x17, 0x45, 0xc7, 0xd8, 0xdf, 0x07, 0xd8, 0x8f,
0x8a, 0x8c, 0xb7, 0xb2, 0x90, 0x2a, 0x8e, 0xd6, 0xfa, 0xf2, 0xb9, 0x27, 0x99, 0xb0, 0x32, 0x34, 0x18, 0xa2, 0x8f, 0x3c, 0x1f, 0x25, 0xd6, 0x1b, 0x26, 0xa4, 0x8a, 0xa3, 0xa5, 0x75, 0x79, 0xdd,
0x66, 0x1f, 0x56, 0x2d, 0x12, 0xf3, 0x74, 0xf4, 0xb2, 0x1e, 0xa9, 0x75, 0x0d, 0xb5, 0x4e, 0x20, 0x93, 0x0d, 0x38, 0x06, 0x8d, 0xbd, 0x0e, 0x8b, 0x0e, 0x49, 0x79, 0x3a, 0x7a, 0x51, 0x7f, 0xa9,
0x2d, 0x35, 0x67, 0x1e, 0xe8, 0x16, 0x36, 0x65, 0xa7, 0xb6, 0xa8, 0x0f, 0x35, 0xac, 0x71, 0x2a, 0x79, 0x2d, 0x35, 0x4f, 0x20, 0x1d, 0x35, 0x66, 0xef, 0xe9, 0x16, 0x36, 0x67, 0xa7, 0x96, 0x68,
0xab, 0xcc, 0x8b, 0x4e, 0x49, 0xcc, 0x0f, 0x61, 0x5d, 0x72, 0x92, 0x9c, 0x35, 0x9b, 0x97, 0x61, 0x1d, 0x1a, 0x58, 0xe3, 0x54, 0x56, 0x99, 0x16, 0x9d, 0x93, 0xd8, 0xef, 0xc3, 0xb2, 0xe4, 0x24,
0x95, 0x6a, 0x35, 0x0a, 0xe9, 0x3b, 0x8f, 0x22, 0x52, 0x73, 0xdc, 0x1f, 0x9f, 0xe1, 0x88, 0xa5, 0x39, 0x6b, 0x36, 0x2f, 0xc2, 0x22, 0xd5, 0x6a, 0x94, 0xf2, 0x7b, 0x1e, 0x45, 0xa4, 0xc6, 0xb8,
0x86, 0x68, 0x7f, 0xac, 0x43, 0x87, 0x4f, 0xe4, 0x78, 0x9a, 0x1f, 0x43, 0xe3, 0x91, 0x35, 0xf8, 0x3f, 0x3e, 0xc1, 0x09, 0xcb, 0x0d, 0xd1, 0xfe, 0x58, 0x86, 0x1e, 0x1f, 0x28, 0xf0, 0xb4, 0x3f,
0x1c, 0xe1, 0xd1, 0x78, 0xc8, 0xb3, 0xe7, 0xff, 0xe5, 0x61, 0x65, 0xb0, 0xa1, 0xb4, 0xcd, 0x4c, 0x84, 0xd6, 0xa6, 0x33, 0xfc, 0x14, 0xe1, 0xd1, 0xc9, 0x31, 0xcf, 0x9e, 0xdf, 0x2b, 0xc2, 0xca,
0x59, 0x39, 0x3a, 0xf3, 0x13, 0xd8, 0x7c, 0xe4, 0x79, 0x59, 0x94, 0xd6, 0xfa, 0x2d, 0xa8, 0x05, 0x60, 0x4b, 0x69, 0x6b, 0x0c, 0x39, 0x05, 0x3a, 0xfb, 0x23, 0x58, 0xdd, 0x0c, 0x02, 0x13, 0xa5,
0x19, 0x76, 0x99, 0x3b, 0x2b, 0x47, 0x9d, 0x12, 0x99, 0x3f, 0x81, 0xf5, 0x27, 0xc1, 0x04, 0x07, 0xb5, 0x7e, 0x03, 0x1a, 0x91, 0xc1, 0xce, 0x38, 0xb3, 0x0a, 0xd4, 0x39, 0x91, 0xfd, 0x33, 0x58,
0x68, 0x77, 0x70, 0x72, 0x84, 0x92, 0x5c, 0x64, 0x40, 0x99, 0xd7, 0x6c, 0x82, 0x47, 0xd5, 0x12, 0x7e, 0x18, 0x8d, 0x71, 0x84, 0xb6, 0x87, 0x47, 0x07, 0x28, 0xcb, 0x45, 0x16, 0x54, 0x79, 0xcd,
0x63, 0x1e, 0x9c, 0xc1, 0xd0, 0x76, 0xc3, 0x38, 0x52, 0x8f, 0x3d, 0xab, 0xc1, 0x70, 0x37, 0x8c, 0x26, 0x78, 0xd4, 0x1d, 0xf1, 0xcd, 0x83, 0x33, 0x3a, 0x76, 0xfd, 0x38, 0x4d, 0xd4, 0x65, 0xcf,
0x23, 0x7e, 0xb9, 0xf0, 0xe2, 0x82, 0x04, 0x93, 0xa9, 0x88, 0xd0, 0xaa, 0x55, 0x71, 0xc3, 0xf8, 0x62, 0x74, 0xbc, 0x1d, 0xa7, 0x09, 0x3f, 0x5c, 0x78, 0x71, 0x41, 0xa2, 0xf1, 0x85, 0x88, 0xd0,
0x49, 0x30, 0x99, 0x9a, 0xff, 0x2b, 0x3a, 0x70, 0x84, 0x3c, 0xcb, 0x09, 0x3c, 0xe2, 0x3f, 0x46, 0xba, 0x53, 0xf3, 0xe3, 0xf4, 0x61, 0x34, 0xbe, 0xb0, 0xbf, 0x2b, 0x3a, 0x70, 0x84, 0x02, 0xc7,
0xe7, 0x19, 0x09, 0x49, 0xb7, 0xa7, 0x33, 0xd1, 0xb7, 0x05, 0x68, 0x3c, 0x1a, 0xa1, 0x80, 0x3d, 0x8b, 0x02, 0x12, 0x3e, 0x40, 0x67, 0x86, 0x84, 0xac, 0xdb, 0xd3, 0x99, 0xe8, 0xeb, 0x12, 0xb4,
0x46, 0xcc, 0xc1, 0x13, 0xd1, 0xd1, 0x9d, 0x23, 0x1a, 0x61, 0x12, 0xa8, 0x70, 0xd3, 0x20, 0x6f, 0x36, 0x47, 0x28, 0x62, 0x0f, 0x10, 0xf3, 0xf0, 0x58, 0x74, 0x74, 0x67, 0x88, 0x26, 0x98, 0x44,
0xc8, 0x71, 0x80, 0x99, 0xed, 0x39, 0xc8, 0x27, 0x81, 0xe0, 0x52, 0xb5, 0x80, 0xa3, 0x1e, 0x0b, 0x2a, 0xdc, 0x34, 0xc8, 0x1b, 0x72, 0x1c, 0x61, 0xe6, 0x06, 0x1e, 0x0a, 0x49, 0x24, 0xb8, 0xd4,
0x8c, 0xf1, 0x2a, 0xb4, 0xe5, 0x63, 0x9c, 0x3d, 0x76, 0x02, 0x6f, 0xc2, 0x03, 0xbd, 0x24, 0x42, 0x1d, 0xe0, 0xa8, 0x07, 0x02, 0x63, 0xbd, 0x0c, 0x5d, 0x79, 0x19, 0xe7, 0x9e, 0x78, 0x51, 0x30,
0xb3, 0x25, 0xd1, 0x07, 0x0a, 0x6b, 0xbc, 0x06, 0x6b, 0x2a, 0x0c, 0x53, 0xca, 0xb2, 0xa0, 0x6c, 0xe6, 0x81, 0x5e, 0x11, 0xa1, 0xd9, 0x91, 0xe8, 0x3d, 0x85, 0xb5, 0x5e, 0x81, 0x25, 0x15, 0x86,
0x2b, 0x7c, 0x8e, 0x34, 0x0e, 0x43, 0x42, 0x59, 0x64, 0x47, 0xc8, 0x75, 0x89, 0x1f, 0xaa, 0x76, 0x39, 0x65, 0x55, 0x50, 0x76, 0x15, 0xbe, 0x40, 0x9a, 0xc6, 0x31, 0xa1, 0x2c, 0x71, 0x13, 0xe4,
0xa8, 0xad, 0xf1, 0xc7, 0x12, 0x6d, 0x8e, 0x60, 0x7d, 0x9f, 0xdb, 0xa9, 0x2c, 0x49, 0x8f, 0x55, 0xfb, 0x24, 0x8c, 0x55, 0x3b, 0xd4, 0xd5, 0xf8, 0x43, 0x89, 0xb6, 0x47, 0xb0, 0xbc, 0xcb, 0xed,
0xcb, 0x47, 0xbe, 0x3d, 0x9c, 0x10, 0xf7, 0xcc, 0xe6, 0xc9, 0x51, 0x79, 0x98, 0x17, 0x5c, 0x3b, 0x54, 0x96, 0xe4, 0xdb, 0xaa, 0x13, 0xa2, 0xd0, 0x3d, 0x1e, 0x13, 0xff, 0xd4, 0xe5, 0xc9, 0x51,
0x1c, 0x79, 0x8c, 0xbf, 0x11, 0x9d, 0x3f, 0xa7, 0x1a, 0x13, 0x16, 0x4e, 0xe2, 0x91, 0x1d, 0x52, 0x79, 0x98, 0x17, 0x5c, 0x5b, 0x1c, 0x79, 0x88, 0xbf, 0x12, 0x9d, 0x3f, 0xa7, 0x3a, 0x21, 0x2c,
0x32, 0x44, 0xca, 0xc4, 0xb6, 0x8f, 0xfc, 0x03, 0x89, 0x1f, 0x70, 0xb4, 0xf9, 0xa7, 0x02, 0x6c, 0x1e, 0xa7, 0x23, 0x37, 0xa6, 0xe4, 0x18, 0x29, 0x13, 0xbb, 0x21, 0x0a, 0xf7, 0x24, 0x7e, 0xc8,
0xe4, 0x25, 0xa9, 0x54, 0xbf, 0x0d, 0x1b, 0x79, 0x51, 0xea, 0xfa, 0x97, 0xe5, 0x65, 0x27, 0x2b, 0xd1, 0xf6, 0x9f, 0x4b, 0xb0, 0x52, 0x94, 0xa4, 0x52, 0xfd, 0x06, 0xac, 0x14, 0x45, 0xa9, 0xe3,
0x50, 0x16, 0x02, 0x0f, 0xa0, 0x29, 0xde, 0x6b, 0x6d, 0x4f, 0x72, 0xca, 0x17, 0x3d, 0xd9, 0x7d, 0x5f, 0x96, 0x97, 0x3d, 0x53, 0xa0, 0x2c, 0x04, 0xee, 0x41, 0x5b, 0x5c, 0xdd, 0xba, 0x81, 0xe4,
0xb1, 0x1a, 0x4e, 0x76, 0x97, 0xde, 0x87, 0x5b, 0xca, 0x7c, 0x7b, 0x5e, 0x6d, 0x79, 0x20, 0x36, 0x54, 0x2c, 0x7a, 0xcc, 0x75, 0x71, 0x5a, 0x9e, 0xb9, 0x4a, 0xef, 0xc2, 0x0d, 0x65, 0xbe, 0x3b,
0x15, 0xc1, 0xd1, 0x8c, 0xf6, 0x9f, 0x41, 0x37, 0x45, 0xed, 0x4c, 0x05, 0x32, 0x3d, 0xcc, 0xeb, 0xad, 0xb6, 0xdc, 0x10, 0xab, 0x8a, 0xe0, 0x60, 0x42, 0xfb, 0x4f, 0xa0, 0x9f, 0xa3, 0xb6, 0x2e,
0x33, 0xc6, 0x3e, 0xf2, 0x3c, 0x2a, 0xa2, 0xa4, 0x6c, 0x2d, 0x9a, 0x32, 0x1f, 0xc2, 0xcd, 0x63, 0x04, 0x32, 0xdf, 0xcc, 0xcb, 0x13, 0xc6, 0x6e, 0x06, 0x01, 0x15, 0x51, 0x52, 0x75, 0x66, 0x0d,
0xc4, 0xa4, 0x37, 0x1c, 0xa6, 0x3a, 0x11, 0xc9, 0x6c, 0x0d, 0x4a, 0xc7, 0xc8, 0x15, 0xc6, 0x97, 0xd9, 0xf7, 0xe1, 0xfa, 0x21, 0x62, 0xd2, 0x1b, 0x1e, 0x53, 0x9d, 0x88, 0x64, 0xb6, 0x04, 0x95,
0x2c, 0x3e, 0xe4, 0x07, 0xf0, 0x24, 0x42, 0xae, 0xb0, 0xb2, 0x64, 0x89, 0xb1, 0xf9, 0x87, 0x02, 0x43, 0xe4, 0x0b, 0xe3, 0x2b, 0x0e, 0xff, 0xe4, 0x1b, 0xf0, 0x28, 0x41, 0xbe, 0xb0, 0xb2, 0xe2,
0x54, 0x54, 0x72, 0xe6, 0x17, 0x8c, 0x47, 0xf1, 0x39, 0xa2, 0xea, 0xe8, 0x29, 0xc8, 0x78, 0x05, 0x88, 0x6f, 0xfb, 0x4f, 0x25, 0xa8, 0xa9, 0xe4, 0xcc, 0x0f, 0x98, 0x80, 0xe2, 0x33, 0x44, 0xd5,
0x5a, 0x72, 0x64, 0x93, 0x90, 0x61, 0x92, 0xa4, 0xfc, 0xa6, 0xc4, 0x3e, 0x91, 0x48, 0xf1, 0xf8, 0xd6, 0x53, 0x90, 0xf5, 0x12, 0x74, 0xe4, 0x97, 0x4b, 0x62, 0x86, 0x49, 0x96, 0xf2, 0xdb, 0x12,
0x26, 0x9e, 0xbf, 0x54, 0xa7, 0xa9, 0x20, 0x8e, 0x3f, 0x8d, 0x78, 0x84, 0x8b, 0x14, 0x5f, 0xb3, 0xfb, 0x50, 0x22, 0xc5, 0xe5, 0x9b, 0xb8, 0xfe, 0x52, 0x9d, 0xa6, 0x82, 0x38, 0xfe, 0x51, 0xc2,
0x14, 0xc4, 0x8f, 0xba, 0xe6, 0xb7, 0x22, 0xf8, 0x69, 0x90, 0x1f, 0x75, 0x9f, 0xc4, 0x01, 0xb3, 0x23, 0x5c, 0xa4, 0xf8, 0x86, 0xa3, 0x20, 0xbe, 0xd5, 0x35, 0xbf, 0x05, 0xc1, 0x4f, 0x83, 0x7c,
0x43, 0x82, 0x03, 0xa6, 0x72, 0x3a, 0x08, 0xd4, 0x80, 0x63, 0xcc, 0x5f, 0x16, 0x60, 0x55, 0x3e, 0xab, 0x87, 0x24, 0x8d, 0x98, 0x1b, 0x13, 0x1c, 0x31, 0x95, 0xd3, 0x41, 0xa0, 0x86, 0x1c, 0x63,
0x40, 0xf3, 0xde, 0x36, 0xb9, 0x59, 0x8b, 0x58, 0x54, 0x29, 0x42, 0x96, 0xbc, 0x4d, 0xc5, 0x98, 0xff, 0xba, 0x04, 0x8b, 0xf2, 0x02, 0x9a, 0xf7, 0xb6, 0xd9, 0xc9, 0x5a, 0xc6, 0xa2, 0x4a, 0x11,
0xc7, 0xf1, 0xb9, 0x2f, 0xef, 0x07, 0xa5, 0xda, 0xb9, 0x2f, 0x2e, 0x86, 0x57, 0xa0, 0x95, 0x5e, 0xb2, 0xe4, 0x69, 0x2a, 0xbe, 0x79, 0x1c, 0x9f, 0x85, 0xf2, 0x7c, 0x50, 0xaa, 0x9d, 0x85, 0xe2,
0xd0, 0x62, 0x5e, 0xaa, 0xd8, 0x4c, 0xb0, 0x82, 0x6c, 0xa9, 0xa6, 0xe6, 0x0f, 0x79, 0x4b, 0x9f, 0x60, 0x78, 0x09, 0x3a, 0xf9, 0x01, 0x2d, 0xc6, 0xa5, 0x8a, 0xed, 0x0c, 0x2b, 0xc8, 0xe6, 0x6a,
0x3c, 0xbe, 0xae, 0x41, 0x29, 0x4e, 0x94, 0xe1, 0x43, 0x8e, 0x19, 0x25, 0x57, 0x3b, 0x1f, 0x1a, 0x6a, 0xff, 0x98, 0xb7, 0xf4, 0xd9, 0xe5, 0xeb, 0x12, 0x54, 0xd2, 0x4c, 0x19, 0xfe, 0xc9, 0x31,
0x77, 0xa1, 0xe5, 0x78, 0x1e, 0xe6, 0xcb, 0x9d, 0xc9, 0x3e, 0xf6, 0x92, 0x20, 0xcd, 0x63, 0xcd, 0xa3, 0xec, 0x68, 0xe7, 0x9f, 0xd6, 0x6d, 0xe8, 0x78, 0x41, 0x80, 0xf9, 0x74, 0x6f, 0xbc, 0x8b,
0xbf, 0x14, 0xa0, 0xbd, 0x4b, 0xc2, 0xe9, 0xc7, 0x78, 0x82, 0x32, 0x19, 0x44, 0x28, 0xa9, 0x6e, 0x83, 0x2c, 0x48, 0x8b, 0x58, 0xfb, 0x6f, 0x25, 0xe8, 0x6e, 0x93, 0xf8, 0xe2, 0x43, 0x3c, 0x46,
0x76, 0x3e, 0xe6, 0xd5, 0xea, 0x29, 0x9e, 0x20, 0x19, 0x5a, 0x72, 0x67, 0xab, 0x1c, 0x21, 0xc2, 0x46, 0x06, 0x11, 0x4a, 0xaa, 0x93, 0x9d, 0x7f, 0xf3, 0x6a, 0xf5, 0x11, 0x1e, 0x23, 0x19, 0x5a,
0x4a, 0x4f, 0x26, 0xcf, 0x6e, 0x4d, 0x39, 0x79, 0x44, 0x3c, 0x51, 0x97, 0x7b, 0x98, 0xda, 0xc9, 0x72, 0x65, 0xeb, 0x1c, 0x21, 0xc2, 0x4a, 0x0f, 0x66, 0xd7, 0x6e, 0x6d, 0x39, 0x78, 0x40, 0x02,
0x23, 0x5b, 0xd3, 0xaa, 0x78, 0x98, 0x8a, 0x29, 0x65, 0xc8, 0x8a, 0x78, 0x44, 0xcd, 0x1a, 0xb2, 0x51, 0x97, 0x07, 0x98, 0xba, 0xd9, 0x25, 0x5b, 0xdb, 0xa9, 0x05, 0x98, 0x8a, 0x21, 0x65, 0xc8,
0x2a, 0x31, 0xdc, 0x90, 0x4d, 0x58, 0x25, 0xa7, 0xa7, 0x11, 0x62, 0xa2, 0x82, 0x2e, 0x59, 0x0a, 0x82, 0xb8, 0x44, 0x35, 0x0d, 0x59, 0x94, 0x18, 0x6e, 0xc8, 0x2a, 0x2c, 0x92, 0x47, 0x8f, 0x12,
0x4a, 0xd2, 0x5c, 0x35, 0x93, 0xe6, 0x36, 0xc0, 0xd8, 0x47, 0xec, 0xc9, 0x93, 0xa3, 0xbd, 0x73, 0xc4, 0x44, 0x05, 0x5d, 0x71, 0x14, 0x94, 0xa5, 0xb9, 0xba, 0x91, 0xe6, 0x56, 0xc0, 0xda, 0x45,
0x14, 0x30, 0x7d, 0x3b, 0xbc, 0x09, 0x55, 0x8d, 0xfa, 0x77, 0x9e, 0x27, 0x5f, 0x87, 0xd6, 0x23, 0xec, 0xe1, 0xc3, 0x83, 0x9d, 0x33, 0x14, 0x31, 0x7d, 0x3a, 0xbc, 0x0e, 0x75, 0x8d, 0xfa, 0x6f,
0xcf, 0x3b, 0x7e, 0xea, 0x84, 0xda, 0x1f, 0x5d, 0xa8, 0x0c, 0x76, 0x0f, 0x07, 0xd2, 0x25, 0x25, 0xae, 0x27, 0x5f, 0x85, 0xce, 0x66, 0x10, 0x1c, 0x3e, 0xf6, 0x62, 0xed, 0x8f, 0x3e, 0xd4, 0x86,
0x6e, 0x80, 0x02, 0xf9, 0x6d, 0xb4, 0x8f, 0xd8, 0x11, 0x62, 0x14, 0xbb, 0xc9, 0x6d, 0x74, 0x07, 0xdb, 0xfb, 0x43, 0xe9, 0x92, 0x0a, 0x37, 0x40, 0x81, 0xfc, 0x34, 0xda, 0x45, 0xec, 0x00, 0x31,
0x2a, 0x0a, 0xc3, 0x57, 0xfa, 0x72, 0xa8, 0xd3, 0xac, 0x02, 0xef, 0xff, 0x71, 0x4d, 0x65, 0x64, 0x8a, 0xfd, 0xec, 0x34, 0xba, 0x05, 0x35, 0x85, 0xe1, 0x33, 0x43, 0xf9, 0xa9, 0xd3, 0xac, 0x02,
0xd5, 0xdc, 0x1b, 0xfb, 0xd0, 0x9e, 0xf9, 0x12, 0x63, 0xa8, 0xd7, 0x9e, 0xc5, 0x1f, 0x68, 0x7a, 0xed, 0x1f, 0x80, 0xf5, 0x23, 0x5e, 0x57, 0x21, 0x59, 0x54, 0x2b, 0x49, 0xaf, 0x42, 0xef, 0x4c,
0x9b, 0x7d, 0xf9, 0x65, 0xa7, 0xaf, 0xbf, 0xec, 0xf4, 0xf7, 0xfc, 0x90, 0x4d, 0x8d, 0x3d, 0x68, 0x60, 0x5d, 0x59, 0x70, 0x18, 0xcb, 0xd0, 0x95, 0x03, 0x22, 0x06, 0x85, 0xec, 0x23, 0x58, 0x96,
0xe5, 0xbf, 0x59, 0x18, 0xcf, 0xeb, 0xe2, 0x68, 0xc1, 0x97, 0x8c, 0xa5, 0x6c, 0xf6, 0xa1, 0x3d, 0x65, 0xa0, 0xe4, 0x73, 0x05, 0x16, 0xdc, 0x87, 0xd9, 0x7a, 0x56, 0x1d, 0xf1, 0x7d, 0xf7, 0x2f,
0xf3, 0xf9, 0x42, 0xeb, 0xb3, 0xf8, 0xab, 0xc6, 0x52, 0x46, 0x0f, 0xa1, 0x9e, 0xf9, 0x5e, 0x61, 0x3d, 0x75, 0x54, 0xa8, 0x5b, 0x07, 0x6b, 0x17, 0xba, 0x13, 0x4f, 0x44, 0x96, 0xba, 0x86, 0x9a,
0x74, 0x25, 0x93, 0xf9, 0x4f, 0x18, 0x4b, 0x19, 0xec, 0x42, 0x33, 0xf7, 0x09, 0xc1, 0xe8, 0x29, 0xfd, 0x72, 0x34, 0x58, 0x5d, 0x97, 0x4f, 0x4e, 0xeb, 0xfa, 0xc9, 0x69, 0x7d, 0x27, 0x8c, 0xd9,
0x7b, 0x16, 0x7c, 0x57, 0x58, 0xca, 0x64, 0x07, 0xea, 0x99, 0x97, 0x7c, 0xad, 0xc5, 0xfc, 0xe7, 0x85, 0xb5, 0x03, 0x9d, 0xe2, 0x63, 0x8a, 0xf5, 0xac, 0xae, 0xda, 0x66, 0x3c, 0xb1, 0xcc, 0x65,
0x82, 0xde, 0xad, 0x05, 0x33, 0x2a, 0xf1, 0xef, 0x43, 0x7b, 0xe6, 0x79, 0x5f, 0xbb, 0x64, 0xf1, 0xb3, 0x0b, 0xdd, 0x89, 0x77, 0x15, 0xad, 0xcf, 0xec, 0xe7, 0x96, 0xb9, 0x8c, 0xee, 0x43, 0xd3,
0xab, 0xff, 0x52, 0x65, 0x3e, 0x15, 0x5b, 0x94, 0xe9, 0xde, 0x32, 0x5b, 0x34, 0xff, 0x98, 0xdf, 0x78, 0x48, 0xb1, 0xfa, 0x92, 0xc9, 0xf4, 0xdb, 0xca, 0x5c, 0x06, 0xdb, 0xd0, 0x2e, 0xbc, 0x6d,
0x7b, 0x61, 0xf1, 0xa4, 0xd2, 0x6a, 0x0f, 0x5a, 0xf9, 0x77, 0x7c, 0xcd, 0x6c, 0xe1, 0xeb, 0xfe, 0x58, 0x03, 0x65, 0xcf, 0x8c, 0x07, 0x8f, 0xb9, 0x4c, 0xb6, 0xa0, 0x69, 0x3c, 0x31, 0x68, 0x2d,
0xe5, 0xfb, 0x9d, 0x7b, 0xd2, 0x4f, 0xf7, 0x7b, 0xd1, 0x4b, 0xff, 0x52, 0x46, 0x8f, 0x00, 0x54, 0xa6, 0xdf, 0x31, 0x06, 0x37, 0x66, 0x8c, 0xa8, 0x13, 0x69, 0x17, 0xba, 0x13, 0xef, 0x0e, 0xda,
0xaf, 0xe6, 0xe1, 0x20, 0x71, 0xf4, 0x5c, 0x8f, 0x98, 0x38, 0x7a, 0x41, 0x5f, 0xf7, 0x10, 0x40, 0x25, 0xb3, 0x9f, 0x23, 0xe6, 0x2a, 0xf3, 0xb1, 0x58, 0x22, 0xa3, 0xad, 0x34, 0x96, 0x68, 0xfa,
0xb6, 0x58, 0x1e, 0x89, 0x99, 0x71, 0x53, 0xab, 0x31, 0xd3, 0xd7, 0xf5, 0xba, 0xf3, 0x13, 0x73, 0x95, 0x61, 0xf0, 0xdc, 0xec, 0x41, 0xa5, 0xd5, 0x0e, 0x74, 0x8a, 0x0f, 0x0c, 0x9a, 0xd9, 0xcc,
0x0c, 0x10, 0xa5, 0x57, 0x61, 0xf0, 0x11, 0x40, 0xda, 0xba, 0x69, 0x06, 0x73, 0xcd, 0xdc, 0x25, 0x67, 0x87, 0xcb, 0xd7, 0xbb, 0xf0, 0xd6, 0x90, 0xaf, 0xf7, 0xac, 0x27, 0x88, 0xb9, 0x8c, 0x36,
0x3e, 0x68, 0x64, 0x1b, 0x35, 0x43, 0xd9, 0xba, 0xa0, 0x79, 0xbb, 0x84, 0x45, 0x7b, 0xa6, 0x10, 0x01, 0x54, 0x13, 0x19, 0xe0, 0x28, 0x73, 0xf4, 0x54, 0xf3, 0x9a, 0x39, 0x7a, 0x46, 0xc3, 0x79,
0xcf, 0x1f, 0xb6, 0xd9, 0xfa, 0xbc, 0x37, 0x57, 0x8c, 0x1b, 0x0f, 0xa0, 0x91, 0xad, 0xc0, 0xb5, 0x1f, 0x40, 0xf6, 0x7e, 0x01, 0x49, 0x99, 0x75, 0x5d, 0xab, 0x31, 0xd1, 0x70, 0x0e, 0xfa, 0xd3,
0x16, 0x0b, 0xaa, 0xf2, 0x5e, 0xae, 0x0a, 0x37, 0x1e, 0x42, 0x2b, 0x5f, 0x7d, 0xeb, 0x23, 0xb5, 0x03, 0x53, 0x0c, 0x10, 0xa5, 0x57, 0x61, 0xf0, 0x01, 0x40, 0xde, 0x53, 0x6a, 0x06, 0x53, 0x5d,
0xb0, 0x26, 0xef, 0xa9, 0xb7, 0xa5, 0x0c, 0xf9, 0x3b, 0x00, 0x69, 0x95, 0xae, 0xdd, 0x37, 0x57, 0xe6, 0x25, 0x3e, 0x68, 0x99, 0x1d, 0xa4, 0xa5, 0x6c, 0x9d, 0xd1, 0x55, 0x5e, 0xc2, 0xa2, 0x3b,
0xb7, 0xcf, 0x48, 0xdd, 0x87, 0xf6, 0x4c, 0xf5, 0xad, 0x2d, 0x5e, 0x5c, 0x94, 0x2f, 0x75, 0xdd, 0xd1, 0x21, 0x14, 0x37, 0xdb, 0x64, 0xe3, 0x30, 0x98, 0xea, 0x12, 0xac, 0x7b, 0xd0, 0x32, 0x5b,
0xbb, 0x00, 0x69, 0x56, 0xd6, 0xd2, 0xe7, 0xf2, 0x74, 0xaf, 0xa9, 0xdf, 0xdd, 0x24, 0xdd, 0x2e, 0x03, 0xad, 0xc5, 0x8c, 0x76, 0x61, 0x50, 0x68, 0x0f, 0xac, 0xfb, 0xd0, 0x29, 0xb6, 0x05, 0x7a,
0x34, 0x73, 0xad, 0xa9, 0x4e, 0x33, 0x8b, 0xfa, 0xd5, 0xcb, 0x92, 0x6f, 0xbe, 0x8f, 0xd3, 0x9e, 0x4b, 0xcd, 0x6c, 0x16, 0x06, 0xea, 0xd2, 0xcb, 0x20, 0x7f, 0x0b, 0x20, 0x6f, 0x1f, 0xb4, 0xfb,
0x5b, 0xd8, 0xdd, 0x5d, 0x76, 0x7e, 0xb2, 0xcd, 0x83, 0xde, 0xb9, 0x05, 0x0d, 0xc5, 0xf7, 0xc4, 0xa6, 0x1a, 0x8a, 0x09, 0xa9, 0xbb, 0xd0, 0x9d, 0x68, 0x0b, 0xb4, 0xc5, 0xb3, 0xbb, 0x85, 0xb9,
0x73, 0xb6, 0x41, 0xc8, 0xc4, 0xf3, 0x82, 0xbe, 0x61, 0x29, 0xa3, 0x03, 0x68, 0xef, 0xeb, 0xda, 0xae, 0x7b, 0x1b, 0x20, 0x3f, 0x2e, 0xb4, 0xf4, 0xa9, 0x03, 0x64, 0xd0, 0xd6, 0x17, 0x82, 0x92,
0x4f, 0xd5, 0xa5, 0x4a, 0x9d, 0x05, 0x75, 0x78, 0xaf, 0xb7, 0x68, 0x4a, 0x05, 0xd5, 0xa7, 0xd0, 0x6e, 0x1b, 0xda, 0x85, 0x9e, 0x59, 0xa7, 0x99, 0x59, 0x8d, 0xf4, 0x65, 0xc9, 0xb7, 0xd8, 0x60,
0x99, 0xab, 0x49, 0x8d, 0xad, 0xe4, 0xf5, 0x73, 0x61, 0xb1, 0xba, 0x54, 0xad, 0x43, 0x58, 0x9b, 0x6a, 0xcf, 0xcd, 0x6c, 0x3b, 0x2f, 0xdb, 0x3f, 0x66, 0x57, 0xa3, 0x57, 0x6e, 0x46, 0xa7, 0xf3,
0x2d, 0x49, 0x8d, 0x17, 0x55, 0xa2, 0x5c, 0x5c, 0xaa, 0x2e, 0x65, 0xf5, 0x3e, 0x54, 0x75, 0x09, 0x2d, 0xf1, 0x6c, 0x76, 0x2e, 0x46, 0x3c, 0xcf, 0x68, 0x68, 0xe6, 0x32, 0xda, 0x83, 0xee, 0xae,
0x64, 0xa8, 0x57, 0xe6, 0x99, 0x92, 0x68, 0xe9, 0xd2, 0x07, 0x50, 0xcf, 0x54, 0x1c, 0x3a, 0xdb, 0x2e, 0x4a, 0x55, 0xc1, 0xac, 0xd4, 0x99, 0xd1, 0x20, 0x0c, 0x06, 0xb3, 0x86, 0x54, 0x50, 0x7d,
0xcd, 0x17, 0x21, 0x3d, 0xf5, 0x28, 0x9c, 0x50, 0x3e, 0x80, 0x8a, 0xaa, 0x32, 0x8c, 0x8d, 0xe4, 0x0c, 0xbd, 0xa9, 0x62, 0xd9, 0x5a, 0xcb, 0xae, 0x65, 0x67, 0x56, 0xd1, 0x73, 0xd5, 0xda, 0x87,
0x90, 0x67, 0x8a, 0x8e, 0x65, 0x12, 0x77, 0x2e, 0xbe, 0xfd, 0x6e, 0xeb, 0xb9, 0xbf, 0x7d, 0xb7, 0xa5, 0xc9, 0x5a, 0xd9, 0x7a, 0x5e, 0x25, 0xca, 0xd9, 0x35, 0xf4, 0x5c, 0x56, 0xef, 0x42, 0x5d,
0xf5, 0xdc, 0x2f, 0x9e, 0x6d, 0x15, 0xbe, 0x7d, 0xb6, 0x55, 0xf8, 0xeb, 0xb3, 0xad, 0xc2, 0x3f, 0xd7, 0x66, 0x96, 0xba, 0xfe, 0x9e, 0xa8, 0xd5, 0xe6, 0x4e, 0xbd, 0x07, 0x4d, 0xa3, 0x14, 0xd2,
0x9e, 0x6d, 0x15, 0x7e, 0xf4, 0xd3, 0xff, 0xf0, 0x9f, 0x25, 0x34, 0x0e, 0x18, 0xf6, 0xd1, 0xf6, 0xd9, 0x6e, 0xba, 0x3a, 0x1a, 0xa8, 0xdb, 0xea, 0x8c, 0xf2, 0x1e, 0xd4, 0x54, 0xf9, 0x63, 0xad,
0x39, 0xa6, 0x2c, 0x33, 0x15, 0x9e, 0x8d, 0xe4, 0xdf, 0x4b, 0x32, 0xff, 0x3a, 0xe1, 0x0a, 0x0e, 0x64, 0x9b, 0xdc, 0xa8, 0x86, 0x2e, 0xdb, 0x61, 0xbb, 0x88, 0x19, 0x45, 0x8d, 0x16, 0x3a, 0x5d,
0x57, 0x05, 0xfc, 0xce, 0xbf, 0x02, 0x00, 0x00, 0xff, 0xff, 0x52, 0x45, 0x13, 0x9c, 0xc2, 0x22, 0xe7, 0xe8, 0x14, 0x5b, 0x18, 0x51, 0x6b, 0xb1, 0x09, 0x2d, 0xb3, 0xac, 0xd1, 0x4b, 0x3a, 0xa3,
0x00, 0x00, 0xd4, 0x99, 0xa7, 0xc9, 0xd6, 0xf9, 0xd7, 0xdf, 0xac, 0x3d, 0xf3, 0x8f, 0x6f, 0xd6, 0x9e, 0xf9,
0xd5, 0x93, 0xb5, 0xd2, 0xd7, 0x4f, 0xd6, 0x4a, 0x7f, 0x7f, 0xb2, 0x56, 0xfa, 0xd7, 0x93, 0xb5,
0xd2, 0x4f, 0x7e, 0xfe, 0x3f, 0xfe, 0x0f, 0x87, 0xa6, 0x11, 0xc3, 0x21, 0xda, 0x38, 0xc3, 0x94,
0x19, 0x43, 0xf1, 0xe9, 0x48, 0xfe, 0x19, 0xc7, 0xf8, 0x8f, 0x0e, 0xd7, 0xf2, 0x78, 0x51, 0xc0,
0x6f, 0xfd, 0x27, 0x00, 0x00, 0xff, 0xff, 0x5c, 0x4e, 0xa6, 0xdc, 0xf0, 0x23, 0x00, 0x00,
} }
func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) { func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) {
@ -5446,6 +5533,79 @@ func (m *Metrics) MarshalToSizedBuffer(dAtA []byte) (int, error) {
return len(dAtA) - i, nil return len(dAtA) - i, nil
} }
func (m *VolumeStatsRequest) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *VolumeStatsRequest) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *VolumeStatsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.XXX_unrecognized != nil {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if len(m.VolumeGuestPath) > 0 {
i -= len(m.VolumeGuestPath)
copy(dAtA[i:], m.VolumeGuestPath)
i = encodeVarintAgent(dAtA, i, uint64(len(m.VolumeGuestPath)))
i--
dAtA[i] = 0xa
}
return len(dAtA) - i, nil
}
func (m *ResizeVolumeRequest) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalToSizedBuffer(dAtA[:size])
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *ResizeVolumeRequest) MarshalTo(dAtA []byte) (int, error) {
size := m.Size()
return m.MarshalToSizedBuffer(dAtA[:size])
}
func (m *ResizeVolumeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) {
i := len(dAtA)
_ = i
var l int
_ = l
if m.XXX_unrecognized != nil {
i -= len(m.XXX_unrecognized)
copy(dAtA[i:], m.XXX_unrecognized)
}
if m.Size_ != 0 {
i = encodeVarintAgent(dAtA, i, uint64(m.Size_))
i--
dAtA[i] = 0x10
}
if len(m.VolumeGuestPath) > 0 {
i -= len(m.VolumeGuestPath)
copy(dAtA[i:], m.VolumeGuestPath)
i = encodeVarintAgent(dAtA, i, uint64(len(m.VolumeGuestPath)))
i--
dAtA[i] = 0xa
}
return len(dAtA) - i, nil
}
func encodeVarintAgent(dAtA []byte, offset int, v uint64) int { func encodeVarintAgent(dAtA []byte, offset int, v uint64) int {
offset -= sovAgent(v) offset -= sovAgent(v)
base := offset base := offset
@ -6737,6 +6897,41 @@ func (m *Metrics) Size() (n int) {
return n return n
} }
func (m *VolumeStatsRequest) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.VolumeGuestPath)
if l > 0 {
n += 1 + l + sovAgent(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func (m *ResizeVolumeRequest) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.VolumeGuestPath)
if l > 0 {
n += 1 + l + sovAgent(uint64(l))
}
if m.Size_ != 0 {
n += 1 + sovAgent(uint64(m.Size_))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func sovAgent(x uint64) (n int) { func sovAgent(x uint64) (n int) {
return (math_bits.Len64(x|1) + 6) / 7 return (math_bits.Len64(x|1) + 6) / 7
} }
@ -7551,6 +7746,29 @@ func (this *Metrics) String() string {
}, "") }, "")
return s return s
} }
func (this *VolumeStatsRequest) String() string {
if this == nil {
return "nil"
}
s := strings.Join([]string{`&VolumeStatsRequest{`,
`VolumeGuestPath:` + fmt.Sprintf("%v", this.VolumeGuestPath) + `,`,
`XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`,
`}`,
}, "")
return s
}
func (this *ResizeVolumeRequest) String() string {
if this == nil {
return "nil"
}
s := strings.Join([]string{`&ResizeVolumeRequest{`,
`VolumeGuestPath:` + fmt.Sprintf("%v", this.VolumeGuestPath) + `,`,
`Size_:` + fmt.Sprintf("%v", this.Size_) + `,`,
`XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`,
`}`,
}, "")
return s
}
func valueToStringAgent(v interface{}) string { func valueToStringAgent(v interface{}) string {
rv := reflect.ValueOf(v) rv := reflect.ValueOf(v)
if rv.IsNil() { if rv.IsNil() {
@ -7592,6 +7810,8 @@ type AgentServiceService interface {
CopyFile(ctx context.Context, req *CopyFileRequest) (*types.Empty, error) CopyFile(ctx context.Context, req *CopyFileRequest) (*types.Empty, error)
GetOOMEvent(ctx context.Context, req *GetOOMEventRequest) (*OOMEvent, error) GetOOMEvent(ctx context.Context, req *GetOOMEventRequest) (*OOMEvent, error)
AddSwap(ctx context.Context, req *AddSwapRequest) (*types.Empty, error) AddSwap(ctx context.Context, req *AddSwapRequest) (*types.Empty, error)
GetVolumeStats(ctx context.Context, req *VolumeStatsRequest) (*VolumeStatsResponse, error)
ResizeVolume(ctx context.Context, req *ResizeVolumeRequest) (*types.Empty, error)
} }
func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc AgentServiceService) { func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc AgentServiceService) {
@ -7813,6 +8033,20 @@ func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc Ag
} }
return svc.AddSwap(ctx, &req) return svc.AddSwap(ctx, &req)
}, },
"GetVolumeStats": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
var req VolumeStatsRequest
if err := unmarshal(&req); err != nil {
return nil, err
}
return svc.GetVolumeStats(ctx, &req)
},
"ResizeVolume": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) {
var req ResizeVolumeRequest
if err := unmarshal(&req); err != nil {
return nil, err
}
return svc.ResizeVolume(ctx, &req)
},
}) })
} }
@ -8073,6 +8307,22 @@ func (c *agentServiceClient) AddSwap(ctx context.Context, req *AddSwapRequest) (
} }
return &resp, nil return &resp, nil
} }
func (c *agentServiceClient) GetVolumeStats(ctx context.Context, req *VolumeStatsRequest) (*VolumeStatsResponse, error) {
var resp VolumeStatsResponse
if err := c.client.Call(ctx, "grpc.AgentService", "GetVolumeStats", req, &resp); err != nil {
return nil, err
}
return &resp, nil
}
func (c *agentServiceClient) ResizeVolume(ctx context.Context, req *ResizeVolumeRequest) (*types.Empty, error) {
var resp types.Empty
if err := c.client.Call(ctx, "grpc.AgentService", "ResizeVolume", req, &resp); err != nil {
return nil, err
}
return &resp, nil
}
func (m *CreateContainerRequest) Unmarshal(dAtA []byte) error { func (m *CreateContainerRequest) Unmarshal(dAtA []byte) error {
l := len(dAtA) l := len(dAtA)
iNdEx := 0 iNdEx := 0
@ -15399,6 +15649,191 @@ func (m *Metrics) Unmarshal(dAtA []byte) error {
} }
return nil return nil
} }
func (m *VolumeStatsRequest) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowAgent
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: VolumeStatsRequest: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: VolumeStatsRequest: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field VolumeGuestPath", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowAgent
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthAgent
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthAgent
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.VolumeGuestPath = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipAgent(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthAgent
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *ResizeVolumeRequest) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowAgent
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: ResizeVolumeRequest: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: ResizeVolumeRequest: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field VolumeGuestPath", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowAgent
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthAgent
}
postIndex := iNdEx + intStringLen
if postIndex < 0 {
return ErrInvalidLengthAgent
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.VolumeGuestPath = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType)
}
m.Size_ = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowAgent
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Size_ |= uint64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipAgent(dAtA[iNdEx:])
if err != nil {
return err
}
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthAgent
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipAgent(dAtA []byte) (n int, err error) { func skipAgent(dAtA []byte) (n int, err error) {
l := len(dAtA) l := len(dAtA)
iNdEx := 0 iNdEx := 0

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,585 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: github.com/kata-containers/kata-containers/src/libs/protocols/protos/csi.proto
package grpc
import (
fmt "fmt"
_ "github.com/gogo/protobuf/gogoproto"
github_com_gogo_protobuf_jsonpb "github.com/gogo/protobuf/jsonpb"
github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto"
proto "github.com/gogo/protobuf/proto"
math "math"
math_rand "math/rand"
testing "testing"
time "time"
)
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
func TestVolumeStatsResponseProto(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeStatsResponse(popr, false)
dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
msg := &VolumeStatsResponse{}
if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
littlefuzz := make([]byte, len(dAtA))
copy(littlefuzz, dAtA)
for i := range dAtA {
dAtA[i] = byte(popr.Intn(256))
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
if len(littlefuzz) > 0 {
fuzzamount := 100
for i := 0; i < fuzzamount; i++ {
littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
}
// shouldn't panic
_ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg)
}
}
func TestVolumeStatsResponseMarshalTo(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeStatsResponse(popr, false)
size := p.Size()
dAtA := make([]byte, size)
for i := range dAtA {
dAtA[i] = byte(popr.Intn(256))
}
_, err := p.MarshalTo(dAtA)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
msg := &VolumeStatsResponse{}
if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
for i := range dAtA {
dAtA[i] = byte(popr.Intn(256))
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
}
func BenchmarkVolumeStatsResponseProtoMarshal(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
pops := make([]*VolumeStatsResponse, 10000)
for i := 0; i < 10000; i++ {
pops[i] = NewPopulatedVolumeStatsResponse(popr, false)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000])
if err != nil {
panic(err)
}
total += len(dAtA)
}
b.SetBytes(int64(total / b.N))
}
func BenchmarkVolumeStatsResponseProtoUnmarshal(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
datas := make([][]byte, 10000)
for i := 0; i < 10000; i++ {
dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedVolumeStatsResponse(popr, false))
if err != nil {
panic(err)
}
datas[i] = dAtA
}
msg := &VolumeStatsResponse{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
total += len(datas[i%10000])
if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil {
panic(err)
}
}
b.SetBytes(int64(total / b.N))
}
func TestVolumeUsageProto(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeUsage(popr, false)
dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
msg := &VolumeUsage{}
if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
littlefuzz := make([]byte, len(dAtA))
copy(littlefuzz, dAtA)
for i := range dAtA {
dAtA[i] = byte(popr.Intn(256))
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
if len(littlefuzz) > 0 {
fuzzamount := 100
for i := 0; i < fuzzamount; i++ {
littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
}
// shouldn't panic
_ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg)
}
}
func TestVolumeUsageMarshalTo(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeUsage(popr, false)
size := p.Size()
dAtA := make([]byte, size)
for i := range dAtA {
dAtA[i] = byte(popr.Intn(256))
}
_, err := p.MarshalTo(dAtA)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
msg := &VolumeUsage{}
if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
for i := range dAtA {
dAtA[i] = byte(popr.Intn(256))
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
}
func BenchmarkVolumeUsageProtoMarshal(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
pops := make([]*VolumeUsage, 10000)
for i := 0; i < 10000; i++ {
pops[i] = NewPopulatedVolumeUsage(popr, false)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000])
if err != nil {
panic(err)
}
total += len(dAtA)
}
b.SetBytes(int64(total / b.N))
}
func BenchmarkVolumeUsageProtoUnmarshal(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
datas := make([][]byte, 10000)
for i := 0; i < 10000; i++ {
dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedVolumeUsage(popr, false))
if err != nil {
panic(err)
}
datas[i] = dAtA
}
msg := &VolumeUsage{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
total += len(datas[i%10000])
if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil {
panic(err)
}
}
b.SetBytes(int64(total / b.N))
}
func TestVolumeConditionProto(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeCondition(popr, false)
dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
msg := &VolumeCondition{}
if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
littlefuzz := make([]byte, len(dAtA))
copy(littlefuzz, dAtA)
for i := range dAtA {
dAtA[i] = byte(popr.Intn(256))
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
if len(littlefuzz) > 0 {
fuzzamount := 100
for i := 0; i < fuzzamount; i++ {
littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256))
littlefuzz = append(littlefuzz, byte(popr.Intn(256)))
}
// shouldn't panic
_ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg)
}
}
func TestVolumeConditionMarshalTo(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeCondition(popr, false)
size := p.Size()
dAtA := make([]byte, size)
for i := range dAtA {
dAtA[i] = byte(popr.Intn(256))
}
_, err := p.MarshalTo(dAtA)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
msg := &VolumeCondition{}
if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
for i := range dAtA {
dAtA[i] = byte(popr.Intn(256))
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
}
func BenchmarkVolumeConditionProtoMarshal(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
pops := make([]*VolumeCondition, 10000)
for i := 0; i < 10000; i++ {
pops[i] = NewPopulatedVolumeCondition(popr, false)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000])
if err != nil {
panic(err)
}
total += len(dAtA)
}
b.SetBytes(int64(total / b.N))
}
func BenchmarkVolumeConditionProtoUnmarshal(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
datas := make([][]byte, 10000)
for i := 0; i < 10000; i++ {
dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedVolumeCondition(popr, false))
if err != nil {
panic(err)
}
datas[i] = dAtA
}
msg := &VolumeCondition{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
total += len(datas[i%10000])
if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil {
panic(err)
}
}
b.SetBytes(int64(total / b.N))
}
func TestVolumeStatsResponseJSON(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeStatsResponse(popr, true)
marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{}
jsondata, err := marshaler.MarshalToString(p)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
msg := &VolumeStatsResponse{}
err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p)
}
}
func TestVolumeUsageJSON(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeUsage(popr, true)
marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{}
jsondata, err := marshaler.MarshalToString(p)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
msg := &VolumeUsage{}
err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p)
}
}
func TestVolumeConditionJSON(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeCondition(popr, true)
marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{}
jsondata, err := marshaler.MarshalToString(p)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
msg := &VolumeCondition{}
err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p)
}
}
func TestVolumeStatsResponseProtoText(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeStatsResponse(popr, true)
dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p)
msg := &VolumeStatsResponse{}
if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
}
func TestVolumeStatsResponseProtoCompactText(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeStatsResponse(popr, true)
dAtA := github_com_gogo_protobuf_proto.CompactTextString(p)
msg := &VolumeStatsResponse{}
if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
}
func TestVolumeUsageProtoText(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeUsage(popr, true)
dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p)
msg := &VolumeUsage{}
if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
}
func TestVolumeUsageProtoCompactText(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeUsage(popr, true)
dAtA := github_com_gogo_protobuf_proto.CompactTextString(p)
msg := &VolumeUsage{}
if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
}
func TestVolumeConditionProtoText(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeCondition(popr, true)
dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p)
msg := &VolumeCondition{}
if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
}
func TestVolumeConditionProtoCompactText(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeCondition(popr, true)
dAtA := github_com_gogo_protobuf_proto.CompactTextString(p)
msg := &VolumeCondition{}
if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
if !p.Equal(msg) {
t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p)
}
}
func TestVolumeStatsResponseSize(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeStatsResponse(popr, true)
size2 := github_com_gogo_protobuf_proto.Size(p)
dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
size := p.Size()
if len(dAtA) != size {
t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA))
}
if size2 != size {
t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2)
}
size3 := github_com_gogo_protobuf_proto.Size(p)
if size3 != size {
t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3)
}
}
func BenchmarkVolumeStatsResponseSize(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
pops := make([]*VolumeStatsResponse, 1000)
for i := 0; i < 1000; i++ {
pops[i] = NewPopulatedVolumeStatsResponse(popr, false)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
total += pops[i%1000].Size()
}
b.SetBytes(int64(total / b.N))
}
func TestVolumeUsageSize(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeUsage(popr, true)
size2 := github_com_gogo_protobuf_proto.Size(p)
dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
size := p.Size()
if len(dAtA) != size {
t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA))
}
if size2 != size {
t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2)
}
size3 := github_com_gogo_protobuf_proto.Size(p)
if size3 != size {
t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3)
}
}
func BenchmarkVolumeUsageSize(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
pops := make([]*VolumeUsage, 1000)
for i := 0; i < 1000; i++ {
pops[i] = NewPopulatedVolumeUsage(popr, false)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
total += pops[i%1000].Size()
}
b.SetBytes(int64(total / b.N))
}
func TestVolumeConditionSize(t *testing.T) {
seed := time.Now().UnixNano()
popr := math_rand.New(math_rand.NewSource(seed))
p := NewPopulatedVolumeCondition(popr, true)
size2 := github_com_gogo_protobuf_proto.Size(p)
dAtA, err := github_com_gogo_protobuf_proto.Marshal(p)
if err != nil {
t.Fatalf("seed = %d, err = %v", seed, err)
}
size := p.Size()
if len(dAtA) != size {
t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA))
}
if size2 != size {
t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2)
}
size3 := github_com_gogo_protobuf_proto.Size(p)
if size3 != size {
t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3)
}
}
func BenchmarkVolumeConditionSize(b *testing.B) {
popr := math_rand.New(math_rand.NewSource(616))
total := 0
pops := make([]*VolumeCondition, 1000)
for i := 0; i < 1000; i++ {
pops[i] = NewPopulatedVolumeCondition(popr, false)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
total += pops[i%1000].Size()
}
b.SetBytes(int64(total / b.N))
}
func TestVolumeStatsResponseStringer(t *testing.T) {
popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
p := NewPopulatedVolumeStatsResponse(popr, false)
s1 := p.String()
s2 := fmt.Sprintf("%v", p)
if s1 != s2 {
t.Fatalf("String want %v got %v", s1, s2)
}
}
func TestVolumeUsageStringer(t *testing.T) {
popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
p := NewPopulatedVolumeUsage(popr, false)
s1 := p.String()
s2 := fmt.Sprintf("%v", p)
if s1 != s2 {
t.Fatalf("String want %v got %v", s1, s2)
}
}
func TestVolumeConditionStringer(t *testing.T) {
popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano()))
p := NewPopulatedVolumeCondition(popr, false)
s1 := p.String()
s2 := fmt.Sprintf("%v", p)
if s1 != s2 {
t.Fatalf("String want %v got %v", s1, s2)
}
}
//These tests are generated by github.com/gogo/protobuf/plugin/testgen

View File

@ -231,3 +231,11 @@ func (p *HybridVSockTTRPCMockImp) GetMetrics(ctx context.Context, req *pb.GetMet
func (p *HybridVSockTTRPCMockImp) AddSwap(ctx context.Context, req *pb.AddSwapRequest) (*gpb.Empty, error) { func (p *HybridVSockTTRPCMockImp) AddSwap(ctx context.Context, req *pb.AddSwapRequest) (*gpb.Empty, error) {
return &gpb.Empty{}, nil return &gpb.Empty{}, nil
} }
func (p *HybridVSockTTRPCMockImp) GetVolumeStats(ctx context.Context, req *pb.VolumeStatsRequest) (*pb.VolumeStatsResponse, error) {
return &pb.VolumeStatsResponse{}, nil
}
func (p *HybridVSockTTRPCMockImp) ResizeVolume(ctx context.Context, req *pb.ResizeVolumeRequest) (*gpb.Empty, error) {
return &gpb.Empty{}, nil
}

View File

@ -254,3 +254,10 @@ func (s *Sandbox) GetAgentURL() (string, error) {
func (s *Sandbox) GetHypervisorPid() (int, error) { func (s *Sandbox) GetHypervisorPid() (int, error) {
return 0, nil return 0, nil
} }
func (s *Sandbox) GuestVolumeStats(ctx context.Context, path string) ([]byte, error) {
return nil, nil
}
func (s *Sandbox) ResizeGuestVolume(ctx context.Context, path string, size uint64) error {
return nil
}

View File

@ -2257,6 +2257,43 @@ func (s *Sandbox) GetAgentURL() (string, error) {
return s.agent.getAgentURL() return s.agent.getAgentURL()
} }
// GuestVolumeStats return the filesystem stat of a given volume in the guest.
func (s *Sandbox) GuestVolumeStats(ctx context.Context, volumePath string) ([]byte, error) {
guestMountPath, err := s.guestMountPath(volumePath)
if err != nil {
return nil, err
}
return s.agent.getGuestVolumeStats(ctx, guestMountPath)
}
// ResizeGuestVolume resizes a volume in the guest.
func (s *Sandbox) ResizeGuestVolume(ctx context.Context, volumePath string, size uint64) error {
// TODO: https://github.com/kata-containers/kata-containers/issues/3694.
guestMountPath, err := s.guestMountPath(volumePath)
if err != nil {
return err
}
return s.agent.resizeGuestVolume(ctx, guestMountPath, size)
}
func (s *Sandbox) guestMountPath(volumePath string) (string, error) {
// verify the device even exists
if _, err := os.Stat(volumePath); err != nil {
s.Logger().WithError(err).WithField("volume", volumePath).Error("Cannot get stats for volume that doesn't exist")
return "", err
}
// verify that we have a mount in this sandbox who's source maps to this
for _, c := range s.containers {
for _, m := range c.mounts {
if volumePath == m.Source {
return m.GuestDeviceMount, nil
}
}
}
return "", fmt.Errorf("mount %s not found in sandbox", volumePath)
}
// getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets' // getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets'
// cpus and mems as a string in canonical linux CPU/mems list format // cpus and mems as a string in canonical linux CPU/mems list format
func (s *Sandbox) getSandboxCPUSet() (string, string, error) { func (s *Sandbox) getSandboxCPUSet() (string, string, error) {

View File

@ -171,6 +171,11 @@ static AGENT_CMDS: &[AgentCmd] = &[
st: ServiceType::Agent, st: ServiceType::Agent,
fp: agent_cmd_sandbox_get_oom_event, fp: agent_cmd_sandbox_get_oom_event,
}, },
AgentCmd {
name: "GetVolumeStats",
st: ServiceType::Agent,
fp: agent_cmd_sandbox_get_volume_stats,
},
AgentCmd { AgentCmd {
name: "ListInterfaces", name: "ListInterfaces",
st: ServiceType::Agent, st: ServiceType::Agent,
@ -1641,6 +1646,29 @@ fn agent_cmd_sandbox_get_oom_event(
Ok(()) Ok(())
} }
fn agent_cmd_sandbox_get_volume_stats(
ctx: &Context,
client: &AgentServiceClient,
_health: &HealthClient,
_options: &mut Options,
args: &str,
) -> Result<()> {
let req: VolumeStatsRequest = utils::make_request(args)?;
let ctx = clone_context(ctx);
debug!(sl!(), "sending request"; "request" => format!("{:?}", req));
let reply = client
.get_volume_stats(ctx, &req)
.map_err(|e| anyhow!(e).context(ERR_API_FAILED))?;
info!(sl!(), "response received";
"response" => format!("{:?}", reply));
Ok(())
}
fn agent_cmd_sandbox_copy_file( fn agent_cmd_sandbox_copy_file(
ctx: &Context, ctx: &Context,
client: &AgentServiceClient, client: &AgentServiceClient,