From fa326b4e0f2c3af642f8c8a7e6b0086fce960c5a Mon Sep 17 00:00:00 2001 From: Feng Wang Date: Fri, 14 Jan 2022 12:42:45 -0800 Subject: [PATCH 1/9] runtime: augment kata-runtime CLI to support direct-assigned volume Add commands to add, remove, resize and get stats of a direct-assigned volume. These commands are expected to be consumed by CSI. Fixes: #3454 Signed-off-by: Feng Wang --- src/runtime/cmd/kata-runtime/kata-volume.go | 108 ++++++++++++++++++++ src/runtime/cmd/kata-runtime/main.go | 1 + 2 files changed, 109 insertions(+) create mode 100644 src/runtime/cmd/kata-runtime/kata-volume.go diff --git a/src/runtime/cmd/kata-runtime/kata-volume.go b/src/runtime/cmd/kata-runtime/kata-volume.go new file mode 100644 index 0000000000..bbe5d5713f --- /dev/null +++ b/src/runtime/cmd/kata-runtime/kata-volume.go @@ -0,0 +1,108 @@ +// Copyright (c) 2022 Databricks Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" + "github.com/urfave/cli" +) + +var volumeSubCmds = []cli.Command{ + addCommand, + removeCommand, + statsCommand, + resizeCommand, +} + +var ( + mountInfo string + volumePath string + size uint64 +) + +var kataVolumeCommand = cli.Command{ + Name: "direct-volume", + Usage: "directly assign a volume to Kata Containers to manage", + Subcommands: volumeSubCmds, + Action: func(context *cli.Context) { + cli.ShowSubcommandHelp(context) + }, +} + +var addCommand = cli.Command{ + Name: "add", + Usage: "add a direct assigned block volume device to the Kata Containers runtime", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "volume-path", + Usage: "the target volume path the volume is published to", + Destination: &volumePath, + }, + cli.StringFlag{ + Name: "mount-info", + Usage: "the mount info for the Kata Containers runtime to manage the volume", + Destination: &mountInfo, + }, + }, + Action: func(c *cli.Context) error { + return volume.Add(volumePath, mountInfo) + }, +} + +var removeCommand = cli.Command{ + Name: "remove", + Usage: "remove a direct assigned block volume device from the Kata Containers runtime", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "volume-path", + Usage: "the target volume path the volume is published to", + Destination: &volumePath, + }, + }, + Action: func(c *cli.Context) error { + return volume.Remove(volumePath) + }, +} + +var statsCommand = cli.Command{ + Name: "stats", + Usage: "get the filesystem stat of a direct assigned volume", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "volume-path", + Usage: "the target volume path the volume is published to", + Destination: &volumePath, + }, + }, + Action: func(c *cli.Context) (string, error) { + stats, err := volume.Stats(volumePath) + if err != nil { + return "", err + } + + return string(stats), nil + }, +} + +var resizeCommand = cli.Command{ + Name: "resize", + Usage: "resize a direct assigned block volume", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "volume-path", + Usage: "the target volume path the volume is published to", + Destination: &volumePath, + }, + cli.Uint64Flag{ + Name: "size", + Usage: "the new size of the volume", + Destination: &size, + }, + }, + Action: func(c *cli.Context) error { + return volume.Resize(volumePath, size) + }, +} diff --git a/src/runtime/cmd/kata-runtime/main.go b/src/runtime/cmd/kata-runtime/main.go index 17309fe83c..def7431f08 100644 --- a/src/runtime/cmd/kata-runtime/main.go +++ b/src/runtime/cmd/kata-runtime/main.go @@ -124,6 +124,7 @@ var runtimeCommands = []cli.Command{ kataExecCLICommand, kataMetricsCLICommand, factoryCLICommand, + kataVolumeCommand, } // runtimeBeforeSubcommands is the function to run before command-line From 6e0090abb557f1cc863e3c1550adbbb4cf59170d Mon Sep 17 00:00:00 2001 From: Feng Wang Date: Fri, 14 Jan 2022 12:56:22 -0800 Subject: [PATCH 2/9] runtime: persist direct volume mount info In the direct assigned volume scenario, Kata Containers persists the information required for managing the volume inside the guest on host filesystem. Fixes: #3454 Signed-off-by: Feng Wang --- src/runtime/pkg/direct-volume/utils.go | 108 ++++++++++++++++++++ src/runtime/pkg/direct-volume/utils_test.go | 94 +++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 src/runtime/pkg/direct-volume/utils.go create mode 100644 src/runtime/pkg/direct-volume/utils_test.go diff --git a/src/runtime/pkg/direct-volume/utils.go b/src/runtime/pkg/direct-volume/utils.go new file mode 100644 index 0000000000..275b0508f8 --- /dev/null +++ b/src/runtime/pkg/direct-volume/utils.go @@ -0,0 +1,108 @@ +// Copyright (c) 2022 Databricks Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package volume + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +const ( + mountInfoFileName = "mountInfo.json" +) + +var kataDirectVolumeRootPath = "/run/kata-containers/shared/direct-volumes" + +// MountInfo contains the information needed by Kata to consume a host block device and mount it as a filesystem inside the guest VM. +type MountInfo struct { + // The type of the volume (ie. block) + VolumeType string `json:"volume-type"` + // The device backing the volume. + Device string `json:"device"` + // The filesystem type to be mounted on the volume. + FsType string `json:"fstype"` + // Additional metadata to pass to the agent regarding this volume. + Metadata map[string]string `json:"metadata,omitempty"` + // Additional mount options. + Options []string `json:"options,omitempty"` +} + +// Add writes the mount info of a direct volume into a filesystem path known to Kata Container. +func Add(volumePath string, mountInfo string) error { + volumeDir := filepath.Join(kataDirectVolumeRootPath, volumePath) + stat, err := os.Stat(volumeDir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if stat != nil && !stat.IsDir() { + return fmt.Errorf("%s should be a directory", volumeDir) + } + if errors.Is(err, os.ErrNotExist) { + if err := os.MkdirAll(volumeDir, 0700); err != nil { + return err + } + } + var deserialized MountInfo + if err := json.Unmarshal([]byte(mountInfo), &deserialized); err != nil { + return err + } + + return ioutil.WriteFile(filepath.Join(volumeDir, mountInfoFileName), []byte(mountInfo), 0600) +} + +// Remove deletes the direct volume path including all the files inside it. +func Remove(volumePath string) error { + // Find the base of the volume path to delete the whole volume path + base := strings.SplitN(volumePath, string(os.PathSeparator), 2)[0] + return os.RemoveAll(filepath.Join(kataDirectVolumeRootPath, base)) +} + +// VolumeMountInfo retrieves the mount info of a direct volume. +func VolumeMountInfo(volumePath string) (*MountInfo, error) { + mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, volumePath, mountInfoFileName) + if _, err := os.Stat(mountInfoFilePath); err != nil { + return nil, err + } + buf, err := ioutil.ReadFile(mountInfoFilePath) + if err != nil { + return nil, err + } + var mountInfo MountInfo + if err := json.Unmarshal(buf, &mountInfo); err != nil { + return nil, err + } + return &mountInfo, nil +} + +// RecordSandboxId associates a sandbox id with a direct volume. +func RecordSandboxId(sandboxId string, volumePath string) error { + mountInfoFilePath := filepath.Join(kataDirectVolumeRootPath, volumePath, mountInfoFileName) + if _, err := os.Stat(mountInfoFilePath); err != nil { + return err + } + + return ioutil.WriteFile(filepath.Join(kataDirectVolumeRootPath, volumePath, sandboxId), []byte(""), 0600) +} + +func GetSandboxIdForVolume(volumePath string) (string, error) { + files, err := ioutil.ReadDir(filepath.Join(kataDirectVolumeRootPath, volumePath)) + if err != nil { + return "", err + } + // Find the id of the first sandbox. + // We expect a direct-assigned volume is associated with only a sandbox at a time. + for _, file := range files { + if file.Name() != mountInfoFileName { + return file.Name(), nil + } + } + return "", fmt.Errorf("no sandbox found for %s", volumePath) +} diff --git a/src/runtime/pkg/direct-volume/utils_test.go b/src/runtime/pkg/direct-volume/utils_test.go new file mode 100644 index 0000000000..6ca80dab81 --- /dev/null +++ b/src/runtime/pkg/direct-volume/utils_test.go @@ -0,0 +1,94 @@ +// Copyright (c) 2022 Databricks Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package volume + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/kata-containers/kata-containers/src/runtime/pkg/uuid" + "github.com/stretchr/testify/assert" +) + +func TestAdd(t *testing.T) { + var err error + kataDirectVolumeRootPath, err = os.MkdirTemp(os.TempDir(), "add-test") + assert.Nil(t, err) + defer os.RemoveAll(kataDirectVolumeRootPath) + var volumePath = "/a/b/c" + var basePath = "a" + actual := MountInfo{ + VolumeType: "block", + Device: "/dev/sda", + FsType: "ext4", + Options: []string{"journal_dev", "noload"}, + } + buf, err := json.Marshal(actual) + assert.Nil(t, err) + + // Add the mount info + assert.Nil(t, Add(volumePath, string(buf))) + + // Validate the mount info + expected, err := VolumeMountInfo(volumePath) + assert.Nil(t, err) + assert.Equal(t, expected.Device, actual.Device) + assert.Equal(t, expected.FsType, actual.FsType) + assert.Equal(t, expected.Options, actual.Options) + + // Remove the file + err = Remove(volumePath) + assert.Nil(t, err) + _, err = os.Stat(filepath.Join(kataDirectVolumeRootPath, basePath)) + assert.True(t, errors.Is(err, os.ErrNotExist)) + + // Test invalid mount info json + assert.Error(t, Add(volumePath, "{invalid json}")) +} + +func TestRecordSandboxId(t *testing.T) { + var err error + kataDirectVolumeRootPath, err = os.MkdirTemp(os.TempDir(), "recordSanboxId-test") + assert.Nil(t, err) + defer os.RemoveAll(kataDirectVolumeRootPath) + + var volumePath = "/a/b/c" + mntInfo := MountInfo{ + VolumeType: "block", + Device: "/dev/sda", + FsType: "ext4", + Options: []string{"journal_dev", "noload"}, + } + buf, err := json.Marshal(mntInfo) + assert.Nil(t, err) + + // Add the mount info + assert.Nil(t, Add(volumePath, string(buf))) + + sandboxId := uuid.Generate().String() + err = RecordSandboxId(sandboxId, volumePath) + assert.Nil(t, err) + + id, err := GetSandboxIdForVolume(volumePath) + assert.Nil(t, err) + assert.Equal(t, sandboxId, id) +} + +func TestRecordSandboxIdNoMountInfoFile(t *testing.T) { + var err error + kataDirectVolumeRootPath, err = os.MkdirTemp(os.TempDir(), "recordSanboxId-test") + assert.Nil(t, err) + defer os.RemoveAll(kataDirectVolumeRootPath) + + var volumePath = "/a/b/c" + sandboxId := uuid.Generate().String() + err = RecordSandboxId(sandboxId, volumePath) + assert.Error(t, err) + assert.True(t, errors.Is(err, os.ErrNotExist)) +} From e9b5a255022834cb0915e10fa70c9e56491bae39 Mon Sep 17 00:00:00 2001 From: Feng Wang Date: Fri, 14 Jan 2022 13:08:28 -0800 Subject: [PATCH 3/9] runtime: add stat and resize APIs to containerd-shim-v2 To query fs stats and resize fs, the requests need to be passed to kata agent through containerd-shim-v2. So we're adding to rest APIs on the shim management endpoint. Also refactor shim management client to its own go file. Fixes: #3454 Signed-off-by: Feng Wang --- src/runtime/cmd/kata-runtime/kata-exec.go | 4 +- src/runtime/cmd/kata-runtime/kata-volume.go | 41 +++++++++- .../pkg/containerd-shim-v2/shim_management.go | 66 +++++++++++++++- src/runtime/pkg/kata-monitor/metrics.go | 6 +- src/runtime/pkg/kata-monitor/monitor.go | 4 +- src/runtime/pkg/kata-monitor/shim_client.go | 52 ------------ .../shimclient/shim_management_client.go | 79 +++++++++++++++++++ 7 files changed, 189 insertions(+), 63 deletions(-) create mode 100644 src/runtime/pkg/utils/shimclient/shim_management_client.go diff --git a/src/runtime/cmd/kata-runtime/kata-exec.go b/src/runtime/cmd/kata-runtime/kata-exec.go index 456134a5fc..0a7b80d702 100644 --- a/src/runtime/cmd/kata-runtime/kata-exec.go +++ b/src/runtime/cmd/kata-runtime/kata-exec.go @@ -19,8 +19,8 @@ import ( "time" "github.com/containerd/console" - kataMonitor "github.com/kata-containers/kata-containers/src/runtime/pkg/kata-monitor" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient" clientUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/client" "github.com/pkg/errors" "github.com/urfave/cli" @@ -154,7 +154,7 @@ func (s *iostream) Read(data []byte) (n int, err error) { } func getConn(sandboxID string, port uint64) (net.Conn, error) { - client, err := kataMonitor.BuildShimClient(sandboxID, defaultTimeout) + client, err := shimclient.BuildShimClient(sandboxID, defaultTimeout) if err != nil { return nil, err } diff --git a/src/runtime/cmd/kata-runtime/kata-volume.go b/src/runtime/cmd/kata-runtime/kata-volume.go index bbe5d5713f..1477091299 100644 --- a/src/runtime/cmd/kata-runtime/kata-volume.go +++ b/src/runtime/cmd/kata-runtime/kata-volume.go @@ -6,7 +6,13 @@ package main import ( + "encoding/json" + "net/url" + + containerdshim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2" "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient" + "github.com/urfave/cli" ) @@ -78,7 +84,7 @@ var statsCommand = cli.Command{ }, }, Action: func(c *cli.Context) (string, error) { - stats, err := volume.Stats(volumePath) + stats, err := Stats(volumePath) if err != nil { return "", err } @@ -103,6 +109,37 @@ var resizeCommand = cli.Command{ }, }, Action: func(c *cli.Context) error { - return volume.Resize(volumePath, size) + return Resize(volumePath, size) }, } + +// Stats retrieves the filesystem stats of the direct volume inside the guest. +func Stats(volumePath string) ([]byte, error) { + sandboxId, err := volume.GetSandboxIdForVolume(volumePath) + if err != nil { + return nil, err + } + urlSafeDevicePath := url.PathEscape(volumePath) + body, err := shimclient.DoGet(sandboxId, defaultTimeout, containerdshim.DirectVolumeStatUrl+"/"+urlSafeDevicePath) + if err != nil { + return nil, err + } + return body, nil +} + +// Resize resizes a direct volume inside the guest. +func Resize(volumePath string, size uint64) error { + sandboxId, err := volume.GetSandboxIdForVolume(volumePath) + if err != nil { + return err + } + resizeReq := containerdshim.ResizeRequest{ + VolumePath: volumePath, + Size: size, + } + encoded, err := json.Marshal(resizeReq) + if err != nil { + return err + } + return shimclient.DoPost(sandboxId, defaultTimeout, containerdshim.DirectVolumeResizeUrl, encoded) +} diff --git a/src/runtime/pkg/containerd-shim-v2/shim_management.go b/src/runtime/pkg/containerd-shim-v2/shim_management.go index 5ccf03304c..b5ad03eed2 100644 --- a/src/runtime/pkg/containerd-shim-v2/shim_management.go +++ b/src/runtime/pkg/containerd-shim-v2/shim_management.go @@ -7,26 +7,33 @@ package containerdshim import ( "context" + "encoding/json" "expvar" "fmt" "io" + "io/ioutil" "net/http" "net/http/pprof" + "net/url" "path/filepath" "strconv" "strings" + "google.golang.org/grpc/codes" + cdshim "github.com/containerd/containerd/runtime/v2/shim" + mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" "github.com/opencontainers/runtime-spec/specs-go" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" +) - "google.golang.org/grpc/codes" - - mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" +const ( + DirectVolumeStatUrl = "/direct-volume/stats" + DirectVolumeResizeUrl = "/direct-volume/resize" ) var ( @@ -34,6 +41,11 @@ var ( shimMgtLog = shimLog.WithField("subsystem", "shim-management") ) +type ResizeRequest struct { + VolumePath string + Size uint64 +} + // agentURL returns URL for agent func (s *service) agentURL(w http.ResponseWriter, r *http.Request) { url, err := s.sandbox.GetAgentURL() @@ -126,6 +138,52 @@ func decodeAgentMetrics(body string) []*dto.MetricFamily { return list } +func (s *service) serveVolumeStats(w http.ResponseWriter, r *http.Request) { + volumePath, err := url.PathUnescape(strings.TrimPrefix(r.URL.Path, DirectVolumeStatUrl)) + if err != nil { + shimMgtLog.WithError(err).Error("failed to unescape the volume stat url path") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + + buf, err := s.sandbox.GuestVolumeStats(context.Background(), volumePath) + if err != nil { + shimMgtLog.WithError(err).WithField("volume-path", volumePath).Error("failed to get volume stats") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + w.Write(buf) +} + +func (s *service) serveVolumeResize(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + shimMgtLog.WithError(err).Error("failed to read request body") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + var resizeReq ResizeRequest + err = json.Unmarshal(body, &resizeReq) + if err != nil { + shimMgtLog.WithError(err).Error("failed to unmarshal the http request body") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + + err = s.sandbox.ResizeGuestVolume(context.Background(), resizeReq.VolumePath, resizeReq.Size) + if err != nil { + shimMgtLog.WithError(err).WithField("volume-path", resizeReq.VolumePath).Error("failed to resize the volume") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + w.Write([]byte("")) +} + func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec) { // metrics socket will under sandbox's bundle path metricsAddress := SocketAddress(s.id) @@ -148,6 +206,8 @@ func (s *service) startManagementServer(ctx context.Context, ociSpec *specs.Spec m := http.NewServeMux() m.Handle("/metrics", http.HandlerFunc(s.serveMetrics)) m.Handle("/agent-url", http.HandlerFunc(s.agentURL)) + m.Handle(DirectVolumeStatUrl, http.HandlerFunc(s.serveVolumeStats)) + m.Handle(DirectVolumeResizeUrl, http.HandlerFunc(s.serveVolumeResize)) s.mountPprofHandle(m, ociSpec) // register shim metrics diff --git a/src/runtime/pkg/kata-monitor/metrics.go b/src/runtime/pkg/kata-monitor/metrics.go index 84098c88ff..7249906cea 100644 --- a/src/runtime/pkg/kata-monitor/metrics.go +++ b/src/runtime/pkg/kata-monitor/metrics.go @@ -16,7 +16,7 @@ import ( "time" mutils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" - + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/expfmt" @@ -224,7 +224,7 @@ func (km *KataMonitor) aggregateSandboxMetrics(encoder expfmt.Encoder) error { } func getParsedMetrics(sandboxID string, sandboxMetadata sandboxCRIMetadata) ([]*dto.MetricFamily, error) { - body, err := doGet(sandboxID, defaultTimeout, "metrics") + body, err := shimclient.DoGet(sandboxID, defaultTimeout, "metrics") if err != nil { return nil, err } @@ -234,7 +234,7 @@ func getParsedMetrics(sandboxID string, sandboxMetadata sandboxCRIMetadata) ([]* // GetSandboxMetrics will get sandbox's metrics from shim func GetSandboxMetrics(sandboxID string) (string, error) { - body, err := doGet(sandboxID, defaultTimeout, "metrics") + body, err := shimclient.DoGet(sandboxID, defaultTimeout, "metrics") if err != nil { return "", err } diff --git a/src/runtime/pkg/kata-monitor/monitor.go b/src/runtime/pkg/kata-monitor/monitor.go index 9004cf103c..ed3ea5c089 100644 --- a/src/runtime/pkg/kata-monitor/monitor.go +++ b/src/runtime/pkg/kata-monitor/monitor.go @@ -14,6 +14,8 @@ import ( "sync" "time" + "github.com/kata-containers/kata-containers/src/runtime/pkg/utils/shimclient" + "github.com/fsnotify/fsnotify" "github.com/sirupsen/logrus" ) @@ -180,7 +182,7 @@ func (km *KataMonitor) GetAgentURL(w http.ResponseWriter, r *http.Request) { return } - data, err := doGet(sandboxID, defaultTimeout, "agent-url") + data, err := shimclient.DoGet(sandboxID, defaultTimeout, "agent-url") if err != nil { commonServeError(w, http.StatusBadRequest, err) return diff --git a/src/runtime/pkg/kata-monitor/shim_client.go b/src/runtime/pkg/kata-monitor/shim_client.go index bdb62d401b..388ac6fff5 100644 --- a/src/runtime/pkg/kata-monitor/shim_client.go +++ b/src/runtime/pkg/kata-monitor/shim_client.go @@ -7,13 +7,9 @@ package katamonitor import ( "fmt" - "io" - "net" "net/http" "time" - cdshim "github.com/containerd/containerd/runtime/v2/shim" - shim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2" ) @@ -40,51 +36,3 @@ func getSandboxIDFromReq(r *http.Request) (string, error) { func getSandboxFS() string { return shim.GetSandboxesStoragePath() } - -// BuildShimClient builds and returns an http client for communicating with the provided sandbox -func BuildShimClient(sandboxID string, timeout time.Duration) (*http.Client, error) { - return buildUnixSocketClient(shim.SocketAddress(sandboxID), timeout) -} - -// buildUnixSocketClient build http client for Unix socket -func buildUnixSocketClient(socketAddr string, timeout time.Duration) (*http.Client, error) { - transport := &http.Transport{ - DisableKeepAlives: true, - Dial: func(proto, addr string) (conn net.Conn, err error) { - return cdshim.AnonDialer(socketAddr, timeout) - }, - } - - client := &http.Client{ - Transport: transport, - } - - if timeout > 0 { - client.Timeout = timeout - } - - return client, nil -} - -func doGet(sandboxID string, timeoutInSeconds time.Duration, urlPath string) ([]byte, error) { - client, err := BuildShimClient(sandboxID, timeoutInSeconds) - if err != nil { - return nil, err - } - - resp, err := client.Get(fmt.Sprintf("http://shim/%s", urlPath)) - if err != nil { - return nil, err - } - - defer func() { - resp.Body.Close() - }() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - return body, nil -} diff --git a/src/runtime/pkg/utils/shimclient/shim_management_client.go b/src/runtime/pkg/utils/shimclient/shim_management_client.go new file mode 100644 index 0000000000..c9ed3ad691 --- /dev/null +++ b/src/runtime/pkg/utils/shimclient/shim_management_client.go @@ -0,0 +1,79 @@ +// Copyright (c) 2022 Databricks Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package shimclient + +import ( + "bytes" + "fmt" + "io" + "net" + "net/http" + "time" + + cdshim "github.com/containerd/containerd/runtime/v2/shim" + shim "github.com/kata-containers/kata-containers/src/runtime/pkg/containerd-shim-v2" +) + +// BuildShimClient builds and returns an http client for communicating with the provided sandbox +func BuildShimClient(sandboxID string, timeout time.Duration) (*http.Client, error) { + return buildUnixSocketClient(shim.SocketAddress(sandboxID), timeout) +} + +// buildUnixSocketClient build http client for Unix socket +func buildUnixSocketClient(socketAddr string, timeout time.Duration) (*http.Client, error) { + transport := &http.Transport{ + DisableKeepAlives: true, + Dial: func(proto, addr string) (conn net.Conn, err error) { + return cdshim.AnonDialer(socketAddr, timeout) + }, + } + + client := &http.Client{ + Transport: transport, + } + + if timeout > 0 { + client.Timeout = timeout + } + + return client, nil +} + +func DoGet(sandboxID string, timeoutInSeconds time.Duration, urlPath string) ([]byte, error) { + client, err := BuildShimClient(sandboxID, timeoutInSeconds) + if err != nil { + return nil, err + } + + resp, err := client.Get(fmt.Sprintf("http://shim/%s", urlPath)) + if err != nil { + return nil, err + } + + defer func() { + resp.Body.Close() + }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return body, nil +} + +func DoPost(sandboxID string, timeoutInSeconds time.Duration, urlPath string, payload []byte) error { + client, err := BuildShimClient(sandboxID, timeoutInSeconds) + if err != nil { + return err + } + + resp, err := client.Post(fmt.Sprintf("http://shim/%s", urlPath), "application/json", bytes.NewBuffer(payload)) + defer func() { + resp.Body.Close() + }() + return err +} From 4e00c2377ccde3df07c69939a438b57b6596b4f0 Mon Sep 17 00:00:00 2001 From: Feng Wang Date: Fri, 14 Jan 2022 13:16:10 -0800 Subject: [PATCH 4/9] agent: add grpc interface for stat and resize operations Add GetVolumeStats and ResizeVolume APIs for the runtime to query stat and resize fs in the guest. Fixes: #3454 Signed-off-by: Feng Wang --- .../samples/configuration-all-endpoints.toml | 2 + src/libs/protocols/build.rs | 1 + .../protocols/hack/update-generated-proto.sh | 4 +- src/libs/protocols/protos/agent.proto | 14 + src/libs/protocols/protos/csi.proto | 60 + src/libs/protocols/protos/oci.proto | 2 - src/libs/protocols/src/lib.rs | 1 + .../pkg/agent/protocols/grpc/agent.pb.go | 811 +++++++++--- .../pkg/agent/protocols/grpc/csi.pb.go | 1164 +++++++++++++++++ .../pkg/agent/protocols/grpc/csipb_test.go | 585 +++++++++ src/runtime/virtcontainers/pkg/mock/mock.go | 8 + 11 files changed, 2461 insertions(+), 191 deletions(-) create mode 100644 src/libs/protocols/protos/csi.proto create mode 100644 src/runtime/virtcontainers/pkg/agent/protocols/grpc/csi.pb.go create mode 100644 src/runtime/virtcontainers/pkg/agent/protocols/grpc/csipb_test.go diff --git a/src/agent/samples/configuration-all-endpoints.toml b/src/agent/samples/configuration-all-endpoints.toml index 1c8b013672..8e6dd9c3c5 100644 --- a/src/agent/samples/configuration-all-endpoints.toml +++ b/src/agent/samples/configuration-all-endpoints.toml @@ -25,6 +25,7 @@ allowed = [ "ReadStreamRequest", "RemoveContainerRequest", "ReseedRandomDevRequest", + "ResizeVolumeRequest", "ResumeContainerRequest", "SetGuestDateTimeRequest", "SignalProcessRequest", @@ -34,6 +35,7 @@ allowed = [ "UpdateContainerRequest", "UpdateInterfaceRequest", "UpdateRoutesRequest", + "VolumeStatsRequest", "WaitProcessRequest", "WriteStreamRequest" ] diff --git a/src/libs/protocols/build.rs b/src/libs/protocols/build.rs index 39654882be..4a43f36777 100644 --- a/src/libs/protocols/build.rs +++ b/src/libs/protocols/build.rs @@ -95,6 +95,7 @@ fn real_main() -> Result<(), std::io::Error> { let protos = vec![ "protos/agent.proto", + "protos/csi.proto", "protos/google/protobuf/empty.proto", "protos/health.proto", "protos/oci.proto", diff --git a/src/libs/protocols/hack/update-generated-proto.sh b/src/libs/protocols/hack/update-generated-proto.sh index 4befd0f09f..64952b03c4 100755 --- a/src/libs/protocols/hack/update-generated-proto.sh +++ b/src/libs/protocols/hack/update-generated-proto.sh @@ -51,6 +51,8 @@ generate_go_sources() { --gogottrpc_out=plugins=ttrpc+fieldpath,\ import_path=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\ \ +Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/csi.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\ +\ Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/types.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols,\ \ Mgithub.com/kata-containers/kata-containers/src/libs/protocols/protos/oci.proto=github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc,\ @@ -69,7 +71,7 @@ if [ "$(basename $(pwd))" != "agent" ]; then fi # Protocol buffer files required to generate golang/rust bindings. -proto_files_list=(agent.proto health.proto oci.proto types.proto) +proto_files_list=(agent.proto csi.proto health.proto oci.proto types.proto) if [ "$1" = "" ]; then show_usage "${proto_files_list[@]}" diff --git a/src/libs/protocols/protos/agent.proto b/src/libs/protocols/protos/agent.proto index ede77ec0f6..d0de9cf390 100644 --- a/src/libs/protocols/protos/agent.proto +++ b/src/libs/protocols/protos/agent.proto @@ -12,6 +12,7 @@ option go_package = "github.com/kata-containers/kata-containers/src/runtime/virt package grpc; import "oci.proto"; +import "csi.proto"; import "types.proto"; import "google/protobuf/empty.proto"; @@ -65,6 +66,8 @@ service AgentService { rpc CopyFile(CopyFileRequest) returns (google.protobuf.Empty); rpc GetOOMEvent(GetOOMEventRequest) returns (OOMEvent); rpc AddSwap(AddSwapRequest) returns (google.protobuf.Empty); + rpc GetVolumeStats(VolumeStatsRequest) returns (VolumeStatsResponse); + rpc ResizeVolume(ResizeVolumeRequest) returns (google.protobuf.Empty); } message CreateContainerRequest { @@ -505,3 +508,14 @@ message GetMetricsRequest {} message Metrics { string metrics = 1; } + +message VolumeStatsRequest { + // The volume path on the guest outside the container + string volume_guest_path = 1; +} + +message ResizeVolumeRequest { + // Full VM guest path of the volume (outside the container) + string volume_guest_path = 1; + uint64 size = 2; +} diff --git a/src/libs/protocols/protos/csi.proto b/src/libs/protocols/protos/csi.proto new file mode 100644 index 0000000000..e6da50c8b8 --- /dev/null +++ b/src/libs/protocols/protos/csi.proto @@ -0,0 +1,60 @@ +// Copyright (c) 2022 Databricks Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +syntax = "proto3"; +option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"; + +package grpc; +import "gogo/protobuf/gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; +option (gogoproto.populate_all) = true; +option (gogoproto.testgen_all) = true; +option (gogoproto.benchgen_all) = true; + +// This should be kept in sync with CSI NodeGetVolumeStatsResponse (https://github.com/container-storage-interface/spec/blob/v1.5.0/csi.proto) +message VolumeStatsResponse { + // This field is OPTIONAL. + repeated VolumeUsage usage = 1; + // Information about the current condition of the volume. + // This field is OPTIONAL. + // This field MUST be specified if the VOLUME_CONDITION node + // capability is supported. + VolumeCondition volume_condition = 2; +} +message VolumeUsage { + enum Unit { + UNKNOWN = 0; + BYTES = 1; + INODES = 2; + } + // The available capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + uint64 available = 1; + + // The total capacity in specified Unit. This field is REQUIRED. + // The value of this field MUST NOT be negative. + uint64 total = 2; + + // The used capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + uint64 used = 3; + + // Units by which values are measured. This field is REQUIRED. + Unit unit = 4; +} + +// VolumeCondition represents the current condition of a volume. +message VolumeCondition { + + // Normal volumes are available for use and operating optimally. + // An abnormal volume does not meet these criteria. + // This field is REQUIRED. + bool abnormal = 1; + + // The message describing the condition of the volume. + // This field is REQUIRED. + string message = 2; +} diff --git a/src/libs/protocols/protos/oci.proto b/src/libs/protocols/protos/oci.proto index 5de9b7d19a..aa0db01238 100644 --- a/src/libs/protocols/protos/oci.proto +++ b/src/libs/protocols/protos/oci.proto @@ -6,11 +6,9 @@ // syntax = "proto3"; - option go_package = "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"; package grpc; - import "gogo/protobuf/gogoproto/gogo.proto"; option (gogoproto.equal_all) = true; diff --git a/src/libs/protocols/src/lib.rs b/src/libs/protocols/src/lib.rs index 652d541e8f..14298e52d9 100644 --- a/src/libs/protocols/src/lib.rs +++ b/src/libs/protocols/src/lib.rs @@ -7,6 +7,7 @@ pub mod agent; pub mod agent_ttrpc; +pub mod csi; pub mod empty; pub mod health; pub mod health_ttrpc; diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go index e4e79dee15..437c9c8174 100644 --- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go @@ -2415,6 +2415,87 @@ func (m *Metrics) XXX_DiscardUnknown() { var xxx_messageInfo_Metrics proto.InternalMessageInfo +type VolumeStatsRequest struct { + // The volume path on the guest outside the container + VolumeGuestPath string `protobuf:"bytes,1,opt,name=volume_guest_path,json=volumeGuestPath,proto3" json:"volume_guest_path,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeStatsRequest) Reset() { *m = VolumeStatsRequest{} } +func (*VolumeStatsRequest) ProtoMessage() {} +func (*VolumeStatsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_712ce9a559fda969, []int{56} +} +func (m *VolumeStatsRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *VolumeStatsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_VolumeStatsRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *VolumeStatsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeStatsRequest.Merge(m, src) +} +func (m *VolumeStatsRequest) XXX_Size() int { + return m.Size() +} +func (m *VolumeStatsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeStatsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeStatsRequest proto.InternalMessageInfo + +type ResizeVolumeRequest struct { + // Full VM guest path of the volume (outside the container) + VolumeGuestPath string `protobuf:"bytes,1,opt,name=volume_guest_path,json=volumeGuestPath,proto3" json:"volume_guest_path,omitempty"` + Size_ uint64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ResizeVolumeRequest) Reset() { *m = ResizeVolumeRequest{} } +func (*ResizeVolumeRequest) ProtoMessage() {} +func (*ResizeVolumeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_712ce9a559fda969, []int{57} +} +func (m *ResizeVolumeRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ResizeVolumeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ResizeVolumeRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ResizeVolumeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ResizeVolumeRequest.Merge(m, src) +} +func (m *ResizeVolumeRequest) XXX_Size() int { + return m.Size() +} +func (m *ResizeVolumeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ResizeVolumeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ResizeVolumeRequest proto.InternalMessageInfo + func init() { proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest") proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest") @@ -2474,6 +2555,8 @@ func init() { proto.RegisterType((*AddSwapRequest)(nil), "grpc.AddSwapRequest") proto.RegisterType((*GetMetricsRequest)(nil), "grpc.GetMetricsRequest") proto.RegisterType((*Metrics)(nil), "grpc.Metrics") + proto.RegisterType((*VolumeStatsRequest)(nil), "grpc.VolumeStatsRequest") + proto.RegisterType((*ResizeVolumeRequest)(nil), "grpc.ResizeVolumeRequest") } func init() { @@ -2481,194 +2564,198 @@ func init() { } var fileDescriptor_712ce9a559fda969 = []byte{ - // 2978 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0x4b, 0x6f, 0x23, 0xc7, - 0xd1, 0xe6, 0x43, 0x22, 0x59, 0x7c, 0x89, 0x23, 0xad, 0x96, 0x4b, 0xdb, 0xfa, 0xd6, 0xb3, 0xf6, - 0x7a, 0x6d, 0x7f, 0xa6, 0xec, 0xb5, 0xf1, 0xad, 0x1f, 0xf0, 0xb7, 0x58, 0x69, 0x65, 0x49, 0xb6, - 0xe5, 0x65, 0x46, 0x16, 0x1c, 0x24, 0x48, 0x06, 0xc3, 0x99, 0x16, 0xd9, 0x16, 0x67, 0x7a, 0xdc, - 0xd3, 0xa3, 0x15, 0x1d, 0x20, 0xc8, 0x29, 0xb9, 0xe5, 0x98, 0x5b, 0xfe, 0x40, 0x90, 0x5b, 0x80, - 0x5c, 0x72, 0xcd, 0xc1, 0xc8, 0x29, 0xc7, 0x9c, 0x82, 0x78, 0x7f, 0x42, 0x7e, 0x41, 0xd0, 0xaf, - 0x79, 0xf0, 0x21, 0x27, 0x82, 0x80, 0x5c, 0x88, 0xae, 0xea, 0xea, 0x7a, 0x75, 0x57, 0x75, 0x55, - 0x0f, 0x61, 0x30, 0xc2, 0x6c, 0x1c, 0x0f, 0xfb, 0x2e, 0xf1, 0xb7, 0xcf, 0x1c, 0xe6, 0xbc, 0xe9, - 0x92, 0x80, 0x39, 0x38, 0x40, 0x34, 0x9a, 0x83, 0x23, 0xea, 0x6e, 0x4f, 0xf0, 0x30, 0xda, 0x0e, - 0x29, 0x61, 0xc4, 0x25, 0x13, 0x35, 0x8a, 0xb6, 0x9d, 0x11, 0x0a, 0x58, 0x5f, 0x00, 0x46, 0x79, - 0x44, 0x43, 0xb7, 0x57, 0x23, 0x2e, 0x96, 0x88, 0x5e, 0x9d, 0x4d, 0x43, 0x14, 0x29, 0xe0, 0xf9, - 0x11, 0x21, 0xa3, 0x09, 0x92, 0x0b, 0x87, 0xf1, 0xe9, 0x36, 0xf2, 0x43, 0x36, 0x95, 0x93, 0xe6, - 0x6f, 0x8b, 0xb0, 0xb9, 0x4b, 0x91, 0xc3, 0xd0, 0xae, 0x96, 0x6a, 0xa1, 0xaf, 0x63, 0x14, 0x31, - 0xe3, 0x25, 0x68, 0x24, 0x9a, 0xd8, 0xd8, 0xeb, 0x16, 0x6e, 0x17, 0xee, 0xd5, 0xac, 0x7a, 0x82, - 0x3b, 0xf4, 0x8c, 0x9b, 0x50, 0x41, 0x17, 0xc8, 0xe5, 0xb3, 0x45, 0x31, 0xbb, 0xca, 0xc1, 0x43, - 0xcf, 0x78, 0x1b, 0xea, 0x11, 0xa3, 0x38, 0x18, 0xd9, 0x71, 0x84, 0x68, 0xb7, 0x74, 0xbb, 0x70, - 0xaf, 0x7e, 0x7f, 0xad, 0xcf, 0xf5, 0xec, 0x1f, 0x8b, 0x89, 0x93, 0x08, 0x51, 0x0b, 0xa2, 0x64, - 0x6c, 0xdc, 0x85, 0x8a, 0x87, 0xce, 0xb1, 0x8b, 0xa2, 0x6e, 0xf9, 0x76, 0xe9, 0x5e, 0xfd, 0x7e, - 0x43, 0x92, 0x3f, 0x16, 0x48, 0x4b, 0x4f, 0x1a, 0xaf, 0x41, 0x35, 0x62, 0x84, 0x3a, 0x23, 0x14, - 0x75, 0x57, 0x04, 0x61, 0x53, 0xf3, 0x15, 0x58, 0x2b, 0x99, 0x36, 0x5e, 0x80, 0xd2, 0x93, 0xdd, - 0xc3, 0xee, 0xaa, 0x90, 0x0e, 0x8a, 0x2a, 0x44, 0xae, 0xc5, 0xd1, 0xc6, 0x1d, 0x68, 0x46, 0x4e, - 0xe0, 0x0d, 0xc9, 0x85, 0x1d, 0x62, 0x2f, 0x88, 0xba, 0x95, 0xdb, 0x85, 0x7b, 0x55, 0xab, 0xa1, - 0x90, 0x03, 0x8e, 0x33, 0x3f, 0x80, 0x1b, 0xc7, 0xcc, 0xa1, 0xec, 0x0a, 0xde, 0x31, 0x4f, 0x60, - 0xd3, 0x42, 0x3e, 0x39, 0xbf, 0x92, 0x6b, 0xbb, 0x50, 0x61, 0xd8, 0x47, 0x24, 0x66, 0xc2, 0xb5, - 0x4d, 0x4b, 0x83, 0xe6, 0xef, 0x0b, 0x60, 0xec, 0x5d, 0x20, 0x77, 0x40, 0x89, 0x8b, 0xa2, 0xe8, - 0xbf, 0xb4, 0x5d, 0xaf, 0x42, 0x25, 0x94, 0x0a, 0x74, 0xcb, 0x82, 0x5c, 0xed, 0x82, 0xd6, 0x4a, - 0xcf, 0x9a, 0x5f, 0xc1, 0xc6, 0x31, 0x1e, 0x05, 0xce, 0xe4, 0x1a, 0xf5, 0xdd, 0x84, 0xd5, 0x48, - 0xf0, 0x14, 0xaa, 0x36, 0x2d, 0x05, 0x99, 0x03, 0x30, 0xbe, 0x74, 0x30, 0xbb, 0x3e, 0x49, 0xe6, - 0x9b, 0xb0, 0x9e, 0xe3, 0x18, 0x85, 0x24, 0x88, 0x90, 0x50, 0x80, 0x39, 0x2c, 0x8e, 0x04, 0xb3, - 0x15, 0x4b, 0x41, 0x26, 0x81, 0xcd, 0x93, 0xd0, 0xbb, 0x62, 0x34, 0xdd, 0x87, 0x1a, 0x45, 0x11, - 0x89, 0x29, 0x8f, 0x81, 0xa2, 0x70, 0xea, 0x86, 0x74, 0xea, 0x67, 0x38, 0x88, 0x2f, 0x2c, 0x3d, - 0x67, 0xa5, 0x64, 0xea, 0x7c, 0xb2, 0xe8, 0x2a, 0xe7, 0xf3, 0x03, 0xb8, 0x31, 0x70, 0xe2, 0xe8, - 0x2a, 0xba, 0x9a, 0x1f, 0xf2, 0xb3, 0x1d, 0xc5, 0xfe, 0x95, 0x16, 0xff, 0xae, 0x00, 0xd5, 0xdd, - 0x30, 0x3e, 0x89, 0x9c, 0x11, 0x32, 0xfe, 0x07, 0xea, 0x8c, 0x30, 0x67, 0x62, 0xc7, 0x1c, 0x14, - 0xe4, 0x65, 0x0b, 0x04, 0x4a, 0x12, 0xbc, 0x04, 0x8d, 0x10, 0x51, 0x37, 0x8c, 0x15, 0x45, 0xf1, - 0x76, 0xe9, 0x5e, 0xd9, 0xaa, 0x4b, 0x9c, 0x24, 0xe9, 0xc3, 0xba, 0x98, 0xb3, 0x71, 0x60, 0x9f, - 0x21, 0x1a, 0xa0, 0x89, 0x4f, 0x3c, 0x24, 0x0e, 0x47, 0xd9, 0xea, 0x88, 0xa9, 0xc3, 0xe0, 0xd3, - 0x64, 0xc2, 0x78, 0x1d, 0x3a, 0x09, 0x3d, 0x3f, 0xf1, 0x82, 0xba, 0x2c, 0xa8, 0xdb, 0x8a, 0xfa, - 0x44, 0xa1, 0xcd, 0x9f, 0x43, 0xeb, 0x8b, 0x31, 0x25, 0x8c, 0x4d, 0x70, 0x30, 0x7a, 0xec, 0x30, - 0x87, 0x87, 0x66, 0x88, 0x28, 0x26, 0x5e, 0xa4, 0xb4, 0xd5, 0xa0, 0xf1, 0x06, 0x74, 0x98, 0xa4, - 0x45, 0x9e, 0xad, 0x69, 0x8a, 0x82, 0x66, 0x2d, 0x99, 0x18, 0x28, 0xe2, 0x57, 0xa0, 0x95, 0x12, - 0xf3, 0xe0, 0x56, 0xfa, 0x36, 0x13, 0xec, 0x17, 0xd8, 0x47, 0xe6, 0xb9, 0xf0, 0x95, 0xd8, 0x64, - 0xe3, 0x0d, 0xa8, 0xa5, 0x7e, 0x28, 0x88, 0x13, 0xd2, 0x92, 0x27, 0x44, 0xbb, 0xd3, 0xaa, 0x26, - 0x4e, 0xf9, 0x08, 0xda, 0x2c, 0x51, 0xdc, 0xf6, 0x1c, 0xe6, 0xe4, 0x0f, 0x55, 0xde, 0x2a, 0xab, - 0xc5, 0x72, 0xb0, 0xf9, 0x21, 0xd4, 0x06, 0xd8, 0x8b, 0xa4, 0xe0, 0x2e, 0x54, 0xdc, 0x98, 0x52, - 0x14, 0x30, 0x6d, 0xb2, 0x02, 0x8d, 0x0d, 0x58, 0x99, 0x60, 0x1f, 0x33, 0x65, 0xa6, 0x04, 0x4c, - 0x02, 0x70, 0x84, 0x7c, 0x42, 0xa7, 0xc2, 0x61, 0x1b, 0xb0, 0x92, 0xdd, 0x5c, 0x09, 0x18, 0xcf, - 0x43, 0xcd, 0x77, 0x2e, 0x92, 0x4d, 0xe5, 0x33, 0x55, 0xdf, 0xb9, 0x90, 0xca, 0x77, 0xa1, 0x72, - 0xea, 0xe0, 0x89, 0x1b, 0x30, 0xe5, 0x15, 0x0d, 0xa6, 0x02, 0xcb, 0x59, 0x81, 0x7f, 0x2e, 0x42, - 0x5d, 0x4a, 0x94, 0x0a, 0x6f, 0xc0, 0x8a, 0xeb, 0xb8, 0xe3, 0x44, 0xa4, 0x00, 0x8c, 0xbb, 0x5a, - 0x91, 0x62, 0x36, 0xc3, 0xa5, 0x9a, 0x6a, 0xd5, 0xb6, 0x01, 0xa2, 0xa7, 0x4e, 0xa8, 0x74, 0x2b, - 0x2d, 0x21, 0xae, 0x71, 0x1a, 0xa9, 0xee, 0x3b, 0xd0, 0x90, 0xe7, 0x4e, 0x2d, 0x29, 0x2f, 0x59, - 0x52, 0x97, 0x54, 0x72, 0xd1, 0x1d, 0x68, 0xc6, 0x11, 0xb2, 0xc7, 0x18, 0x51, 0x87, 0xba, 0xe3, - 0x69, 0x77, 0x45, 0x5e, 0x40, 0x71, 0x84, 0x0e, 0x34, 0xce, 0xb8, 0x0f, 0x2b, 0x3c, 0xb7, 0x44, - 0xdd, 0x55, 0x71, 0xd7, 0xbd, 0x90, 0x65, 0x29, 0x4c, 0xed, 0x8b, 0xdf, 0xbd, 0x80, 0xd1, 0xa9, - 0x25, 0x49, 0x7b, 0xef, 0x01, 0xa4, 0x48, 0x63, 0x0d, 0x4a, 0x67, 0x68, 0xaa, 0xe2, 0x90, 0x0f, - 0xb9, 0x73, 0xce, 0x9d, 0x49, 0xac, 0xbd, 0x2e, 0x81, 0x0f, 0x8a, 0xef, 0x15, 0x4c, 0x17, 0xda, - 0x3b, 0x93, 0x33, 0x4c, 0x32, 0xcb, 0x37, 0x60, 0xc5, 0x77, 0xbe, 0x22, 0x54, 0x7b, 0x52, 0x00, - 0x02, 0x8b, 0x03, 0x42, 0x35, 0x0b, 0x01, 0x18, 0x2d, 0x28, 0x92, 0x50, 0xf8, 0xab, 0x66, 0x15, - 0x49, 0x98, 0x0a, 0x2a, 0x67, 0x04, 0x99, 0x7f, 0x2f, 0x03, 0xa4, 0x52, 0x0c, 0x0b, 0x7a, 0x98, - 0xd8, 0x11, 0xa2, 0xfc, 0x7e, 0xb7, 0x87, 0x53, 0x86, 0x22, 0x9b, 0x22, 0x37, 0xa6, 0x11, 0x3e, - 0xe7, 0xfb, 0xc7, 0xcd, 0xbe, 0x21, 0xcd, 0x9e, 0xd1, 0xcd, 0xba, 0x89, 0xc9, 0xb1, 0x5c, 0xb7, - 0xc3, 0x97, 0x59, 0x7a, 0x95, 0x71, 0x08, 0x37, 0x52, 0x9e, 0x5e, 0x86, 0x5d, 0xf1, 0x32, 0x76, - 0xeb, 0x09, 0x3b, 0x2f, 0x65, 0xb5, 0x07, 0xeb, 0x98, 0xd8, 0x5f, 0xc7, 0x28, 0xce, 0x31, 0x2a, - 0x5d, 0xc6, 0xa8, 0x83, 0xc9, 0x0f, 0xc4, 0x82, 0x94, 0xcd, 0x00, 0x6e, 0x65, 0xac, 0xe4, 0xe1, - 0x9e, 0x61, 0x56, 0xbe, 0x8c, 0xd9, 0x66, 0xa2, 0x15, 0xcf, 0x07, 0x29, 0xc7, 0x4f, 0x60, 0x13, - 0x13, 0xfb, 0xa9, 0x83, 0xd9, 0x2c, 0xbb, 0x95, 0xef, 0x31, 0x92, 0xdf, 0x68, 0x79, 0x5e, 0xd2, - 0x48, 0x1f, 0xd1, 0x51, 0xce, 0xc8, 0xd5, 0xef, 0x31, 0xf2, 0x48, 0x2c, 0x48, 0xd9, 0x3c, 0x82, - 0x0e, 0x26, 0xb3, 0xda, 0x54, 0x2e, 0x63, 0xd2, 0xc6, 0x24, 0xaf, 0xc9, 0x0e, 0x74, 0x22, 0xe4, - 0x32, 0x42, 0xb3, 0x87, 0xa0, 0x7a, 0x19, 0x8b, 0x35, 0x45, 0x9f, 0xf0, 0x30, 0x7f, 0x0c, 0x8d, - 0x83, 0x78, 0x84, 0xd8, 0x64, 0x98, 0x24, 0x83, 0x6b, 0xcb, 0x3f, 0xe6, 0x3f, 0x8b, 0x50, 0xdf, - 0x1d, 0x51, 0x12, 0x87, 0xb9, 0x9c, 0x2c, 0x83, 0x74, 0x36, 0x27, 0x0b, 0x12, 0x91, 0x93, 0x25, - 0xf1, 0xbb, 0xd0, 0xf0, 0x45, 0xe8, 0x2a, 0x7a, 0x99, 0x87, 0x3a, 0x73, 0x41, 0x6d, 0xd5, 0xfd, - 0x4c, 0x32, 0xeb, 0x03, 0x84, 0xd8, 0x8b, 0xd4, 0x1a, 0x99, 0x8e, 0xda, 0xaa, 0xdc, 0xd2, 0x29, - 0xda, 0xaa, 0x85, 0x49, 0xb6, 0x7e, 0x1b, 0xea, 0x43, 0xee, 0x24, 0xb5, 0x20, 0x97, 0x8c, 0x52, - 0xef, 0x59, 0x30, 0x4c, 0x83, 0xf0, 0x00, 0x9a, 0x63, 0xe9, 0x32, 0xb5, 0x48, 0x9e, 0xa1, 0x3b, - 0xca, 0x92, 0xd4, 0xde, 0x7e, 0xd6, 0xb3, 0x72, 0x03, 0x1a, 0xe3, 0x0c, 0xaa, 0x77, 0x0c, 0x9d, - 0x39, 0x92, 0x05, 0x39, 0xe8, 0x5e, 0x36, 0x07, 0xd5, 0xef, 0x1b, 0x52, 0x50, 0x76, 0x65, 0x36, - 0x2f, 0xfd, 0xba, 0x08, 0x8d, 0xcf, 0x11, 0x7b, 0x4a, 0xe8, 0x99, 0xd4, 0xd7, 0x80, 0x72, 0xe0, - 0xf8, 0x48, 0x71, 0x14, 0x63, 0xe3, 0x16, 0x54, 0xe9, 0x85, 0x4c, 0x20, 0x6a, 0x3f, 0x2b, 0xf4, - 0x42, 0x24, 0x06, 0xe3, 0x45, 0x00, 0x7a, 0x61, 0x87, 0x8e, 0x7b, 0x86, 0x94, 0x07, 0xcb, 0x56, - 0x8d, 0x5e, 0x0c, 0x24, 0x82, 0x1f, 0x05, 0x7a, 0x61, 0x23, 0x4a, 0x09, 0x8d, 0x54, 0xae, 0xaa, - 0xd2, 0x8b, 0x3d, 0x01, 0xab, 0xb5, 0x1e, 0x25, 0x61, 0x88, 0x3c, 0x91, 0xa3, 0xc5, 0xda, 0xc7, - 0x12, 0xc1, 0xa5, 0x32, 0x2d, 0x75, 0x55, 0x4a, 0x65, 0xa9, 0x54, 0x96, 0x4a, 0xad, 0xc8, 0x95, - 0x2c, 0x2b, 0x95, 0x25, 0x52, 0xab, 0x52, 0x2a, 0xcb, 0x48, 0x65, 0xa9, 0xd4, 0x9a, 0x5e, 0xab, - 0xa4, 0x9a, 0xbf, 0x2a, 0xc0, 0xe6, 0x6c, 0xe1, 0xa7, 0x6a, 0xd3, 0x77, 0xa1, 0xe1, 0x8a, 0xfd, - 0xca, 0x9d, 0xc9, 0xce, 0xdc, 0x4e, 0x5a, 0x75, 0x37, 0x73, 0x8c, 0x1f, 0x40, 0x33, 0x90, 0x0e, - 0x4e, 0x8e, 0x66, 0x29, 0xdd, 0x97, 0xac, 0xef, 0xad, 0x46, 0x90, 0x81, 0x4c, 0x0f, 0x8c, 0x2f, - 0x29, 0x66, 0xe8, 0x98, 0x51, 0xe4, 0xf8, 0xd7, 0x51, 0xdd, 0x1b, 0x50, 0x16, 0xd5, 0x0a, 0xdf, - 0xa6, 0x86, 0x25, 0xc6, 0xe6, 0xab, 0xb0, 0x9e, 0x93, 0xa2, 0x6c, 0x5d, 0x83, 0xd2, 0x04, 0x05, - 0x82, 0x7b, 0xd3, 0xe2, 0x43, 0xd3, 0x81, 0x8e, 0x85, 0x1c, 0xef, 0xfa, 0xb4, 0x51, 0x22, 0x4a, - 0xa9, 0x88, 0x7b, 0x60, 0x64, 0x45, 0x28, 0x55, 0xb4, 0xd6, 0x85, 0x8c, 0xd6, 0x4f, 0xa0, 0xb3, - 0x3b, 0x21, 0x11, 0x3a, 0x66, 0x1e, 0x0e, 0xae, 0xa3, 0x1d, 0xf9, 0x19, 0xac, 0x7f, 0xc1, 0xa6, - 0x5f, 0x72, 0x66, 0x11, 0xfe, 0x06, 0x5d, 0x93, 0x7d, 0x94, 0x3c, 0xd5, 0xf6, 0x51, 0xf2, 0x94, - 0x37, 0x37, 0x2e, 0x99, 0xc4, 0x7e, 0x20, 0x42, 0xa1, 0x69, 0x29, 0xc8, 0xdc, 0x81, 0x86, 0xac, - 0xa1, 0x8f, 0x88, 0x17, 0x4f, 0xd0, 0xc2, 0x18, 0xdc, 0x02, 0x08, 0x1d, 0xea, 0xf8, 0x88, 0x21, - 0x2a, 0xcf, 0x50, 0xcd, 0xca, 0x60, 0xcc, 0xdf, 0x14, 0x61, 0x43, 0xbe, 0x37, 0x1c, 0xcb, 0x36, - 0x5b, 0x9b, 0xd0, 0x83, 0xea, 0x98, 0x44, 0x2c, 0xc3, 0x30, 0x81, 0xb9, 0x8a, 0xbc, 0x3f, 0x97, - 0xdc, 0xf8, 0x30, 0xf7, 0x08, 0x50, 0xba, 0xfc, 0x11, 0x60, 0xae, 0xcd, 0x2f, 0xcf, 0xb7, 0xf9, - 0x3c, 0xda, 0x34, 0x11, 0x96, 0x31, 0x5e, 0xb3, 0x6a, 0x0a, 0x73, 0xe8, 0x19, 0x77, 0xa1, 0x3d, - 0xe2, 0x5a, 0xda, 0x63, 0x42, 0xce, 0xec, 0xd0, 0x61, 0x63, 0x11, 0xea, 0x35, 0xab, 0x29, 0xd0, - 0x07, 0x84, 0x9c, 0x0d, 0x1c, 0x36, 0x36, 0xde, 0x87, 0x96, 0x2a, 0x03, 0x7d, 0xe1, 0xa2, 0x48, - 0x5d, 0x7e, 0x2a, 0x8a, 0xb2, 0xde, 0xb3, 0x9a, 0x67, 0x19, 0x28, 0x32, 0x6f, 0xc2, 0x8d, 0xc7, - 0x28, 0x62, 0x94, 0x4c, 0xf3, 0x8e, 0x31, 0xff, 0x1f, 0xe0, 0x30, 0x60, 0x88, 0x9e, 0x3a, 0x2e, - 0x8a, 0x8c, 0xb7, 0xb2, 0x90, 0x2a, 0x8e, 0xd6, 0xfa, 0xf2, 0xb9, 0x27, 0x99, 0xb0, 0x32, 0x34, - 0x66, 0x1f, 0x56, 0x2d, 0x12, 0xf3, 0x74, 0xf4, 0xb2, 0x1e, 0xa9, 0x75, 0x0d, 0xb5, 0x4e, 0x20, - 0x2d, 0x35, 0x67, 0x1e, 0xe8, 0x16, 0x36, 0x65, 0xa7, 0xb6, 0xa8, 0x0f, 0x35, 0xac, 0x71, 0x2a, - 0xab, 0xcc, 0x8b, 0x4e, 0x49, 0xcc, 0x0f, 0x61, 0x5d, 0x72, 0x92, 0x9c, 0x35, 0x9b, 0x97, 0x61, - 0x95, 0x6a, 0x35, 0x0a, 0xe9, 0x3b, 0x8f, 0x22, 0x52, 0x73, 0xdc, 0x1f, 0x9f, 0xe1, 0x88, 0xa5, - 0x86, 0x68, 0x7f, 0xac, 0x43, 0x87, 0x4f, 0xe4, 0x78, 0x9a, 0x1f, 0x43, 0xe3, 0x91, 0x35, 0xf8, - 0x1c, 0xe1, 0xd1, 0x78, 0xc8, 0xb3, 0xe7, 0xff, 0xe5, 0x61, 0x65, 0xb0, 0xa1, 0xb4, 0xcd, 0x4c, - 0x59, 0x39, 0x3a, 0xf3, 0x13, 0xd8, 0x7c, 0xe4, 0x79, 0x59, 0x94, 0xd6, 0xfa, 0x2d, 0xa8, 0x05, - 0x19, 0x76, 0x99, 0x3b, 0x2b, 0x47, 0x9d, 0x12, 0x99, 0x3f, 0x81, 0xf5, 0x27, 0xc1, 0x04, 0x07, - 0x68, 0x77, 0x70, 0x72, 0x84, 0x92, 0x5c, 0x64, 0x40, 0x99, 0xd7, 0x6c, 0x82, 0x47, 0xd5, 0x12, - 0x63, 0x1e, 0x9c, 0xc1, 0xd0, 0x76, 0xc3, 0x38, 0x52, 0x8f, 0x3d, 0xab, 0xc1, 0x70, 0x37, 0x8c, - 0x23, 0x7e, 0xb9, 0xf0, 0xe2, 0x82, 0x04, 0x93, 0xa9, 0x88, 0xd0, 0xaa, 0x55, 0x71, 0xc3, 0xf8, - 0x49, 0x30, 0x99, 0x9a, 0xff, 0x2b, 0x3a, 0x70, 0x84, 0x3c, 0xcb, 0x09, 0x3c, 0xe2, 0x3f, 0x46, - 0xe7, 0x19, 0x09, 0x49, 0xb7, 0xa7, 0x33, 0xd1, 0xb7, 0x05, 0x68, 0x3c, 0x1a, 0xa1, 0x80, 0x3d, - 0x46, 0xcc, 0xc1, 0x13, 0xd1, 0xd1, 0x9d, 0x23, 0x1a, 0x61, 0x12, 0xa8, 0x70, 0xd3, 0x20, 0x6f, - 0xc8, 0x71, 0x80, 0x99, 0xed, 0x39, 0xc8, 0x27, 0x81, 0xe0, 0x52, 0xb5, 0x80, 0xa3, 0x1e, 0x0b, - 0x8c, 0xf1, 0x2a, 0xb4, 0xe5, 0x63, 0x9c, 0x3d, 0x76, 0x02, 0x6f, 0xc2, 0x03, 0xbd, 0x24, 0x42, - 0xb3, 0x25, 0xd1, 0x07, 0x0a, 0x6b, 0xbc, 0x06, 0x6b, 0x2a, 0x0c, 0x53, 0xca, 0xb2, 0xa0, 0x6c, - 0x2b, 0x7c, 0x8e, 0x34, 0x0e, 0x43, 0x42, 0x59, 0x64, 0x47, 0xc8, 0x75, 0x89, 0x1f, 0xaa, 0x76, - 0xa8, 0xad, 0xf1, 0xc7, 0x12, 0x6d, 0x8e, 0x60, 0x7d, 0x9f, 0xdb, 0xa9, 0x2c, 0x49, 0x8f, 0x55, - 0xcb, 0x47, 0xbe, 0x3d, 0x9c, 0x10, 0xf7, 0xcc, 0xe6, 0xc9, 0x51, 0x79, 0x98, 0x17, 0x5c, 0x3b, - 0x1c, 0x79, 0x8c, 0xbf, 0x11, 0x9d, 0x3f, 0xa7, 0x1a, 0x13, 0x16, 0x4e, 0xe2, 0x91, 0x1d, 0x52, - 0x32, 0x44, 0xca, 0xc4, 0xb6, 0x8f, 0xfc, 0x03, 0x89, 0x1f, 0x70, 0xb4, 0xf9, 0xa7, 0x02, 0x6c, - 0xe4, 0x25, 0xa9, 0x54, 0xbf, 0x0d, 0x1b, 0x79, 0x51, 0xea, 0xfa, 0x97, 0xe5, 0x65, 0x27, 0x2b, - 0x50, 0x16, 0x02, 0x0f, 0xa0, 0x29, 0xde, 0x6b, 0x6d, 0x4f, 0x72, 0xca, 0x17, 0x3d, 0xd9, 0x7d, - 0xb1, 0x1a, 0x4e, 0x76, 0x97, 0xde, 0x87, 0x5b, 0xca, 0x7c, 0x7b, 0x5e, 0x6d, 0x79, 0x20, 0x36, - 0x15, 0xc1, 0xd1, 0x8c, 0xf6, 0x9f, 0x41, 0x37, 0x45, 0xed, 0x4c, 0x05, 0x32, 0x3d, 0xcc, 0xeb, - 0x33, 0xc6, 0x3e, 0xf2, 0x3c, 0x2a, 0xa2, 0xa4, 0x6c, 0x2d, 0x9a, 0x32, 0x1f, 0xc2, 0xcd, 0x63, - 0xc4, 0xa4, 0x37, 0x1c, 0xa6, 0x3a, 0x11, 0xc9, 0x6c, 0x0d, 0x4a, 0xc7, 0xc8, 0x15, 0xc6, 0x97, - 0x2c, 0x3e, 0xe4, 0x07, 0xf0, 0x24, 0x42, 0xae, 0xb0, 0xb2, 0x64, 0x89, 0xb1, 0xf9, 0x87, 0x02, - 0x54, 0x54, 0x72, 0xe6, 0x17, 0x8c, 0x47, 0xf1, 0x39, 0xa2, 0xea, 0xe8, 0x29, 0xc8, 0x78, 0x05, - 0x5a, 0x72, 0x64, 0x93, 0x90, 0x61, 0x92, 0xa4, 0xfc, 0xa6, 0xc4, 0x3e, 0x91, 0x48, 0xf1, 0xf8, - 0x26, 0x9e, 0xbf, 0x54, 0xa7, 0xa9, 0x20, 0x8e, 0x3f, 0x8d, 0x78, 0x84, 0x8b, 0x14, 0x5f, 0xb3, - 0x14, 0xc4, 0x8f, 0xba, 0xe6, 0xb7, 0x22, 0xf8, 0x69, 0x90, 0x1f, 0x75, 0x9f, 0xc4, 0x01, 0xb3, - 0x43, 0x82, 0x03, 0xa6, 0x72, 0x3a, 0x08, 0xd4, 0x80, 0x63, 0xcc, 0x5f, 0x16, 0x60, 0x55, 0x3e, - 0x40, 0xf3, 0xde, 0x36, 0xb9, 0x59, 0x8b, 0x58, 0x54, 0x29, 0x42, 0x96, 0xbc, 0x4d, 0xc5, 0x98, - 0xc7, 0xf1, 0xb9, 0x2f, 0xef, 0x07, 0xa5, 0xda, 0xb9, 0x2f, 0x2e, 0x86, 0x57, 0xa0, 0x95, 0x5e, - 0xd0, 0x62, 0x5e, 0xaa, 0xd8, 0x4c, 0xb0, 0x82, 0x6c, 0xa9, 0xa6, 0xe6, 0x0f, 0x79, 0x4b, 0x9f, - 0x3c, 0xbe, 0xae, 0x41, 0x29, 0x4e, 0x94, 0xe1, 0x43, 0x8e, 0x19, 0x25, 0x57, 0x3b, 0x1f, 0x1a, - 0x77, 0xa1, 0xe5, 0x78, 0x1e, 0xe6, 0xcb, 0x9d, 0xc9, 0x3e, 0xf6, 0x92, 0x20, 0xcd, 0x63, 0xcd, - 0xbf, 0x14, 0xa0, 0xbd, 0x4b, 0xc2, 0xe9, 0xc7, 0x78, 0x82, 0x32, 0x19, 0x44, 0x28, 0xa9, 0x6e, - 0x76, 0x3e, 0xe6, 0xd5, 0xea, 0x29, 0x9e, 0x20, 0x19, 0x5a, 0x72, 0x67, 0xab, 0x1c, 0x21, 0xc2, - 0x4a, 0x4f, 0x26, 0xcf, 0x6e, 0x4d, 0x39, 0x79, 0x44, 0x3c, 0x51, 0x97, 0x7b, 0x98, 0xda, 0xc9, - 0x23, 0x5b, 0xd3, 0xaa, 0x78, 0x98, 0x8a, 0x29, 0x65, 0xc8, 0x8a, 0x78, 0x44, 0xcd, 0x1a, 0xb2, - 0x2a, 0x31, 0xdc, 0x90, 0x4d, 0x58, 0x25, 0xa7, 0xa7, 0x11, 0x62, 0xa2, 0x82, 0x2e, 0x59, 0x0a, - 0x4a, 0xd2, 0x5c, 0x35, 0x93, 0xe6, 0x36, 0xc0, 0xd8, 0x47, 0xec, 0xc9, 0x93, 0xa3, 0xbd, 0x73, - 0x14, 0x30, 0x7d, 0x3b, 0xbc, 0x09, 0x55, 0x8d, 0xfa, 0x77, 0x9e, 0x27, 0x5f, 0x87, 0xd6, 0x23, - 0xcf, 0x3b, 0x7e, 0xea, 0x84, 0xda, 0x1f, 0x5d, 0xa8, 0x0c, 0x76, 0x0f, 0x07, 0xd2, 0x25, 0x25, - 0x6e, 0x80, 0x02, 0xf9, 0x6d, 0xb4, 0x8f, 0xd8, 0x11, 0x62, 0x14, 0xbb, 0xc9, 0x6d, 0x74, 0x07, - 0x2a, 0x0a, 0xc3, 0x57, 0xfa, 0x72, 0xa8, 0xd3, 0xac, 0x02, 0xef, 0xff, 0x71, 0x4d, 0x65, 0x64, - 0xd5, 0xdc, 0x1b, 0xfb, 0xd0, 0x9e, 0xf9, 0x12, 0x63, 0xa8, 0xd7, 0x9e, 0xc5, 0x1f, 0x68, 0x7a, - 0x9b, 0x7d, 0xf9, 0x65, 0xa7, 0xaf, 0xbf, 0xec, 0xf4, 0xf7, 0xfc, 0x90, 0x4d, 0x8d, 0x3d, 0x68, - 0xe5, 0xbf, 0x59, 0x18, 0xcf, 0xeb, 0xe2, 0x68, 0xc1, 0x97, 0x8c, 0xa5, 0x6c, 0xf6, 0xa1, 0x3d, - 0xf3, 0xf9, 0x42, 0xeb, 0xb3, 0xf8, 0xab, 0xc6, 0x52, 0x46, 0x0f, 0xa1, 0x9e, 0xf9, 0x5e, 0x61, - 0x74, 0x25, 0x93, 0xf9, 0x4f, 0x18, 0x4b, 0x19, 0xec, 0x42, 0x33, 0xf7, 0x09, 0xc1, 0xe8, 0x29, - 0x7b, 0x16, 0x7c, 0x57, 0x58, 0xca, 0x64, 0x07, 0xea, 0x99, 0x97, 0x7c, 0xad, 0xc5, 0xfc, 0xe7, - 0x82, 0xde, 0xad, 0x05, 0x33, 0x2a, 0xf1, 0xef, 0x43, 0x7b, 0xe6, 0x79, 0x5f, 0xbb, 0x64, 0xf1, - 0xab, 0xff, 0x52, 0x65, 0x3e, 0x15, 0x5b, 0x94, 0xe9, 0xde, 0x32, 0x5b, 0x34, 0xff, 0x98, 0xdf, - 0x7b, 0x61, 0xf1, 0xa4, 0xd2, 0x6a, 0x0f, 0x5a, 0xf9, 0x77, 0x7c, 0xcd, 0x6c, 0xe1, 0xeb, 0xfe, - 0xe5, 0xfb, 0x9d, 0x7b, 0xd2, 0x4f, 0xf7, 0x7b, 0xd1, 0x4b, 0xff, 0x52, 0x46, 0x8f, 0x00, 0x54, - 0xaf, 0xe6, 0xe1, 0x20, 0x71, 0xf4, 0x5c, 0x8f, 0x98, 0x38, 0x7a, 0x41, 0x5f, 0xf7, 0x10, 0x40, - 0xb6, 0x58, 0x1e, 0x89, 0x99, 0x71, 0x53, 0xab, 0x31, 0xd3, 0xd7, 0xf5, 0xba, 0xf3, 0x13, 0x73, - 0x0c, 0x10, 0xa5, 0x57, 0x61, 0xf0, 0x11, 0x40, 0xda, 0xba, 0x69, 0x06, 0x73, 0xcd, 0xdc, 0x25, - 0x3e, 0x68, 0x64, 0x1b, 0x35, 0x43, 0xd9, 0xba, 0xa0, 0x79, 0xbb, 0x84, 0x45, 0x7b, 0xa6, 0x10, - 0xcf, 0x1f, 0xb6, 0xd9, 0xfa, 0xbc, 0x37, 0x57, 0x8c, 0x1b, 0x0f, 0xa0, 0x91, 0xad, 0xc0, 0xb5, - 0x16, 0x0b, 0xaa, 0xf2, 0x5e, 0xae, 0x0a, 0x37, 0x1e, 0x42, 0x2b, 0x5f, 0x7d, 0xeb, 0x23, 0xb5, - 0xb0, 0x26, 0xef, 0xa9, 0xb7, 0xa5, 0x0c, 0xf9, 0x3b, 0x00, 0x69, 0x95, 0xae, 0xdd, 0x37, 0x57, - 0xb7, 0xcf, 0x48, 0xdd, 0x87, 0xf6, 0x4c, 0xf5, 0xad, 0x2d, 0x5e, 0x5c, 0x94, 0x2f, 0x75, 0xdd, - 0xbb, 0x00, 0x69, 0x56, 0xd6, 0xd2, 0xe7, 0xf2, 0x74, 0xaf, 0xa9, 0xdf, 0xdd, 0x24, 0xdd, 0x2e, - 0x34, 0x73, 0xad, 0xa9, 0x4e, 0x33, 0x8b, 0xfa, 0xd5, 0xcb, 0x92, 0x6f, 0xbe, 0x8f, 0xd3, 0x9e, - 0x5b, 0xd8, 0xdd, 0x5d, 0x76, 0x7e, 0xb2, 0xcd, 0x83, 0xde, 0xb9, 0x05, 0x0d, 0xc5, 0xf7, 0xc4, - 0x73, 0xb6, 0x41, 0xc8, 0xc4, 0xf3, 0x82, 0xbe, 0x61, 0x29, 0xa3, 0x03, 0x68, 0xef, 0xeb, 0xda, - 0x4f, 0xd5, 0xa5, 0x4a, 0x9d, 0x05, 0x75, 0x78, 0xaf, 0xb7, 0x68, 0x4a, 0x05, 0xd5, 0xa7, 0xd0, - 0x99, 0xab, 0x49, 0x8d, 0xad, 0xe4, 0xf5, 0x73, 0x61, 0xb1, 0xba, 0x54, 0xad, 0x43, 0x58, 0x9b, - 0x2d, 0x49, 0x8d, 0x17, 0x55, 0xa2, 0x5c, 0x5c, 0xaa, 0x2e, 0x65, 0xf5, 0x3e, 0x54, 0x75, 0x09, - 0x64, 0xa8, 0x57, 0xe6, 0x99, 0x92, 0x68, 0xe9, 0xd2, 0x07, 0x50, 0xcf, 0x54, 0x1c, 0x3a, 0xdb, - 0xcd, 0x17, 0x21, 0x3d, 0xf5, 0x28, 0x9c, 0x50, 0x3e, 0x80, 0x8a, 0xaa, 0x32, 0x8c, 0x8d, 0xe4, - 0x90, 0x67, 0x8a, 0x8e, 0x65, 0x12, 0x77, 0x2e, 0xbe, 0xfd, 0x6e, 0xeb, 0xb9, 0xbf, 0x7d, 0xb7, - 0xf5, 0xdc, 0x2f, 0x9e, 0x6d, 0x15, 0xbe, 0x7d, 0xb6, 0x55, 0xf8, 0xeb, 0xb3, 0xad, 0xc2, 0x3f, - 0x9e, 0x6d, 0x15, 0x7e, 0xf4, 0xd3, 0xff, 0xf0, 0x9f, 0x25, 0x34, 0x0e, 0x18, 0xf6, 0xd1, 0xf6, - 0x39, 0xa6, 0x2c, 0x33, 0x15, 0x9e, 0x8d, 0xe4, 0xdf, 0x4b, 0x32, 0xff, 0x3a, 0xe1, 0x0a, 0x0e, - 0x57, 0x05, 0xfc, 0xce, 0xbf, 0x02, 0x00, 0x00, 0xff, 0xff, 0x52, 0x45, 0x13, 0x9c, 0xc2, 0x22, - 0x00, 0x00, + // 3055 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x1a, 0xcb, 0x72, 0x24, 0x47, + 0xd1, 0xf3, 0x90, 0x66, 0x26, 0xe7, 0xa5, 0x69, 0x69, 0xb5, 0xb3, 0x63, 0x5b, 0xac, 0x7b, 0xed, + 0xf5, 0xda, 0xc6, 0x92, 0xbd, 0x76, 0xb0, 0x7e, 0x84, 0x59, 0x24, 0xad, 0x2c, 0xc9, 0xb6, 0xbc, + 0x43, 0xcb, 0xc2, 0x04, 0x04, 0x74, 0xb4, 0xba, 0x6b, 0x47, 0x65, 0x4d, 0x77, 0xb5, 0xab, 0xab, + 0xb5, 0x92, 0x89, 0x20, 0x38, 0xc1, 0x8d, 0x23, 0x37, 0x7e, 0x80, 0xe0, 0xc6, 0x91, 0x0b, 0x07, + 0x0e, 0x0e, 0x4e, 0x1c, 0x39, 0x11, 0x78, 0x3f, 0x81, 0x2f, 0x20, 0xea, 0xd5, 0x5d, 0x3d, 0x0f, + 0x19, 0x14, 0x1b, 0xc1, 0x65, 0xa2, 0x33, 0x2b, 0x2b, 0x5f, 0x55, 0x99, 0x95, 0x59, 0x35, 0x30, + 0x1c, 0x61, 0x76, 0x92, 0x1e, 0xaf, 0xfb, 0x24, 0xdc, 0x38, 0xf5, 0x98, 0xf7, 0xba, 0x4f, 0x22, + 0xe6, 0xe1, 0x08, 0xd1, 0x64, 0x0a, 0x4e, 0xa8, 0xbf, 0x31, 0xc6, 0xc7, 0xc9, 0x46, 0x4c, 0x09, + 0x23, 0x3e, 0x19, 0xab, 0xaf, 0x64, 0xc3, 0x1b, 0xa1, 0x88, 0xad, 0x0b, 0xc0, 0xaa, 0x8e, 0x68, + 0xec, 0x0f, 0x1a, 0xc4, 0xc7, 0x12, 0x31, 0x68, 0xf8, 0x89, 0xfe, 0x6c, 0xb2, 0x8b, 0x18, 0x25, + 0x0a, 0x78, 0x76, 0x44, 0xc8, 0x68, 0x8c, 0x24, 0x8f, 0xe3, 0xf4, 0xd1, 0x06, 0x0a, 0x63, 0x76, + 0x21, 0x07, 0xed, 0xdf, 0x97, 0x61, 0x75, 0x9b, 0x22, 0x8f, 0xa1, 0x6d, 0xad, 0x80, 0x83, 0xbe, + 0x4c, 0x51, 0xc2, 0xac, 0x17, 0xa0, 0x95, 0x29, 0xe5, 0xe2, 0xa0, 0x5f, 0xba, 0x59, 0xba, 0xd3, + 0x70, 0x9a, 0x19, 0x6e, 0x3f, 0xb0, 0xae, 0x43, 0x0d, 0x9d, 0x23, 0x9f, 0x8f, 0x96, 0xc5, 0xe8, + 0x22, 0x07, 0xf7, 0x03, 0xeb, 0x4d, 0x68, 0x26, 0x8c, 0xe2, 0x68, 0xe4, 0xa6, 0x09, 0xa2, 0xfd, + 0xca, 0xcd, 0xd2, 0x9d, 0xe6, 0xdd, 0xa5, 0x75, 0xae, 0xf2, 0xfa, 0xa1, 0x18, 0x38, 0x4a, 0x10, + 0x75, 0x20, 0xc9, 0xbe, 0xad, 0xdb, 0x50, 0x0b, 0xd0, 0x19, 0xf6, 0x51, 0xd2, 0xaf, 0xde, 0xac, + 0xdc, 0x69, 0xde, 0x6d, 0x49, 0xf2, 0x07, 0x02, 0xe9, 0xe8, 0x41, 0xeb, 0x15, 0xa8, 0x27, 0x8c, + 0x50, 0x6f, 0x84, 0x92, 0xfe, 0x82, 0x20, 0x6c, 0x6b, 0xbe, 0x02, 0xeb, 0x64, 0xc3, 0xd6, 0x73, + 0x50, 0x79, 0xb8, 0xbd, 0xdf, 0x5f, 0x14, 0xd2, 0x41, 0x51, 0xc5, 0xc8, 0x77, 0x38, 0xda, 0xba, + 0x05, 0xed, 0xc4, 0x8b, 0x82, 0x63, 0x72, 0xee, 0xc6, 0x38, 0x88, 0x92, 0x7e, 0xed, 0x66, 0xe9, + 0x4e, 0xdd, 0x69, 0x29, 0xe4, 0x90, 0xe3, 0xec, 0xf7, 0xe0, 0xda, 0x21, 0xf3, 0x28, 0xbb, 0x82, + 0x77, 0xec, 0x23, 0x58, 0x75, 0x50, 0x48, 0xce, 0xae, 0xe4, 0xda, 0x3e, 0xd4, 0x18, 0x0e, 0x11, + 0x49, 0x99, 0x70, 0x6d, 0xdb, 0xd1, 0xa0, 0xfd, 0xc7, 0x12, 0x58, 0x3b, 0xe7, 0xc8, 0x1f, 0x52, + 0xe2, 0xa3, 0x24, 0xf9, 0x3f, 0x2d, 0xd7, 0xcb, 0x50, 0x8b, 0xa5, 0x02, 0xfd, 0xaa, 0x20, 0x57, + 0xab, 0xa0, 0xb5, 0xd2, 0xa3, 0xf6, 0x17, 0xb0, 0x72, 0x88, 0x47, 0x91, 0x37, 0x7e, 0x8a, 0xfa, + 0xae, 0xc2, 0x62, 0x22, 0x78, 0x0a, 0x55, 0xdb, 0x8e, 0x82, 0xec, 0x21, 0x58, 0x9f, 0x7b, 0x98, + 0x3d, 0x3d, 0x49, 0xf6, 0xeb, 0xb0, 0x5c, 0xe0, 0x98, 0xc4, 0x24, 0x4a, 0x90, 0x50, 0x80, 0x79, + 0x2c, 0x4d, 0x04, 0xb3, 0x05, 0x47, 0x41, 0x36, 0x81, 0xd5, 0xa3, 0x38, 0xb8, 0x62, 0x34, 0xdd, + 0x85, 0x06, 0x45, 0x09, 0x49, 0x29, 0x8f, 0x81, 0xb2, 0x70, 0xea, 0x8a, 0x74, 0xea, 0x27, 0x38, + 0x4a, 0xcf, 0x1d, 0x3d, 0xe6, 0xe4, 0x64, 0x6a, 0x7f, 0xb2, 0xe4, 0x2a, 0xfb, 0xf3, 0x3d, 0xb8, + 0x36, 0xf4, 0xd2, 0xe4, 0x2a, 0xba, 0xda, 0xef, 0xf3, 0xbd, 0x9d, 0xa4, 0xe1, 0x95, 0x26, 0xff, + 0xa1, 0x04, 0xf5, 0xed, 0x38, 0x3d, 0x4a, 0xbc, 0x11, 0xb2, 0xbe, 0x03, 0x4d, 0x46, 0x98, 0x37, + 0x76, 0x53, 0x0e, 0x0a, 0xf2, 0xaa, 0x03, 0x02, 0x25, 0x09, 0x5e, 0x80, 0x56, 0x8c, 0xa8, 0x1f, + 0xa7, 0x8a, 0xa2, 0x7c, 0xb3, 0x72, 0xa7, 0xea, 0x34, 0x25, 0x4e, 0x92, 0xac, 0xc3, 0xb2, 0x18, + 0x73, 0x71, 0xe4, 0x9e, 0x22, 0x1a, 0xa1, 0x71, 0x48, 0x02, 0x24, 0x36, 0x47, 0xd5, 0xe9, 0x89, + 0xa1, 0xfd, 0xe8, 0xe3, 0x6c, 0xc0, 0x7a, 0x15, 0x7a, 0x19, 0x3d, 0xdf, 0xf1, 0x82, 0xba, 0x2a, + 0xa8, 0xbb, 0x8a, 0xfa, 0x48, 0xa1, 0xed, 0x5f, 0x42, 0xe7, 0xb3, 0x13, 0x4a, 0x18, 0x1b, 0xe3, + 0x68, 0xf4, 0xc0, 0x63, 0x1e, 0x0f, 0xcd, 0x18, 0x51, 0x4c, 0x82, 0x44, 0x69, 0xab, 0x41, 0xeb, + 0x35, 0xe8, 0x31, 0x49, 0x8b, 0x02, 0x57, 0xd3, 0x94, 0x05, 0xcd, 0x52, 0x36, 0x30, 0x54, 0xc4, + 0x2f, 0x41, 0x27, 0x27, 0xe6, 0xc1, 0xad, 0xf4, 0x6d, 0x67, 0xd8, 0xcf, 0x70, 0x88, 0xec, 0x33, + 0xe1, 0x2b, 0xb1, 0xc8, 0xd6, 0x6b, 0xd0, 0xc8, 0xfd, 0x50, 0x12, 0x3b, 0xa4, 0x23, 0x77, 0x88, + 0x76, 0xa7, 0x53, 0xcf, 0x9c, 0xf2, 0x01, 0x74, 0x59, 0xa6, 0xb8, 0x1b, 0x78, 0xcc, 0x2b, 0x6e, + 0xaa, 0xa2, 0x55, 0x4e, 0x87, 0x15, 0x60, 0xfb, 0x7d, 0x68, 0x0c, 0x71, 0x90, 0x48, 0xc1, 0x7d, + 0xa8, 0xf9, 0x29, 0xa5, 0x28, 0x62, 0xda, 0x64, 0x05, 0x5a, 0x2b, 0xb0, 0x30, 0xc6, 0x21, 0x66, + 0xca, 0x4c, 0x09, 0xd8, 0x04, 0xe0, 0x00, 0x85, 0x84, 0x5e, 0x08, 0x87, 0xad, 0xc0, 0x82, 0xb9, + 0xb8, 0x12, 0xb0, 0x9e, 0x85, 0x46, 0xe8, 0x9d, 0x67, 0x8b, 0xca, 0x47, 0xea, 0xa1, 0x77, 0x2e, + 0x95, 0xef, 0x43, 0xed, 0x91, 0x87, 0xc7, 0x7e, 0xc4, 0x94, 0x57, 0x34, 0x98, 0x0b, 0xac, 0x9a, + 0x02, 0xff, 0x5a, 0x86, 0xa6, 0x94, 0x28, 0x15, 0x5e, 0x81, 0x05, 0xdf, 0xf3, 0x4f, 0x32, 0x91, + 0x02, 0xb0, 0x6e, 0x6b, 0x45, 0xca, 0x66, 0x86, 0xcb, 0x35, 0xd5, 0xaa, 0x6d, 0x00, 0x24, 0x8f, + 0xbd, 0x58, 0xe9, 0x56, 0x99, 0x43, 0xdc, 0xe0, 0x34, 0x52, 0xdd, 0xb7, 0xa0, 0x25, 0xf7, 0x9d, + 0x9a, 0x52, 0x9d, 0x33, 0xa5, 0x29, 0xa9, 0xe4, 0xa4, 0x5b, 0xd0, 0x4e, 0x13, 0xe4, 0x9e, 0x60, + 0x44, 0x3d, 0xea, 0x9f, 0x5c, 0xf4, 0x17, 0xe4, 0x01, 0x94, 0x26, 0x68, 0x4f, 0xe3, 0xac, 0xbb, + 0xb0, 0xc0, 0x73, 0x4b, 0xd2, 0x5f, 0x14, 0x67, 0xdd, 0x73, 0x26, 0x4b, 0x61, 0xea, 0xba, 0xf8, + 0xdd, 0x89, 0x18, 0xbd, 0x70, 0x24, 0xe9, 0xe0, 0x1d, 0x80, 0x1c, 0x69, 0x2d, 0x41, 0xe5, 0x14, + 0x5d, 0xa8, 0x38, 0xe4, 0x9f, 0xdc, 0x39, 0x67, 0xde, 0x38, 0xd5, 0x5e, 0x97, 0xc0, 0x7b, 0xe5, + 0x77, 0x4a, 0xb6, 0x0f, 0xdd, 0xad, 0xf1, 0x29, 0x26, 0xc6, 0xf4, 0x15, 0x58, 0x08, 0xbd, 0x2f, + 0x08, 0xd5, 0x9e, 0x14, 0x80, 0xc0, 0xe2, 0x88, 0x50, 0xcd, 0x42, 0x00, 0x56, 0x07, 0xca, 0x24, + 0x16, 0xfe, 0x6a, 0x38, 0x65, 0x12, 0xe7, 0x82, 0xaa, 0x86, 0x20, 0xfb, 0x9f, 0x55, 0x80, 0x5c, + 0x8a, 0xe5, 0xc0, 0x00, 0x13, 0x37, 0x41, 0x94, 0x9f, 0xef, 0xee, 0xf1, 0x05, 0x43, 0x89, 0x4b, + 0x91, 0x9f, 0xd2, 0x04, 0x9f, 0xf1, 0xf5, 0xe3, 0x66, 0x5f, 0x93, 0x66, 0x4f, 0xe8, 0xe6, 0x5c, + 0xc7, 0xe4, 0x50, 0xce, 0xdb, 0xe2, 0xd3, 0x1c, 0x3d, 0xcb, 0xda, 0x87, 0x6b, 0x39, 0xcf, 0xc0, + 0x60, 0x57, 0xbe, 0x8c, 0xdd, 0x72, 0xc6, 0x2e, 0xc8, 0x59, 0xed, 0xc0, 0x32, 0x26, 0xee, 0x97, + 0x29, 0x4a, 0x0b, 0x8c, 0x2a, 0x97, 0x31, 0xea, 0x61, 0xf2, 0x43, 0x31, 0x21, 0x67, 0x33, 0x84, + 0x1b, 0x86, 0x95, 0x3c, 0xdc, 0x0d, 0x66, 0xd5, 0xcb, 0x98, 0xad, 0x66, 0x5a, 0xf1, 0x7c, 0x90, + 0x73, 0xfc, 0x08, 0x56, 0x31, 0x71, 0x1f, 0x7b, 0x98, 0x4d, 0xb2, 0x5b, 0xf8, 0x16, 0x23, 0xf9, + 0x89, 0x56, 0xe4, 0x25, 0x8d, 0x0c, 0x11, 0x1d, 0x15, 0x8c, 0x5c, 0xfc, 0x16, 0x23, 0x0f, 0xc4, + 0x84, 0x9c, 0xcd, 0x26, 0xf4, 0x30, 0x99, 0xd4, 0xa6, 0x76, 0x19, 0x93, 0x2e, 0x26, 0x45, 0x4d, + 0xb6, 0xa0, 0x97, 0x20, 0x9f, 0x11, 0x6a, 0x6e, 0x82, 0xfa, 0x65, 0x2c, 0x96, 0x14, 0x7d, 0xc6, + 0xc3, 0xfe, 0x29, 0xb4, 0xf6, 0xd2, 0x11, 0x62, 0xe3, 0xe3, 0x2c, 0x19, 0x3c, 0xb5, 0xfc, 0x63, + 0xff, 0xbb, 0x0c, 0xcd, 0xed, 0x11, 0x25, 0x69, 0x5c, 0xc8, 0xc9, 0x32, 0x48, 0x27, 0x73, 0xb2, + 0x20, 0x11, 0x39, 0x59, 0x12, 0xbf, 0x0d, 0xad, 0x50, 0x84, 0xae, 0xa2, 0x97, 0x79, 0xa8, 0x37, + 0x15, 0xd4, 0x4e, 0x33, 0x34, 0x92, 0xd9, 0x3a, 0x40, 0x8c, 0x83, 0x44, 0xcd, 0x91, 0xe9, 0xa8, + 0xab, 0xca, 0x2d, 0x9d, 0xa2, 0x9d, 0x46, 0x9c, 0x65, 0xeb, 0x37, 0xa1, 0x79, 0xcc, 0x9d, 0xa4, + 0x26, 0x14, 0x92, 0x51, 0xee, 0x3d, 0x07, 0x8e, 0xf3, 0x20, 0xdc, 0x83, 0xf6, 0x89, 0x74, 0x99, + 0x9a, 0x24, 0xf7, 0xd0, 0x2d, 0x65, 0x49, 0x6e, 0xef, 0xba, 0xe9, 0x59, 0xb9, 0x00, 0xad, 0x13, + 0x03, 0x35, 0x38, 0x84, 0xde, 0x14, 0xc9, 0x8c, 0x1c, 0x74, 0xc7, 0xcc, 0x41, 0xcd, 0xbb, 0x96, + 0x14, 0x64, 0xce, 0x34, 0xf3, 0xd2, 0x6f, 0xcb, 0xd0, 0xfa, 0x14, 0xb1, 0xc7, 0x84, 0x9e, 0x4a, + 0x7d, 0x2d, 0xa8, 0x46, 0x5e, 0x88, 0x14, 0x47, 0xf1, 0x6d, 0xdd, 0x80, 0x3a, 0x3d, 0x97, 0x09, + 0x44, 0xad, 0x67, 0x8d, 0x9e, 0x8b, 0xc4, 0x60, 0x3d, 0x0f, 0x40, 0xcf, 0xdd, 0xd8, 0xf3, 0x4f, + 0x91, 0xf2, 0x60, 0xd5, 0x69, 0xd0, 0xf3, 0xa1, 0x44, 0xf0, 0xad, 0x40, 0xcf, 0x5d, 0x44, 0x29, + 0xa1, 0x89, 0xca, 0x55, 0x75, 0x7a, 0xbe, 0x23, 0x60, 0x35, 0x37, 0xa0, 0x24, 0x8e, 0x51, 0x20, + 0x72, 0xb4, 0x98, 0xfb, 0x40, 0x22, 0xb8, 0x54, 0xa6, 0xa5, 0x2e, 0x4a, 0xa9, 0x2c, 0x97, 0xca, + 0x72, 0xa9, 0x35, 0x39, 0x93, 0x99, 0x52, 0x59, 0x26, 0xb5, 0x2e, 0xa5, 0x32, 0x43, 0x2a, 0xcb, + 0xa5, 0x36, 0xf4, 0x5c, 0x25, 0xd5, 0xfe, 0x4d, 0x09, 0x56, 0x27, 0x0b, 0x3f, 0x55, 0x9b, 0xbe, + 0x0d, 0x2d, 0x5f, 0xac, 0x57, 0x61, 0x4f, 0xf6, 0xa6, 0x56, 0xd2, 0x69, 0xfa, 0xc6, 0x36, 0xbe, + 0x07, 0xed, 0x48, 0x3a, 0x38, 0xdb, 0x9a, 0x95, 0x7c, 0x5d, 0x4c, 0xdf, 0x3b, 0xad, 0xc8, 0x80, + 0xec, 0x00, 0xac, 0xcf, 0x29, 0x66, 0xe8, 0x90, 0x51, 0xe4, 0x85, 0x4f, 0xa3, 0xba, 0xb7, 0xa0, + 0x2a, 0xaa, 0x15, 0xbe, 0x4c, 0x2d, 0x47, 0x7c, 0xdb, 0x2f, 0xc3, 0x72, 0x41, 0x8a, 0xb2, 0x75, + 0x09, 0x2a, 0x63, 0x14, 0x09, 0xee, 0x6d, 0x87, 0x7f, 0xda, 0x1e, 0xf4, 0x1c, 0xe4, 0x05, 0x4f, + 0x4f, 0x1b, 0x25, 0xa2, 0x92, 0x8b, 0xb8, 0x03, 0x96, 0x29, 0x42, 0xa9, 0xa2, 0xb5, 0x2e, 0x19, + 0x5a, 0x3f, 0x84, 0xde, 0xf6, 0x98, 0x24, 0xe8, 0x90, 0x05, 0x38, 0x7a, 0x1a, 0xed, 0xc8, 0x2f, + 0x60, 0xf9, 0x33, 0x76, 0xf1, 0x39, 0x67, 0x96, 0xe0, 0xaf, 0xd0, 0x53, 0xb2, 0x8f, 0x92, 0xc7, + 0xda, 0x3e, 0x4a, 0x1e, 0xf3, 0xe6, 0xc6, 0x27, 0xe3, 0x34, 0x8c, 0x44, 0x28, 0xb4, 0x1d, 0x05, + 0xd9, 0x5b, 0xd0, 0x92, 0x35, 0xf4, 0x01, 0x09, 0xd2, 0x31, 0x9a, 0x19, 0x83, 0x6b, 0x00, 0xb1, + 0x47, 0xbd, 0x10, 0x31, 0x44, 0xe5, 0x1e, 0x6a, 0x38, 0x06, 0xc6, 0xfe, 0x5d, 0x19, 0x56, 0xe4, + 0x7d, 0xc3, 0xa1, 0x6c, 0xb3, 0xb5, 0x09, 0x03, 0xa8, 0x9f, 0x90, 0x84, 0x19, 0x0c, 0x33, 0x98, + 0xab, 0xc8, 0xfb, 0x73, 0xc9, 0x8d, 0x7f, 0x16, 0x2e, 0x01, 0x2a, 0x97, 0x5f, 0x02, 0x4c, 0xb5, + 0xf9, 0xd5, 0xe9, 0x36, 0x9f, 0x47, 0x9b, 0x26, 0xc2, 0x32, 0xc6, 0x1b, 0x4e, 0x43, 0x61, 0xf6, + 0x03, 0xeb, 0x36, 0x74, 0x47, 0x5c, 0x4b, 0xf7, 0x84, 0x90, 0x53, 0x37, 0xf6, 0xd8, 0x89, 0x08, + 0xf5, 0x86, 0xd3, 0x16, 0xe8, 0x3d, 0x42, 0x4e, 0x87, 0x1e, 0x3b, 0xb1, 0xde, 0x85, 0x8e, 0x2a, + 0x03, 0x43, 0xe1, 0xa2, 0x44, 0x1d, 0x7e, 0x2a, 0x8a, 0x4c, 0xef, 0x39, 0xed, 0x53, 0x03, 0x4a, + 0xec, 0xeb, 0x70, 0xed, 0x01, 0x4a, 0x18, 0x25, 0x17, 0x45, 0xc7, 0xd8, 0xdf, 0x07, 0xd8, 0x8f, + 0x18, 0xa2, 0x8f, 0x3c, 0x1f, 0x25, 0xd6, 0x1b, 0x26, 0xa4, 0x8a, 0xa3, 0xa5, 0x75, 0x79, 0xdd, + 0x93, 0x0d, 0x38, 0x06, 0x8d, 0xbd, 0x0e, 0x8b, 0x0e, 0x49, 0x79, 0x3a, 0x7a, 0x51, 0x7f, 0xa9, + 0x79, 0x2d, 0x35, 0x4f, 0x20, 0x1d, 0x35, 0x66, 0xef, 0xe9, 0x16, 0x36, 0x67, 0xa7, 0x96, 0x68, + 0x1d, 0x1a, 0x58, 0xe3, 0x54, 0x56, 0x99, 0x16, 0x9d, 0x93, 0xd8, 0xef, 0xc3, 0xb2, 0xe4, 0x24, + 0x39, 0x6b, 0x36, 0x2f, 0xc2, 0x22, 0xd5, 0x6a, 0x94, 0xf2, 0x7b, 0x1e, 0x45, 0xa4, 0xc6, 0xb8, + 0x3f, 0x3e, 0xc1, 0x09, 0xcb, 0x0d, 0xd1, 0xfe, 0x58, 0x86, 0x1e, 0x1f, 0x28, 0xf0, 0xb4, 0x3f, + 0x84, 0xd6, 0xa6, 0x33, 0xfc, 0x14, 0xe1, 0xd1, 0xc9, 0x31, 0xcf, 0x9e, 0xdf, 0x2b, 0xc2, 0xca, + 0x60, 0x4b, 0x69, 0x6b, 0x0c, 0x39, 0x05, 0x3a, 0xfb, 0x23, 0x58, 0xdd, 0x0c, 0x02, 0x13, 0xa5, + 0xb5, 0x7e, 0x03, 0x1a, 0x91, 0xc1, 0xce, 0x38, 0xb3, 0x0a, 0xd4, 0x39, 0x91, 0xfd, 0x33, 0x58, + 0x7e, 0x18, 0x8d, 0x71, 0x84, 0xb6, 0x87, 0x47, 0x07, 0x28, 0xcb, 0x45, 0x16, 0x54, 0x79, 0xcd, + 0x26, 0x78, 0xd4, 0x1d, 0xf1, 0xcd, 0x83, 0x33, 0x3a, 0x76, 0xfd, 0x38, 0x4d, 0xd4, 0x65, 0xcf, + 0x62, 0x74, 0xbc, 0x1d, 0xa7, 0x09, 0x3f, 0x5c, 0x78, 0x71, 0x41, 0xa2, 0xf1, 0x85, 0x88, 0xd0, + 0xba, 0x53, 0xf3, 0xe3, 0xf4, 0x61, 0x34, 0xbe, 0xb0, 0xbf, 0x2b, 0x3a, 0x70, 0x84, 0x02, 0xc7, + 0x8b, 0x02, 0x12, 0x3e, 0x40, 0x67, 0x86, 0x84, 0xac, 0xdb, 0xd3, 0x99, 0xe8, 0xeb, 0x12, 0xb4, + 0x36, 0x47, 0x28, 0x62, 0x0f, 0x10, 0xf3, 0xf0, 0x58, 0x74, 0x74, 0x67, 0x88, 0x26, 0x98, 0x44, + 0x2a, 0xdc, 0x34, 0xc8, 0x1b, 0x72, 0x1c, 0x61, 0xe6, 0x06, 0x1e, 0x0a, 0x49, 0x24, 0xb8, 0xd4, + 0x1d, 0xe0, 0xa8, 0x07, 0x02, 0x63, 0xbd, 0x0c, 0x5d, 0x79, 0x19, 0xe7, 0x9e, 0x78, 0x51, 0x30, + 0xe6, 0x81, 0x5e, 0x11, 0xa1, 0xd9, 0x91, 0xe8, 0x3d, 0x85, 0xb5, 0x5e, 0x81, 0x25, 0x15, 0x86, + 0x39, 0x65, 0x55, 0x50, 0x76, 0x15, 0xbe, 0x40, 0x9a, 0xc6, 0x31, 0xa1, 0x2c, 0x71, 0x13, 0xe4, + 0xfb, 0x24, 0x8c, 0x55, 0x3b, 0xd4, 0xd5, 0xf8, 0x43, 0x89, 0xb6, 0x47, 0xb0, 0xbc, 0xcb, 0xed, + 0x54, 0x96, 0xe4, 0xdb, 0xaa, 0x13, 0xa2, 0xd0, 0x3d, 0x1e, 0x13, 0xff, 0xd4, 0xe5, 0xc9, 0x51, + 0x79, 0x98, 0x17, 0x5c, 0x5b, 0x1c, 0x79, 0x88, 0xbf, 0x12, 0x9d, 0x3f, 0xa7, 0x3a, 0x21, 0x2c, + 0x1e, 0xa7, 0x23, 0x37, 0xa6, 0xe4, 0x18, 0x29, 0x13, 0xbb, 0x21, 0x0a, 0xf7, 0x24, 0x7e, 0xc8, + 0xd1, 0xf6, 0x9f, 0x4b, 0xb0, 0x52, 0x94, 0xa4, 0x52, 0xfd, 0x06, 0xac, 0x14, 0x45, 0xa9, 0xe3, + 0x5f, 0x96, 0x97, 0x3d, 0x53, 0xa0, 0x2c, 0x04, 0xee, 0x41, 0x5b, 0x5c, 0xdd, 0xba, 0x81, 0xe4, + 0x54, 0x2c, 0x7a, 0xcc, 0x75, 0x71, 0x5a, 0x9e, 0xb9, 0x4a, 0xef, 0xc2, 0x0d, 0x65, 0xbe, 0x3b, + 0xad, 0xb6, 0xdc, 0x10, 0xab, 0x8a, 0xe0, 0x60, 0x42, 0xfb, 0x4f, 0xa0, 0x9f, 0xa3, 0xb6, 0x2e, + 0x04, 0x32, 0xdf, 0xcc, 0xcb, 0x13, 0xc6, 0x6e, 0x06, 0x01, 0x15, 0x51, 0x52, 0x75, 0x66, 0x0d, + 0xd9, 0xf7, 0xe1, 0xfa, 0x21, 0x62, 0xd2, 0x1b, 0x1e, 0x53, 0x9d, 0x88, 0x64, 0xb6, 0x04, 0x95, + 0x43, 0xe4, 0x0b, 0xe3, 0x2b, 0x0e, 0xff, 0xe4, 0x1b, 0xf0, 0x28, 0x41, 0xbe, 0xb0, 0xb2, 0xe2, + 0x88, 0x6f, 0xfb, 0x4f, 0x25, 0xa8, 0xa9, 0xe4, 0xcc, 0x0f, 0x98, 0x80, 0xe2, 0x33, 0x44, 0xd5, + 0xd6, 0x53, 0x90, 0xf5, 0x12, 0x74, 0xe4, 0x97, 0x4b, 0x62, 0x86, 0x49, 0x96, 0xf2, 0xdb, 0x12, + 0xfb, 0x50, 0x22, 0xc5, 0xe5, 0x9b, 0xb8, 0xfe, 0x52, 0x9d, 0xa6, 0x82, 0x38, 0xfe, 0x51, 0xc2, + 0x23, 0x5c, 0xa4, 0xf8, 0x86, 0xa3, 0x20, 0xbe, 0xd5, 0x35, 0xbf, 0x05, 0xc1, 0x4f, 0x83, 0x7c, + 0xab, 0x87, 0x24, 0x8d, 0x98, 0x1b, 0x13, 0x1c, 0x31, 0x95, 0xd3, 0x41, 0xa0, 0x86, 0x1c, 0x63, + 0xff, 0xba, 0x04, 0x8b, 0xf2, 0x02, 0x9a, 0xf7, 0xb6, 0xd9, 0xc9, 0x5a, 0xc6, 0xa2, 0x4a, 0x11, + 0xb2, 0xe4, 0x69, 0x2a, 0xbe, 0x79, 0x1c, 0x9f, 0x85, 0xf2, 0x7c, 0x50, 0xaa, 0x9d, 0x85, 0xe2, + 0x60, 0x78, 0x09, 0x3a, 0xf9, 0x01, 0x2d, 0xc6, 0xa5, 0x8a, 0xed, 0x0c, 0x2b, 0xc8, 0xe6, 0x6a, + 0x6a, 0xff, 0x98, 0xb7, 0xf4, 0xd9, 0xe5, 0xeb, 0x12, 0x54, 0xd2, 0x4c, 0x19, 0xfe, 0xc9, 0x31, + 0xa3, 0xec, 0x68, 0xe7, 0x9f, 0xd6, 0x6d, 0xe8, 0x78, 0x41, 0x80, 0xf9, 0x74, 0x6f, 0xbc, 0x8b, + 0x83, 0x2c, 0x48, 0x8b, 0x58, 0xfb, 0x6f, 0x25, 0xe8, 0x6e, 0x93, 0xf8, 0xe2, 0x43, 0x3c, 0x46, + 0x46, 0x06, 0x11, 0x4a, 0xaa, 0x93, 0x9d, 0x7f, 0xf3, 0x6a, 0xf5, 0x11, 0x1e, 0x23, 0x19, 0x5a, + 0x72, 0x65, 0xeb, 0x1c, 0x21, 0xc2, 0x4a, 0x0f, 0x66, 0xd7, 0x6e, 0x6d, 0x39, 0x78, 0x40, 0x02, + 0x51, 0x97, 0x07, 0x98, 0xba, 0xd9, 0x25, 0x5b, 0xdb, 0xa9, 0x05, 0x98, 0x8a, 0x21, 0x65, 0xc8, + 0x82, 0xb8, 0x44, 0x35, 0x0d, 0x59, 0x94, 0x18, 0x6e, 0xc8, 0x2a, 0x2c, 0x92, 0x47, 0x8f, 0x12, + 0xc4, 0x44, 0x05, 0x5d, 0x71, 0x14, 0x94, 0xa5, 0xb9, 0xba, 0x91, 0xe6, 0x56, 0xc0, 0xda, 0x45, + 0xec, 0xe1, 0xc3, 0x83, 0x9d, 0x33, 0x14, 0x31, 0x7d, 0x3a, 0xbc, 0x0e, 0x75, 0x8d, 0xfa, 0x6f, + 0xae, 0x27, 0x5f, 0x85, 0xce, 0x66, 0x10, 0x1c, 0x3e, 0xf6, 0x62, 0xed, 0x8f, 0x3e, 0xd4, 0x86, + 0xdb, 0xfb, 0x43, 0xe9, 0x92, 0x0a, 0x37, 0x40, 0x81, 0xfc, 0x34, 0xda, 0x45, 0xec, 0x00, 0x31, + 0x8a, 0xfd, 0xec, 0x34, 0xba, 0x05, 0x35, 0x85, 0xe1, 0x33, 0x43, 0xf9, 0xa9, 0xd3, 0xac, 0x02, + 0xed, 0x1f, 0x80, 0xf5, 0x23, 0x5e, 0x57, 0x21, 0x59, 0x54, 0x2b, 0x49, 0xaf, 0x42, 0xef, 0x4c, + 0x60, 0x5d, 0x59, 0x70, 0x18, 0xcb, 0xd0, 0x95, 0x03, 0x22, 0x06, 0x85, 0xec, 0x23, 0x58, 0x96, + 0x65, 0xa0, 0xe4, 0x73, 0x05, 0x16, 0xdc, 0x87, 0xd9, 0x7a, 0x56, 0x1d, 0xf1, 0x7d, 0xf7, 0x2f, + 0x3d, 0x75, 0x54, 0xa8, 0x5b, 0x07, 0x6b, 0x17, 0xba, 0x13, 0x4f, 0x44, 0x96, 0xba, 0x86, 0x9a, + 0xfd, 0x72, 0x34, 0x58, 0x5d, 0x97, 0x4f, 0x4e, 0xeb, 0xfa, 0xc9, 0x69, 0x7d, 0x27, 0x8c, 0xd9, + 0x85, 0xb5, 0x03, 0x9d, 0xe2, 0x63, 0x8a, 0xf5, 0xac, 0xae, 0xda, 0x66, 0x3c, 0xb1, 0xcc, 0x65, + 0xb3, 0x0b, 0xdd, 0x89, 0x77, 0x15, 0xad, 0xcf, 0xec, 0xe7, 0x96, 0xb9, 0x8c, 0xee, 0x43, 0xd3, + 0x78, 0x48, 0xb1, 0xfa, 0x92, 0xc9, 0xf4, 0xdb, 0xca, 0x5c, 0x06, 0xdb, 0xd0, 0x2e, 0xbc, 0x6d, + 0x58, 0x03, 0x65, 0xcf, 0x8c, 0x07, 0x8f, 0xb9, 0x4c, 0xb6, 0xa0, 0x69, 0x3c, 0x31, 0x68, 0x2d, + 0xa6, 0xdf, 0x31, 0x06, 0x37, 0x66, 0x8c, 0xa8, 0x13, 0x69, 0x17, 0xba, 0x13, 0xef, 0x0e, 0xda, + 0x25, 0xb3, 0x9f, 0x23, 0xe6, 0x2a, 0xf3, 0xb1, 0x58, 0x22, 0xa3, 0xad, 0x34, 0x96, 0x68, 0xfa, + 0x95, 0x61, 0xf0, 0xdc, 0xec, 0x41, 0xa5, 0xd5, 0x0e, 0x74, 0x8a, 0x0f, 0x0c, 0x9a, 0xd9, 0xcc, + 0x67, 0x87, 0xcb, 0xd7, 0xbb, 0xf0, 0xd6, 0x90, 0xaf, 0xf7, 0xac, 0x27, 0x88, 0xb9, 0x8c, 0x36, + 0x01, 0x54, 0x13, 0x19, 0xe0, 0x28, 0x73, 0xf4, 0x54, 0xf3, 0x9a, 0x39, 0x7a, 0x46, 0xc3, 0x79, + 0x1f, 0x40, 0xf6, 0x7e, 0x01, 0x49, 0x99, 0x75, 0x5d, 0xab, 0x31, 0xd1, 0x70, 0x0e, 0xfa, 0xd3, + 0x03, 0x53, 0x0c, 0x10, 0xa5, 0x57, 0x61, 0xf0, 0x01, 0x40, 0xde, 0x53, 0x6a, 0x06, 0x53, 0x5d, + 0xe6, 0x25, 0x3e, 0x68, 0x99, 0x1d, 0xa4, 0xa5, 0x6c, 0x9d, 0xd1, 0x55, 0x5e, 0xc2, 0xa2, 0x3b, + 0xd1, 0x21, 0x14, 0x37, 0xdb, 0x64, 0xe3, 0x30, 0x98, 0xea, 0x12, 0xac, 0x7b, 0xd0, 0x32, 0x5b, + 0x03, 0xad, 0xc5, 0x8c, 0x76, 0x61, 0x50, 0x68, 0x0f, 0xac, 0xfb, 0xd0, 0x29, 0xb6, 0x05, 0x7a, + 0x4b, 0xcd, 0x6c, 0x16, 0x06, 0xea, 0xd2, 0xcb, 0x20, 0x7f, 0x0b, 0x20, 0x6f, 0x1f, 0xb4, 0xfb, + 0xa6, 0x1a, 0x8a, 0x09, 0xa9, 0xbb, 0xd0, 0x9d, 0x68, 0x0b, 0xb4, 0xc5, 0xb3, 0xbb, 0x85, 0xb9, + 0xae, 0x7b, 0x1b, 0x20, 0x3f, 0x2e, 0xb4, 0xf4, 0xa9, 0x03, 0x64, 0xd0, 0xd6, 0x17, 0x82, 0x92, + 0x6e, 0x1b, 0xda, 0x85, 0x9e, 0x59, 0xa7, 0x99, 0x59, 0x8d, 0xf4, 0x65, 0xc9, 0xb7, 0xd8, 0x60, + 0x6a, 0xcf, 0xcd, 0x6c, 0x3b, 0x2f, 0xdb, 0x3f, 0x66, 0x57, 0xa3, 0x57, 0x6e, 0x46, 0xa7, 0xf3, + 0x2d, 0xf1, 0x6c, 0x76, 0x2e, 0x46, 0x3c, 0xcf, 0x68, 0x68, 0xe6, 0x32, 0xda, 0x83, 0xee, 0xae, + 0x2e, 0x4a, 0x55, 0xc1, 0xac, 0xd4, 0x99, 0xd1, 0x20, 0x0c, 0x06, 0xb3, 0x86, 0x54, 0x50, 0x7d, + 0x0c, 0xbd, 0xa9, 0x62, 0xd9, 0x5a, 0xcb, 0xae, 0x65, 0x67, 0x56, 0xd1, 0x73, 0xd5, 0xda, 0x87, + 0xa5, 0xc9, 0x5a, 0xd9, 0x7a, 0x5e, 0x25, 0xca, 0xd9, 0x35, 0xf4, 0x5c, 0x56, 0xef, 0x42, 0x5d, + 0xd7, 0x66, 0x96, 0xba, 0xfe, 0x9e, 0xa8, 0xd5, 0xe6, 0x4e, 0xbd, 0x07, 0x4d, 0xa3, 0x14, 0xd2, + 0xd9, 0x6e, 0xba, 0x3a, 0x1a, 0xa8, 0xdb, 0xea, 0x8c, 0xf2, 0x1e, 0xd4, 0x54, 0xf9, 0x63, 0xad, + 0x64, 0x9b, 0xdc, 0xa8, 0x86, 0x2e, 0xdb, 0x61, 0xbb, 0x88, 0x19, 0x45, 0x8d, 0x16, 0x3a, 0x5d, + 0xe7, 0xe8, 0x14, 0x5b, 0x18, 0x51, 0x6b, 0xb1, 0x09, 0x2d, 0xb3, 0xac, 0xd1, 0x4b, 0x3a, 0xa3, + 0xd4, 0x99, 0xa7, 0xc9, 0xd6, 0xf9, 0xd7, 0xdf, 0xac, 0x3d, 0xf3, 0x8f, 0x6f, 0xd6, 0x9e, 0xf9, + 0xd5, 0x93, 0xb5, 0xd2, 0xd7, 0x4f, 0xd6, 0x4a, 0x7f, 0x7f, 0xb2, 0x56, 0xfa, 0xd7, 0x93, 0xb5, + 0xd2, 0x4f, 0x7e, 0xfe, 0x3f, 0xfe, 0x0f, 0x87, 0xa6, 0x11, 0xc3, 0x21, 0xda, 0x38, 0xc3, 0x94, + 0x19, 0x43, 0xf1, 0xe9, 0x48, 0xfe, 0x19, 0xc7, 0xf8, 0x8f, 0x0e, 0xd7, 0xf2, 0x78, 0x51, 0xc0, + 0x6f, 0xfd, 0x27, 0x00, 0x00, 0xff, 0xff, 0x5c, 0x4e, 0xa6, 0xdc, 0xf0, 0x23, 0x00, 0x00, } func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) { @@ -5446,6 +5533,79 @@ func (m *Metrics) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *VolumeStatsRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeStatsRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *VolumeStatsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.VolumeGuestPath) > 0 { + i -= len(m.VolumeGuestPath) + copy(dAtA[i:], m.VolumeGuestPath) + i = encodeVarintAgent(dAtA, i, uint64(len(m.VolumeGuestPath))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ResizeVolumeRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ResizeVolumeRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ResizeVolumeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Size_ != 0 { + i = encodeVarintAgent(dAtA, i, uint64(m.Size_)) + i-- + dAtA[i] = 0x10 + } + if len(m.VolumeGuestPath) > 0 { + i -= len(m.VolumeGuestPath) + copy(dAtA[i:], m.VolumeGuestPath) + i = encodeVarintAgent(dAtA, i, uint64(len(m.VolumeGuestPath))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func encodeVarintAgent(dAtA []byte, offset int, v uint64) int { offset -= sovAgent(v) base := offset @@ -6737,6 +6897,41 @@ func (m *Metrics) Size() (n int) { return n } +func (m *VolumeStatsRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.VolumeGuestPath) + if l > 0 { + n += 1 + l + sovAgent(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ResizeVolumeRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.VolumeGuestPath) + if l > 0 { + n += 1 + l + sovAgent(uint64(l)) + } + if m.Size_ != 0 { + n += 1 + sovAgent(uint64(m.Size_)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovAgent(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -7551,6 +7746,29 @@ func (this *Metrics) String() string { }, "") return s } +func (this *VolumeStatsRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&VolumeStatsRequest{`, + `VolumeGuestPath:` + fmt.Sprintf("%v", this.VolumeGuestPath) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} +func (this *ResizeVolumeRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ResizeVolumeRequest{`, + `VolumeGuestPath:` + fmt.Sprintf("%v", this.VolumeGuestPath) + `,`, + `Size_:` + fmt.Sprintf("%v", this.Size_) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} func valueToStringAgent(v interface{}) string { rv := reflect.ValueOf(v) if rv.IsNil() { @@ -7592,6 +7810,8 @@ type AgentServiceService interface { CopyFile(ctx context.Context, req *CopyFileRequest) (*types.Empty, error) GetOOMEvent(ctx context.Context, req *GetOOMEventRequest) (*OOMEvent, error) AddSwap(ctx context.Context, req *AddSwapRequest) (*types.Empty, error) + GetVolumeStats(ctx context.Context, req *VolumeStatsRequest) (*VolumeStatsResponse, error) + ResizeVolume(ctx context.Context, req *ResizeVolumeRequest) (*types.Empty, error) } func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc AgentServiceService) { @@ -7813,6 +8033,20 @@ func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc Ag } return svc.AddSwap(ctx, &req) }, + "GetVolumeStats": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req VolumeStatsRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.GetVolumeStats(ctx, &req) + }, + "ResizeVolume": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req ResizeVolumeRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.ResizeVolume(ctx, &req) + }, }) } @@ -8073,6 +8307,22 @@ func (c *agentServiceClient) AddSwap(ctx context.Context, req *AddSwapRequest) ( } return &resp, nil } + +func (c *agentServiceClient) GetVolumeStats(ctx context.Context, req *VolumeStatsRequest) (*VolumeStatsResponse, error) { + var resp VolumeStatsResponse + if err := c.client.Call(ctx, "grpc.AgentService", "GetVolumeStats", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *agentServiceClient) ResizeVolume(ctx context.Context, req *ResizeVolumeRequest) (*types.Empty, error) { + var resp types.Empty + if err := c.client.Call(ctx, "grpc.AgentService", "ResizeVolume", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} func (m *CreateContainerRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -15399,6 +15649,191 @@ func (m *Metrics) Unmarshal(dAtA []byte) error { } return nil } +func (m *VolumeStatsRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeStatsRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeStatsRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field VolumeGuestPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthAgent + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthAgent + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.VolumeGuestPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipAgent(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ResizeVolumeRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ResizeVolumeRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ResizeVolumeRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field VolumeGuestPath", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthAgent + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthAgent + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.VolumeGuestPath = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType) + } + m.Size_ = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size_ |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipAgent(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipAgent(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/csi.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/csi.pb.go new file mode 100644 index 0000000000..6ae4f148ed --- /dev/null +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/csi.pb.go @@ -0,0 +1,1164 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: github.com/kata-containers/kata-containers/src/libs/protocols/protos/csi.proto + +package grpc + +import ( + bytes "bytes" + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" + reflect "reflect" + strings "strings" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +type VolumeUsage_Unit int32 + +const ( + VolumeUsage_UNKNOWN VolumeUsage_Unit = 0 + VolumeUsage_BYTES VolumeUsage_Unit = 1 + VolumeUsage_INODES VolumeUsage_Unit = 2 +) + +var VolumeUsage_Unit_name = map[int32]string{ + 0: "UNKNOWN", + 1: "BYTES", + 2: "INODES", +} + +var VolumeUsage_Unit_value = map[string]int32{ + "UNKNOWN": 0, + "BYTES": 1, + "INODES": 2, +} + +func (x VolumeUsage_Unit) String() string { + return proto.EnumName(VolumeUsage_Unit_name, int32(x)) +} + +func (VolumeUsage_Unit) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_5658ecc8e79d3214, []int{1, 0} +} + +// This should be kept in sync with CSI NodeGetVolumeStatsResponse (https://github.com/container-storage-interface/spec/blob/v1.5.0/csi.proto) +type VolumeStatsResponse struct { + // This field is OPTIONAL. + Usage []*VolumeUsage `protobuf:"bytes,1,rep,name=usage,proto3" json:"usage,omitempty"` + // Information about the current condition of the volume. + // This field is OPTIONAL. + // This field MUST be specified if the VOLUME_CONDITION node + // capability is supported. + VolumeCondition *VolumeCondition `protobuf:"bytes,2,opt,name=volume_condition,json=volumeCondition,proto3" json:"volume_condition,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeStatsResponse) Reset() { *m = VolumeStatsResponse{} } +func (*VolumeStatsResponse) ProtoMessage() {} +func (*VolumeStatsResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_5658ecc8e79d3214, []int{0} +} +func (m *VolumeStatsResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *VolumeStatsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_VolumeStatsResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *VolumeStatsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeStatsResponse.Merge(m, src) +} +func (m *VolumeStatsResponse) XXX_Size() int { + return m.Size() +} +func (m *VolumeStatsResponse) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeStatsResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeStatsResponse proto.InternalMessageInfo + +type VolumeUsage struct { + // The available capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + Available uint64 `protobuf:"varint,1,opt,name=available,proto3" json:"available,omitempty"` + // The total capacity in specified Unit. This field is REQUIRED. + // The value of this field MUST NOT be negative. + Total uint64 `protobuf:"varint,2,opt,name=total,proto3" json:"total,omitempty"` + // The used capacity in specified Unit. This field is OPTIONAL. + // The value of this field MUST NOT be negative. + Used uint64 `protobuf:"varint,3,opt,name=used,proto3" json:"used,omitempty"` + // Units by which values are measured. This field is REQUIRED. + Unit VolumeUsage_Unit `protobuf:"varint,4,opt,name=unit,proto3,enum=grpc.VolumeUsage_Unit" json:"unit,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeUsage) Reset() { *m = VolumeUsage{} } +func (*VolumeUsage) ProtoMessage() {} +func (*VolumeUsage) Descriptor() ([]byte, []int) { + return fileDescriptor_5658ecc8e79d3214, []int{1} +} +func (m *VolumeUsage) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *VolumeUsage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_VolumeUsage.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *VolumeUsage) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeUsage.Merge(m, src) +} +func (m *VolumeUsage) XXX_Size() int { + return m.Size() +} +func (m *VolumeUsage) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeUsage.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeUsage proto.InternalMessageInfo + +// VolumeCondition represents the current condition of a volume. +type VolumeCondition struct { + // Normal volumes are available for use and operating optimally. + // An abnormal volume does not meet these criteria. + // This field is REQUIRED. + Abnormal bool `protobuf:"varint,1,opt,name=abnormal,proto3" json:"abnormal,omitempty"` + // The message describing the condition of the volume. + // This field is REQUIRED. + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VolumeCondition) Reset() { *m = VolumeCondition{} } +func (*VolumeCondition) ProtoMessage() {} +func (*VolumeCondition) Descriptor() ([]byte, []int) { + return fileDescriptor_5658ecc8e79d3214, []int{2} +} +func (m *VolumeCondition) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *VolumeCondition) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_VolumeCondition.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *VolumeCondition) XXX_Merge(src proto.Message) { + xxx_messageInfo_VolumeCondition.Merge(m, src) +} +func (m *VolumeCondition) XXX_Size() int { + return m.Size() +} +func (m *VolumeCondition) XXX_DiscardUnknown() { + xxx_messageInfo_VolumeCondition.DiscardUnknown(m) +} + +var xxx_messageInfo_VolumeCondition proto.InternalMessageInfo + +func init() { + proto.RegisterEnum("grpc.VolumeUsage_Unit", VolumeUsage_Unit_name, VolumeUsage_Unit_value) + proto.RegisterType((*VolumeStatsResponse)(nil), "grpc.VolumeStatsResponse") + proto.RegisterType((*VolumeUsage)(nil), "grpc.VolumeUsage") + proto.RegisterType((*VolumeCondition)(nil), "grpc.VolumeCondition") +} + +func init() { + proto.RegisterFile("github.com/kata-containers/kata-containers/src/libs/protocols/protos/csi.proto", fileDescriptor_5658ecc8e79d3214) +} + +var fileDescriptor_5658ecc8e79d3214 = []byte{ + // 421 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x92, 0xbf, 0x6e, 0xd4, 0x40, + 0x10, 0xc6, 0x6f, 0x12, 0x5f, 0x92, 0x9b, 0x93, 0xc8, 0xb1, 0xfc, 0x91, 0x15, 0xa1, 0xd5, 0xc9, + 0x0d, 0xa7, 0x48, 0xd8, 0xd2, 0xf1, 0x02, 0x28, 0x10, 0x21, 0x84, 0xe4, 0x48, 0x7b, 0x1c, 0x08, + 0x0a, 0xd0, 0xda, 0x59, 0xcc, 0x2a, 0xf6, 0xae, 0xe5, 0x5d, 0xbb, 0x4e, 0xc9, 0xa3, 0xd0, 0xd0, + 0x53, 0x52, 0xa6, 0xa4, 0xa4, 0xe4, 0xfc, 0x14, 0x94, 0xc8, 0x6b, 0x02, 0x86, 0x54, 0xe9, 0xbe, + 0xdf, 0xf7, 0xcd, 0xfe, 0x99, 0xd1, 0x60, 0x9c, 0x49, 0xfb, 0xa1, 0x4e, 0xc2, 0x54, 0x17, 0xd1, + 0x19, 0xb7, 0xfc, 0x41, 0xaa, 0x95, 0xe5, 0x52, 0x89, 0xca, 0x5c, 0x61, 0x53, 0xa5, 0x51, 0x2e, + 0x13, 0x13, 0x95, 0x95, 0xb6, 0x3a, 0xd5, 0xf9, 0x6f, 0x65, 0xa2, 0xd4, 0xc8, 0xd0, 0x49, 0xe2, + 0x65, 0x55, 0x99, 0x1e, 0x04, 0x99, 0xce, 0x74, 0x1f, 0x26, 0xf5, 0xfb, 0xa8, 0x23, 0x07, 0x4e, + 0xf5, 0x95, 0xc1, 0x39, 0xe0, 0xad, 0x97, 0x3a, 0xaf, 0x0b, 0xb1, 0xb2, 0xdc, 0x1a, 0x26, 0x4c, + 0xa9, 0x95, 0x11, 0xe4, 0x3e, 0x8e, 0x6b, 0xc3, 0x33, 0xe1, 0xc3, 0x7c, 0x7b, 0x31, 0x5d, 0xde, + 0x0c, 0xbb, 0x1b, 0xc3, 0xbe, 0x72, 0xdd, 0x05, 0xac, 0xcf, 0xc9, 0x23, 0x9c, 0x35, 0xce, 0x7d, + 0x97, 0x6a, 0x75, 0x2a, 0xad, 0xd4, 0xca, 0xdf, 0x9a, 0xc3, 0x62, 0xba, 0xbc, 0x33, 0x3c, 0xf3, + 0xf8, 0x32, 0x64, 0xfb, 0xcd, 0xbf, 0x46, 0xf0, 0x19, 0x70, 0x3a, 0xb8, 0x98, 0xdc, 0xc3, 0x09, + 0x6f, 0xb8, 0xcc, 0x79, 0x92, 0x77, 0xcf, 0xc3, 0xc2, 0x63, 0x7f, 0x0d, 0x72, 0x1b, 0xc7, 0x56, + 0x5b, 0x9e, 0xbb, 0x47, 0x3c, 0xd6, 0x03, 0x21, 0xe8, 0xd5, 0x46, 0x9c, 0xfa, 0xdb, 0xce, 0x74, + 0x9a, 0x1c, 0xa2, 0x57, 0x2b, 0x69, 0x7d, 0x6f, 0x0e, 0x8b, 0x1b, 0xcb, 0xbb, 0x57, 0x3a, 0x08, + 0xd7, 0x4a, 0x5a, 0xe6, 0x6a, 0x82, 0x43, 0xf4, 0x3a, 0x22, 0x53, 0xdc, 0x5d, 0xc7, 0xcf, 0xe3, + 0x93, 0x57, 0xf1, 0x6c, 0x44, 0x26, 0x38, 0x3e, 0x7a, 0xfd, 0xe2, 0x78, 0x35, 0x03, 0x82, 0xb8, + 0xf3, 0x2c, 0x3e, 0x79, 0x72, 0xbc, 0x9a, 0x6d, 0x05, 0x4f, 0x71, 0xff, 0xbf, 0x9e, 0xc8, 0x01, + 0xee, 0xf1, 0x44, 0xe9, 0xaa, 0xe0, 0xb9, 0xfb, 0xf1, 0x1e, 0xfb, 0xc3, 0xc4, 0xc7, 0xdd, 0x42, + 0x18, 0x37, 0xcb, 0xee, 0xcb, 0x13, 0x76, 0x89, 0x47, 0x1f, 0xe1, 0x62, 0x43, 0x47, 0xdf, 0x37, + 0x74, 0xf4, 0x73, 0x43, 0xe1, 0xbc, 0xa5, 0xf0, 0xa9, 0xa5, 0xf0, 0xa5, 0xa5, 0xf0, 0xb5, 0xa5, + 0x70, 0xd1, 0x52, 0xf8, 0xd6, 0x52, 0xf8, 0xd1, 0x52, 0x78, 0xf3, 0xf6, 0x9a, 0x0b, 0x52, 0xd5, + 0xca, 0xca, 0x42, 0x44, 0x8d, 0xac, 0xec, 0x20, 0x2a, 0xcf, 0xb2, 0x88, 0x67, 0x42, 0xd9, 0xc1, + 0xf2, 0x74, 0x63, 0x49, 0x76, 0x1c, 0x3f, 0xfc, 0x15, 0x00, 0x00, 0xff, 0xff, 0xce, 0x52, 0xbe, + 0x85, 0x89, 0x02, 0x00, 0x00, +} + +func (this *VolumeStatsResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*VolumeStatsResponse) + if !ok { + that2, ok := that.(VolumeStatsResponse) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if len(this.Usage) != len(that1.Usage) { + return false + } + for i := range this.Usage { + if !this.Usage[i].Equal(that1.Usage[i]) { + return false + } + } + if !this.VolumeCondition.Equal(that1.VolumeCondition) { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (this *VolumeUsage) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*VolumeUsage) + if !ok { + that2, ok := that.(VolumeUsage) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Available != that1.Available { + return false + } + if this.Total != that1.Total { + return false + } + if this.Used != that1.Used { + return false + } + if this.Unit != that1.Unit { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (this *VolumeCondition) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*VolumeCondition) + if !ok { + that2, ok := that.(VolumeCondition) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Abnormal != that1.Abnormal { + return false + } + if this.Message != that1.Message { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} +func (m *VolumeStatsResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeStatsResponse) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *VolumeStatsResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.VolumeCondition != nil { + { + size, err := m.VolumeCondition.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintCsi(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + if len(m.Usage) > 0 { + for iNdEx := len(m.Usage) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Usage[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintCsi(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *VolumeUsage) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeUsage) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *VolumeUsage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.Unit != 0 { + i = encodeVarintCsi(dAtA, i, uint64(m.Unit)) + i-- + dAtA[i] = 0x20 + } + if m.Used != 0 { + i = encodeVarintCsi(dAtA, i, uint64(m.Used)) + i-- + dAtA[i] = 0x18 + } + if m.Total != 0 { + i = encodeVarintCsi(dAtA, i, uint64(m.Total)) + i-- + dAtA[i] = 0x10 + } + if m.Available != 0 { + i = encodeVarintCsi(dAtA, i, uint64(m.Available)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *VolumeCondition) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *VolumeCondition) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *VolumeCondition) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Message) > 0 { + i -= len(m.Message) + copy(dAtA[i:], m.Message) + i = encodeVarintCsi(dAtA, i, uint64(len(m.Message))) + i-- + dAtA[i] = 0x12 + } + if m.Abnormal { + i-- + if m.Abnormal { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func encodeVarintCsi(dAtA []byte, offset int, v uint64) int { + offset -= sovCsi(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func NewPopulatedVolumeStatsResponse(r randyCsi, easy bool) *VolumeStatsResponse { + this := &VolumeStatsResponse{} + if r.Intn(5) != 0 { + v1 := r.Intn(5) + this.Usage = make([]*VolumeUsage, v1) + for i := 0; i < v1; i++ { + this.Usage[i] = NewPopulatedVolumeUsage(r, easy) + } + } + if r.Intn(5) != 0 { + this.VolumeCondition = NewPopulatedVolumeCondition(r, easy) + } + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedCsi(r, 3) + } + return this +} + +func NewPopulatedVolumeUsage(r randyCsi, easy bool) *VolumeUsage { + this := &VolumeUsage{} + this.Available = uint64(uint64(r.Uint32())) + this.Total = uint64(uint64(r.Uint32())) + this.Used = uint64(uint64(r.Uint32())) + this.Unit = VolumeUsage_Unit([]int32{0, 1, 2}[r.Intn(3)]) + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedCsi(r, 5) + } + return this +} + +func NewPopulatedVolumeCondition(r randyCsi, easy bool) *VolumeCondition { + this := &VolumeCondition{} + this.Abnormal = bool(bool(r.Intn(2) == 0)) + this.Message = string(randStringCsi(r)) + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedCsi(r, 3) + } + return this +} + +type randyCsi interface { + Float32() float32 + Float64() float64 + Int63() int64 + Int31() int32 + Uint32() uint32 + Intn(n int) int +} + +func randUTF8RuneCsi(r randyCsi) rune { + ru := r.Intn(62) + if ru < 10 { + return rune(ru + 48) + } else if ru < 36 { + return rune(ru + 55) + } + return rune(ru + 61) +} +func randStringCsi(r randyCsi) string { + v2 := r.Intn(100) + tmps := make([]rune, v2) + for i := 0; i < v2; i++ { + tmps[i] = randUTF8RuneCsi(r) + } + return string(tmps) +} +func randUnrecognizedCsi(r randyCsi, maxFieldNumber int) (dAtA []byte) { + l := r.Intn(5) + for i := 0; i < l; i++ { + wire := r.Intn(4) + if wire == 3 { + wire = 5 + } + fieldNumber := maxFieldNumber + r.Intn(100) + dAtA = randFieldCsi(dAtA, r, fieldNumber, wire) + } + return dAtA +} +func randFieldCsi(dAtA []byte, r randyCsi, fieldNumber int, wire int) []byte { + key := uint32(fieldNumber)<<3 | uint32(wire) + switch wire { + case 0: + dAtA = encodeVarintPopulateCsi(dAtA, uint64(key)) + v3 := r.Int63() + if r.Intn(2) == 0 { + v3 *= -1 + } + dAtA = encodeVarintPopulateCsi(dAtA, uint64(v3)) + case 1: + dAtA = encodeVarintPopulateCsi(dAtA, uint64(key)) + dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) + case 2: + dAtA = encodeVarintPopulateCsi(dAtA, uint64(key)) + ll := r.Intn(100) + dAtA = encodeVarintPopulateCsi(dAtA, uint64(ll)) + for j := 0; j < ll; j++ { + dAtA = append(dAtA, byte(r.Intn(256))) + } + default: + dAtA = encodeVarintPopulateCsi(dAtA, uint64(key)) + dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) + } + return dAtA +} +func encodeVarintPopulateCsi(dAtA []byte, v uint64) []byte { + for v >= 1<<7 { + dAtA = append(dAtA, uint8(uint64(v)&0x7f|0x80)) + v >>= 7 + } + dAtA = append(dAtA, uint8(v)) + return dAtA +} +func (m *VolumeStatsResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Usage) > 0 { + for _, e := range m.Usage { + l = e.Size() + n += 1 + l + sovCsi(uint64(l)) + } + } + if m.VolumeCondition != nil { + l = m.VolumeCondition.Size() + n += 1 + l + sovCsi(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *VolumeUsage) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Available != 0 { + n += 1 + sovCsi(uint64(m.Available)) + } + if m.Total != 0 { + n += 1 + sovCsi(uint64(m.Total)) + } + if m.Used != 0 { + n += 1 + sovCsi(uint64(m.Used)) + } + if m.Unit != 0 { + n += 1 + sovCsi(uint64(m.Unit)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *VolumeCondition) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Abnormal { + n += 2 + } + l = len(m.Message) + if l > 0 { + n += 1 + l + sovCsi(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovCsi(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozCsi(x uint64) (n int) { + return sovCsi(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *VolumeStatsResponse) String() string { + if this == nil { + return "nil" + } + repeatedStringForUsage := "[]*VolumeUsage{" + for _, f := range this.Usage { + repeatedStringForUsage += strings.Replace(f.String(), "VolumeUsage", "VolumeUsage", 1) + "," + } + repeatedStringForUsage += "}" + s := strings.Join([]string{`&VolumeStatsResponse{`, + `Usage:` + repeatedStringForUsage + `,`, + `VolumeCondition:` + strings.Replace(this.VolumeCondition.String(), "VolumeCondition", "VolumeCondition", 1) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} +func (this *VolumeUsage) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&VolumeUsage{`, + `Available:` + fmt.Sprintf("%v", this.Available) + `,`, + `Total:` + fmt.Sprintf("%v", this.Total) + `,`, + `Used:` + fmt.Sprintf("%v", this.Used) + `,`, + `Unit:` + fmt.Sprintf("%v", this.Unit) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} +func (this *VolumeCondition) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&VolumeCondition{`, + `Abnormal:` + fmt.Sprintf("%v", this.Abnormal) + `,`, + `Message:` + fmt.Sprintf("%v", this.Message) + `,`, + `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, + `}`, + }, "") + return s +} +func valueToStringCsi(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *VolumeStatsResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeStatsResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeStatsResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Usage", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthCsi + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthCsi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Usage = append(m.Usage, &VolumeUsage{}) + if err := m.Usage[len(m.Usage)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field VolumeCondition", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthCsi + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthCsi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.VolumeCondition == nil { + m.VolumeCondition = &VolumeCondition{} + } + if err := m.VolumeCondition.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipCsi(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthCsi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VolumeUsage) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeUsage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeUsage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Available", wireType) + } + m.Available = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Available |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Total", wireType) + } + m.Total = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Total |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Used", wireType) + } + m.Used = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Used |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Unit", wireType) + } + m.Unit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Unit |= VolumeUsage_Unit(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipCsi(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthCsi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *VolumeCondition) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: VolumeCondition: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: VolumeCondition: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Abnormal", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Abnormal = bool(v != 0) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCsi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCsi + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthCsi + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Message = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipCsi(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthCsi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipCsi(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCsi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCsi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCsi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthCsi + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupCsi + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthCsi + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthCsi = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowCsi = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupCsi = fmt.Errorf("proto: unexpected end of group") +) diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/csipb_test.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/csipb_test.go new file mode 100644 index 0000000000..87346ddf1d --- /dev/null +++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/csipb_test.go @@ -0,0 +1,585 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: github.com/kata-containers/kata-containers/src/libs/protocols/protos/csi.proto + +package grpc + +import ( + fmt "fmt" + _ "github.com/gogo/protobuf/gogoproto" + github_com_gogo_protobuf_jsonpb "github.com/gogo/protobuf/jsonpb" + github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" + proto "github.com/gogo/protobuf/proto" + math "math" + math_rand "math/rand" + testing "testing" + time "time" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +func TestVolumeStatsResponseProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeStatsResponse(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeStatsResponse{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestVolumeStatsResponseMarshalTo(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeStatsResponse(popr, false) + size := p.Size() + dAtA := make([]byte, size) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + _, err := p.MarshalTo(dAtA) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeStatsResponse{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func BenchmarkVolumeStatsResponseProtoMarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*VolumeStatsResponse, 10000) + for i := 0; i < 10000; i++ { + pops[i] = NewPopulatedVolumeStatsResponse(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000]) + if err != nil { + panic(err) + } + total += len(dAtA) + } + b.SetBytes(int64(total / b.N)) +} + +func BenchmarkVolumeStatsResponseProtoUnmarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + datas := make([][]byte, 10000) + for i := 0; i < 10000; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedVolumeStatsResponse(popr, false)) + if err != nil { + panic(err) + } + datas[i] = dAtA + } + msg := &VolumeStatsResponse{} + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += len(datas[i%10000]) + if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil { + panic(err) + } + } + b.SetBytes(int64(total / b.N)) +} + +func TestVolumeUsageProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeUsage(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeUsage{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestVolumeUsageMarshalTo(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeUsage(popr, false) + size := p.Size() + dAtA := make([]byte, size) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + _, err := p.MarshalTo(dAtA) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeUsage{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func BenchmarkVolumeUsageProtoMarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*VolumeUsage, 10000) + for i := 0; i < 10000; i++ { + pops[i] = NewPopulatedVolumeUsage(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000]) + if err != nil { + panic(err) + } + total += len(dAtA) + } + b.SetBytes(int64(total / b.N)) +} + +func BenchmarkVolumeUsageProtoUnmarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + datas := make([][]byte, 10000) + for i := 0; i < 10000; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedVolumeUsage(popr, false)) + if err != nil { + panic(err) + } + datas[i] = dAtA + } + msg := &VolumeUsage{} + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += len(datas[i%10000]) + if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil { + panic(err) + } + } + b.SetBytes(int64(total / b.N)) +} + +func TestVolumeConditionProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeCondition(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeCondition{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestVolumeConditionMarshalTo(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeCondition(popr, false) + size := p.Size() + dAtA := make([]byte, size) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + _, err := p.MarshalTo(dAtA) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeCondition{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func BenchmarkVolumeConditionProtoMarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*VolumeCondition, 10000) + for i := 0; i < 10000; i++ { + pops[i] = NewPopulatedVolumeCondition(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000]) + if err != nil { + panic(err) + } + total += len(dAtA) + } + b.SetBytes(int64(total / b.N)) +} + +func BenchmarkVolumeConditionProtoUnmarshal(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + datas := make([][]byte, 10000) + for i := 0; i < 10000; i++ { + dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedVolumeCondition(popr, false)) + if err != nil { + panic(err) + } + datas[i] = dAtA + } + msg := &VolumeCondition{} + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += len(datas[i%10000]) + if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil { + panic(err) + } + } + b.SetBytes(int64(total / b.N)) +} + +func TestVolumeStatsResponseJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeStatsResponse(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeStatsResponse{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestVolumeUsageJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeUsage(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeUsage{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestVolumeConditionJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeCondition(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &VolumeCondition{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} +func TestVolumeStatsResponseProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeStatsResponse(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &VolumeStatsResponse{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestVolumeStatsResponseProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeStatsResponse(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &VolumeStatsResponse{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestVolumeUsageProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeUsage(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &VolumeUsage{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestVolumeUsageProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeUsage(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &VolumeUsage{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestVolumeConditionProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeCondition(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &VolumeCondition{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestVolumeConditionProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeCondition(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &VolumeCondition{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestVolumeStatsResponseSize(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeStatsResponse(popr, true) + size2 := github_com_gogo_protobuf_proto.Size(p) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + size := p.Size() + if len(dAtA) != size { + t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA)) + } + if size2 != size { + t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2) + } + size3 := github_com_gogo_protobuf_proto.Size(p) + if size3 != size { + t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3) + } +} + +func BenchmarkVolumeStatsResponseSize(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*VolumeStatsResponse, 1000) + for i := 0; i < 1000; i++ { + pops[i] = NewPopulatedVolumeStatsResponse(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += pops[i%1000].Size() + } + b.SetBytes(int64(total / b.N)) +} + +func TestVolumeUsageSize(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeUsage(popr, true) + size2 := github_com_gogo_protobuf_proto.Size(p) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + size := p.Size() + if len(dAtA) != size { + t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA)) + } + if size2 != size { + t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2) + } + size3 := github_com_gogo_protobuf_proto.Size(p) + if size3 != size { + t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3) + } +} + +func BenchmarkVolumeUsageSize(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*VolumeUsage, 1000) + for i := 0; i < 1000; i++ { + pops[i] = NewPopulatedVolumeUsage(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += pops[i%1000].Size() + } + b.SetBytes(int64(total / b.N)) +} + +func TestVolumeConditionSize(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVolumeCondition(popr, true) + size2 := github_com_gogo_protobuf_proto.Size(p) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + size := p.Size() + if len(dAtA) != size { + t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA)) + } + if size2 != size { + t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2) + } + size3 := github_com_gogo_protobuf_proto.Size(p) + if size3 != size { + t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3) + } +} + +func BenchmarkVolumeConditionSize(b *testing.B) { + popr := math_rand.New(math_rand.NewSource(616)) + total := 0 + pops := make([]*VolumeCondition, 1000) + for i := 0; i < 1000; i++ { + pops[i] = NewPopulatedVolumeCondition(popr, false) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + total += pops[i%1000].Size() + } + b.SetBytes(int64(total / b.N)) +} + +func TestVolumeStatsResponseStringer(t *testing.T) { + popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) + p := NewPopulatedVolumeStatsResponse(popr, false) + s1 := p.String() + s2 := fmt.Sprintf("%v", p) + if s1 != s2 { + t.Fatalf("String want %v got %v", s1, s2) + } +} +func TestVolumeUsageStringer(t *testing.T) { + popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) + p := NewPopulatedVolumeUsage(popr, false) + s1 := p.String() + s2 := fmt.Sprintf("%v", p) + if s1 != s2 { + t.Fatalf("String want %v got %v", s1, s2) + } +} +func TestVolumeConditionStringer(t *testing.T) { + popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) + p := NewPopulatedVolumeCondition(popr, false) + s1 := p.String() + s2 := fmt.Sprintf("%v", p) + if s1 != s2 { + t.Fatalf("String want %v got %v", s1, s2) + } +} + +//These tests are generated by github.com/gogo/protobuf/plugin/testgen diff --git a/src/runtime/virtcontainers/pkg/mock/mock.go b/src/runtime/virtcontainers/pkg/mock/mock.go index 4efdcb0add..d01fefcbd9 100644 --- a/src/runtime/virtcontainers/pkg/mock/mock.go +++ b/src/runtime/virtcontainers/pkg/mock/mock.go @@ -218,3 +218,11 @@ func (p *HybridVSockTTRPCMockImp) GetMetrics(ctx context.Context, req *pb.GetMet func (p *HybridVSockTTRPCMockImp) AddSwap(ctx context.Context, req *pb.AddSwapRequest) (*gpb.Empty, error) { return &gpb.Empty{}, nil } + +func (p *HybridVSockTTRPCMockImp) GetVolumeStats(ctx context.Context, req *pb.VolumeStatsRequest) (*pb.VolumeStatsResponse, error) { + return &pb.VolumeStatsResponse{}, nil +} + +func (p *HybridVSockTTRPCMockImp) ResizeVolume(ctx context.Context, req *pb.ResizeVolumeRequest) (*gpb.Empty, error) { + return &gpb.Empty{}, nil +} From c39281ad65f853e791c36a11092679c8b6dea871 Mon Sep 17 00:00:00 2001 From: Feng Wang Date: Fri, 14 Jan 2022 13:21:41 -0800 Subject: [PATCH 5/9] runtime: update container creation to work with direct assigned volumes During the container creation, it will parse the mount info file of the direct assigned volumes and update the in memory mount object. Fixes: #3454 Signed-off-by: Feng Wang --- src/runtime/virtcontainers/container.go | 23 ++++ src/runtime/virtcontainers/kata_agent.go | 124 +++++++++--------- src/runtime/virtcontainers/kata_agent_test.go | 18 ++- src/runtime/virtcontainers/mount.go | 11 +- 4 files changed, 106 insertions(+), 70 deletions(-) diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 82aae79e37..1a4cb8e839 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -15,6 +15,7 @@ import ( "syscall" "time" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager" @@ -611,6 +612,28 @@ func (c *Container) createBlockDevices(ctx context.Context) error { continue } + // Handle directly assigned volume. Update the mount info based on the mount info json. + mntInfo, e := volume.VolumeMountInfo(m.Source) + if e != nil && !os.IsNotExist(e) { + c.Logger().WithError(e).WithField("mount-source", m.Source). + Error("failed to parse the mount info file for a direct assigned volume") + continue + } + + if mntInfo != nil { + // Write out sandbox info file on the mount source to allow CSI to communicate with the runtime + if err := volume.RecordSandboxId(c.sandboxID, m.Source); err != nil { + c.Logger().WithError(err).Error("error writing sandbox info") + } + + c.mounts[i].Source = mntInfo.Device + c.mounts[i].Type = mntInfo.FsType + c.mounts[i].Options = mntInfo.Options + m.Source = mntInfo.Device + m.Type = mntInfo.FsType + m.Options = mntInfo.Options + } + var stat unix.Stat_t if err := unix.Stat(m.Source, &stat); err != nil { return fmt.Errorf("stat %q failed: %v", m.Source, err) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 47a1c1cc3f..71781e4ade 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -882,35 +882,6 @@ func (k *kataAgent) removeIgnoredOCIMount(spec *specs.Spec, ignoredMounts map[st return nil } -func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages []*grpc.Storage) error { - ociMounts := spec.Mounts - var index int - var m specs.Mount - - for i, v := range volumeStorages { - for index, m = range ociMounts { - if m.Destination != v.MountPoint { - continue - } - - // Create a temporary location to mount the Storage. Mounting to the correct location - // will be handled by the OCI mount structure. - filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination)) - path := filepath.Join(kataGuestSandboxStorageDir(), filename) - - k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path) - ociMounts[index].Source = path - volumeStorages[i].MountPoint = path - - break - } - if index == len(ociMounts) { - return fmt.Errorf("OCI mount not found for block volume %s", v.MountPoint) - } - } - return nil -} - func (k *kataAgent) constrainGRPCSpec(grpcSpec *grpc.Spec, passSeccomp bool, stripVfio bool) { // Disable Hooks since they have been handled on the host and there is // no reason to send them to the agent. It would make no sense to try @@ -1247,19 +1218,13 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co // Append container devices for block devices passed with --device. ctrDevices = k.appendDevices(ctrDevices, c) - // Handle all the volumes that are block device files. - // Note this call modifies the list of container devices to make sure - // all hotplugged devices are unplugged, so this needs be done - // after devices passed with --device are handled. - volumeStorages, err := k.handleBlockVolumes(c) + // Block based volumes will require some adjustments in the OCI spec, and creation of + // storage objects to pass to the agent. + volumeStorages, err := k.handleBlkOCIMounts(c, ociSpec) if err != nil { return nil, err } - if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil { - return nil, err - } - ctrStorages = append(ctrStorages, volumeStorages...) grpcSpec, err := grpc.OCItoGRPC(ociSpec) @@ -1522,16 +1487,46 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, m Mount, device api.D vol.Options = []string{"bind"} vol.MountPoint = m.Destination + // Assign the type from the mount, if it's specified (e.g. direct assigned volume) + if m.Type != "" { + vol.Fstype = m.Type + vol.Options = m.Options + } + return vol, nil } -// handleBlockVolumes handles volumes that are block devices files -// by passing the block devices as Storage to the agent. -func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { +func (k *kataAgent) createBlkStorageObject(c *Container, m Mount) (*grpc.Storage, error) { + var vol *grpc.Storage + + id := m.BlockDeviceID + device := c.sandbox.devManager.GetDeviceByID(id) + if device == nil { + k.Logger().WithField("device", id).Error("failed to find device by id") + return nil, fmt.Errorf("Failed to find device by id (id=%s)", id) + } + + var err error + switch device.DeviceType() { + case config.DeviceBlock: + vol, err = k.handleDeviceBlockVolume(c, m, device) + case config.VhostUserBlk: + vol, err = k.handleVhostUserBlkVolume(c, m, device) + default: + return nil, fmt.Errorf("Unknown device type") + } + + return vol, err +} + +// handleBlkOCIMounts will create a unique destination mountpoint in the guest for each volume in the +// given container and will update the OCI spec to utilize this mount point as the new source for the +// container volume. The container mount structure is updated to store the guest destination mountpoint. +func (k *kataAgent) handleBlkOCIMounts(c *Container, spec *specs.Spec) ([]*grpc.Storage, error) { var volumeStorages []*grpc.Storage - for _, m := range c.mounts { + for i, m := range c.mounts { id := m.BlockDeviceID if len(id) == 0 { @@ -1542,29 +1537,36 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { // device is detached with detachDevices() for a container. c.devices = append(c.devices, ContainerDevice{ID: id, ContainerPath: m.Destination}) - var vol *grpc.Storage - - device := c.sandbox.devManager.GetDeviceByID(id) - if device == nil { - k.Logger().WithField("device", id).Error("failed to find device by id") - return nil, fmt.Errorf("Failed to find device by id (id=%s)", id) - } - - var err error - switch device.DeviceType() { - case config.DeviceBlock: - vol, err = k.handleDeviceBlockVolume(c, m, device) - case config.VhostUserBlk: - vol, err = k.handleVhostUserBlkVolume(c, m, device) - default: - k.Logger().Error("Unknown device type") - continue - } - + // Create Storage structure + vol, err := k.createBlkStorageObject(c, m) if vol == nil || err != nil { return nil, err } + // The device will be mounted at a unique location within the VM. Mounting + // to the container specific location is handled within the OCI spec. Let's ensure that + // the storage mount point is unique, and that this is utilized as the source in the OCI + // spec. + filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(vol.MountPoint)) + path := filepath.Join(kataGuestSandboxStorageDir(), filename) + + // Update applicable OCI mount source + for idx, ociMount := range spec.Mounts { + if ociMount.Destination != vol.MountPoint { + continue + } + k.Logger().WithFields(logrus.Fields{ + "original-source": ociMount.Source, + "new-source": path, + }).Debug("Replacing OCI mount source") + spec.Mounts[idx].Source = path + break + } + + // Update storage mountpoint, and save guest device mount path to container mount struct: + vol.MountPoint = path + c.mounts[i].GuestDeviceMount = path + volumeStorages = append(volumeStorages, vol) } diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index 16dd9dcfe1..9f861e932b 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -398,24 +398,28 @@ func TestHandleBlockVolume(t *testing.T) { containers[c.id].sandbox = &sandbox containers[c.id].mounts = mounts - volumeStorages, err := k.handleBlockVolumes(c) + vStorage, err := k.createBlkStorageObject(c, vMount) + assert.Nil(t, err, "Error while handling block volumes") + bStorage, err := k.createBlkStorageObject(c, bMount) + assert.Nil(t, err, "Error while handling block volumes") + dStorage, err := k.createBlkStorageObject(c, dMount) assert.Nil(t, err, "Error while handling block volumes") - vStorage := &pb.Storage{ + vStorageExpected := &pb.Storage{ MountPoint: vDestination, Fstype: "bind", Options: []string{"bind"}, Driver: kataBlkDevType, Source: vPCIPath.String(), } - bStorage := &pb.Storage{ + bStorageExpected := &pb.Storage{ MountPoint: bDestination, Fstype: "bind", Options: []string{"bind"}, Driver: kataBlkDevType, Source: bPCIPath.String(), } - dStorage := &pb.Storage{ + dStorageExpected := &pb.Storage{ MountPoint: dDestination, Fstype: "ext4", Options: []string{"ro"}, @@ -423,9 +427,9 @@ func TestHandleBlockVolume(t *testing.T) { Source: dPCIPath.String(), } - assert.Equal(t, vStorage, volumeStorages[0], "Error while handle VhostUserBlk type block volume") - assert.Equal(t, bStorage, volumeStorages[1], "Error while handle BlockDevice type block volume") - assert.Equal(t, dStorage, volumeStorages[2], "Error while handle direct BlockDevice type block volume") + assert.Equal(t, vStorage, vStorageExpected, "Error while handle VhostUserBlk type block volume") + assert.Equal(t, bStorage, bStorageExpected, "Error while handle BlockDevice type block volume") + assert.Equal(t, dStorage, dStorageExpected, "Error while handle direct BlockDevice type block volume") } func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) { diff --git a/src/runtime/virtcontainers/mount.go b/src/runtime/virtcontainers/mount.go index c2879f2135..0e83bc6899 100644 --- a/src/runtime/virtcontainers/mount.go +++ b/src/runtime/virtcontainers/mount.go @@ -172,7 +172,7 @@ func getDeviceForPath(path string) (device, error) { }, nil } - // We get the mount point by recursively peforming stat on the path + // We get the mount point by recursively performing stat on the path // The point where the device changes indicates the mountpoint for { if mountPoint == "/" { @@ -326,7 +326,9 @@ func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string // Mount describes a container mount. type Mount struct { - Source string + // Source is the source of the mount. + Source string + // Destination is the destination of the mount (within the container). Destination string // Type specifies the type of filesystem to mount. @@ -335,6 +337,11 @@ type Mount struct { // HostPath used to store host side bind mount path HostPath string + // GuestDeviceMount represents the path within the VM that the device + // is mounted. Only relevant for block devices. This is tracked in the event + // runtime wants to query the agent for mount stats. + GuestDeviceMount string + // BlockDeviceID represents block device that is attached to the // VM in case this mount is a block device file or a directory // backed by a block device. From ea51ef1c409bc05ab1d8a3c5bc31a3b803616c31 Mon Sep 17 00:00:00 2001 From: Feng Wang Date: Fri, 14 Jan 2022 13:23:55 -0800 Subject: [PATCH 6/9] runtime: forward the stat and resize requests from shimv2 to kata agent Translate the volume path from host-known path to guest-known path and forward the request to kata agent. Fixes: #3454 Signed-off-by: Feng Wang --- src/runtime/virtcontainers/agent.go | 6 +++ src/runtime/virtcontainers/interfaces.go | 3 ++ src/runtime/virtcontainers/kata_agent.go | 27 ++++++++++++++ src/runtime/virtcontainers/mock_agent.go | 8 ++++ .../virtcontainers/pkg/vcmock/sandbox.go | 7 ++++ src/runtime/virtcontainers/sandbox.go | 37 +++++++++++++++++++ 6 files changed, 88 insertions(+) diff --git a/src/runtime/virtcontainers/agent.go b/src/runtime/virtcontainers/agent.go index a7888552b5..0d6d6b7951 100644 --- a/src/runtime/virtcontainers/agent.go +++ b/src/runtime/virtcontainers/agent.go @@ -190,4 +190,10 @@ type agent interface { // getAgentMetrics get metrics of agent and guest through agent getAgentMetrics(context.Context, *grpc.GetMetricsRequest) (*grpc.Metrics, error) + + //getGuestVolumeStats get the filesystem stats of a volume specified by the volume mount path on the guest. + getGuestVolumeStats(ctx context.Context, volumeGuestPath string) ([]byte, error) + + // resizeGuestVolume resizes a volume specified by the volume mount path on the guest. + resizeGuestVolume(ctx context.Context, volumeGuestPath string, size uint64) error } diff --git a/src/runtime/virtcontainers/interfaces.go b/src/runtime/virtcontainers/interfaces.go index 85c9d9fe61..bb3935be06 100644 --- a/src/runtime/virtcontainers/interfaces.go +++ b/src/runtime/virtcontainers/interfaces.go @@ -76,6 +76,9 @@ type VCSandbox interface { UpdateRuntimeMetrics() error GetAgentMetrics(ctx context.Context) (string, error) GetAgentURL() (string, error) + + GuestVolumeStats(ctx context.Context, volumePath string) ([]byte, error) + ResizeGuestVolume(ctx context.Context, volumePath string, size uint64) error } // VCContainer is the Container interface diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 71781e4ade..6bc8097b7b 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -138,6 +138,8 @@ const ( grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" grpcGetMetricsRequest = "grpc.GetMetricsRequest" grpcAddSwapRequest = "grpc.AddSwapRequest" + grpcVolumeStatsRequest = "grpc.VolumeStatsRequest" + grpcResizeVolumeRequest = "grpc.ResizeVolumeRequest" ) // newKataAgent returns an agent from an agent type. @@ -1952,6 +1954,12 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { k.reqHandlers[grpcAddSwapRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { return k.client.AgentServiceClient.AddSwap(ctx, req.(*grpc.AddSwapRequest)) } + k.reqHandlers[grpcVolumeStatsRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { + return k.client.AgentServiceClient.GetVolumeStats(ctx, req.(*grpc.VolumeStatsRequest)) + } + k.reqHandlers[grpcResizeVolumeRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { + return k.client.AgentServiceClient.ResizeVolume(ctx, req.(*grpc.ResizeVolumeRequest)) + } } func (k *kataAgent) getReqContext(ctx context.Context, reqName string) (newCtx context.Context, cancel context.CancelFunc) { @@ -2169,3 +2177,22 @@ func (k *kataAgent) getAgentMetrics(ctx context.Context, req *grpc.GetMetricsReq return resp.(*grpc.Metrics), nil } + +func (k *kataAgent) getGuestVolumeStats(ctx context.Context, volumeGuestPath string) ([]byte, error) { + result, err := k.sendReq(ctx, &grpc.VolumeStatsRequest{VolumeGuestPath: volumeGuestPath}) + if err != nil { + return nil, err + } + + buf, err := json.Marshal(result.(*grpc.VolumeStatsResponse)) + if err != nil { + return nil, err + } + + return buf, nil +} + +func (k *kataAgent) resizeGuestVolume(ctx context.Context, volumeGuestPath string, size uint64) error { + _, err := k.sendReq(ctx, &grpc.ResizeVolumeRequest{VolumeGuestPath: volumeGuestPath, Size_: size}) + return err +} diff --git a/src/runtime/virtcontainers/mock_agent.go b/src/runtime/virtcontainers/mock_agent.go index 393597397a..99df3537b4 100644 --- a/src/runtime/virtcontainers/mock_agent.go +++ b/src/runtime/virtcontainers/mock_agent.go @@ -242,3 +242,11 @@ func (n *mockAgent) getOOMEvent(ctx context.Context) (string, error) { func (n *mockAgent) getAgentMetrics(ctx context.Context, req *grpc.GetMetricsRequest) (*grpc.Metrics, error) { return nil, nil } + +func (n *mockAgent) getGuestVolumeStats(ctx context.Context, volumeGuestPath string) ([]byte, error) { + return nil, nil +} + +func (n *mockAgent) resizeGuestVolume(ctx context.Context, volumeGuestPath string, size uint64) error { + return nil +} diff --git a/src/runtime/virtcontainers/pkg/vcmock/sandbox.go b/src/runtime/virtcontainers/pkg/vcmock/sandbox.go index 17fdeb4bf7..b3b5f7a8d1 100644 --- a/src/runtime/virtcontainers/pkg/vcmock/sandbox.go +++ b/src/runtime/virtcontainers/pkg/vcmock/sandbox.go @@ -254,3 +254,10 @@ func (s *Sandbox) GetAgentURL() (string, error) { func (s *Sandbox) GetHypervisorPid() (int, error) { return 0, nil } + +func (s *Sandbox) GuestVolumeStats(ctx context.Context, path string) ([]byte, error) { + return nil, nil +} +func (s *Sandbox) ResizeGuestVolume(ctx context.Context, path string, size uint64) error { + return nil +} diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 8c72d90cec..9e1ac02b50 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -2257,6 +2257,43 @@ func (s *Sandbox) GetAgentURL() (string, error) { return s.agent.getAgentURL() } +// GuestVolumeStats return the filesystem stat of a given volume in the guest. +func (s *Sandbox) GuestVolumeStats(ctx context.Context, volumePath string) ([]byte, error) { + guestMountPath, err := s.guestMountPath(volumePath) + if err != nil { + return nil, err + } + return s.agent.getGuestVolumeStats(ctx, guestMountPath) +} + +// ResizeGuestVolume resizes a volume in the guest. +func (s *Sandbox) ResizeGuestVolume(ctx context.Context, volumePath string, size uint64) error { + // TODO: https://github.com/kata-containers/kata-containers/issues/3694. + guestMountPath, err := s.guestMountPath(volumePath) + if err != nil { + return err + } + return s.agent.resizeGuestVolume(ctx, guestMountPath, size) +} + +func (s *Sandbox) guestMountPath(volumePath string) (string, error) { + // verify the device even exists + if _, err := os.Stat(volumePath); err != nil { + s.Logger().WithError(err).WithField("volume", volumePath).Error("Cannot get stats for volume that doesn't exist") + return "", err + } + + // verify that we have a mount in this sandbox who's source maps to this + for _, c := range s.containers { + for _, m := range c.mounts { + if volumePath == m.Source { + return m.GuestDeviceMount, nil + } + } + } + return "", fmt.Errorf("mount %s not found in sandbox", volumePath) +} + // getSandboxCPUSet returns the union of each of the sandbox's containers' CPU sets' // cpus and mems as a string in canonical linux CPU/mems list format func (s *Sandbox) getSandboxCPUSet() (string, string, error) { From 27fb490228187efde5181d23ab1f173a8df05739 Mon Sep 17 00:00:00 2001 From: shuochen0311 Date: Wed, 19 Jan 2022 09:02:18 -0800 Subject: [PATCH 7/9] agent: add get volume stats handler in agent retrieve the stats of direct-assigned volumes from the guest Fixes: #3454 Signed-off-by: shuochen0311 --- src/agent/Cargo.lock | 71 ++++++++++++++++++++++++ src/agent/Cargo.toml | 1 + src/agent/src/rpc.rs | 91 ++++++++++++++++++++++++++++++- src/tools/agent-ctl/src/client.rs | 28 ++++++++++ 4 files changed, 189 insertions(+), 2 deletions(-) diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index ce0bc5f1ad..5c9862e253 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -214,6 +214,12 @@ dependencies = [ "syn", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "crc32fast" version = "1.3.0" @@ -233,6 +239,30 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97242a70df9b89a65d0b6df3c4bf5b9ce03c5b7309019777fbde37e7537f8762" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.5" @@ -565,6 +595,7 @@ dependencies = [ "slog", "slog-scope", "slog-stdlog", + "sysinfo", "tempfile", "thiserror", "tokio", @@ -1238,6 +1269,31 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.10" @@ -1518,6 +1574,21 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "sysinfo" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e757000a4bed2b1be9be65a3f418b9696adf30bb419214c73997422de73a591" +dependencies = [ + "cfg-if 1.0.0", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + [[package]] name = "take_mut" version = "0.2.2" diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index e8f0f828b3..521fbd5dc6 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -20,6 +20,7 @@ scopeguard = "1.0.0" thiserror = "1.0.26" regex = "1.5.4" serial_test = "0.5.1" +sysinfo = "0.23.0" # Async helpers async-trait = "0.1.42" diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index 276746b9fe..cb188c60d5 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -23,9 +23,10 @@ use oci::{LinuxNamespace, Root, Spec}; use protobuf::{Message, RepeatedField, SingularPtrField}; use protocols::agent::{ AddSwapRequest, AgentDetails, CopyFileRequest, GuestDetailsResponse, Interfaces, Metrics, - OOMEvent, ReadStreamResponse, Routes, StatsContainerResponse, WaitProcessResponse, - WriteStreamResponse, + OOMEvent, ReadStreamResponse, Routes, StatsContainerResponse, VolumeStatsRequest, + WaitProcessResponse, WriteStreamResponse, }; +use protocols::csi::{VolumeCondition, VolumeStatsResponse, VolumeUsage, VolumeUsage_Unit}; use protocols::empty::Empty; use protocols::health::{ HealthCheckResponse, HealthCheckResponse_ServingStatus, VersionCheckResponse, @@ -43,6 +44,8 @@ use nix::sys::stat; use nix::unistd::{self, Pid}; use rustjail::process::ProcessOperations; +use sysinfo::{DiskExt, System, SystemExt}; + use crate::device::{ add_devices, get_virtio_blk_pci_device_name, update_device_cgroup, update_env_pci, }; @@ -68,6 +71,7 @@ use tracing::instrument; use libc::{self, c_char, c_ushort, pid_t, winsize, TIOCSWINSZ}; use std::convert::TryFrom; use std::fs; +use std::os::unix::fs::MetadataExt; use std::os::unix::prelude::PermissionsExt; use std::process::{Command, Stdio}; use std::time::Duration; @@ -1254,6 +1258,47 @@ impl protocols::agent_ttrpc::AgentService for AgentService { Err(ttrpc_error!(ttrpc::Code::INTERNAL, "")) } + async fn get_volume_stats( + &self, + ctx: &TtrpcContext, + req: VolumeStatsRequest, + ) -> ttrpc::Result { + trace_rpc_call!(ctx, "get_volume_stats", req); + is_allowed!(req); + + info!(sl!(), "get volume stats!"); + let mut resp = VolumeStatsResponse::new(); + + let mut condition = VolumeCondition::new(); + + match File::open(&req.volume_guest_path) { + Ok(_) => { + condition.abnormal = false; + condition.message = String::from("OK"); + } + Err(e) => { + info!(sl!(), "failed to open the volume"); + return Err(ttrpc_error!(ttrpc::Code::INTERNAL, e)); + } + }; + + let mut usage_vec = Vec::new(); + + // to get volume capacity stats + get_volume_capacity_stats(&req.volume_guest_path) + .map(|u| usage_vec.push(u)) + .map_err(|e| ttrpc_error!(ttrpc::Code::INTERNAL, e))?; + + // to get volume inode stats + get_volume_inode_stats(&req.volume_guest_path) + .map(|u| usage_vec.push(u)) + .map_err(|e| ttrpc_error!(ttrpc::Code::INTERNAL, e))?; + + resp.usage = RepeatedField::from_vec(usage_vec); + resp.volume_condition = SingularPtrField::some(condition); + Ok(resp) + } + async fn add_swap( &self, ctx: &TtrpcContext, @@ -1341,6 +1386,48 @@ fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> { Ok((size, plug)) } +fn get_volume_capacity_stats(path: &str) -> Result { + let mut usage = VolumeUsage::new(); + + let s = System::new(); + for disk in s.disks() { + if let Some(v) = disk.name().to_str() { + if v.to_string().eq(path) { + usage.available = disk.available_space(); + usage.total = disk.total_space(); + usage.used = usage.total - usage.available; + usage.unit = VolumeUsage_Unit::BYTES; // bytes + break; + } + } else { + return Err(anyhow!(nix::Error::EINVAL)); + } + } + + Ok(usage) +} + +fn get_volume_inode_stats(path: &str) -> Result { + let mut usage = VolumeUsage::new(); + + let s = System::new(); + for disk in s.disks() { + if let Some(v) = disk.name().to_str() { + if v.to_string().eq(path) { + let meta = fs::metadata(disk.mount_point())?; + let inode = meta.ino(); + usage.used = inode; + usage.unit = VolumeUsage_Unit::INODES; + break; + } + } else { + return Err(anyhow!(nix::Error::EINVAL)); + } + } + + Ok(usage) +} + pub fn have_seccomp() -> bool { if cfg!(feature = "seccomp") { return true; diff --git a/src/tools/agent-ctl/src/client.rs b/src/tools/agent-ctl/src/client.rs index a27b88cc89..a4f1e45a82 100644 --- a/src/tools/agent-ctl/src/client.rs +++ b/src/tools/agent-ctl/src/client.rs @@ -171,6 +171,11 @@ static AGENT_CMDS: &[AgentCmd] = &[ st: ServiceType::Agent, fp: agent_cmd_sandbox_get_oom_event, }, + AgentCmd { + name: "GetVolumeStats", + st: ServiceType::Agent, + fp: agent_cmd_sandbox_get_volume_stats, + }, AgentCmd { name: "ListInterfaces", st: ServiceType::Agent, @@ -1641,6 +1646,29 @@ fn agent_cmd_sandbox_get_oom_event( Ok(()) } +fn agent_cmd_sandbox_get_volume_stats( + ctx: &Context, + client: &AgentServiceClient, + _health: &HealthClient, + _options: &mut Options, + args: &str, +) -> Result<()> { + let req: VolumeStatsRequest = utils::make_request(args)?; + + let ctx = clone_context(ctx); + + debug!(sl!(), "sending request"; "request" => format!("{:?}", req)); + + let reply = client + .get_volume_stats(ctx, &req) + .map_err(|e| anyhow!(e).context(ERR_API_FAILED))?; + + info!(sl!(), "response received"; + "response" => format!("{:?}", reply)); + + Ok(()) +} + fn agent_cmd_sandbox_copy_file( ctx: &Context, client: &AgentServiceClient, From f905161bbb9d560baf76ef07ff7a7e90fdfe687a Mon Sep 17 00:00:00 2001 From: Feng Wang Date: Mon, 28 Feb 2022 20:44:31 -0800 Subject: [PATCH 8/9] runtime: mount direct-assigned block device fs only once Mount the direct-assigned block device fs only once and keep a refcount in the guest. Also use the ro flag inside the options field to determine whether the block device and filesystem should be mounted as ro Fixes: #3454 Signed-off-by: Feng Wang --- src/agent/src/mount.rs | 24 +++++++++--------------- src/agent/src/rpc.rs | 4 +--- src/runtime/virtcontainers/container.go | 10 ++++++++++ src/runtime/virtcontainers/kata_agent.go | 12 ++++++++---- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/agent/src/mount.rs b/src/agent/src/mount.rs index d7dbc08ef4..fbead0953a 100644 --- a/src/agent/src/mount.rs +++ b/src/agent/src/mount.rs @@ -193,13 +193,6 @@ async fn ephemeral_storage_handler( storage: &Storage, sandbox: Arc>, ) -> Result { - let mut sb = sandbox.lock().await; - let new_storage = sb.set_sandbox_storage(&storage.mount_point); - - if !new_storage { - return Ok("".to_string()); - } - // hugetlbfs if storage.fstype == FS_TYPE_HUGETLB { return handle_hugetlbfs_storage(logger, storage).await; @@ -255,13 +248,6 @@ async fn local_storage_handler( storage: &Storage, sandbox: Arc>, ) -> Result { - let mut sb = sandbox.lock().await; - let new_storage = sb.set_sandbox_storage(&storage.mount_point); - - if !new_storage { - return Ok("".to_string()); - } - fs::create_dir_all(&storage.mount_point).context(format!( "failed to create dir all {:?}", &storage.mount_point @@ -401,7 +387,7 @@ fn get_pagesize_and_size_from_option(options: &[String]) -> Result<(u64, u64)> { async fn virtiommio_blk_storage_handler( logger: &Logger, storage: &Storage, - _sandbox: Arc>, + sandbox: Arc>, ) -> Result { //The source path is VmPath common_storage_handler(logger, storage) @@ -641,6 +627,14 @@ pub async fn add_storages( "subsystem" => "storage", "storage-type" => handler_name.to_owned())); + { + let mut sb = sandbox.lock().await; + let new_storage = sb.set_sandbox_storage(&storage.mount_point); + if !new_storage { + continue; + } + } + let res = match handler_name.as_str() { DRIVER_BLK_TYPE => virtio_blk_storage_handler(&logger, &storage, sandbox.clone()).await, DRIVER_BLK_CCW_TYPE => { diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs index cb188c60d5..1f4729b7d0 100644 --- a/src/agent/src/rpc.rs +++ b/src/agent/src/rpc.rs @@ -51,7 +51,7 @@ use crate::device::{ }; use crate::linux_abi::*; use crate::metrics::get_metrics; -use crate::mount::{add_storages, baremount, remove_mounts, STORAGE_HANDLER_LIST}; +use crate::mount::{add_storages, baremount, STORAGE_HANDLER_LIST}; use crate::namespace::{NSTYPEIPC, NSTYPEPID, NSTYPEUTS}; use crate::network::setup_guest_dns; use crate::pci; @@ -287,8 +287,6 @@ impl AgentService { // Find the sandbox storage used by this container let mounts = sandbox.container_mounts.get(&cid); if let Some(mounts) = mounts { - remove_mounts(mounts)?; - for m in mounts.iter() { if sandbox.storages.get(m).is_some() { cmounts.push(m.to_string()); diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 1a4cb8e839..809ba88e3d 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -626,12 +626,22 @@ func (c *Container) createBlockDevices(ctx context.Context) error { c.Logger().WithError(err).Error("error writing sandbox info") } + readonly := false + for _, flag := range mntInfo.Options { + if flag == "ro" { + readonly = true + break + } + } + c.mounts[i].Source = mntInfo.Device c.mounts[i].Type = mntInfo.FsType c.mounts[i].Options = mntInfo.Options + c.mounts[i].ReadOnly = readonly m.Source = mntInfo.Device m.Type = mntInfo.FsType m.Options = mntInfo.Options + m.ReadOnly = readonly } var stat unix.Stat_t diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 6bc8097b7b..339b2220b2 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -6,6 +6,7 @@ package virtcontainers import ( + b64 "encoding/base64" "encoding/json" "errors" "fmt" @@ -1545,11 +1546,14 @@ func (k *kataAgent) handleBlkOCIMounts(c *Container, spec *specs.Spec) ([]*grpc. return nil, err } - // The device will be mounted at a unique location within the VM. Mounting + // Each device will be mounted at a unique location within the VM only once. Mounting // to the container specific location is handled within the OCI spec. Let's ensure that - // the storage mount point is unique, and that this is utilized as the source in the OCI - // spec. - filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(vol.MountPoint)) + // the storage mount point is unique for each device. This is then utilized as the source + // in the OCI spec. If multiple containers mount the same block device, it's refcounted inside + // the guest by Kata agent. + filename := b64.StdEncoding.EncodeToString([]byte(vol.Source)) + // Make the base64 encoding path safe. + filename = strings.ReplaceAll(filename, "/", "_") path := filepath.Join(kataGuestSandboxStorageDir(), filename) // Update applicable OCI mount source From e76519af83f94e8541f6c4fe252ab657c55a7688 Mon Sep 17 00:00:00 2001 From: Feng Wang Date: Fri, 4 Mar 2022 09:20:12 -0800 Subject: [PATCH 9/9] runtime: small refactor to improve readability Remove some confusing/duplicate code so it's more readable Fixes: #3454 Signed-off-by: Feng Wang --- src/runtime/virtcontainers/container.go | 32 +++++++++++-------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 809ba88e3d..b95e72ddc4 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -599,30 +599,30 @@ func (c *Container) createBlockDevices(ctx context.Context) error { } // iterate all mounts and create block device if it's block based. - for i, m := range c.mounts { - if len(m.BlockDeviceID) > 0 { + for i := range c.mounts { + if len(c.mounts[i].BlockDeviceID) > 0 { // Non-empty m.BlockDeviceID indicates there's already one device // associated with the mount,so no need to create a new device for it // and we only create block device for bind mount continue } - if m.Type != "bind" { + if c.mounts[i].Type != "bind" { // We only handle for bind-mounts continue } // Handle directly assigned volume. Update the mount info based on the mount info json. - mntInfo, e := volume.VolumeMountInfo(m.Source) + mntInfo, e := volume.VolumeMountInfo(c.mounts[i].Source) if e != nil && !os.IsNotExist(e) { - c.Logger().WithError(e).WithField("mount-source", m.Source). + c.Logger().WithError(e).WithField("mount-source", c.mounts[i].Source). Error("failed to parse the mount info file for a direct assigned volume") continue } if mntInfo != nil { // Write out sandbox info file on the mount source to allow CSI to communicate with the runtime - if err := volume.RecordSandboxId(c.sandboxID, m.Source); err != nil { + if err := volume.RecordSandboxId(c.sandboxID, c.mounts[i].Source); err != nil { c.Logger().WithError(err).Error("error writing sandbox info") } @@ -638,15 +638,11 @@ func (c *Container) createBlockDevices(ctx context.Context) error { c.mounts[i].Type = mntInfo.FsType c.mounts[i].Options = mntInfo.Options c.mounts[i].ReadOnly = readonly - m.Source = mntInfo.Device - m.Type = mntInfo.FsType - m.Options = mntInfo.Options - m.ReadOnly = readonly } var stat unix.Stat_t - if err := unix.Stat(m.Source, &stat); err != nil { - return fmt.Errorf("stat %q failed: %v", m.Source, err) + if err := unix.Stat(c.mounts[i].Source, &stat); err != nil { + return fmt.Errorf("stat %q failed: %v", c.mounts[i].Source, err) } var di *config.DeviceInfo @@ -656,17 +652,17 @@ func (c *Container) createBlockDevices(ctx context.Context) error { // instead of passing this as a shared mount. if stat.Mode&unix.S_IFBLK == unix.S_IFBLK { di = &config.DeviceInfo{ - HostPath: m.Source, - ContainerPath: m.Destination, + HostPath: c.mounts[i].Source, + ContainerPath: c.mounts[i].Destination, DevType: "b", Major: int64(unix.Major(uint64(stat.Rdev))), Minor: int64(unix.Minor(uint64(stat.Rdev))), - ReadOnly: m.ReadOnly, + ReadOnly: c.mounts[i].ReadOnly, } // Check whether source can be used as a pmem device - } else if di, err = config.PmemDeviceInfo(m.Source, m.Destination); err != nil { + } else if di, err = config.PmemDeviceInfo(c.mounts[i].Source, c.mounts[i].Destination); err != nil { c.Logger().WithError(err). - WithField("mount-source", m.Source). + WithField("mount-source", c.mounts[i].Source). Debug("no loop device") } @@ -675,7 +671,7 @@ func (c *Container) createBlockDevices(ctx context.Context) error { if err != nil { // Do not return an error, try to create // devices for other mounts - c.Logger().WithError(err).WithField("mount-source", m.Source). + c.Logger().WithError(err).WithField("mount-source", c.mounts[i].Source). Error("device manager failed to create new device") continue