diff --git a/.gitignore b/.gitignore index 29f801ccdc..58b1b58a28 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /cli/coverage.html /cli/config-generated.go /cli/config/configuration.toml +/kata-netmon /virtcontainers/hack/virtc/virtc /virtcontainers/hook/mock/hook /virtcontainers/shim/mock/shim diff --git a/Gopkg.lock b/Gopkg.lock index a7a9c88e9b..1866ed7a63 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -130,14 +130,14 @@ revision = "032705ba6aae05a9bf41e296cf89c8529cffb822" [[projects]] - digest = "1:01c37fcb6e2a1fe1321a97faaef74c66ac531ea292ca3f929b7189cc400b1d47" + digest = "1:aec1ed6dbfffe247e5a8a9e3102206fe97552077e10ea44e3e35dce98ab6e5aa" name = "github.com/kata-containers/agent" packages = [ "protocols/client", "protocols/grpc", ] pruneopts = "NUT" - revision = "46396d205bf096db4e69fcfa319525858ce8050c" + revision = "7c95a50ef97052bf7f5566dcca53d6611f7458ac" [[projects]] digest = "1:04054595e5c5a35d1553a7f3464d18577caf597445d643992998643df56d4afd" diff --git a/Gopkg.toml b/Gopkg.toml index c5ea29b91f..5bf281f7e6 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -56,7 +56,7 @@ [[constraint]] name = "github.com/kata-containers/agent" - revision = "46396d205bf096db4e69fcfa319525858ce8050c" + revision = "7c95a50ef97052bf7f5566dcca53d6611f7458ac" [[constraint]] name = "github.com/containerd/cri-containerd" diff --git a/Makefile b/Makefile index fa01aa8ca0..96b26dc6eb 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,7 @@ SCRIPTS := # list of binaries to install BINLIST := +BINLIBEXECLIST := BIN_PREFIX = $(PROJECT_TYPE) PROJECT_DIR = $(PROJECT_TAG) @@ -49,6 +50,11 @@ TARGET = $(BIN_PREFIX)-runtime TARGET_OUTPUT = $(CURDIR)/$(TARGET) BINLIST += $(TARGET) +NETMON_DIR = netmon +NETMON_TARGET = $(PROJECT_TYPE)-netmon +NETMON_TARGET_OUTPUT = $(CURDIR)/$(NETMON_TARGET) +BINLIBEXECLIST += $(NETMON_TARGET) + DESTDIR := / installing = $(findstring install,$(MAKECMDGOALS)) @@ -110,6 +116,9 @@ SHIMPATH := $(PKGLIBEXECDIR)/$(SHIMCMD) PROXYCMD := $(BIN_PREFIX)-proxy PROXYPATH := $(PKGLIBEXECDIR)/$(PROXYCMD) +NETMONCMD := $(BIN_PREFIX)-netmon +NETMONPATH := $(PKGLIBEXECDIR)/$(NETMONCMD) + # Default number of vCPUs DEFVCPUS := 1 # Default maximum number of vCPUs @@ -183,6 +192,7 @@ USER_VARS += PROJECT_NAME USER_VARS += PROJECT_PREFIX USER_VARS += PROJECT_TYPE USER_VARS += PROXYPATH +USER_VARS += NETMONPATH USER_VARS += QEMUBINDIR USER_VARS += QEMUCMD USER_VARS += QEMUPATH @@ -225,7 +235,12 @@ define SHOW_ARCH $(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)","")) endef -all: runtime +all: runtime netmon + +netmon: $(NETMON_TARGET_OUTPUT) + +$(NETMON_TARGET_OUTPUT): $(SOURCES) + $(QUIET_BUILD)(cd $(NETMON_DIR) && go build -i -o $@ -ldflags "-X main.version=$(VERSION)") runtime: $(TARGET_OUTPUT) $(CONFIG) .DEFAULT: default @@ -308,6 +323,7 @@ var defaultRuntimeConfiguration = "$(CONFIG_PATH)" var defaultSysConfRuntimeConfiguration = "$(SYSCONFIG)" var defaultProxyPath = "$(PROXYPATH)" +var defaultNetmonPath = "$(NETMONPATH)" endef export GENERATED_CODE @@ -362,6 +378,7 @@ $(GENERATED_FILES): %: %.in Makefile VERSION -e "s|@LOCALSTATEDIR@|$(LOCALSTATEDIR)|g" \ -e "s|@PKGLIBEXECDIR@|$(PKGLIBEXECDIR)|g" \ -e "s|@PROXYPATH@|$(PROXYPATH)|g" \ + -e "s|@NETMONPATH@|$(NETMONPATH)|g" \ -e "s|@PROJECT_BUG_URL@|$(PROJECT_BUG_URL)|g" \ -e "s|@PROJECT_URL@|$(PROJECT_URL)|g" \ -e "s|@PROJECT_NAME@|$(PROJECT_NAME)|g" \ @@ -405,11 +422,14 @@ check-go-static: coverage: $(QUIET_TEST).ci/go-test.sh html-coverage -install: default runtime install-scripts install-completions install-config install-bin +install: default runtime install-scripts install-completions install-config install-bin install-bin-libexec install-bin: $(BINLIST) $(foreach f,$(BINLIST),$(call INSTALL_EXEC,$f,$(BINDIR))) +install-bin-libexec: $(BINLIBEXECLIST) + $(foreach f,$(BINLIBEXECLIST),$(call INSTALL_EXEC,$f,$(PKGLIBEXECDIR))) + install-config: $(CONFIG) $(QUIET_INST)install --mode 0644 -D $(CONFIG) $(DESTDIR)/$(CONFIG_PATH) @@ -420,7 +440,7 @@ install-completions: $(QUIET_INST)install --mode 0644 -D $(BASH_COMPLETIONS) $(DESTDIR)/$(BASH_COMPLETIONSDIR)/$(notdir $(BASH_COMPLETIONS)); clean: - $(QUIET_CLEAN)rm -f $(TARGET) $(CONFIG) $(GENERATED_GO_FILES) $(GENERATED_FILES) $(COLLECT_SCRIPT) + $(QUIET_CLEAN)rm -f $(TARGET) $(NETMON_TARGET) $(CONFIG) $(GENERATED_GO_FILES) $(GENERATED_FILES) $(COLLECT_SCRIPT) show-usage: show-header @printf "• Overview:\n" @@ -483,6 +503,8 @@ show-summary: show-header @printf "\tbinaries to install :\n" @printf \ "$(foreach b,$(sort $(BINLIST)),$(shell printf "\\t - $(shell readlink -m $(DESTDIR)/$(BINDIR)/$(b))\\\n"))" + @printf \ + "$(foreach b,$(sort $(BINLIBEXECLIST)),$(shell printf "\\t - $(shell readlink -m $(DESTDIR)/$(PKGLIBEXECDIR)/$(b))\\\n"))" @printf \ "$(foreach s,$(sort $(SCRIPTS)),$(shell printf "\\t - $(shell readlink -m $(DESTDIR)/$(BINDIR)/$(s))\\\n"))" @printf "\tconfig to install (CONFIG) : %s\n" $(CONFIG) diff --git a/cli/config.go b/cli/config.go index 2e463c4885..7c69fb0ffe 100644 --- a/cli/config.go +++ b/cli/config.go @@ -64,6 +64,7 @@ type tomlConfig struct { Agent map[string]agent Runtime runtime Factory factory + Netmon netmon } type factory struct { @@ -116,6 +117,12 @@ type shim struct { type agent struct { } +type netmon struct { + Path string `toml:"path"` + Debug bool `toml:"enable_debug"` + Enable bool `toml:"enable_netmon"` +} + func (h hypervisor) path() (string, error) { p := h.Path @@ -302,6 +309,22 @@ func (s shim) debug() bool { return s.Debug } +func (n netmon) enable() bool { + return n.Enable +} + +func (n netmon) path() string { + if n.Path == "" { + return defaultNetmonPath + } + + return n.Path +} + +func (n netmon) debug() bool { + return n.Debug +} + func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { hypervisor, err := h.path() if err != nil { @@ -464,6 +487,12 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run } config.FactoryConfig = fConfig + config.NetmonConfig = vc.NetmonConfig{ + Path: tomlConf.Netmon.path(), + Debug: tomlConf.Netmon.debug(), + Enable: tomlConf.Netmon.enable(), + } + return nil } diff --git a/cli/config/configuration.toml.in b/cli/config/configuration.toml.in index 842ff16b0a..305fc275f6 100644 --- a/cli/config/configuration.toml.in +++ b/cli/config/configuration.toml.in @@ -181,6 +181,21 @@ path = "@SHIMPATH@" # There is no field for this section. The goal is only to be able to # specify which type of agent the user wants to use. +[netmon] +# If enabled, the network monitoring process gets started when the +# sandbox is created. This allows for the detection of some additional +# network being added to the existing network namespace, after the +# sandbox has been created. +# (default: disabled) +#enable_netmon = true + +# Specify the path to the netmon binary. +path = "@NETMONPATH@" + +# If enabled, netmon messages will be sent to the system log +# (default: disabled) +#enable_debug = true + [runtime] # If enabled, the runtime will log additional debug messages to the # system log diff --git a/cli/config_test.go b/cli/config_test.go index 10290d89ed..c835639661 100644 --- a/cli/config_test.go +++ b/cli/config_test.go @@ -30,6 +30,7 @@ var ( proxyDebug = false runtimeDebug = false shimDebug = false + netmonDebug = false ) type testRuntimeConfig struct { @@ -41,7 +42,7 @@ type testRuntimeConfig struct { LogPath string } -func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath string, disableBlock bool, blockDeviceDriver string, enableIOThreads bool, hotplugVFIOOnRootBus bool) string { +func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, netmonPath, logPath string, disableBlock bool, blockDeviceDriver string, enableIOThreads bool, hotplugVFIOOnRootBus bool) string { return ` # Runtime configuration file @@ -71,6 +72,10 @@ func makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath [agent.kata] + [netmon] + path = "` + netmonPath + `" + enable_debug = ` + strconv.FormatBool(netmonDebug) + ` + [runtime] enable_debug = ` + strconv.FormatBool(runtimeDebug) } @@ -103,6 +108,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf imagePath := path.Join(dir, "image") shimPath := path.Join(dir, "shim") proxyPath := path.Join(dir, "proxy") + netmonPath := path.Join(dir, "netmon") logDir := path.Join(dir, "logs") logPath := path.Join(logDir, "runtime.log") machineType := "machineType" @@ -111,7 +117,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf enableIOThreads := true hotplugVFIOOnRootBus := true - runtimeConfigFileData := makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, logPath, disableBlockDevice, blockDeviceDriver, enableIOThreads, hotplugVFIOOnRootBus) + runtimeConfigFileData := makeRuntimeConfigFileData(hypervisor, hypervisorPath, kernelPath, imagePath, kernelParams, machineType, shimPath, proxyPath, netmonPath, logPath, disableBlockDevice, blockDeviceDriver, enableIOThreads, hotplugVFIOOnRootBus) configPath := path.Join(dir, "runtime.toml") err = createConfig(configPath, runtimeConfigFileData) @@ -165,6 +171,12 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf Path: shimPath, } + netmonConfig := vc.NetmonConfig{ + Path: netmonPath, + Debug: false, + Enable: false, + } + runtimeConfig := oci.RuntimeConfig{ HypervisorType: defaultHypervisor, HypervisorConfig: hypervisorConfig, @@ -177,6 +189,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf ShimType: defaultShim, ShimConfig: shimConfig, + + NetmonConfig: netmonConfig, } config = testRuntimeConfig{ @@ -482,11 +496,11 @@ func TestMinimalRuntimeConfig(t *testing.T) { proxyPath := path.Join(dir, "proxy") hypervisorPath := path.Join(dir, "hypervisor") defaultHypervisorPath = hypervisorPath + netmonPath := path.Join(dir, "netmon") imagePath := path.Join(dir, "image.img") initrdPath := path.Join(dir, "initrd.img") - hypervisorPath = path.Join(dir, "hypervisor") kernelPath := path.Join(dir, "kernel") savedDefaultImagePath := defaultImagePath @@ -525,6 +539,9 @@ func TestMinimalRuntimeConfig(t *testing.T) { path = "` + shimPath + `" [agent.kata] + + [netmon] + path = "` + netmonPath + `" ` configPath := path.Join(dir, "runtime.toml") @@ -553,6 +570,11 @@ func TestMinimalRuntimeConfig(t *testing.T) { t.Error(err) } + err = createEmptyFile(netmonPath) + if err != nil { + t.Error(err) + } + _, config, err = loadConfiguration(configPath, false) if err != nil { t.Fatal(err) @@ -584,6 +606,12 @@ func TestMinimalRuntimeConfig(t *testing.T) { Path: shimPath, } + expectedNetmonConfig := vc.NetmonConfig{ + Path: netmonPath, + Debug: false, + Enable: false, + } + expectedConfig := oci.RuntimeConfig{ HypervisorType: defaultHypervisor, HypervisorConfig: expectedHypervisorConfig, @@ -596,6 +624,8 @@ func TestMinimalRuntimeConfig(t *testing.T) { ShimType: defaultShim, ShimConfig: expectedShimConfig, + + NetmonConfig: expectedNetmonConfig, } if reflect.DeepEqual(config, expectedConfig) == false { diff --git a/cli/kata-env.go b/cli/kata-env.go index 595ce19f1a..e2a24c7460 100644 --- a/cli/kata-env.go +++ b/cli/kata-env.go @@ -25,7 +25,7 @@ import ( // // XXX: Increment for every change to the output format // (meaning any change to the EnvInfo type). -const formatVersion = "1.0.15" +const formatVersion = "1.0.16" // MetaInfo stores information on the format of the output itself type MetaInfo struct { @@ -123,6 +123,14 @@ type HostInfo struct { SupportVSocks bool } +// NetmonInfo stores netmon details +type NetmonInfo struct { + Version string + Path string + Debug bool + Enable bool +} + // EnvInfo collects all information that will be displayed by the // env command. // @@ -138,6 +146,7 @@ type EnvInfo struct { Shim ShimInfo Agent AgentInfo Host HostInfo + Netmon NetmonInfo } func getMetaInfo() MetaInfo { @@ -241,6 +250,22 @@ func getProxyInfo(config oci.RuntimeConfig) (ProxyInfo, error) { return proxy, nil } +func getNetmonInfo(config oci.RuntimeConfig) (NetmonInfo, error) { + version, err := getCommandVersion(defaultNetmonPath) + if err != nil { + version = unknown + } + + netmon := NetmonInfo{ + Version: version, + Path: config.NetmonConfig.Path, + Debug: config.NetmonConfig.Debug, + Enable: config.NetmonConfig.Enable, + } + + return netmon, nil +} + func getCommandVersion(cmd string) (string, error) { return runCommand([]string{cmd, "--version"}) } @@ -309,6 +334,8 @@ func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err e proxy, _ := getProxyInfo(config) + netmon, _ := getNetmonInfo(config) + shim, err := getShimInfo(config) if err != nil { return EnvInfo{}, err @@ -342,6 +369,7 @@ func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err e Shim: shim, Agent: agent, Host: host, + Netmon: netmon, } return env, nil diff --git a/cli/kata-env_test.go b/cli/kata-env_test.go index c49d989956..c3a888e8b3 100644 --- a/cli/kata-env_test.go +++ b/cli/kata-env_test.go @@ -31,6 +31,7 @@ import ( const testProxyURL = "file:///proxyURL" const testProxyVersion = "proxy version 0.1" const testShimVersion = "shim version 0.1" +const testNetmonVersion = "netmon version 0.1" const testHypervisorVersion = "QEMU emulator version 2.7.0+git.741f430a96-6.1, Copyright (c) 2003-2016 Fabrice Bellard and the QEMU Project developers" // makeVersionBinary creates a shell script with the specified file @@ -61,6 +62,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC machineType := "machineType" shimPath := filepath.Join(prefixDir, "shim") proxyPath := filepath.Join(prefixDir, "proxy") + netmonPath := filepath.Join(prefixDir, "netmon") disableBlock := true blockStorageDriver := "virtio-scsi" enableIOThreads := true @@ -68,6 +70,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC // override defaultProxyPath = proxyPath + defaultNetmonPath = netmonPath filesToCreate := []string{ hypervisorPath, @@ -93,6 +96,11 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC return "", oci.RuntimeConfig{}, err } + err = makeVersionBinary(netmonPath, testNetmonVersion) + if err != nil { + return "", oci.RuntimeConfig{}, err + } + err = makeVersionBinary(hypervisorPath, testHypervisorVersion) if err != nil { return "", oci.RuntimeConfig{}, err @@ -107,6 +115,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC machineType, shimPath, testProxyURL, + netmonPath, logPath, disableBlock, blockStorageDriver, @@ -137,6 +146,15 @@ func getExpectedProxyDetails(config oci.RuntimeConfig) (ProxyInfo, error) { }, nil } +func getExpectedNetmonDetails(config oci.RuntimeConfig) (NetmonInfo, error) { + return NetmonInfo{ + Version: testNetmonVersion, + Path: config.NetmonConfig.Path, + Debug: config.NetmonConfig.Debug, + Enable: config.NetmonConfig.Enable, + }, nil +} + func getExpectedShimDetails(config oci.RuntimeConfig) (ShimInfo, error) { shimConfig, ok := config.ShimConfig.(vc.ShimConfig) if !ok { @@ -303,6 +321,11 @@ func getExpectedSettings(config oci.RuntimeConfig, tmpdir, configFile string) (E return EnvInfo{}, err } + netmon, err := getExpectedNetmonDetails(config) + if err != nil { + return EnvInfo{}, err + } + hypervisor := getExpectedHypervisor(config) kernel := getExpectedKernel(config) image := getExpectedImage(config) @@ -317,6 +340,7 @@ func getExpectedSettings(config oci.RuntimeConfig, tmpdir, configFile string) (E Shim: shim, Agent: agent, Host: host, + Netmon: netmon, } return env, nil @@ -608,6 +632,50 @@ func TestEnvGetProxyInfoNoVersion(t *testing.T) { assert.Equal(t, expectedProxy, proxy) } +func TestEnvGetNetmonInfo(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "") + if err != nil { + panic(err) + } + defer os.RemoveAll(tmpdir) + + _, config, err := makeRuntimeConfig(tmpdir) + assert.NoError(t, err) + + expectedNetmon, err := getExpectedNetmonDetails(config) + assert.NoError(t, err) + + netmon, err := getNetmonInfo(config) + assert.NoError(t, err) + + assert.Equal(t, expectedNetmon, netmon) +} + +func TestEnvGetNetmonInfoNoVersion(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "") + if err != nil { + panic(err) + } + defer os.RemoveAll(tmpdir) + + _, config, err := makeRuntimeConfig(tmpdir) + assert.NoError(t, err) + + expectedNetmon, err := getExpectedNetmonDetails(config) + assert.NoError(t, err) + + // remove the netmon ensuring its version cannot be queried + err = os.Remove(defaultNetmonPath) + assert.NoError(t, err) + + expectedNetmon.Version = unknown + + netmon, err := getNetmonInfo(config) + assert.NoError(t, err) + + assert.Equal(t, expectedNetmon, netmon) +} + func TestEnvGetShimInfo(t *testing.T) { tmpdir, err := ioutil.TempDir("", "") if err != nil { diff --git a/netmon/netmon.go b/netmon/netmon.go new file mode 100644 index 0000000000..9ba6d0ecca --- /dev/null +++ b/netmon/netmon.go @@ -0,0 +1,661 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log/syslog" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/sirupsen/logrus" + lSyslog "github.com/sirupsen/logrus/hooks/syslog" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +// The following types and structures have to be kept in sync with the +// description of the agent protocol. Those definitions need to be in their +// own separate package so that they can be imported directly from this code. +// The reason for not importing them now, is because importing the whole agent +// protocol adds up too much overhead because of the grpc protocol involved. + +// IPFamily define the IP address family type. +type IPFamily int32 + +// IPAddress describes the IP address format expected by Kata API. +type IPAddress struct { + Family IPFamily `json:"family,omitempty"` + Address string `json:"address,omitempty"` + Mask string `json:"mask,omitempty"` +} + +// Interface describes the network interface format expected by Kata API. +type Interface struct { + Device string `json:"device,omitempty"` + Name string `json:"name,omitempty"` + IPAddresses []*IPAddress `json:"IPAddresses,omitempty"` + Mtu uint64 `json:"mtu,omitempty"` + HwAddr string `json:"hwAddr,omitempty"` + PciAddr string `json:"pciAddr,omitempty"` +} + +// Route describes the network route format expected by Kata API. +type Route struct { + Dest string `json:"dest,omitempty"` + Gateway string `json:"gateway,omitempty"` + Device string `json:"device,omitempty"` + Source string `json:"source,omitempty"` + Scope uint32 `json:"scope,omitempty"` +} + +const ( + netmonName = "kata-netmon" + + kataCmd = "kata-network" + kataCLIAddIfaceCmd = "add-iface" + kataCLIDelIfaceCmd = "del-iface" + kataCLIUpdtRoutesCmd = "update-routes" + + kataSuffix = "kata" + + // sharedFile is the name of the file that will be used to share + // the data between this process and the kata-runtime process + // responsible for updating the network. + sharedFile = "shared.json" + storageFilePerm = os.FileMode(0640) + storageDirPerm = os.FileMode(0750) +) + +var ( + // version is the netmon version. This variable is populated at build time. + version = "unknown" + + // For simplicity the code will only focus on IPv4 addresses for now. + netlinkFamily = netlink.FAMILY_V4 + + storageParentPath = "/var/run/kata-containers/netmon/sbs" +) + +type netmonParams struct { + sandboxID string + runtimePath string + debug bool + logLevel string +} + +type netmon struct { + netmonParams + + storagePath string + sharedFile string + + netIfaces map[int]Interface + + linkUpdateCh chan netlink.LinkUpdate + linkDoneCh chan struct{} + + rtUpdateCh chan netlink.RouteUpdate + rtDoneCh chan struct{} + + netHandler *netlink.Handle +} + +var netmonLog = logrus.New() + +func printVersion() { + fmt.Printf("%s version %s\n", netmonName, version) +} + +const componentDescription = `is a network monitoring process that is intended to be started in the +appropriate network namespace so that it can listen to any event related to +link and routes. Whenever a new interface or route is created/updated, it is +responsible for calling into the kata-runtime CLI to ask for the actual +creation/update of the given interface or route. +` + +func printComponentDescription() { + fmt.Printf("\n%s %s\n", netmonName, componentDescription) +} + +func parseOptions() netmonParams { + var version, help bool + + params := netmonParams{} + + flag.BoolVar(&help, "h", false, "describe component usage") + flag.BoolVar(&help, "help", false, "") + flag.BoolVar(¶ms.debug, "d", false, "enable debug mode") + flag.BoolVar(&version, "v", false, "display program version and exit") + flag.BoolVar(&version, "version", false, "") + flag.StringVar(¶ms.sandboxID, "s", "", "sandbox id (required)") + flag.StringVar(¶ms.runtimePath, "r", "", "runtime path (required)") + flag.StringVar(¶ms.logLevel, "log", "warn", + "log messages above specified level: debug, warn, error, fatal or panic") + + flag.Parse() + + if help { + printComponentDescription() + flag.PrintDefaults() + os.Exit(0) + } + + if version { + printVersion() + os.Exit(0) + } + + if params.sandboxID == "" { + fmt.Fprintf(os.Stderr, "Error: sandbox id is empty, one must be provided\n") + flag.PrintDefaults() + os.Exit(1) + } + + if params.runtimePath == "" { + fmt.Fprintf(os.Stderr, "Error: runtime path is empty, one must be provided\n") + flag.PrintDefaults() + os.Exit(1) + } + + return params +} + +func newNetmon(params netmonParams) (*netmon, error) { + handler, err := netlink.NewHandle(netlinkFamily) + if err != nil { + return nil, err + } + + n := &netmon{ + netmonParams: params, + storagePath: filepath.Join(storageParentPath, params.sandboxID), + sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile), + netIfaces: make(map[int]Interface), + linkUpdateCh: make(chan netlink.LinkUpdate), + linkDoneCh: make(chan struct{}), + rtUpdateCh: make(chan netlink.RouteUpdate), + rtDoneCh: make(chan struct{}), + netHandler: handler, + } + + if err := os.MkdirAll(n.storagePath, storageDirPerm); err != nil { + return nil, err + } + + return n, nil +} + +func (n *netmon) cleanup() { + os.RemoveAll(n.storagePath) + n.netHandler.Delete() + close(n.linkDoneCh) + close(n.rtDoneCh) +} + +func (n *netmon) logger() *logrus.Entry { + fields := logrus.Fields{ + "name": netmonName, + "pid": os.Getpid(), + "source": "netmon", + } + + if n.sandboxID != "" { + fields["sandbox"] = n.sandboxID + } + + return netmonLog.WithFields(fields) +} + +func (n *netmon) setupLogger() error { + level, err := logrus.ParseLevel(n.logLevel) + if err != nil { + return err + } + + netmonLog.SetLevel(level) + + netmonLog.Formatter = &logrus.TextFormatter{TimestampFormat: time.RFC3339Nano} + + hook, err := lSyslog.NewSyslogHook("", "", syslog.LOG_INFO|syslog.LOG_USER, netmonName) + if err != nil { + return err + } + + netmonLog.AddHook(hook) + + announceFields := logrus.Fields{ + "runtime-path": n.runtimePath, + "debug": n.debug, + "log-level": n.logLevel, + } + + n.logger().WithFields(announceFields).Info("announce") + + return nil +} + +func (n *netmon) listenNetlinkEvents() error { + if err := netlink.LinkSubscribe(n.linkUpdateCh, n.linkDoneCh); err != nil { + return err + } + + return netlink.RouteSubscribe(n.rtUpdateCh, n.rtDoneCh) +} + +// convertInterface converts a link and its IP addresses as defined by netlink +// package, into the Interface structure format expected by kata-runtime to +// describe an interface and its associated IP addresses. +func convertInterface(linkAttrs *netlink.LinkAttrs, addrs []netlink.Addr) Interface { + if linkAttrs == nil { + netmonLog.Warn("Link attributes are nil") + return Interface{} + } + + var ipAddrs []*IPAddress + + for _, addr := range addrs { + if addr.IPNet == nil { + continue + } + + netMask, _ := addr.Mask.Size() + + ipAddr := &IPAddress{ + Family: IPFamily(netlinkFamily), + Address: addr.IP.String(), + Mask: fmt.Sprintf("%d", netMask), + } + + ipAddrs = append(ipAddrs, ipAddr) + } + + iface := Interface{ + Device: linkAttrs.Name, + Name: linkAttrs.Name, + IPAddresses: ipAddrs, + Mtu: uint64(linkAttrs.MTU), + HwAddr: linkAttrs.HardwareAddr.String(), + } + + netmonLog.WithField("interface", iface).Debug("Interface converted") + + return iface +} + +// convertRoutes converts a list of routes as defined by netlink package, +// into a list of Route structure format expected by kata-runtime to +// describe a set of routes. +func convertRoutes(netRoutes []netlink.Route) []Route { + var routes []Route + + // Ignore routes with IPv6 addresses as this is not supported + // by Kata yet. + for _, netRoute := range netRoutes { + dst := "" + if netRoute.Dst != nil { + if netRoute.Dst.IP.To4() != nil { + dst = netRoute.Dst.IP.String() + } else { + netmonLog.WithField("destination", netRoute.Dst.IP.String()).Warn("Not IPv4 format") + } + } + + src := "" + if netRoute.Src.To4() != nil { + src = netRoute.Src.String() + } else { + netmonLog.WithField("source", netRoute.Src.String()).Warn("Not IPv4 format") + } + + gw := "" + if netRoute.Gw.To4() != nil { + gw = netRoute.Gw.String() + } else { + netmonLog.WithField("gateway", netRoute.Gw.String()).Warn("Not IPv4 format") + } + + dev := "" + iface, err := net.InterfaceByIndex(netRoute.LinkIndex) + if err == nil { + dev = iface.Name + } + + route := Route{ + Dest: dst, + Gateway: gw, + Device: dev, + Source: src, + Scope: uint32(netRoute.Scope), + } + + routes = append(routes, route) + } + + netmonLog.WithField("routes", routes).Debug("Routes converted") + + return routes +} + +// scanNetwork lists all the interfaces it can find inside the current +// network namespace, and store them in-memory to keep track of them. +func (n *netmon) scanNetwork() error { + links, err := n.netHandler.LinkList() + if err != nil { + return err + } + + for _, link := range links { + addrs, err := n.netHandler.AddrList(link, netlinkFamily) + if err != nil { + return err + } + + linkAttrs := link.Attrs() + if linkAttrs == nil { + continue + } + + iface := convertInterface(linkAttrs, addrs) + n.netIfaces[linkAttrs.Index] = iface + } + + n.logger().Debug("Network scanned") + + return nil +} + +func (n *netmon) storeDataToSend(data interface{}) error { + // Marshal the data structure into a JSON bytes array. + jsonArray, err := json.Marshal(data) + if err != nil { + return err + } + + // Store the JSON bytes array at the specified path. + return ioutil.WriteFile(n.sharedFile, jsonArray, storageFilePerm) +} + +func (n *netmon) execKataCmd(subCmd string) error { + execCmd := exec.Command(n.runtimePath, kataCmd, subCmd, n.sandboxID, n.sharedFile) + + n.logger().WithField("command", execCmd).Debug("Running runtime command") + + // Make use of Run() to ensure the kata-runtime process has correctly + // terminated before to go further. + if err := execCmd.Run(); err != nil { + return err + } + + // Remove the shared file after the command returned. At this point + // we know the content of the file is not going to be used anymore, + // and the file path can be reused for further commands. + return os.Remove(n.sharedFile) +} + +func (n *netmon) addInterfaceCLI(iface Interface) error { + if err := n.storeDataToSend(iface); err != nil { + return err + } + + return n.execKataCmd(kataCLIAddIfaceCmd) +} + +func (n *netmon) delInterfaceCLI(iface Interface) error { + if err := n.storeDataToSend(iface); err != nil { + return err + } + + return n.execKataCmd(kataCLIDelIfaceCmd) +} + +func (n *netmon) updateRoutesCLI(routes []Route) error { + if err := n.storeDataToSend(routes); err != nil { + return err + } + + return n.execKataCmd(kataCLIUpdtRoutesCmd) +} + +func (n *netmon) updateRoutes() error { + // Get all the routes. + netlinkRoutes, err := n.netHandler.RouteList(nil, netlinkFamily) + if err != nil { + return err + } + + // Translate them into Route structures. + routes := convertRoutes(netlinkRoutes) + + // Update the routes through the Kata CLI. + return n.updateRoutesCLI(routes) +} + +func (n *netmon) handleRTMNewAddr(ev netlink.LinkUpdate) error { + n.logger().Debug("Interface update not supported") + return nil +} + +func (n *netmon) handleRTMDelAddr(ev netlink.LinkUpdate) error { + n.logger().Debug("Interface update not supported") + return nil +} + +func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error { + // NEWLINK might be a lot of different things. We're interested in + // adding the interface (both to our list and by calling into the + // Kata CLI API) only if this has the flags UP and RUNNING, meaning + // we don't expect any further change on the interface, and that we + // are ready to add it. + + linkAttrs := ev.Link.Attrs() + if linkAttrs == nil { + n.logger().Warn("The link attributes are nil") + return nil + } + + // First, ignore if the interface name contains "kata". This way we + // are preventing from adding interfaces created by Kata Containers. + if strings.HasSuffix(linkAttrs.Name, kataSuffix) { + n.logger().Debugf("Ignore the interface %s because found %q", + linkAttrs.Name, kataSuffix) + return nil + } + + // Check if the interface exist in the internal list. + if _, exist := n.netIfaces[int(ev.Index)]; exist { + n.logger().Debugf("Ignoring interface %s because already exist", + linkAttrs.Name) + return nil + } + + // Now, check if the interface has been enabled to UP and RUNNING. + if (ev.Flags&unix.IFF_UP) != unix.IFF_UP || + (ev.Flags&unix.IFF_RUNNING) != unix.IFF_RUNNING { + n.logger().Debugf("Ignore the interface %s because not UP and RUNNING", + linkAttrs.Name) + return nil + } + + // Get the list of IP addresses associated with this interface. + addrs, err := n.netHandler.AddrList(ev.Link, netlinkFamily) + if err != nil { + return err + } + + // Convert the interfaces in the appropriate structure format. + iface := convertInterface(linkAttrs, addrs) + + // Add the interface through the Kata CLI. + if err := n.addInterfaceCLI(iface); err != nil { + return err + } + + // Add the interface to the internal list. + n.netIfaces[linkAttrs.Index] = iface + + // Complete by updating the routes. + return n.updateRoutes() +} + +func (n *netmon) handleRTMDelLink(ev netlink.LinkUpdate) error { + // It can only delete if identical interface is found in the internal + // list of interfaces. Otherwise, the deletion will be ignored. + linkAttrs := ev.Link.Attrs() + if linkAttrs == nil { + n.logger().Warn("Link attributes are nil") + return nil + } + + // First, ignore if the interface name contains "kata". This way we + // are preventing from deleting interfaces created by Kata Containers. + if strings.Contains(linkAttrs.Name, kataSuffix) { + n.logger().Debugf("Ignore the interface %s because found %q", + linkAttrs.Name, kataSuffix) + return nil + } + + // Check if the interface exist in the internal list. + iface, exist := n.netIfaces[int(ev.Index)] + if !exist { + n.logger().Debugf("Ignoring interface %s because not found", + linkAttrs.Name) + return nil + } + + if err := n.delInterfaceCLI(iface); err != nil { + return err + } + + // Delete the interface from the internal list. + delete(n.netIfaces, linkAttrs.Index) + + // Complete by updating the routes. + return n.updateRoutes() +} + +func (n *netmon) handleRTMNewRoute(ev netlink.RouteUpdate) error { + // Add the route through updateRoutes(), only if the route refer to an + // interface that already exists in the internal list of interfaces. + if _, exist := n.netIfaces[ev.Route.LinkIndex]; !exist { + n.logger().Debugf("Ignoring route %+v since interface %d not found", + ev.Route, ev.Route.LinkIndex) + return nil + } + + return n.updateRoutes() +} + +func (n *netmon) handleRTMDelRoute(ev netlink.RouteUpdate) error { + // Remove the route through updateRoutes(), only if the route refer to + // an interface that already exists in the internal list of interfaces. + return n.updateRoutes() +} + +func (n *netmon) handleLinkEvent(ev netlink.LinkUpdate) error { + n.logger().Debug("handleLinkEvent: netlink event received") + + switch ev.Header.Type { + case unix.NLMSG_DONE: + n.logger().Debug("NLMSG_DONE") + return nil + case unix.NLMSG_ERROR: + n.logger().Error("NLMSG_ERROR") + return fmt.Errorf("Error while listening on netlink socket") + case unix.RTM_NEWADDR: + n.logger().Debug("RTM_NEWADDR") + return n.handleRTMNewAddr(ev) + case unix.RTM_DELADDR: + n.logger().Debug("RTM_DELADDR") + return n.handleRTMDelAddr(ev) + case unix.RTM_NEWLINK: + n.logger().Debug("RTM_NEWLINK") + return n.handleRTMNewLink(ev) + case unix.RTM_DELLINK: + n.logger().Debug("RTM_DELLINK") + return n.handleRTMDelLink(ev) + default: + n.logger().Warnf("Unknown msg type %v", ev.Header.Type) + } + + return nil +} + +func (n *netmon) handleRouteEvent(ev netlink.RouteUpdate) error { + n.logger().Debug("handleRouteEvent: netlink event received") + + switch ev.Type { + case unix.RTM_NEWROUTE: + n.logger().Debug("RTM_NEWROUTE") + return n.handleRTMNewRoute(ev) + case unix.RTM_DELROUTE: + n.logger().Debug("RTM_DELROUTE") + return n.handleRTMDelRoute(ev) + default: + n.logger().Warnf("Unknown msg type %v", ev.Type) + } + + return nil +} + +func (n *netmon) handleEvents() (err error) { + for { + select { + case ev := <-n.linkUpdateCh: + if err = n.handleLinkEvent(ev); err != nil { + return err + } + case ev := <-n.rtUpdateCh: + if err = n.handleRouteEvent(ev); err != nil { + return err + } + } + } +} + +func main() { + // Parse parameters. + params := parseOptions() + + // Create netmon handler. + n, err := newNetmon(params) + if err != nil { + netmonLog.WithError(err).Fatal("newNetmon()") + os.Exit(1) + } + defer n.cleanup() + + // Init logger. + if err := n.setupLogger(); err != nil { + netmonLog.WithError(err).Fatal("setupLogger()") + os.Exit(1) + } + + // Scan the current interfaces. + if err := n.scanNetwork(); err != nil { + n.logger().WithError(err).Fatal("scanNetwork()") + os.Exit(1) + } + + // Subscribe to the link listener. + if err := n.listenNetlinkEvents(); err != nil { + n.logger().WithError(err).Fatal("listenNetlinkEvents()") + os.Exit(1) + } + + // Go into the main loop. + if err := n.handleEvents(); err != nil { + n.logger().WithError(err).Fatal("handleEvents()") + os.Exit(1) + } +} diff --git a/netmon/netmon_test.go b/netmon/netmon_test.go new file mode 100644 index 0000000000..823aa60b3f --- /dev/null +++ b/netmon/netmon_test.go @@ -0,0 +1,632 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package main + +import ( + "encoding/json" + "io/ioutil" + "net" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + "golang.org/x/sys/unix" +) + +const ( + testSandboxID = "123456789" + testRuntimePath = "/foo/bar/test-runtime" + testLogLevel = "info" + testStorageParentPath = "/tmp/netmon" + testSharedFile = "foo-shared.json" + testWrongNetlinkFamily = -1 + testIfaceName = "test_eth0" + testMTU = 12345 + testHwAddr = "02:00:ca:fe:00:48" + testIPAddress = "192.168.0.15" + testIPAddressWithMask = "192.168.0.15/32" + testScope = 1 + testTxQLen = -1 + testIfaceIndex = 5 +) + +func skipUnlessRoot(t *testing.T) { + if os.Getuid() != 0 { + t.Skip("Test disabled as requires root user") + } +} + +func TestNewNetmon(t *testing.T) { + skipUnlessRoot(t) + + // Override storageParentPath + savedStorageParentPath := storageParentPath + storageParentPath = testStorageParentPath + defer func() { + storageParentPath = savedStorageParentPath + }() + + params := netmonParams{ + sandboxID: testSandboxID, + runtimePath: testRuntimePath, + debug: true, + logLevel: testLogLevel, + } + + expected := &netmon{ + netmonParams: params, + storagePath: filepath.Join(storageParentPath, params.sandboxID), + sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile), + } + + os.RemoveAll(expected.storagePath) + + got, err := newNetmon(params) + assert.Nil(t, err) + assert.True(t, reflect.DeepEqual(expected.netmonParams, got.netmonParams), + "Got %+v\nExpected %+v", got.netmonParams, expected.netmonParams) + assert.True(t, reflect.DeepEqual(expected.storagePath, got.storagePath), + "Got %+v\nExpected %+v", got.storagePath, expected.storagePath) + assert.True(t, reflect.DeepEqual(expected.sharedFile, got.sharedFile), + "Got %+v\nExpected %+v", got.sharedFile, expected.sharedFile) + + _, err = os.Stat(got.storagePath) + assert.Nil(t, err) + + os.RemoveAll(got.storagePath) +} + +func TestNewNetmonErrorWrongFamilyType(t *testing.T) { + // Override netlinkFamily + savedNetlinkFamily := netlinkFamily + netlinkFamily = testWrongNetlinkFamily + defer func() { + netlinkFamily = savedNetlinkFamily + }() + + n, err := newNetmon(netmonParams{}) + assert.NotNil(t, err) + assert.Nil(t, n) +} + +func TestCleanup(t *testing.T) { + skipUnlessRoot(t) + + // Override storageParentPath + savedStorageParentPath := storageParentPath + storageParentPath = testStorageParentPath + defer func() { + storageParentPath = savedStorageParentPath + }() + + handler, err := netlink.NewHandle(netlinkFamily) + assert.Nil(t, err) + + n := &netmon{ + storagePath: filepath.Join(storageParentPath, testSandboxID), + linkDoneCh: make(chan struct{}), + rtDoneCh: make(chan struct{}), + netHandler: handler, + } + + err = os.MkdirAll(n.storagePath, storageDirPerm) + assert.Nil(t, err) + _, err = os.Stat(n.storagePath) + assert.Nil(t, err) + + n.cleanup() + + _, err = os.Stat(n.storagePath) + assert.NotNil(t, err) + _, ok := (<-n.linkDoneCh) + assert.False(t, ok) + _, ok = (<-n.rtDoneCh) + assert.False(t, ok) +} + +func TestLogger(t *testing.T) { + fields := logrus.Fields{ + "name": netmonName, + "pid": os.Getpid(), + "source": "netmon", + "sandbox": testSandboxID, + } + + expected := netmonLog.WithFields(fields) + + n := &netmon{ + netmonParams: netmonParams{ + sandboxID: testSandboxID, + }, + } + + got := n.logger() + assert.True(t, reflect.DeepEqual(*expected, *got), + "Got %+v\nExpected %+v", *got, *expected) +} + +func TestConvertInterface(t *testing.T) { + hwAddr, err := net.ParseMAC(testHwAddr) + assert.Nil(t, err) + + addrs := []netlink.Addr{ + { + IPNet: &net.IPNet{ + IP: net.ParseIP(testIPAddress), + }, + }, + } + + linkAttrs := &netlink.LinkAttrs{ + Name: testIfaceName, + MTU: testMTU, + HardwareAddr: hwAddr, + } + + expected := Interface{ + Device: testIfaceName, + Name: testIfaceName, + Mtu: uint64(testMTU), + HwAddr: testHwAddr, + IPAddresses: []*IPAddress{ + { + Family: IPFamily(netlinkFamily), + Address: testIPAddress, + Mask: "0", + }, + }, + } + + got := convertInterface(linkAttrs, addrs) + assert.True(t, reflect.DeepEqual(expected, got), + "Got %+v\nExpected %+v", got, expected) +} + +func TestConvertRoutes(t *testing.T) { + ip, ipNet, err := net.ParseCIDR(testIPAddressWithMask) + assert.Nil(t, err) + assert.NotNil(t, ipNet) + + routes := []netlink.Route{ + { + Dst: ipNet, + Src: ip, + Gw: ip, + LinkIndex: -1, + Scope: testScope, + }, + } + + expected := []Route{ + { + Dest: testIPAddress, + Gateway: testIPAddress, + Source: testIPAddress, + Scope: uint32(testScope), + }, + } + + got := convertRoutes(routes) + assert.True(t, reflect.DeepEqual(expected, got), + "Got %+v\nExpected %+v", got, expected) +} + +type testTeardownNetwork func() + +func testSetupNetwork(t *testing.T) testTeardownNetwork { + skipUnlessRoot(t) + + // new temporary namespace so we don't pollute the host + // lock thread since the namespace is thread local + runtime.LockOSThread() + var err error + ns, err := netns.New() + if err != nil { + t.Fatal("Failed to create newns", ns) + } + + return func() { + ns.Close() + runtime.UnlockOSThread() + } +} + +func testCreateDummyNetwork(t *testing.T, handler *netlink.Handle) (int, Interface) { + hwAddr, err := net.ParseMAC(testHwAddr) + assert.Nil(t, err) + + link := &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + MTU: testMTU, + TxQLen: testTxQLen, + Name: testIfaceName, + HardwareAddr: hwAddr, + }, + } + + err = handler.LinkAdd(link) + assert.Nil(t, err) + err = handler.LinkSetUp(link) + assert.Nil(t, err) + + attrs := link.Attrs() + assert.NotNil(t, attrs) + + iface := Interface{ + Device: testIfaceName, + Name: testIfaceName, + Mtu: uint64(testMTU), + HwAddr: testHwAddr, + } + + return attrs.Index, iface +} + +func TestScanNetwork(t *testing.T) { + tearDownNetworkCb := testSetupNetwork(t) + defer tearDownNetworkCb() + + handler, err := netlink.NewHandle(netlinkFamily) + assert.Nil(t, err) + assert.NotNil(t, handler) + defer handler.Delete() + + idx, expected := testCreateDummyNetwork(t, handler) + + n := &netmon{ + netIfaces: make(map[int]Interface), + netHandler: handler, + } + + err = n.scanNetwork() + assert.Nil(t, err) + assert.True(t, reflect.DeepEqual(expected, n.netIfaces[idx]), + "Got %+v\nExpected %+v", n.netIfaces[idx], expected) +} + +func TestStoreDataToSend(t *testing.T) { + var got Interface + + expected := Interface{ + Device: testIfaceName, + Name: testIfaceName, + Mtu: uint64(testMTU), + HwAddr: testHwAddr, + } + + n := &netmon{ + sharedFile: filepath.Join(testStorageParentPath, testSharedFile), + } + + err := os.MkdirAll(testStorageParentPath, storageDirPerm) + defer os.RemoveAll(testStorageParentPath) + assert.Nil(t, err) + + err = n.storeDataToSend(expected) + assert.Nil(t, err) + + // Check the file has been created, check the content, and delete it. + _, err = os.Stat(n.sharedFile) + assert.Nil(t, err) + byteArray, err := ioutil.ReadFile(n.sharedFile) + assert.Nil(t, err) + err = json.Unmarshal(byteArray, &got) + assert.Nil(t, err) + assert.True(t, reflect.DeepEqual(expected, got), + "Got %+v\nExpected %+v", got, expected) +} + +func TestExecKataCmdSuccess(t *testing.T) { + trueBinPath, err := exec.LookPath("true") + assert.Nil(t, err) + assert.NotEmpty(t, trueBinPath) + + params := netmonParams{ + runtimePath: trueBinPath, + } + + n := &netmon{ + netmonParams: params, + sharedFile: filepath.Join(testStorageParentPath, testSharedFile), + } + + err = os.MkdirAll(testStorageParentPath, storageDirPerm) + assert.Nil(t, err) + defer os.RemoveAll(testStorageParentPath) + + file, err := os.Create(n.sharedFile) + assert.Nil(t, err) + assert.NotNil(t, file) + file.Close() + + _, err = os.Stat(n.sharedFile) + assert.Nil(t, err) + + err = n.execKataCmd("") + assert.Nil(t, err) + _, err = os.Stat(n.sharedFile) + assert.NotNil(t, err) +} + +func TestExecKataCmdFailure(t *testing.T) { + falseBinPath, err := exec.LookPath("false") + assert.Nil(t, err) + assert.NotEmpty(t, falseBinPath) + + params := netmonParams{ + runtimePath: falseBinPath, + } + + n := &netmon{ + netmonParams: params, + } + + err = n.execKataCmd("") + assert.NotNil(t, err) +} + +func TestActionsCLI(t *testing.T) { + trueBinPath, err := exec.LookPath("true") + assert.Nil(t, err) + assert.NotEmpty(t, trueBinPath) + + params := netmonParams{ + runtimePath: trueBinPath, + } + + n := &netmon{ + netmonParams: params, + sharedFile: filepath.Join(testStorageParentPath, testSharedFile), + } + + err = os.MkdirAll(testStorageParentPath, storageDirPerm) + assert.Nil(t, err) + defer os.RemoveAll(testStorageParentPath) + + // Test addInterfaceCLI + err = n.addInterfaceCLI(Interface{}) + assert.Nil(t, err) + + // Test delInterfaceCLI + err = n.delInterfaceCLI(Interface{}) + assert.Nil(t, err) + + // Test updateRoutesCLI + err = n.updateRoutesCLI([]Route{}) + assert.Nil(t, err) + + tearDownNetworkCb := testSetupNetwork(t) + defer tearDownNetworkCb() + + handler, err := netlink.NewHandle(netlinkFamily) + assert.Nil(t, err) + assert.NotNil(t, handler) + defer handler.Delete() + + n.netHandler = handler + + // Test updateRoutes + err = n.updateRoutes() + assert.Nil(t, err) + + // Test handleRTMDelRoute + err = n.handleRTMDelRoute(netlink.RouteUpdate{}) + assert.Nil(t, err) +} + +func TestHandleRTMNewAddr(t *testing.T) { + n := &netmon{} + + err := n.handleRTMNewAddr(netlink.LinkUpdate{}) + assert.Nil(t, err) +} + +func TestHandleRTMDelAddr(t *testing.T) { + n := &netmon{} + + err := n.handleRTMDelAddr(netlink.LinkUpdate{}) + assert.Nil(t, err) +} + +func TestHandleRTMNewLink(t *testing.T) { + n := &netmon{} + ev := netlink.LinkUpdate{ + Link: &netlink.Dummy{}, + } + + // LinkAttrs is nil + err := n.handleRTMNewLink(ev) + assert.Nil(t, err) + + // Link name contains "kata" suffix + ev = netlink.LinkUpdate{ + Link: &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "foo_kata", + }, + }, + } + err = n.handleRTMNewLink(ev) + assert.Nil(t, err) + + // Interface already exist in list + n.netIfaces = make(map[int]Interface) + n.netIfaces[testIfaceIndex] = Interface{} + ev = netlink.LinkUpdate{ + Link: &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "foo0", + }, + }, + } + ev.Index = testIfaceIndex + err = n.handleRTMNewLink(ev) + assert.Nil(t, err) + + // Flags are not up and running + n.netIfaces = make(map[int]Interface) + ev = netlink.LinkUpdate{ + Link: &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "foo0", + }, + }, + } + ev.Index = testIfaceIndex + err = n.handleRTMNewLink(ev) + assert.Nil(t, err) + + // Invalid link + n.netIfaces = make(map[int]Interface) + ev = netlink.LinkUpdate{ + Link: &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "foo0", + }, + }, + } + ev.Index = testIfaceIndex + ev.Flags = unix.IFF_UP | unix.IFF_RUNNING + handler, err := netlink.NewHandle(netlinkFamily) + assert.Nil(t, err) + assert.NotNil(t, handler) + defer handler.Delete() + n.netHandler = handler + err = n.handleRTMNewLink(ev) + assert.NotNil(t, err) +} + +func TestHandleRTMDelLink(t *testing.T) { + n := &netmon{} + ev := netlink.LinkUpdate{ + Link: &netlink.Dummy{}, + } + + // LinkAttrs is nil + err := n.handleRTMDelLink(ev) + assert.Nil(t, err) + + // Link name contains "kata" suffix + ev = netlink.LinkUpdate{ + Link: &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "foo_kata", + }, + }, + } + err = n.handleRTMDelLink(ev) + assert.Nil(t, err) + + // Interface does not exist in list + n.netIfaces = make(map[int]Interface) + ev = netlink.LinkUpdate{ + Link: &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "foo0", + }, + }, + } + ev.Index = testIfaceIndex + err = n.handleRTMDelLink(ev) + assert.Nil(t, err) +} + +func TestHandleRTMNewRouteIfaceNotFound(t *testing.T) { + n := &netmon{ + netIfaces: make(map[int]Interface), + } + + err := n.handleRTMNewRoute(netlink.RouteUpdate{}) + assert.Nil(t, err) +} + +func TestHandleLinkEvent(t *testing.T) { + n := &netmon{} + ev := netlink.LinkUpdate{} + + // Unknown event + err := n.handleLinkEvent(ev) + assert.Nil(t, err) + + // DONE event + ev.Header.Type = unix.NLMSG_DONE + err = n.handleLinkEvent(ev) + assert.Nil(t, err) + + // ERROR event + ev.Header.Type = unix.NLMSG_ERROR + err = n.handleLinkEvent(ev) + assert.NotNil(t, err) + + // NEWADDR event + ev.Header.Type = unix.RTM_NEWADDR + err = n.handleLinkEvent(ev) + assert.Nil(t, err) + + // DELADDR event + ev.Header.Type = unix.RTM_DELADDR + err = n.handleLinkEvent(ev) + assert.Nil(t, err) + + // NEWLINK event + ev.Header.Type = unix.RTM_NEWLINK + ev.Link = &netlink.Dummy{} + err = n.handleLinkEvent(ev) + assert.Nil(t, err) + + // DELLINK event + ev.Header.Type = unix.RTM_DELLINK + ev.Link = &netlink.Dummy{} + err = n.handleLinkEvent(ev) + assert.Nil(t, err) +} + +func TestHandleRouteEvent(t *testing.T) { + n := &netmon{} + ev := netlink.RouteUpdate{} + + // Unknown event + err := n.handleRouteEvent(ev) + assert.Nil(t, err) + + // RTM_NEWROUTE event + ev.Type = unix.RTM_NEWROUTE + err = n.handleRouteEvent(ev) + assert.Nil(t, err) + + trueBinPath, err := exec.LookPath("true") + assert.Nil(t, err) + assert.NotEmpty(t, trueBinPath) + + n.runtimePath = trueBinPath + n.sharedFile = filepath.Join(testStorageParentPath, testSharedFile) + + err = os.MkdirAll(testStorageParentPath, storageDirPerm) + assert.Nil(t, err) + defer os.RemoveAll(testStorageParentPath) + + tearDownNetworkCb := testSetupNetwork(t) + defer tearDownNetworkCb() + + handler, err := netlink.NewHandle(netlinkFamily) + assert.Nil(t, err) + assert.NotNil(t, handler) + defer handler.Delete() + + n.netHandler = handler + + // RTM_DELROUTE event + ev.Type = unix.RTM_DELROUTE + err = n.handleRouteEvent(ev) + assert.Nil(t, err) +} diff --git a/vendor/github.com/kata-containers/agent/protocols/client/client.go b/vendor/github.com/kata-containers/agent/protocols/client/client.go index 6e386fe129..4da19ece98 100644 --- a/vendor/github.com/kata-containers/agent/protocols/client/client.go +++ b/vendor/github.com/kata-containers/agent/protocols/client/client.go @@ -8,6 +8,7 @@ package client import ( "context" + "fmt" "net" "net/url" "strconv" @@ -31,6 +32,7 @@ const ( ) var defaultDialTimeout = 15 * time.Second +var defaultCloseTimeout = 5 * time.Second // AgentClient is an agent gRPC client connection wrapper for agentgrpc.AgentServiceClient type AgentClient struct { @@ -45,7 +47,26 @@ type yamuxSessionStream struct { } func (y *yamuxSessionStream) Close() error { - return y.session.Close() + waitCh := y.session.CloseChan() + timeout := time.NewTimer(defaultCloseTimeout) + + if err := y.Conn.Close(); err != nil { + return err + } + + if err := y.session.Close(); err != nil { + return err + } + + // block until session is really closed + select { + case <-waitCh: + timeout.Stop() + case <-timeout.C: + return fmt.Errorf("timeout waiting for session close") + } + + return nil } type dialer func(string, time.Duration) (net.Conn, error) diff --git a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go index 07164f1360..ca0c995ef5 100644 --- a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go +++ b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go @@ -1173,6 +1173,10 @@ type Interface struct { IPAddresses []*IPAddress `protobuf:"bytes,3,rep,name=IPAddresses" json:"IPAddresses,omitempty"` Mtu uint64 `protobuf:"varint,4,opt,name=mtu,proto3" json:"mtu,omitempty"` HwAddr string `protobuf:"bytes,5,opt,name=hwAddr,proto3" json:"hwAddr,omitempty"` + // pciAddr is the PCI address in the format "bridgeAddr/deviceAddr". + // Here, bridgeAddr is the address at which the bridge is attached on the root bus, + // while deviceAddr is the address at which the network device is attached on the bridge. + PciAddr string `protobuf:"bytes,6,opt,name=pciAddr,proto3" json:"pciAddr,omitempty"` } func (m *Interface) Reset() { *m = Interface{} } @@ -1215,6 +1219,13 @@ func (m *Interface) GetHwAddr() string { return "" } +func (m *Interface) GetPciAddr() string { + if m != nil { + return m.PciAddr + } + return "" +} + type Interfaces struct { Interfaces []*Interface `protobuf:"bytes,1,rep,name=Interfaces" json:"Interfaces,omitempty"` } @@ -1382,6 +1393,8 @@ type OnlineCPUMemRequest struct { Wait bool `protobuf:"varint,1,opt,name=wait,proto3" json:"wait,omitempty"` // NbCpus specifies the number of CPUs that were added and the agent has to online. NbCpus uint32 `protobuf:"varint,2,opt,name=nb_cpus,json=nbCpus,proto3" json:"nb_cpus,omitempty"` + // CpuOnly specifies whether only online CPU or not. + CpuOnly bool `protobuf:"varint,3,opt,name=cpu_only,json=cpuOnly,proto3" json:"cpu_only,omitempty"` } func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } @@ -1403,6 +1416,13 @@ func (m *OnlineCPUMemRequest) GetNbCpus() uint32 { return 0 } +func (m *OnlineCPUMemRequest) GetCpuOnly() bool { + if m != nil { + return m.CpuOnly + } + return false +} + type ReseedRandomDevRequest struct { // Data specifies the random data used to reseed the guest crng. Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` @@ -3948,6 +3968,12 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintAgent(dAtA, i, uint64(len(m.HwAddr))) i += copy(dAtA[i:], m.HwAddr) } + if len(m.PciAddr) > 0 { + dAtA[i] = 0x32 + i++ + i = encodeVarintAgent(dAtA, i, uint64(len(m.PciAddr))) + i += copy(dAtA[i:], m.PciAddr) + } return i, nil } @@ -4236,6 +4262,16 @@ func (m *OnlineCPUMemRequest) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintAgent(dAtA, i, uint64(m.NbCpus)) } + if m.CpuOnly { + dAtA[i] = 0x18 + i++ + if m.CpuOnly { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -5052,6 +5088,10 @@ func (m *Interface) Size() (n int) { if l > 0 { n += 1 + l + sovAgent(uint64(l)) } + l = len(m.PciAddr) + if l > 0 { + n += 1 + l + sovAgent(uint64(l)) + } return n } @@ -5165,6 +5205,9 @@ func (m *OnlineCPUMemRequest) Size() (n int) { if m.NbCpus != 0 { n += 1 + sovAgent(uint64(m.NbCpus)) } + if m.CpuOnly { + n += 2 + } return n } @@ -9782,6 +9825,35 @@ func (m *Interface) Unmarshal(dAtA []byte) error { } m.HwAddr = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PciAddr", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthAgent + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PciAddr = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipAgent(dAtA[iNdEx:]) @@ -10650,6 +10722,26 @@ func (m *OnlineCPUMemRequest) Unmarshal(dAtA []byte) error { break } } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CpuOnly", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.CpuOnly = bool(v != 0) default: iNdEx = preIndex skippy, err := skipAgent(dAtA[iNdEx:]) @@ -11416,152 +11508,154 @@ var ( func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } var fileDescriptorAgent = []byte{ - // 2352 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0x5f, 0x6f, 0x1c, 0x49, - 0x11, 0x67, 0xff, 0xda, 0x5b, 0xbb, 0x6b, 0x7b, 0xdb, 0x8e, 0xb3, 0xb7, 0x39, 0x82, 0x6f, 0x02, - 0x39, 0x73, 0x47, 0x1c, 0x9d, 0x13, 0xc1, 0x29, 0x51, 0x14, 0x62, 0xc7, 0x38, 0xe6, 0x2e, 0x64, - 0x19, 0xc7, 0x0a, 0x12, 0x0f, 0xab, 0xf1, 0x4c, 0x7b, 0xdd, 0x97, 0x9d, 0xe9, 0xb9, 0xee, 0x1e, - 0xdb, 0xcb, 0x49, 0x3c, 0xf2, 0x09, 0x78, 0xe5, 0x0b, 0x20, 0xde, 0xee, 0x2b, 0xf0, 0xc0, 0x23, - 0x9f, 0x00, 0xa1, 0x7c, 0x01, 0x24, 0x3e, 0x01, 0xea, 0x7f, 0xf3, 0x67, 0xff, 0x38, 0xe0, 0xb3, - 0xc4, 0xcb, 0xee, 0x54, 0x75, 0x75, 0xd5, 0xaf, 0xaa, 0xbb, 0xab, 0xab, 0x0b, 0x9a, 0xde, 0x10, - 0x47, 0x62, 0x2b, 0x66, 0x54, 0x50, 0x54, 0x1d, 0xb2, 0xd8, 0xef, 0x35, 0xa8, 0x4f, 0x34, 0xa3, - 0x77, 0x6b, 0x48, 0xe9, 0x70, 0x84, 0xef, 0x2b, 0xea, 0x38, 0x39, 0xb9, 0x8f, 0xc3, 0x58, 0x8c, - 0xf5, 0xa0, 0xf3, 0xa7, 0x32, 0xac, 0xef, 0x32, 0xec, 0x09, 0xbc, 0x4b, 0x23, 0xe1, 0x91, 0x08, - 0x33, 0x17, 0x7f, 0x9d, 0x60, 0x2e, 0xd0, 0x47, 0xd0, 0xf2, 0x2d, 0x6f, 0x40, 0x82, 0x6e, 0x69, - 0xa3, 0xb4, 0xd9, 0x70, 0x9b, 0x29, 0xef, 0x20, 0x40, 0x37, 0x61, 0x01, 0x5f, 0x60, 0x5f, 0x8e, - 0x96, 0xd5, 0x68, 0x5d, 0x92, 0x07, 0x01, 0xfa, 0x0c, 0x9a, 0x5c, 0x30, 0x12, 0x0d, 0x07, 0x09, - 0xc7, 0xac, 0x5b, 0xd9, 0x28, 0x6d, 0x36, 0xb7, 0x57, 0xb6, 0x24, 0xb4, 0xad, 0x43, 0x35, 0x70, - 0xc4, 0x31, 0x73, 0x81, 0xa7, 0xdf, 0xe8, 0x2e, 0x2c, 0x04, 0xf8, 0x8c, 0xf8, 0x98, 0x77, 0xab, - 0x1b, 0x95, 0xcd, 0xe6, 0x76, 0x4b, 0x8b, 0x3f, 0x57, 0x4c, 0xd7, 0x0e, 0xa2, 0x1f, 0xc3, 0x22, - 0x17, 0x94, 0x79, 0x43, 0xcc, 0xbb, 0x35, 0x25, 0xd8, 0xb6, 0x7a, 0x15, 0xd7, 0x4d, 0x87, 0xd1, - 0x87, 0x50, 0x79, 0xb5, 0x7b, 0xd0, 0xad, 0x2b, 0xeb, 0x60, 0xa4, 0x62, 0xec, 0xbb, 0x92, 0x8d, - 0xee, 0x40, 0x9b, 0x7b, 0x51, 0x70, 0x4c, 0x2f, 0x06, 0x31, 0x09, 0x22, 0xde, 0x5d, 0xd8, 0x28, - 0x6d, 0x2e, 0xba, 0x2d, 0xc3, 0xec, 0x4b, 0x9e, 0xf3, 0x08, 0x6e, 0x1c, 0x0a, 0x8f, 0x89, 0x2b, - 0x44, 0xc7, 0x39, 0x82, 0x75, 0x17, 0x87, 0xf4, 0xec, 0x4a, 0xa1, 0xed, 0xc2, 0x82, 0x20, 0x21, - 0xa6, 0x89, 0x50, 0xa1, 0x6d, 0xbb, 0x96, 0x74, 0xfe, 0x52, 0x02, 0xb4, 0x77, 0x81, 0xfd, 0x3e, - 0xa3, 0x3e, 0xe6, 0xfc, 0xff, 0xb4, 0x5c, 0x1f, 0xc3, 0x42, 0xac, 0x01, 0x74, 0xab, 0x4a, 0xdc, - 0xac, 0x82, 0x45, 0x65, 0x47, 0x9d, 0xaf, 0x60, 0xed, 0x90, 0x0c, 0x23, 0x6f, 0x74, 0x8d, 0x78, - 0xd7, 0xa1, 0xce, 0x95, 0x4e, 0x05, 0xb5, 0xed, 0x1a, 0xca, 0xe9, 0x03, 0x7a, 0xe3, 0x11, 0x71, - 0x7d, 0x96, 0x9c, 0x7b, 0xb0, 0x5a, 0xd0, 0xc8, 0x63, 0x1a, 0x71, 0xac, 0x00, 0x08, 0x4f, 0x24, - 0x5c, 0x29, 0xab, 0xb9, 0x86, 0x72, 0x30, 0xac, 0x7d, 0x49, 0xb8, 0x15, 0xc7, 0xff, 0x0b, 0x84, - 0x75, 0xa8, 0x9f, 0x50, 0x16, 0x7a, 0xc2, 0x22, 0xd0, 0x14, 0x42, 0x50, 0xf5, 0xd8, 0x90, 0x77, - 0x2b, 0x1b, 0x95, 0xcd, 0x86, 0xab, 0xbe, 0xe5, 0xae, 0x9c, 0x30, 0x63, 0x70, 0x7d, 0x04, 0x2d, - 0x13, 0xf7, 0xc1, 0x88, 0x70, 0xa1, 0xec, 0xb4, 0xdc, 0xa6, 0xe1, 0xc9, 0x39, 0x0e, 0x85, 0xf5, - 0xa3, 0x38, 0xb8, 0xe2, 0x81, 0xdf, 0x86, 0x06, 0xc3, 0x9c, 0x26, 0x4c, 0x1e, 0xd3, 0xb2, 0x5a, - 0xf7, 0x35, 0xbd, 0xee, 0x5f, 0x92, 0x28, 0xb9, 0x70, 0xed, 0x98, 0x9b, 0x89, 0x99, 0x23, 0x24, - 0xf8, 0x55, 0x8e, 0xd0, 0x23, 0xb8, 0xd1, 0xf7, 0x12, 0x7e, 0x15, 0xac, 0xce, 0x63, 0x79, 0xfc, - 0x78, 0x12, 0x5e, 0x69, 0xf2, 0x9f, 0x4b, 0xb0, 0xb8, 0x1b, 0x27, 0x47, 0xdc, 0x1b, 0x62, 0xf4, - 0x03, 0x68, 0x0a, 0x2a, 0xbc, 0xd1, 0x20, 0x91, 0xa4, 0x12, 0xaf, 0xba, 0xa0, 0x58, 0x5a, 0x40, - 0x86, 0x1d, 0x33, 0x3f, 0x4e, 0x8c, 0x44, 0x79, 0xa3, 0xb2, 0x59, 0x75, 0x9b, 0x9a, 0xa7, 0x45, - 0xb6, 0x60, 0x55, 0x8d, 0x0d, 0x48, 0x34, 0x78, 0x8b, 0x59, 0x84, 0x47, 0x21, 0x0d, 0xb0, 0xda, - 0xbf, 0x55, 0xb7, 0xa3, 0x86, 0x0e, 0xa2, 0x2f, 0xd2, 0x01, 0xf4, 0x09, 0x74, 0x52, 0x79, 0x79, - 0x28, 0x95, 0x74, 0x55, 0x49, 0x2f, 0x1b, 0xe9, 0x23, 0xc3, 0x76, 0x7e, 0x0f, 0x4b, 0xaf, 0x4f, - 0x19, 0x15, 0x62, 0x44, 0xa2, 0xe1, 0x73, 0x4f, 0x78, 0x32, 0x7b, 0xc4, 0x98, 0x11, 0x1a, 0x70, - 0x83, 0xd6, 0x92, 0xe8, 0x53, 0xe8, 0x08, 0x2d, 0x8b, 0x83, 0x81, 0x95, 0x29, 0x2b, 0x99, 0x95, - 0x74, 0xa0, 0x6f, 0x84, 0x7f, 0x04, 0x4b, 0x99, 0xb0, 0xcc, 0x3f, 0x06, 0x6f, 0x3b, 0xe5, 0xbe, - 0x26, 0x21, 0x76, 0xce, 0x54, 0xac, 0xd4, 0x22, 0xa3, 0x4f, 0xa1, 0x91, 0xc5, 0xa1, 0xa4, 0x76, - 0xc8, 0x92, 0xde, 0x21, 0x36, 0x9c, 0xee, 0x62, 0x1a, 0x94, 0x27, 0xb0, 0x2c, 0x52, 0xe0, 0x83, - 0xc0, 0x13, 0x5e, 0x71, 0x53, 0x15, 0xbd, 0x72, 0x97, 0x44, 0x81, 0x76, 0x1e, 0x43, 0xa3, 0x4f, - 0x02, 0xae, 0x0d, 0x77, 0x61, 0xc1, 0x4f, 0x18, 0xc3, 0x91, 0xb0, 0x2e, 0x1b, 0x12, 0xad, 0x41, - 0x6d, 0x44, 0x42, 0x22, 0x8c, 0x9b, 0x9a, 0x70, 0x28, 0xc0, 0x4b, 0x1c, 0x52, 0x36, 0x56, 0x01, - 0x5b, 0x83, 0x5a, 0x7e, 0x71, 0x35, 0x81, 0x6e, 0x41, 0x23, 0xf4, 0x2e, 0xd2, 0x45, 0x95, 0x23, - 0x8b, 0xa1, 0x77, 0xa1, 0xc1, 0x77, 0x61, 0xe1, 0xc4, 0x23, 0x23, 0x3f, 0x12, 0x26, 0x2a, 0x96, - 0xcc, 0x0c, 0x56, 0xf3, 0x06, 0xff, 0x5a, 0x86, 0xa6, 0xb6, 0xa8, 0x01, 0xaf, 0x41, 0xcd, 0xf7, - 0xfc, 0xd3, 0xd4, 0xa4, 0x22, 0xd0, 0x5d, 0x0b, 0xa4, 0x9c, 0x4f, 0xc2, 0x19, 0x52, 0x0b, 0xed, - 0x3e, 0x00, 0x3f, 0xf7, 0x62, 0x83, 0xad, 0x32, 0x47, 0xb8, 0x21, 0x65, 0x34, 0xdc, 0x07, 0xd0, - 0xd2, 0xfb, 0xce, 0x4c, 0xa9, 0xce, 0x99, 0xd2, 0xd4, 0x52, 0x7a, 0xd2, 0x1d, 0x68, 0x27, 0x1c, - 0x0f, 0x4e, 0x09, 0x66, 0x1e, 0xf3, 0x4f, 0xc7, 0xdd, 0x9a, 0xbe, 0x23, 0x13, 0x8e, 0x5f, 0x58, - 0x1e, 0xda, 0x86, 0x9a, 0x4c, 0x7f, 0xbc, 0x5b, 0x57, 0xd7, 0xf1, 0x87, 0x79, 0x95, 0xca, 0xd5, - 0x2d, 0xf5, 0xbb, 0x17, 0x09, 0x36, 0x76, 0xb5, 0x68, 0xef, 0x73, 0x80, 0x8c, 0x89, 0x56, 0xa0, - 0xf2, 0x16, 0x8f, 0xcd, 0x39, 0x94, 0x9f, 0x32, 0x38, 0x67, 0xde, 0x28, 0xb1, 0x51, 0xd7, 0xc4, - 0xa3, 0xf2, 0xe7, 0x25, 0xc7, 0x87, 0xe5, 0x9d, 0xd1, 0x5b, 0x42, 0x73, 0xd3, 0xd7, 0xa0, 0x16, - 0x7a, 0x5f, 0x51, 0x66, 0x23, 0xa9, 0x08, 0xc5, 0x25, 0x11, 0x65, 0x56, 0x85, 0x22, 0xd0, 0x12, - 0x94, 0x69, 0xac, 0xe2, 0xd5, 0x70, 0xcb, 0x34, 0xce, 0x0c, 0x55, 0x73, 0x86, 0x9c, 0x7f, 0x54, - 0x01, 0x32, 0x2b, 0xc8, 0x85, 0x1e, 0xa1, 0x03, 0x8e, 0x99, 0x2c, 0x41, 0x06, 0xc7, 0x63, 0x81, - 0xf9, 0x80, 0x61, 0x3f, 0x61, 0x9c, 0x9c, 0xc9, 0xf5, 0x93, 0x6e, 0xdf, 0xd0, 0x6e, 0x4f, 0x60, - 0x73, 0x6f, 0x12, 0x7a, 0xa8, 0xe7, 0xed, 0xc8, 0x69, 0xae, 0x9d, 0x85, 0x0e, 0xe0, 0x46, 0xa6, - 0x33, 0xc8, 0xa9, 0x2b, 0x5f, 0xa6, 0x6e, 0x35, 0x55, 0x17, 0x64, 0xaa, 0xf6, 0x60, 0x95, 0xd0, - 0xc1, 0xd7, 0x09, 0x4e, 0x0a, 0x8a, 0x2a, 0x97, 0x29, 0xea, 0x10, 0xfa, 0x6b, 0x35, 0x21, 0x53, - 0xd3, 0x87, 0x0f, 0x72, 0x5e, 0xca, 0xe3, 0x9e, 0x53, 0x56, 0xbd, 0x4c, 0xd9, 0x7a, 0x8a, 0x4a, - 0xe6, 0x83, 0x4c, 0xe3, 0x2f, 0x61, 0x9d, 0xd0, 0xc1, 0xb9, 0x47, 0xc4, 0xa4, 0xba, 0xda, 0x7b, - 0x9c, 0x94, 0x97, 0x6e, 0x51, 0x97, 0x76, 0x32, 0xc4, 0x6c, 0x58, 0x70, 0xb2, 0xfe, 0x1e, 0x27, - 0x5f, 0xaa, 0x09, 0x99, 0x9a, 0x67, 0xd0, 0x21, 0x74, 0x12, 0xcd, 0xc2, 0x65, 0x4a, 0x96, 0x09, - 0x2d, 0x22, 0xd9, 0x81, 0x0e, 0xc7, 0xbe, 0xa0, 0x2c, 0xbf, 0x09, 0x16, 0x2f, 0x53, 0xb1, 0x62, - 0xe4, 0x53, 0x1d, 0xce, 0x6f, 0xa1, 0xf5, 0x22, 0x19, 0x62, 0x31, 0x3a, 0x4e, 0x93, 0xc1, 0xb5, - 0xe5, 0x1f, 0xe7, 0xdf, 0x65, 0x68, 0xee, 0x0e, 0x19, 0x4d, 0xe2, 0x42, 0x4e, 0xd6, 0x87, 0x74, - 0x32, 0x27, 0x2b, 0x11, 0x95, 0x93, 0xb5, 0xf0, 0x43, 0x68, 0x85, 0xea, 0xe8, 0x1a, 0x79, 0x9d, - 0x87, 0x3a, 0x53, 0x87, 0xda, 0x6d, 0x86, 0xb9, 0x64, 0xb6, 0x05, 0x10, 0x93, 0x80, 0x9b, 0x39, - 0x3a, 0x1d, 0x2d, 0x9b, 0x8a, 0xd0, 0xa6, 0x68, 0xb7, 0x11, 0xa7, 0xd9, 0xfa, 0x33, 0x68, 0x1e, - 0xcb, 0x20, 0x99, 0x09, 0x85, 0x64, 0x94, 0x45, 0xcf, 0x85, 0xe3, 0xec, 0x10, 0xbe, 0x80, 0xf6, - 0xa9, 0x0e, 0x99, 0x99, 0xa4, 0xf7, 0xd0, 0x1d, 0xe3, 0x49, 0xe6, 0xef, 0x56, 0x3e, 0xb2, 0x7a, - 0x01, 0x5a, 0xa7, 0x39, 0x56, 0xef, 0x10, 0x3a, 0x53, 0x22, 0x33, 0x72, 0xd0, 0x66, 0x3e, 0x07, - 0x35, 0xb7, 0x91, 0x36, 0x94, 0x9f, 0x99, 0xcf, 0x4b, 0xbf, 0x82, 0xf5, 0xc9, 0x32, 0xc7, 0x14, - 0x65, 0x0f, 0xa1, 0xe5, 0x2b, 0x74, 0x85, 0x15, 0xe8, 0x4c, 0xe1, 0x76, 0x9b, 0x7e, 0x46, 0x38, - 0x01, 0xa0, 0x37, 0x8c, 0x08, 0x7c, 0x28, 0x18, 0xf6, 0xc2, 0xeb, 0xa8, 0x9a, 0x11, 0x54, 0xd5, - 0x15, 0x5b, 0x51, 0x45, 0xa1, 0xfa, 0x76, 0x3e, 0x86, 0xd5, 0x82, 0x15, 0x03, 0x79, 0x05, 0x2a, - 0x23, 0x1c, 0x29, 0xed, 0x6d, 0x57, 0x7e, 0x3a, 0x1e, 0x74, 0x5c, 0xec, 0x05, 0xd7, 0x87, 0xc6, - 0x98, 0xa8, 0x64, 0x26, 0x36, 0x01, 0xe5, 0x4d, 0x18, 0x28, 0x16, 0x75, 0x29, 0x87, 0xfa, 0x15, - 0x74, 0x76, 0x47, 0x94, 0xe3, 0x43, 0x11, 0x90, 0xe8, 0x3a, 0xca, 0xfc, 0x6f, 0x60, 0xf5, 0xb5, - 0x18, 0xbf, 0x91, 0xca, 0x38, 0xf9, 0x1d, 0xbe, 0x26, 0xff, 0x18, 0x3d, 0xb7, 0xfe, 0x31, 0x7a, - 0x2e, 0x2b, 0x7c, 0x9f, 0x8e, 0x92, 0x30, 0x52, 0xdb, 0xbd, 0xed, 0x1a, 0xca, 0xf9, 0xb6, 0x04, - 0x6b, 0xfa, 0x0d, 0x7e, 0xa8, 0x9f, 0x9e, 0xd6, 0x7c, 0x0f, 0x16, 0x4f, 0x29, 0x17, 0x91, 0x17, - 0x62, 0x63, 0x3a, 0xa5, 0xa5, 0x7a, 0xf9, 0x66, 0x2d, 0xab, 0x57, 0x81, 0xfc, 0x2c, 0x3c, 0x8c, - 0x2b, 0x97, 0x3f, 0x8c, 0xa7, 0x9e, 0xbe, 0xd5, 0xe9, 0xa7, 0x2f, 0xfa, 0x3e, 0x80, 0x15, 0x22, - 0x81, 0xba, 0xf8, 0x1b, 0x6e, 0xc3, 0x70, 0x0e, 0x02, 0xe7, 0x26, 0xdc, 0x78, 0x8e, 0xb9, 0x60, - 0x74, 0x5c, 0x44, 0xed, 0x78, 0xd0, 0x38, 0xe8, 0x3f, 0x0b, 0x02, 0x86, 0x39, 0x47, 0x77, 0xa1, - 0x7e, 0xe2, 0x85, 0x64, 0xa4, 0x0f, 0xd6, 0x92, 0xcd, 0x3b, 0x07, 0xfd, 0x5f, 0x28, 0xae, 0x6b, - 0x46, 0x65, 0x32, 0xf3, 0xf4, 0x14, 0x13, 0x46, 0x4b, 0xca, 0xf5, 0x0f, 0x3d, 0xfe, 0xd6, 0x5c, - 0xd9, 0xea, 0xdb, 0xf9, 0x63, 0x09, 0x1a, 0x07, 0x91, 0xc0, 0xec, 0xc4, 0xf3, 0xd5, 0x63, 0x4c, - 0x37, 0x07, 0x4c, 0x90, 0x0c, 0x25, 0x67, 0xaa, 0xd0, 0x69, 0x85, 0xea, 0x5b, 0xe6, 0x9d, 0x14, - 0x5c, 0x1a, 0xa7, 0x65, 0x0b, 0xca, 0x0c, 0xb8, 0x79, 0x19, 0x19, 0xe9, 0x50, 0x24, 0xa6, 0x3e, - 0x90, 0x9f, 0xd2, 0xe0, 0xe9, 0xb9, 0x14, 0x30, 0x51, 0x31, 0x94, 0xf3, 0x04, 0x20, 0x45, 0xc5, - 0x65, 0x85, 0x96, 0x51, 0xa6, 0x48, 0xb0, 0x96, 0x2c, 0xdf, 0xcd, 0x89, 0x38, 0xdf, 0x40, 0xcd, - 0xa5, 0x89, 0xd0, 0x5b, 0x1e, 0x9b, 0xd7, 0x5b, 0xc3, 0x55, 0xdf, 0x32, 0x40, 0x43, 0x4f, 0xe0, - 0x73, 0x6f, 0x6c, 0x03, 0x64, 0xc8, 0x9c, 0xfb, 0x95, 0x82, 0xfb, 0xf2, 0x8d, 0xaa, 0x9e, 0x60, - 0x0a, 0x7a, 0xc3, 0x35, 0x94, 0xbc, 0x6a, 0xb8, 0x4f, 0x63, 0xac, 0xc0, 0xb7, 0x5d, 0x4d, 0x38, - 0xf7, 0xa0, 0xae, 0x8c, 0xcb, 0xcd, 0x61, 0xbe, 0x0c, 0xe6, 0xa6, 0xc6, 0xac, 0x78, 0xae, 0x19, - 0x72, 0xf6, 0xed, 0x2b, 0x32, 0x73, 0xc5, 0x6c, 0xda, 0x7b, 0xd0, 0x20, 0x96, 0x67, 0x52, 0xdd, - 0x94, 0xd7, 0x99, 0x84, 0xf3, 0x1c, 0x56, 0x9f, 0x05, 0xc1, 0x77, 0xd5, 0xb2, 0x6f, 0x5b, 0x2d, - 0xdf, 0x55, 0xd1, 0x63, 0x58, 0xd5, 0x7e, 0x69, 0x3f, 0xad, 0x96, 0x1f, 0x42, 0x9d, 0xd9, 0x98, - 0x94, 0xb2, 0xde, 0x94, 0x11, 0x32, 0x63, 0xf2, 0x48, 0xc8, 0x27, 0x76, 0xb6, 0xa4, 0xf6, 0x48, - 0xac, 0x42, 0x47, 0x0e, 0x14, 0x74, 0x3a, 0x3b, 0xb0, 0xfa, 0x2a, 0x1a, 0x91, 0x08, 0xef, 0xf6, - 0x8f, 0x5e, 0xe2, 0x34, 0xa7, 0x22, 0xa8, 0xca, 0x82, 0x49, 0x19, 0x5a, 0x74, 0xd5, 0xb7, 0x4c, - 0x32, 0xd1, 0xf1, 0xc0, 0x8f, 0x13, 0x6e, 0x9a, 0x41, 0xf5, 0xe8, 0x78, 0x37, 0x4e, 0xb8, 0xf3, - 0x13, 0xf5, 0xc6, 0xc5, 0x38, 0x70, 0xbd, 0x28, 0xa0, 0xe1, 0x73, 0x7c, 0x96, 0x53, 0x93, 0xbe, - 0xa7, 0x6c, 0xda, 0xfc, 0xb6, 0x04, 0x0b, 0x26, 0x19, 0xa8, 0x5d, 0xc3, 0xc8, 0x19, 0x66, 0xe9, - 0xa1, 0x51, 0x94, 0x7c, 0xf2, 0xe9, 0xaf, 0x01, 0x8d, 0x05, 0xa1, 0x69, 0x8a, 0x69, 0x6b, 0xee, - 0x2b, 0xcd, 0xcc, 0x6d, 0xae, 0x4a, 0x61, 0x73, 0xad, 0x43, 0xfd, 0x84, 0x8b, 0x71, 0x9c, 0x6e, - 0x3a, 0x4d, 0xc9, 0xed, 0x6b, 0xf5, 0xd5, 0x94, 0x3e, 0x4b, 0xca, 0xc7, 0x75, 0x48, 0x93, 0x48, - 0x0c, 0x62, 0x4a, 0x22, 0xa1, 0x9a, 0x75, 0x0d, 0x17, 0x14, 0xab, 0x2f, 0x39, 0xce, 0x1f, 0x4a, - 0x50, 0xd7, 0x4d, 0x40, 0x59, 0xbc, 0xa7, 0x59, 0xb8, 0x4c, 0xd4, 0x8d, 0xa6, 0x6c, 0x99, 0x13, - 0xae, 0x2c, 0xdd, 0x84, 0x85, 0xb3, 0x70, 0x10, 0x7b, 0xe2, 0xd4, 0x42, 0x3b, 0x0b, 0xfb, 0x9e, - 0x38, 0x95, 0x9e, 0x65, 0xc9, 0x5c, 0x8d, 0x6b, 0x88, 0xed, 0x94, 0xab, 0xc4, 0xe6, 0x22, 0x75, - 0x7e, 0x23, 0xdf, 0x2c, 0x69, 0x03, 0x6c, 0x05, 0x2a, 0x49, 0x0a, 0x46, 0x7e, 0x4a, 0xce, 0x30, - 0xbd, 0x06, 0xe4, 0x27, 0xba, 0x0b, 0x4b, 0x5e, 0x10, 0x10, 0x39, 0xdd, 0x1b, 0xed, 0x93, 0xc0, - 0x76, 0x71, 0x26, 0xb8, 0x9f, 0xf4, 0x60, 0xd1, 0x66, 0x44, 0x54, 0x87, 0xf2, 0xd9, 0xc3, 0x95, - 0xef, 0xa9, 0xff, 0x9f, 0xae, 0x94, 0xb6, 0xff, 0xd5, 0x86, 0xd6, 0xb3, 0x21, 0x8e, 0x84, 0xa9, - 0xb0, 0xd1, 0x3e, 0x2c, 0x4f, 0x74, 0x6c, 0x91, 0x79, 0x72, 0xcd, 0x6e, 0xe4, 0xf6, 0xd6, 0xb7, - 0x74, 0x07, 0x78, 0xcb, 0x76, 0x80, 0xb7, 0xf6, 0xc2, 0x58, 0x8c, 0xd1, 0x1e, 0x2c, 0x15, 0x7b, - 0x9b, 0xe8, 0x96, 0xbd, 0x30, 0x66, 0x74, 0x3c, 0xe7, 0xaa, 0xd9, 0x87, 0xe5, 0x89, 0x36, 0xa7, - 0xc5, 0x33, 0xbb, 0xfb, 0x39, 0x57, 0xd1, 0x53, 0x68, 0xe6, 0xfa, 0x9a, 0xa8, 0xab, 0x95, 0x4c, - 0xb7, 0x3a, 0xe7, 0x2a, 0xd8, 0x85, 0x76, 0xa1, 0xd5, 0x88, 0x7a, 0xc6, 0x9f, 0x19, 0xfd, 0xc7, - 0xb9, 0x4a, 0x76, 0xa0, 0x99, 0xeb, 0xf8, 0x59, 0x14, 0xd3, 0x6d, 0xc5, 0xde, 0x07, 0x33, 0x46, - 0x4c, 0xcd, 0xf2, 0x02, 0xda, 0x85, 0xfe, 0x9c, 0x05, 0x32, 0xab, 0x37, 0xd8, 0xbb, 0x35, 0x73, - 0xcc, 0x68, 0xda, 0x87, 0xe5, 0x89, 0x6e, 0x9d, 0x0d, 0xee, 0xec, 0x26, 0xde, 0x5c, 0xb7, 0xbe, - 0x50, 0x8b, 0x9d, 0x2b, 0x4f, 0x73, 0x8b, 0x3d, 0xdd, 0x9b, 0xeb, 0x7d, 0x38, 0x7b, 0xd0, 0xa0, - 0xda, 0x83, 0xa5, 0x62, 0x5b, 0xce, 0x2a, 0x9b, 0xd9, 0xac, 0xbb, 0x7c, 0xe7, 0x14, 0x3a, 0x74, - 0xd9, 0xce, 0x99, 0xd5, 0xb8, 0x9b, 0xab, 0xe8, 0x19, 0x80, 0xa9, 0x62, 0x03, 0x12, 0xa5, 0x4b, - 0x36, 0x55, 0x3d, 0xa7, 0x4b, 0x36, 0xa3, 0xe2, 0x7d, 0x0a, 0xa0, 0x8b, 0xcf, 0x80, 0x26, 0x02, - 0xdd, 0xb4, 0x30, 0x26, 0x2a, 0xde, 0x5e, 0x77, 0x7a, 0x60, 0x4a, 0x01, 0x66, 0xec, 0x2a, 0x0a, - 0x9e, 0x00, 0x64, 0x45, 0xad, 0x55, 0x30, 0x55, 0xe6, 0x5e, 0x12, 0x83, 0x56, 0xbe, 0x84, 0x45, - 0xc6, 0xd7, 0x19, 0x65, 0xed, 0x5c, 0x15, 0x8f, 0xa0, 0x95, 0xbf, 0x8b, 0xad, 0x8a, 0x19, 0xf7, - 0x73, 0x6f, 0xf2, 0x0e, 0x45, 0x3f, 0xb7, 0x1b, 0x35, 0x63, 0x15, 0x36, 0xea, 0x7f, 0xa5, 0x61, - 0xe2, 0x0e, 0x2f, 0xe6, 0x91, 0xf7, 0x6b, 0xf8, 0x19, 0xb4, 0xf2, 0x97, 0xb7, 0xc5, 0x3f, 0xe3, - 0x42, 0xef, 0x15, 0x2e, 0x70, 0xf4, 0x14, 0x96, 0x8a, 0x17, 0x37, 0xca, 0x1d, 0xca, 0xa9, 0xeb, - 0xbc, 0xb7, 0x32, 0x61, 0x98, 0xa3, 0x07, 0x00, 0xd9, 0x05, 0x6f, 0xd7, 0x6e, 0xea, 0xca, 0x9f, - 0xb0, 0xba, 0x0b, 0xed, 0x42, 0xd9, 0x6f, 0xb3, 0xc4, 0xac, 0xb7, 0xc0, 0x65, 0x49, 0xbc, 0x58, - 0x86, 0x5b, 0xe8, 0x33, 0x8b, 0xf3, 0xcb, 0x76, 0x4f, 0xbe, 0x18, 0xb1, 0xa1, 0x9b, 0x51, 0xa0, - 0xbc, 0xe7, 0x34, 0xe7, 0x6b, 0x91, 0xdc, 0x69, 0x9e, 0x51, 0xa2, 0xcc, 0x53, 0xb4, 0xd3, 0xfa, - 0xdb, 0xbb, 0xdb, 0xa5, 0xbf, 0xbf, 0xbb, 0x5d, 0xfa, 0xe7, 0xbb, 0xdb, 0xa5, 0xe3, 0xba, 0x1a, - 0x7d, 0xf0, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2b, 0xc8, 0x6c, 0x30, 0xe5, 0x1c, 0x00, 0x00, + // 2381 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0xdb, 0x6e, 0x1b, 0xc9, + 0xd1, 0xfe, 0x79, 0x10, 0x29, 0x16, 0x49, 0x49, 0x6c, 0xc9, 0x32, 0x4d, 0xfb, 0x77, 0xb4, 0xe3, + 0xc4, 0xab, 0xec, 0xc6, 0x32, 0x56, 0x36, 0x92, 0x85, 0x0d, 0xc3, 0xb1, 0x65, 0x45, 0x56, 0x76, + 0x1d, 0x33, 0x23, 0x0b, 0x0e, 0x10, 0x04, 0xc4, 0x68, 0xa6, 0x45, 0xf5, 0x9a, 0x33, 0x3d, 0xdb, + 0xdd, 0x23, 0x89, 0x59, 0x20, 0x97, 0x79, 0x8b, 0xbc, 0x40, 0x10, 0xe4, 0x66, 0x5f, 0x21, 0x17, + 0xb9, 0xcc, 0x13, 0x04, 0x81, 0x5f, 0x20, 0x40, 0x9e, 0x20, 0xe8, 0xd3, 0x1c, 0x78, 0x90, 0x13, + 0xad, 0x80, 0xdc, 0x90, 0x53, 0xd5, 0xd5, 0x55, 0x5f, 0x55, 0x77, 0x57, 0x57, 0x17, 0x34, 0xbd, + 0x21, 0x8e, 0xc4, 0x56, 0xcc, 0xa8, 0xa0, 0xa8, 0x3a, 0x64, 0xb1, 0xdf, 0x6b, 0x50, 0x9f, 0x68, + 0x46, 0xef, 0xe6, 0x90, 0xd2, 0xe1, 0x08, 0xdf, 0x57, 0xd4, 0x51, 0x72, 0x7c, 0x1f, 0x87, 0xb1, + 0x18, 0xeb, 0x41, 0xe7, 0x0f, 0x65, 0x58, 0xdf, 0x61, 0xd8, 0x13, 0x78, 0x87, 0x46, 0xc2, 0x23, + 0x11, 0x66, 0x2e, 0xfe, 0x3a, 0xc1, 0x5c, 0xa0, 0x8f, 0xa0, 0xe5, 0x5b, 0xde, 0x80, 0x04, 0xdd, + 0xd2, 0x46, 0x69, 0xb3, 0xe1, 0x36, 0x53, 0xde, 0x7e, 0x80, 0xae, 0x43, 0x1d, 0x9f, 0x63, 0x5f, + 0x8e, 0x96, 0xd5, 0x68, 0x4d, 0x92, 0xfb, 0x01, 0xfa, 0x0c, 0x9a, 0x5c, 0x30, 0x12, 0x0d, 0x07, + 0x09, 0xc7, 0xac, 0x5b, 0xd9, 0x28, 0x6d, 0x36, 0xb7, 0x57, 0xb6, 0x24, 0xb4, 0xad, 0x03, 0x35, + 0x70, 0xc8, 0x31, 0x73, 0x81, 0xa7, 0xdf, 0xe8, 0x2e, 0xd4, 0x03, 0x7c, 0x4a, 0x7c, 0xcc, 0xbb, + 0xd5, 0x8d, 0xca, 0x66, 0x73, 0xbb, 0xa5, 0xc5, 0x5f, 0x28, 0xa6, 0x6b, 0x07, 0xd1, 0x0f, 0x61, + 0x91, 0x0b, 0xca, 0xbc, 0x21, 0xe6, 0xdd, 0x05, 0x25, 0xd8, 0xb6, 0x7a, 0x15, 0xd7, 0x4d, 0x87, + 0xd1, 0x2d, 0xa8, 0xbc, 0xde, 0xd9, 0xef, 0xd6, 0x94, 0x75, 0x30, 0x52, 0x31, 0xf6, 0x5d, 0xc9, + 0x46, 0x77, 0xa0, 0xcd, 0xbd, 0x28, 0x38, 0xa2, 0xe7, 0x83, 0x98, 0x04, 0x11, 0xef, 0xd6, 0x37, + 0x4a, 0x9b, 0x8b, 0x6e, 0xcb, 0x30, 0xfb, 0x92, 0xe7, 0x3c, 0x82, 0x6b, 0x07, 0xc2, 0x63, 0xe2, + 0x12, 0xd1, 0x71, 0x0e, 0x61, 0xdd, 0xc5, 0x21, 0x3d, 0xbd, 0x54, 0x68, 0xbb, 0x50, 0x17, 0x24, + 0xc4, 0x34, 0x11, 0x2a, 0xb4, 0x6d, 0xd7, 0x92, 0xce, 0x9f, 0x4a, 0x80, 0x76, 0xcf, 0xb1, 0xdf, + 0x67, 0xd4, 0xc7, 0x9c, 0xff, 0x8f, 0x96, 0xeb, 0x63, 0xa8, 0xc7, 0x1a, 0x40, 0xb7, 0xaa, 0xc4, + 0xcd, 0x2a, 0x58, 0x54, 0x76, 0xd4, 0xf9, 0x0a, 0xd6, 0x0e, 0xc8, 0x30, 0xf2, 0x46, 0x57, 0x88, + 0x77, 0x1d, 0x6a, 0x5c, 0xe9, 0x54, 0x50, 0xdb, 0xae, 0xa1, 0x9c, 0x3e, 0xa0, 0xb7, 0x1e, 0x11, + 0x57, 0x67, 0xc9, 0xb9, 0x07, 0xab, 0x05, 0x8d, 0x3c, 0xa6, 0x11, 0xc7, 0x0a, 0x80, 0xf0, 0x44, + 0xc2, 0x95, 0xb2, 0x05, 0xd7, 0x50, 0x0e, 0x86, 0xb5, 0x2f, 0x09, 0xb7, 0xe2, 0xf8, 0xbf, 0x81, + 0xb0, 0x0e, 0xb5, 0x63, 0xca, 0x42, 0x4f, 0x58, 0x04, 0x9a, 0x42, 0x08, 0xaa, 0x1e, 0x1b, 0xf2, + 0x6e, 0x65, 0xa3, 0xb2, 0xd9, 0x70, 0xd5, 0xb7, 0xdc, 0x95, 0x13, 0x66, 0x0c, 0xae, 0x8f, 0xa0, + 0x65, 0xe2, 0x3e, 0x18, 0x11, 0x2e, 0x94, 0x9d, 0x96, 0xdb, 0x34, 0x3c, 0x39, 0xc7, 0xa1, 0xb0, + 0x7e, 0x18, 0x07, 0x97, 0x3c, 0xf0, 0xdb, 0xd0, 0x60, 0x98, 0xd3, 0x84, 0xc9, 0x63, 0x5a, 0x56, + 0xeb, 0xbe, 0xa6, 0xd7, 0xfd, 0x4b, 0x12, 0x25, 0xe7, 0xae, 0x1d, 0x73, 0x33, 0x31, 0x73, 0x84, + 0x04, 0xbf, 0xcc, 0x11, 0x7a, 0x04, 0xd7, 0xfa, 0x5e, 0xc2, 0x2f, 0x83, 0xd5, 0x79, 0x2c, 0x8f, + 0x1f, 0x4f, 0xc2, 0x4b, 0x4d, 0xfe, 0x63, 0x09, 0x16, 0x77, 0xe2, 0xe4, 0x90, 0x7b, 0x43, 0x8c, + 0xbe, 0x07, 0x4d, 0x41, 0x85, 0x37, 0x1a, 0x24, 0x92, 0x54, 0xe2, 0x55, 0x17, 0x14, 0x4b, 0x0b, + 0xc8, 0xb0, 0x63, 0xe6, 0xc7, 0x89, 0x91, 0x28, 0x6f, 0x54, 0x36, 0xab, 0x6e, 0x53, 0xf3, 0xb4, + 0xc8, 0x16, 0xac, 0xaa, 0xb1, 0x01, 0x89, 0x06, 0xef, 0x30, 0x8b, 0xf0, 0x28, 0xa4, 0x01, 0x56, + 0xfb, 0xb7, 0xea, 0x76, 0xd4, 0xd0, 0x7e, 0xf4, 0x45, 0x3a, 0x80, 0x3e, 0x81, 0x4e, 0x2a, 0x2f, + 0x0f, 0xa5, 0x92, 0xae, 0x2a, 0xe9, 0x65, 0x23, 0x7d, 0x68, 0xd8, 0xce, 0xef, 0x60, 0xe9, 0xcd, + 0x09, 0xa3, 0x42, 0x8c, 0x48, 0x34, 0x7c, 0xe1, 0x09, 0x4f, 0x66, 0x8f, 0x18, 0x33, 0x42, 0x03, + 0x6e, 0xd0, 0x5a, 0x12, 0x7d, 0x0a, 0x1d, 0xa1, 0x65, 0x71, 0x30, 0xb0, 0x32, 0x65, 0x25, 0xb3, + 0x92, 0x0e, 0xf4, 0x8d, 0xf0, 0x0f, 0x60, 0x29, 0x13, 0x96, 0xf9, 0xc7, 0xe0, 0x6d, 0xa7, 0xdc, + 0x37, 0x24, 0xc4, 0xce, 0xa9, 0x8a, 0x95, 0x5a, 0x64, 0xf4, 0x29, 0x34, 0xb2, 0x38, 0x94, 0xd4, + 0x0e, 0x59, 0xd2, 0x3b, 0xc4, 0x86, 0xd3, 0x5d, 0x4c, 0x83, 0xf2, 0x04, 0x96, 0x45, 0x0a, 0x7c, + 0x10, 0x78, 0xc2, 0x2b, 0x6e, 0xaa, 0xa2, 0x57, 0xee, 0x92, 0x28, 0xd0, 0xce, 0x63, 0x68, 0xf4, + 0x49, 0xc0, 0xb5, 0xe1, 0x2e, 0xd4, 0xfd, 0x84, 0x31, 0x1c, 0x09, 0xeb, 0xb2, 0x21, 0xd1, 0x1a, + 0x2c, 0x8c, 0x48, 0x48, 0x84, 0x71, 0x53, 0x13, 0x0e, 0x05, 0x78, 0x85, 0x43, 0xca, 0xc6, 0x2a, + 0x60, 0x6b, 0xb0, 0x90, 0x5f, 0x5c, 0x4d, 0xa0, 0x9b, 0xd0, 0x08, 0xbd, 0xf3, 0x74, 0x51, 0xe5, + 0xc8, 0x62, 0xe8, 0x9d, 0x6b, 0xf0, 0x5d, 0xa8, 0x1f, 0x7b, 0x64, 0xe4, 0x47, 0xc2, 0x44, 0xc5, + 0x92, 0x99, 0xc1, 0x6a, 0xde, 0xe0, 0x5f, 0xca, 0xd0, 0xd4, 0x16, 0x35, 0xe0, 0x35, 0x58, 0xf0, + 0x3d, 0xff, 0x24, 0x35, 0xa9, 0x08, 0x74, 0xd7, 0x02, 0x29, 0xe7, 0x93, 0x70, 0x86, 0xd4, 0x42, + 0xbb, 0x0f, 0xc0, 0xcf, 0xbc, 0xd8, 0x60, 0xab, 0xcc, 0x11, 0x6e, 0x48, 0x19, 0x0d, 0xf7, 0x01, + 0xb4, 0xf4, 0xbe, 0x33, 0x53, 0xaa, 0x73, 0xa6, 0x34, 0xb5, 0x94, 0x9e, 0x74, 0x07, 0xda, 0x09, + 0xc7, 0x83, 0x13, 0x82, 0x99, 0xc7, 0xfc, 0x93, 0x71, 0x77, 0x41, 0xdf, 0x91, 0x09, 0xc7, 0x2f, + 0x2d, 0x0f, 0x6d, 0xc3, 0x82, 0x4c, 0x7f, 0xbc, 0x5b, 0x53, 0xd7, 0xf1, 0xad, 0xbc, 0x4a, 0xe5, + 0xea, 0x96, 0xfa, 0xdd, 0x8d, 0x04, 0x1b, 0xbb, 0x5a, 0xb4, 0xf7, 0x39, 0x40, 0xc6, 0x44, 0x2b, + 0x50, 0x79, 0x87, 0xc7, 0xe6, 0x1c, 0xca, 0x4f, 0x19, 0x9c, 0x53, 0x6f, 0x94, 0xd8, 0xa8, 0x6b, + 0xe2, 0x51, 0xf9, 0xf3, 0x92, 0xe3, 0xc3, 0xf2, 0xf3, 0xd1, 0x3b, 0x42, 0x73, 0xd3, 0xd7, 0x60, + 0x21, 0xf4, 0xbe, 0xa2, 0xcc, 0x46, 0x52, 0x11, 0x8a, 0x4b, 0x22, 0xca, 0xac, 0x0a, 0x45, 0xa0, + 0x25, 0x28, 0xd3, 0x58, 0xc5, 0xab, 0xe1, 0x96, 0x69, 0x9c, 0x19, 0xaa, 0xe6, 0x0c, 0x39, 0x7f, + 0xaf, 0x02, 0x64, 0x56, 0x90, 0x0b, 0x3d, 0x42, 0x07, 0x1c, 0x33, 0x59, 0x82, 0x0c, 0x8e, 0xc6, + 0x02, 0xf3, 0x01, 0xc3, 0x7e, 0xc2, 0x38, 0x39, 0x95, 0xeb, 0x27, 0xdd, 0xbe, 0xa6, 0xdd, 0x9e, + 0xc0, 0xe6, 0x5e, 0x27, 0xf4, 0x40, 0xcf, 0x7b, 0x2e, 0xa7, 0xb9, 0x76, 0x16, 0xda, 0x87, 0x6b, + 0x99, 0xce, 0x20, 0xa7, 0xae, 0x7c, 0x91, 0xba, 0xd5, 0x54, 0x5d, 0x90, 0xa9, 0xda, 0x85, 0x55, + 0x42, 0x07, 0x5f, 0x27, 0x38, 0x29, 0x28, 0xaa, 0x5c, 0xa4, 0xa8, 0x43, 0xe8, 0x2f, 0xd5, 0x84, + 0x4c, 0x4d, 0x1f, 0x6e, 0xe4, 0xbc, 0x94, 0xc7, 0x3d, 0xa7, 0xac, 0x7a, 0x91, 0xb2, 0xf5, 0x14, + 0x95, 0xcc, 0x07, 0x99, 0xc6, 0x9f, 0xc3, 0x3a, 0xa1, 0x83, 0x33, 0x8f, 0x88, 0x49, 0x75, 0x0b, + 0x1f, 0x70, 0x52, 0x5e, 0xba, 0x45, 0x5d, 0xda, 0xc9, 0x10, 0xb3, 0x61, 0xc1, 0xc9, 0xda, 0x07, + 0x9c, 0x7c, 0xa5, 0x26, 0x64, 0x6a, 0x9e, 0x41, 0x87, 0xd0, 0x49, 0x34, 0xf5, 0x8b, 0x94, 0x2c, + 0x13, 0x5a, 0x44, 0xf2, 0x1c, 0x3a, 0x1c, 0xfb, 0x82, 0xb2, 0xfc, 0x26, 0x58, 0xbc, 0x48, 0xc5, + 0x8a, 0x91, 0x4f, 0x75, 0x38, 0xbf, 0x86, 0xd6, 0xcb, 0x64, 0x88, 0xc5, 0xe8, 0x28, 0x4d, 0x06, + 0x57, 0x96, 0x7f, 0x9c, 0x7f, 0x95, 0xa1, 0xb9, 0x33, 0x64, 0x34, 0x89, 0x0b, 0x39, 0x59, 0x1f, + 0xd2, 0xc9, 0x9c, 0xac, 0x44, 0x54, 0x4e, 0xd6, 0xc2, 0x0f, 0xa1, 0x15, 0xaa, 0xa3, 0x6b, 0xe4, + 0x75, 0x1e, 0xea, 0x4c, 0x1d, 0x6a, 0xb7, 0x19, 0xe6, 0x92, 0xd9, 0x16, 0x40, 0x4c, 0x02, 0x6e, + 0xe6, 0xe8, 0x74, 0xb4, 0x6c, 0x2a, 0x42, 0x9b, 0xa2, 0xdd, 0x46, 0x9c, 0x66, 0xeb, 0xcf, 0xa0, + 0x79, 0x24, 0x83, 0x64, 0x26, 0x14, 0x92, 0x51, 0x16, 0x3d, 0x17, 0x8e, 0xb2, 0x43, 0xf8, 0x12, + 0xda, 0x27, 0x3a, 0x64, 0x66, 0x92, 0xde, 0x43, 0x77, 0x8c, 0x27, 0x99, 0xbf, 0x5b, 0xf9, 0xc8, + 0xea, 0x05, 0x68, 0x9d, 0xe4, 0x58, 0xbd, 0x03, 0xe8, 0x4c, 0x89, 0xcc, 0xc8, 0x41, 0x9b, 0xf9, + 0x1c, 0xd4, 0xdc, 0x46, 0xda, 0x50, 0x7e, 0x66, 0x3e, 0x2f, 0xfd, 0x02, 0xd6, 0x27, 0xcb, 0x1c, + 0x53, 0x94, 0x3d, 0x84, 0x96, 0xaf, 0xd0, 0x15, 0x56, 0xa0, 0x33, 0x85, 0xdb, 0x6d, 0xfa, 0x19, + 0xe1, 0x04, 0x80, 0xde, 0x32, 0x22, 0xf0, 0x81, 0x60, 0xd8, 0x0b, 0xaf, 0xa2, 0x6a, 0x46, 0x50, + 0x55, 0x57, 0x6c, 0x45, 0x15, 0x85, 0xea, 0xdb, 0xf9, 0x18, 0x56, 0x0b, 0x56, 0x0c, 0xe4, 0x15, + 0xa8, 0x8c, 0x70, 0xa4, 0xb4, 0xb7, 0x5d, 0xf9, 0xe9, 0x78, 0xd0, 0x71, 0xb1, 0x17, 0x5c, 0x1d, + 0x1a, 0x63, 0xa2, 0x92, 0x99, 0xd8, 0x04, 0x94, 0x37, 0x61, 0xa0, 0x58, 0xd4, 0xa5, 0x1c, 0xea, + 0xd7, 0xd0, 0xd9, 0x19, 0x51, 0x8e, 0x0f, 0x44, 0x40, 0xa2, 0xab, 0x28, 0xf3, 0xbf, 0x81, 0xd5, + 0x37, 0x62, 0xfc, 0x56, 0x2a, 0xe3, 0xe4, 0xb7, 0xf8, 0x8a, 0xfc, 0x63, 0xf4, 0xcc, 0xfa, 0xc7, + 0xe8, 0x99, 0xac, 0xf0, 0x7d, 0x3a, 0x4a, 0xc2, 0x48, 0x6d, 0xf7, 0xb6, 0x6b, 0x28, 0xe7, 0xdb, + 0x12, 0xac, 0xe9, 0x37, 0xf8, 0x81, 0x7e, 0x7a, 0x5a, 0xf3, 0x3d, 0x58, 0x3c, 0xa1, 0x5c, 0x44, + 0x5e, 0x88, 0x8d, 0xe9, 0x94, 0x96, 0xea, 0xe5, 0x9b, 0xb5, 0xac, 0x5e, 0x05, 0xf2, 0xb3, 0xf0, + 0x30, 0xae, 0x5c, 0xfc, 0x30, 0x9e, 0x7a, 0xfa, 0x56, 0xa7, 0x9f, 0xbe, 0xe8, 0xff, 0x01, 0xac, + 0x10, 0x09, 0xd4, 0xc5, 0xdf, 0x70, 0x1b, 0x86, 0xb3, 0x1f, 0x38, 0xd7, 0xe1, 0xda, 0x0b, 0xcc, + 0x05, 0xa3, 0xe3, 0x22, 0x6a, 0xc7, 0x83, 0xc6, 0x7e, 0xff, 0x59, 0x10, 0x30, 0xcc, 0x39, 0xba, + 0x0b, 0xb5, 0x63, 0x2f, 0x24, 0x23, 0x7d, 0xb0, 0x96, 0x6c, 0xde, 0xd9, 0xef, 0xff, 0x4c, 0x71, + 0x5d, 0x33, 0x2a, 0x93, 0x99, 0xa7, 0xa7, 0x98, 0x30, 0x5a, 0x52, 0xae, 0x7f, 0xe8, 0xf1, 0x77, + 0xe6, 0xca, 0x56, 0xdf, 0xce, 0x9f, 0x4b, 0xd0, 0xd8, 0x8f, 0x04, 0x66, 0xc7, 0x9e, 0xaf, 0x1e, + 0x63, 0xba, 0x39, 0x60, 0x82, 0x64, 0x28, 0x39, 0x53, 0x85, 0x4e, 0x2b, 0x54, 0xdf, 0x32, 0xef, + 0xa4, 0xe0, 0xd2, 0x38, 0x2d, 0x5b, 0x50, 0x66, 0xc0, 0xcd, 0xcb, 0xc8, 0x48, 0x87, 0x22, 0x31, + 0xf5, 0x81, 0xfc, 0x94, 0x06, 0x4f, 0xce, 0xa4, 0x80, 0x89, 0x8a, 0xa1, 0x54, 0xd5, 0xed, 0x13, + 0x35, 0x50, 0xd3, 0x4e, 0x18, 0xd2, 0x79, 0x02, 0x90, 0xe2, 0xe5, 0xb2, 0x76, 0xcb, 0x28, 0x53, + 0x3e, 0x58, 0x0c, 0x96, 0xef, 0xe6, 0x44, 0x9c, 0x6f, 0x60, 0xc1, 0xa5, 0x89, 0xd0, 0x87, 0x01, + 0x9b, 0x77, 0x5d, 0xc3, 0x55, 0xdf, 0xd2, 0xea, 0xd0, 0x13, 0xf8, 0xcc, 0x1b, 0xdb, 0xd0, 0x19, + 0x32, 0x17, 0x98, 0x4a, 0x21, 0x30, 0xf2, 0xf5, 0xaa, 0x1e, 0x67, 0xca, 0xa9, 0x86, 0x6b, 0x28, + 0x79, 0x09, 0x71, 0x9f, 0xc6, 0x58, 0xb9, 0xd5, 0x76, 0x35, 0xe1, 0xdc, 0x83, 0x9a, 0x32, 0x2e, + 0xb7, 0x8d, 0xf9, 0x32, 0x98, 0x9b, 0x1a, 0xb3, 0xe2, 0xb9, 0x66, 0xc8, 0xd9, 0xb3, 0xef, 0xcb, + 0xcc, 0x15, 0xb3, 0x9d, 0xef, 0x41, 0x83, 0x58, 0x9e, 0x49, 0x82, 0x53, 0x5e, 0x67, 0x12, 0xce, + 0x0b, 0x58, 0x7d, 0x16, 0x04, 0xdf, 0x55, 0xcb, 0x9e, 0x6d, 0xc2, 0x7c, 0x57, 0x45, 0x8f, 0x61, + 0x55, 0xfb, 0xa5, 0xfd, 0xb4, 0x5a, 0xbe, 0x0f, 0x35, 0x66, 0x63, 0x52, 0xca, 0xba, 0x56, 0x46, + 0xc8, 0x8c, 0xc9, 0xc3, 0x22, 0x1f, 0xdf, 0xd9, 0x92, 0xda, 0xc3, 0xb2, 0x0a, 0x1d, 0x39, 0x50, + 0xd0, 0xe9, 0xfc, 0x06, 0x56, 0x5f, 0x47, 0x23, 0x12, 0xe1, 0x9d, 0xfe, 0xe1, 0x2b, 0x9c, 0x66, + 0x5b, 0x04, 0x55, 0x59, 0x4a, 0x29, 0x43, 0x8b, 0xae, 0xfa, 0x96, 0xe9, 0x27, 0x3a, 0x1a, 0xf8, + 0x71, 0xc2, 0x4d, 0x9b, 0xa8, 0x16, 0x1d, 0xed, 0xc4, 0x09, 0x47, 0x37, 0x40, 0x5e, 0xe9, 0x03, + 0x1a, 0x8d, 0xc6, 0x6a, 0xf5, 0x17, 0xdd, 0xba, 0x1f, 0x27, 0xaf, 0xa3, 0xd1, 0xd8, 0xf9, 0x91, + 0x7a, 0x18, 0x63, 0x1c, 0xb8, 0x5e, 0x14, 0xd0, 0xf0, 0x05, 0x3e, 0xcd, 0x59, 0x48, 0x1f, 0x61, + 0x36, 0xd7, 0x7e, 0x5b, 0x82, 0xba, 0xc9, 0x20, 0x6a, 0x43, 0x31, 0x72, 0x8a, 0x59, 0x7a, 0xd2, + 0x14, 0x25, 0xdf, 0x89, 0xfa, 0x6b, 0x40, 0x63, 0x41, 0x68, 0x9a, 0x97, 0xda, 0x9a, 0xfb, 0x5a, + 0x33, 0x73, 0xfb, 0xae, 0x52, 0xd8, 0x77, 0xeb, 0x50, 0x3b, 0xe6, 0x62, 0x1c, 0xa7, 0xfb, 0x51, + 0x53, 0x72, 0x67, 0x5b, 0x7d, 0x0b, 0x4a, 0x9f, 0x25, 0xe5, 0x8b, 0x3c, 0xa4, 0x49, 0x24, 0x06, + 0x31, 0x25, 0x91, 0x30, 0xa7, 0x0d, 0x14, 0xab, 0x2f, 0x39, 0xce, 0xef, 0x4b, 0x50, 0xd3, 0x9d, + 0x43, 0x59, 0xf1, 0xa7, 0xa9, 0xbb, 0x4c, 0xd4, 0x35, 0xa8, 0x6c, 0x99, 0xb4, 0xa0, 0x2c, 0x5d, + 0x87, 0xfa, 0x69, 0x38, 0x88, 0x3d, 0x71, 0x62, 0xa1, 0x9d, 0x86, 0x7d, 0x4f, 0x9c, 0x48, 0xcf, + 0xb2, 0x1b, 0x40, 0x8d, 0x6b, 0x88, 0xed, 0x94, 0xab, 0xc4, 0xe6, 0x22, 0x75, 0x7e, 0x25, 0x1f, + 0x3a, 0x69, 0xd7, 0x6c, 0x05, 0x2a, 0x49, 0x0a, 0x46, 0x7e, 0x4a, 0xce, 0x30, 0xbd, 0x3b, 0xe4, + 0x27, 0xba, 0x0b, 0x4b, 0x5e, 0x10, 0x10, 0x39, 0xdd, 0x1b, 0xed, 0x91, 0xc0, 0xb6, 0x7e, 0x26, + 0xb8, 0x9f, 0xf4, 0x60, 0xd1, 0xa6, 0x51, 0x54, 0x83, 0xf2, 0xe9, 0xc3, 0x95, 0xff, 0x53, 0xff, + 0x3f, 0x5e, 0x29, 0x6d, 0xff, 0xb3, 0x0d, 0xad, 0x67, 0x43, 0x1c, 0x09, 0x53, 0x96, 0xa3, 0x3d, + 0x58, 0x9e, 0x68, 0xf3, 0x22, 0xf3, 0x4e, 0x9b, 0xdd, 0xfd, 0xed, 0xad, 0x6f, 0xe9, 0xb6, 0xf1, + 0x96, 0x6d, 0x1b, 0x6f, 0xed, 0x86, 0xb1, 0x18, 0xa3, 0x5d, 0x58, 0x2a, 0x36, 0x44, 0xd1, 0x4d, + 0x7b, 0xcb, 0xcc, 0x68, 0x93, 0xce, 0x55, 0xb3, 0x07, 0xcb, 0x13, 0xbd, 0x51, 0x8b, 0x67, 0x76, + 0xcb, 0x74, 0xae, 0xa2, 0xa7, 0xd0, 0xcc, 0x35, 0x43, 0x51, 0x57, 0x2b, 0x99, 0xee, 0x8f, 0xce, + 0x55, 0xb0, 0x03, 0xed, 0x42, 0x7f, 0x12, 0xf5, 0x8c, 0x3f, 0x33, 0x9a, 0x96, 0x73, 0x95, 0x3c, + 0x87, 0x66, 0xae, 0x4d, 0x68, 0x51, 0x4c, 0xf7, 0x22, 0x7b, 0x37, 0x66, 0x8c, 0x98, 0x42, 0xe7, + 0x25, 0xb4, 0x0b, 0x4d, 0x3d, 0x0b, 0x64, 0x56, 0x43, 0xb1, 0x77, 0x73, 0xe6, 0x98, 0xd1, 0xb4, + 0x07, 0xcb, 0x13, 0x2d, 0x3e, 0x1b, 0xdc, 0xd9, 0x9d, 0xbf, 0xb9, 0x6e, 0x7d, 0xa1, 0x16, 0x3b, + 0x57, 0xd3, 0xe6, 0x16, 0x7b, 0xba, 0xa1, 0xd7, 0xbb, 0x35, 0x7b, 0xd0, 0xa0, 0xda, 0x85, 0xa5, + 0x62, 0x2f, 0xcf, 0x2a, 0x9b, 0xd9, 0xe1, 0xbb, 0x78, 0xe7, 0x14, 0xda, 0x7a, 0xd9, 0xce, 0x99, + 0xd5, 0xed, 0x9b, 0xab, 0xe8, 0x19, 0x80, 0x29, 0x7d, 0x03, 0x12, 0xa5, 0x4b, 0x36, 0x55, 0x72, + 0xa7, 0x4b, 0x36, 0xa3, 0x4c, 0x7e, 0x0a, 0xa0, 0x2b, 0xd6, 0x80, 0x26, 0x02, 0x5d, 0xb7, 0x30, + 0x26, 0xca, 0xe4, 0x5e, 0x77, 0x7a, 0x60, 0x4a, 0x01, 0x66, 0xec, 0x32, 0x0a, 0x9e, 0x00, 0x64, + 0x95, 0xb0, 0x55, 0x30, 0x55, 0x1b, 0x5f, 0x10, 0x83, 0x56, 0xbe, 0xee, 0x45, 0xc6, 0xd7, 0x19, + 0xb5, 0xf0, 0x5c, 0x15, 0x8f, 0xa0, 0x95, 0xbf, 0xa6, 0xad, 0x8a, 0x19, 0x57, 0x77, 0x6f, 0xf2, + 0x7a, 0x45, 0x3f, 0xb5, 0x1b, 0x35, 0x63, 0x15, 0x36, 0xea, 0x7f, 0xa4, 0x61, 0xe2, 0x7a, 0x2f, + 0xe6, 0x91, 0x0f, 0x6b, 0xf8, 0x09, 0xb4, 0xf2, 0xf7, 0xba, 0xc5, 0x3f, 0xe3, 0xae, 0xef, 0x15, + 0xee, 0x76, 0xf4, 0x14, 0x96, 0x8a, 0x77, 0x3a, 0xca, 0x1d, 0xca, 0xa9, 0x9b, 0xbe, 0xb7, 0x32, + 0x61, 0x98, 0xa3, 0x07, 0x00, 0xd9, 0xdd, 0x6f, 0xd7, 0x6e, 0xaa, 0x1a, 0x98, 0xb0, 0xba, 0x03, + 0xed, 0xc2, 0x5b, 0xc1, 0x66, 0x89, 0x59, 0x0f, 0x88, 0x8b, 0x92, 0x78, 0xb1, 0x76, 0xb7, 0xd0, + 0x67, 0x56, 0xf4, 0x17, 0xed, 0x9e, 0x7c, 0x9d, 0x62, 0x43, 0x37, 0xa3, 0x76, 0xf9, 0xc0, 0x69, + 0xce, 0xd7, 0x22, 0xb9, 0xd3, 0x3c, 0xa3, 0x44, 0x99, 0xa7, 0xe8, 0x79, 0xeb, 0xaf, 0xef, 0x6f, + 0x97, 0xfe, 0xf6, 0xfe, 0x76, 0xe9, 0x1f, 0xef, 0x6f, 0x97, 0x8e, 0x6a, 0x6a, 0xf4, 0xc1, 0xbf, + 0x03, 0x00, 0x00, 0xff, 0xff, 0x9d, 0xb6, 0xaf, 0x46, 0x1a, 0x1d, 0x00, 0x00, } diff --git a/virtcontainers/netmon.go b/virtcontainers/netmon.go new file mode 100644 index 0000000000..f10d73d9f5 --- /dev/null +++ b/virtcontainers/netmon.go @@ -0,0 +1,96 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "fmt" + "os/exec" + "syscall" + + "github.com/sirupsen/logrus" +) + +// NetmonConfig is the structure providing specific configuration +// for the network monitor. +type NetmonConfig struct { + Path string + Debug bool + Enable bool +} + +// netmonParams is the structure providing specific parameters needed +// for the execution of the network monitor binary. +type netmonParams struct { + netmonPath string + debug bool + logLevel string + runtime string + sandboxID string +} + +func netmonLogger() *logrus.Entry { + return virtLog.WithField("subsystem", "netmon") +} + +func prepareNetMonParams(params netmonParams) ([]string, error) { + if params.netmonPath == "" { + return []string{}, fmt.Errorf("Netmon path is empty") + } + if params.runtime == "" { + return []string{}, fmt.Errorf("Netmon runtime path is empty") + } + if params.sandboxID == "" { + return []string{}, fmt.Errorf("Netmon sandbox ID is empty") + } + + args := []string{params.netmonPath, + "-r", params.runtime, + "-s", params.sandboxID, + } + + if params.debug { + args = append(args, "-d") + } + if params.logLevel != "" { + args = append(args, []string{"-log", params.logLevel}...) + } + + return args, nil +} + +func startNetmon(params netmonParams) (int, error) { + args, err := prepareNetMonParams(params) + if err != nil { + return -1, err + } + + cmd := exec.Command(args[0], args[1:]...) + if err := cmd.Start(); err != nil { + return -1, err + } + + return cmd.Process.Pid, nil +} + +func stopNetmon(pid int) error { + if pid <= 0 { + return nil + } + + sig := syscall.SIGKILL + + netmonLogger().WithFields( + logrus.Fields{ + "netmon-pid": pid, + "netmon-signal": sig, + }).Info("Stopping netmon") + + if err := syscall.Kill(pid, sig); err != nil && err != syscall.ESRCH { + return err + } + + return nil +} diff --git a/virtcontainers/netmon_test.go b/virtcontainers/netmon_test.go new file mode 100644 index 0000000000..0a7f3bb53d --- /dev/null +++ b/virtcontainers/netmon_test.go @@ -0,0 +1,61 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + testNetmonPath = "/foo/bar/netmon" + testRuntimePath = "/foo/bar/runtime" +) + +func TestNetmonLogger(t *testing.T) { + got := netmonLogger() + expected := virtLog.WithField("subsystem", "netmon") + assert.True(t, reflect.DeepEqual(expected, got), + "Got %+v\nExpected %+v", got, expected) +} + +func TestPrepareNetMonParams(t *testing.T) { + // Empty netmon path + params := netmonParams{} + got, err := prepareNetMonParams(params) + assert.NotNil(t, err) + assert.Equal(t, got, []string{}) + + // Empty runtime path + params.netmonPath = testNetmonPath + got, err = prepareNetMonParams(params) + assert.NotNil(t, err) + assert.Equal(t, got, []string{}) + + // Empty sandbox ID + params.runtime = testRuntimePath + got, err = prepareNetMonParams(params) + assert.NotNil(t, err) + assert.Equal(t, got, []string{}) + + // Successful case + params.sandboxID = testSandboxID + got, err = prepareNetMonParams(params) + assert.Nil(t, err) + expected := []string{testNetmonPath, + "-r", testRuntimePath, + "-s", testSandboxID} + assert.True(t, reflect.DeepEqual(expected, got), + "Got %+v\nExpected %+v", got, expected) +} + +func TestStopNetmon(t *testing.T) { + pid := -1 + err := stopNetmon(pid) + assert.Nil(t, err) +} diff --git a/virtcontainers/network.go b/virtcontainers/network.go index cd35c6c0fc..89a27a532a 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -142,6 +142,7 @@ type NetworkInterfacePair struct { type NetworkConfig struct { NetNSPath string NetNsCreated bool + NetmonConfig NetmonConfig InterworkingModel NetInterworkingModel } @@ -257,7 +258,7 @@ func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error { return err } - if _, err := h.hotplugAddDevice(*endpoint, netDev); err != nil { + if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil { networkLogger().WithError(err).Error("Error attach virtual ep") return err } @@ -273,11 +274,10 @@ func (endpoint *VirtualEndpoint) HotDetach(h hypervisor, netNsCreated bool, netN if err := doNetNS(netNsPath, func(_ ns.NetNS) error { return xconnectVMNetwork(&(endpoint.NetPair), false, 0, h.hypervisorConfig().DisableVhostNet) }); err != nil { - networkLogger().WithError(err).Error("Error abridging virtual ep") - return err + networkLogger().WithError(err).Warn("Error un-bridging virtual ep") } - if _, err := h.hotplugRemoveDevice(*endpoint, netDev); err != nil { + if _, err := h.hotplugRemoveDevice(endpoint, netDev); err != nil { networkLogger().WithError(err).Error("Error detach virtual ep") return err } @@ -475,6 +475,7 @@ type NetworkNamespace struct { NetNsPath string NetNsCreated bool Endpoints []Endpoint + NetmonPID int } // TypedJSONEndpoint is used as an intermediate representation for @@ -1151,13 +1152,13 @@ func createVirtualNetworkEndpoint(idx int, ifName string, interworkingModel NetI // at the time of hypervisor attach and not here NetPair: NetworkInterfacePair{ ID: uniqueID, - Name: fmt.Sprintf("br%d", idx), + Name: fmt.Sprintf("br%d_kata", idx), VirtIface: NetworkInterface{ Name: fmt.Sprintf("eth%d", idx), HardAddr: hardAddr.String(), }, TAPIface: NetworkInterface{ - Name: fmt.Sprintf("tap%d", idx), + Name: fmt.Sprintf("tap%d_kata", idx), }, NetInterworkingModel: interworkingModel, }, diff --git a/virtcontainers/network_test.go b/virtcontainers/network_test.go index e02ae9d5ed..6a82e8a591 100644 --- a/virtcontainers/network_test.go +++ b/virtcontainers/network_test.go @@ -209,13 +209,13 @@ func TestCreateVirtualNetworkEndpoint(t *testing.T) { expected := &VirtualEndpoint{ NetPair: NetworkInterfacePair{ ID: "uniqueTestID-4", - Name: "br4", + Name: "br4_kata", VirtIface: NetworkInterface{ Name: "eth4", HardAddr: macAddr.String(), }, TAPIface: NetworkInterface{ - Name: "tap4", + Name: "tap4_kata", }, NetInterworkingModel: DefaultNetInterworkingModel, }, @@ -241,13 +241,13 @@ func TestCreateVirtualNetworkEndpointChooseIfaceName(t *testing.T) { expected := &VirtualEndpoint{ NetPair: NetworkInterfacePair{ ID: "uniqueTestID-4", - Name: "br4", + Name: "br4_kata", VirtIface: NetworkInterface{ Name: "eth1", HardAddr: macAddr.String(), }, TAPIface: NetworkInterface{ - Name: "tap4", + Name: "tap4_kata", }, NetInterworkingModel: DefaultNetInterworkingModel, }, diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index 66a809c5ab..665b541239 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -103,6 +103,8 @@ type RuntimeConfig struct { HypervisorType vc.HypervisorType HypervisorConfig vc.HypervisorConfig + NetmonConfig vc.NetmonConfig + AgentType vc.AgentType AgentConfig interface{} @@ -325,6 +327,12 @@ func networkConfig(ocispec CompatOCISpec, config RuntimeConfig) (vc.NetworkConfi } netConf.InterworkingModel = config.InterNetworkModel + netConf.NetmonConfig = vc.NetmonConfig{ + Path: config.NetmonConfig.Path, + Debug: config.NetmonConfig.Debug, + Enable: config.NetmonConfig.Enable, + } + return netConf, nil } diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 4aa2bca705..f33f436086 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -821,7 +821,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error { return nil } -func (q *qemu) hotplugMacvtap(drive VirtualEndpoint) error { +func (q *qemu) hotplugMacvtap(drive *VirtualEndpoint) error { var ( VMFdNames []string VhostFdNames []string @@ -845,7 +845,7 @@ func (q *qemu) hotplugMacvtap(drive VirtualEndpoint) error { return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", drive.NetPair.Name, VMFdNames, VhostFdNames) } -func (q *qemu) hotplugNetDevice(drive VirtualEndpoint, op operation) error { +func (q *qemu) hotplugNetDevice(drive *VirtualEndpoint, op operation) error { err := q.qmpSetup() if err != nil { return err @@ -902,7 +902,7 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati memdev := devInfo.(*memoryDevice) return nil, q.hotplugMemory(memdev, op) case netDev: - device := devInfo.(VirtualEndpoint) + device := devInfo.(*VirtualEndpoint) return nil, q.hotplugNetDevice(device, op) default: return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType) diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index d9aed435b8..7d249d975c 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -961,6 +961,40 @@ func (s *Sandbox) Delete() error { return s.storage.deleteSandboxResources(s.id, nil) } +func (s *Sandbox) startNetworkMonitor() error { + span, _ := s.trace("startNetworkMonitor") + defer span.Finish() + + binPath, err := os.Executable() + if err != nil { + return err + } + + logLevel := "info" + if s.config.NetworkConfig.NetmonConfig.Debug { + logLevel = "debug" + } + + params := netmonParams{ + netmonPath: s.config.NetworkConfig.NetmonConfig.Path, + debug: s.config.NetworkConfig.NetmonConfig.Debug, + logLevel: logLevel, + runtime: binPath, + sandboxID: s.id, + } + + return s.network.run(s.networkNS.NetNsPath, func() error { + pid, err := startNetmon(params) + if err != nil { + return err + } + + s.networkNS.NetmonPID = pid + + return nil + }) +} + func (s *Sandbox) createNetwork() error { span, _ := s.trace("createNetwork") defer span.Finish() @@ -983,6 +1017,12 @@ func (s *Sandbox) createNetwork() error { } } + if s.config.NetworkConfig.NetmonConfig.Enable { + if err := s.startNetworkMonitor(); err != nil { + return err + } + } + // Store the network return s.storage.storeSandboxNetwork(s.id, s.networkNS) } @@ -991,6 +1031,12 @@ func (s *Sandbox) removeNetwork() error { span, _ := s.trace("removeNetwork") defer span.Finish() + if s.config.NetworkConfig.NetmonConfig.Enable { + if err := stopNetmon(s.networkNS.NetmonPID); err != nil { + return err + } + } + // In case there is a factory, the network has been handled through // some API calls to hotplug some interfaces and routes. This means // the removal of the network should follow the same logic. @@ -1056,6 +1102,7 @@ func (s *Sandbox) AddInterface(inf *grpc.Interface) (*grpc.Interface, error) { } // Add network for vm + inf.PciAddr = endpoint.PCIAddr return s.agent.updateInterface(inf) } diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go index ce7d67f276..8cda6303ea 100644 --- a/virtcontainers/sandbox_test.go +++ b/virtcontainers/sandbox_test.go @@ -11,6 +11,7 @@ import ( "fmt" "io/ioutil" "os" + "os/exec" "path/filepath" "reflect" "sync" @@ -23,6 +24,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/device/manager" "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" + "golang.org/x/sys/unix" ) func newHypervisorConfig(kernelParams []Param, hParams []Param) HypervisorConfig { @@ -1722,3 +1724,27 @@ func TestGetNetNs(t *testing.T) { netNs = s.GetNetNs() assert.Equal(t, netNs, expected) } + +func TestStartNetworkMonitor(t *testing.T) { + trueBinPath, err := exec.LookPath("true") + assert.Nil(t, err) + assert.NotEmpty(t, trueBinPath) + + s := &Sandbox{ + id: testSandboxID, + config: &SandboxConfig{ + NetworkConfig: NetworkConfig{ + NetmonConfig: NetmonConfig{ + Path: trueBinPath, + }, + }, + }, + network: &defNetwork{}, + networkNS: NetworkNamespace{ + NetNsPath: fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), unix.Gettid()), + }, + } + + err = s.startNetworkMonitor() + assert.Nil(t, err) +}