diff --git a/Gopkg.lock b/Gopkg.lock index 57f009b89..722770b4f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,23 +2,23 @@ [[projects]] - digest = "1:167b6f65a6656de568092189ae791253939f076df60231fdd64588ac703892a1" + digest = "1:5d72bbcc9c8667b11c3dc3cbe681c5a6f71e5096744c0bf7726ab5c6425d5dc4" name = "github.com/BurntSushi/toml" packages = ["."] pruneopts = "NUT" - revision = "b26d9c308763d68093482582cea63d69be07a0f0" - version = "v0.3.0" + revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005" + version = "v0.3.1" [[projects]] digest = "1:2be791e7b333ff7c06f8fb3dc18a7d70580e9399dbdffd352621d067ff260b6e" name = "github.com/Microsoft/go-winio" packages = ["."] pruneopts = "NUT" - revision = "97e4973ce50b2ff5f09635a57e2b88a037aae829" - version = "v0.4.11" + revision = "1a8911d1ed007260465c3bfbbc785ac6915a0bb8" + version = "v0.4.12" [[projects]] - digest = "1:46df22fa970c4b4384851338ada3a67f32d1c1400d27aa69b09e493dd01bb8bf" + digest = "1:a8e16b4caf3575365c9aa3380d9418f31dd0b810596faebfe3a15c37fabeee4a" name = "github.com/Microsoft/hcsshim" packages = [ ".", @@ -28,6 +28,7 @@ "internal/hcserror", "internal/hns", "internal/interop", + "internal/logfields", "internal/longpath", "internal/mergemaps", "internal/safefile", @@ -37,16 +38,16 @@ "internal/wclayer", ] pruneopts = "NUT" - revision = "4f64a598035b09da04155f7dfd76b63edf04fca1" - version = "v0.8.1" + revision = "f92b8fb9c92e17da496af5a69e3ee13fbe9916e1" + version = "v0.8.6" [[projects]] - digest = "1:d8ebbd207f3d3266d4423ce4860c9f3794956306ded6c7ba312ecc69cdfbf04c" + digest = "1:0a111edd8693fd977f42a0c4f199a0efb13c20aec9da99ad8830c7bb6a87e8d6" name = "github.com/PuerkitoBio/purell" packages = ["."] pruneopts = "NUT" - revision = "0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4" - version = "v1.1.0" + revision = "44968752391892e1b0d0b821ee79e9a85fa13049" + version = "v1.1.1" [[projects]] branch = "master" @@ -84,11 +85,11 @@ [[projects]] branch = "master" - digest = "1:73f5d6c006826b458d9ab942581ed24486a3b3c69f5b3e6794f8bdfe8e306080" + digest = "1:a6f5d43cbf707438b7d6e071c8574a043be3100071092d4c64f4981ed9641e87" name = "github.com/containerd/cgroups" packages = ["."] pruneopts = "NUT" - revision = "5017d4e9a9cf2d4381db99eacd9baf84b95bfb14" + revision = "dbea6f2bd41658b84b00417ceefa416b979cbf10" [[projects]] branch = "master" @@ -131,11 +132,11 @@ [[projects]] branch = "master" - digest = "1:a16d601cf7295c29ccc3e7711be788bca1376e1f22e0826929f8883ed36c7c99" + digest = "1:c34ee53dc499450d386058a6cf7cc979a48724ac83b3fcb14a981abdfc2781b7" name = "github.com/containerd/fifo" packages = ["."] pruneopts = "NUT" - revision = "3d5202aec260678c48179c56f40e6f38a095738c" + revision = "a9fb20d87448d386e6d50b1f2e1fa70dcf0de43c" [[projects]] branch = "master" @@ -143,30 +144,31 @@ name = "github.com/containerd/go-runc" packages = ["."] pruneopts = "NUT" - revision = "5a6d9f37cfa36b15efba46dc7ea349fa9b7143c3" + revision = "7d11b49dc0769f6dbb0d1b19f3d48524d1bad9ad" [[projects]] branch = "master" - digest = "1:1b7f9015632851c458c7b713d1a29c9d763c5a2bd85c4e3731139e198b2df2c8" + digest = "1:25435262330720ca0cade25af7ee7fb96d0cb70cc1ea0c0961694681c12a90e6" name = "github.com/containerd/ttrpc" packages = ["."] pruneopts = "NUT" - revision = "f51df4475b76e0ab0315ee0684bef0703a070e6b" + revision = "69144327078caa5a2f1d5eda8bea6110bf16eeb3" [[projects]] branch = "master" - digest = "1:07ac073876dbf7df80789ba4c2959a969200b34875a34fc13848f76d60b51551" + digest = "1:fda6b4a39771e0c9bdc114f8a5528d3ab5c4eb635a2cfa54ff08ac0fcc8db3a7" name = "github.com/containerd/typeurl" packages = ["."] pruneopts = "NUT" - revision = "461401dc8f19d80baa4b70178935e4501286c00b" + revision = "2a93cfde8c20b23de8eb84a5adbc234ddf7a9e8d" [[projects]] digest = "1:827ed8a74e55981880c4d77f8472d638bceb899188104ba7bf24a9548fd97292" name = "github.com/containernetworking/cni" packages = ["pkg/types"] pruneopts = "NUT" - revision = "384d8c0b5288c25b9f1da901c66ea5155e6c567d" + revision = "a7885cb6f8ab03fba07852ded351e4f5e7a112bf" + version = "v0.6.0" [[projects]] digest = "1:1552ba1ba0d0f3596966cca53601a5f59c257ca7ace87f41708a2480835c5286" @@ -176,20 +178,20 @@ revision = "7f98c94613021d8b57acfa1a2f0c8d0f6fd7ae5a" [[projects]] - digest = "1:c262724e34fd94dbdcc64516cbf638d5a6737edcd63ec4efebc284e92309afe7" + digest = "1:6f614c5193761d29b3184a170380672cafd025306261ce7810e2dd99907184c4" name = "github.com/coreos/go-systemd" packages = ["dbus"] pruneopts = "NUT" - revision = "39ca1b05acc7ad1220e09f133283b8859a8b71ab" - version = "v17" + revision = "9002847aa1425fb6ac49077c0a630b3b67e0fbfd" + version = "v18" [[projects]] - digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" + digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" name = "github.com/davecgh/go-spew" packages = ["spew"] pruneopts = "NUT" - revision = "346938d642f2ec3594ed81d874461961cd0faa76" - version = "v1.1.0" + revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" + version = "v1.1.1" [[projects]] branch = "master" @@ -237,23 +239,23 @@ revision = "20b96f641a5ea98f2f8619ff4f3e061cff4833bd" [[projects]] - digest = "1:da40b9e5973892e2bd37a72c36464b8252a4034522925d920983edaabda03693" + digest = "1:e101aa2e25fac7e82ba4d2f66807eedd4bcf11abc5afcb4a4629a88f9a652b84" name = "github.com/go-openapi/analysis" packages = [ ".", "internal", ] pruneopts = "NUT" - revision = "c701774f4e604d952e4e8c56dee260be696e33c3" - version = "v0.17.2" + revision = "e2f3fdbb7ed0e56e070ccbfb6fc75b288a33c014" + version = "v0.18.0" [[projects]] - digest = "1:02356188d2100454319fc9f17e956bf6db7abd0fd66fef602ffa17d7ceb2808b" + digest = "1:2c25c65f6928a43066d02607fbedd6a1b21322db5f8d1ab92ac77ac3bcc72776" name = "github.com/go-openapi/errors" packages = ["."] pruneopts = "NUT" - revision = "d9664f9fab8994271e573ed69cf2adfc09b7a800" - version = "v0.17.2" + revision = "7a7ff1b7b8020f22574411a32f28b4d168d69237" + version = "v0.18.0" [[projects]] digest = "1:260f7ebefc63024c8dfe2c9f1a2935a89fa4213637a1f522f592f80c001cc441" @@ -261,7 +263,7 @@ packages = ["."] pruneopts = "NUT" revision = "ef5f0afec364d3b9396b7b77b43dbe26bf1f8004" - version = "v0.17.2" + version = "v0.18.0" [[projects]] digest = "1:98abd61947ff5c7c6fcfec5473d02a4821ed3a2dd99a4fbfdb7925b0dd745546" @@ -269,18 +271,18 @@ packages = ["."] pruneopts = "NUT" revision = "8483a886a90412cd6858df4ea3483dce9c8e35a3" - version = "v0.17.2" + version = "v0.18.0" [[projects]] - digest = "1:cc4186672d13bce6e14f7b39c6f51b2f8c5126532a020ced03841e7175886651" + digest = "1:c3cb0571346173abe3b329baf50e5ca05d2d1640cfe2817c65ed377e44c34afc" name = "github.com/go-openapi/loads" packages = ["."] pruneopts = "NUT" - revision = "150d36912387ec2f607be674c5be309ddccc0eed" - version = "v0.17.2" + revision = "74628589c3b94e3526a842d24f46589980f5ab22" + version = "v0.18.0" [[projects]] - digest = "1:4493a5de5ae6e54fcedc502715c0254fd0dec1cd872fa6ca99f28e61cebc4d11" + digest = "1:757a8958779fedcfddafb3ac93f707876db7b4fbc71b76fbc25450b3f057025e" name = "github.com/go-openapi/runtime" packages = [ ".", @@ -293,32 +295,32 @@ "security", ] pruneopts = "NUT" - revision = "231d7876b7019dbcbfc97a7ba764379497b67c1d" - version = "v0.17.2" + revision = "41e24cc66d7af6af39eb9b5a6418e901bcdd333c" + version = "v0.18.0" [[projects]] - digest = "1:dfab391de021809e0041f0ab5648da6b74dd16a685472a1b8c3dc06b3dca1ee2" + digest = "1:4da4ea0a664ba528965683d350f602d0f11464e6bb2e17aad0914723bc25d163" name = "github.com/go-openapi/spec" packages = ["."] pruneopts = "NUT" - revision = "5bae59e25b21498baea7f9d46e9c147ec106a42e" - version = "v0.17.2" + revision = "5b6cdde3200976e3ecceb2868706ee39b6aff3e4" + version = "v0.18.0" [[projects]] - digest = "1:76667bb6bc2df5b95be9efb9ee0b72b3550c639f530232a0b8fcf3347035d987" + digest = "1:4226678c15d6932792564e34e71694cd5f555ddaa09641e0b711fed5c6f1d878" name = "github.com/go-openapi/strfmt" packages = ["."] pruneopts = "NUT" - revision = "edab9990ffc9b4a428f3306ecf4d18a069ca3317" - version = "v0.17.2" + revision = "e471370ae57ac74eaf0afe816a66e4ddd7f1b027" + version = "v0.18.0" [[projects]] - digest = "1:983f95b2fae6fe8fdd361738325ed6090f4f3bd15ce4db745e899fb5b0fdfc46" + digest = "1:dc0f590770e5a6c70ea086232324f7b7dc4857c60eca63ab8ff78e0a5cfcdbf3" name = "github.com/go-openapi/swag" packages = ["."] pruneopts = "NUT" - revision = "5899d5c5e619fda5fa86e14795a835f473ca284c" - version = "v0.17.2" + revision = "1d29f06aebd59ccdf11ae04aa0334ded96e2d909" + version = "v0.18.0" [[projects]] digest = "1:7d7626b94bc5e04d1c23eaa97816181f4ff6218540a6d43379070d6ece9ca467" @@ -326,15 +328,15 @@ packages = ["."] pruneopts = "NUT" revision = "d2eab7d93009e9215fc85b2faa2c2f2a98c2af48" - version = "v0.17.2" + version = "v0.18.0" [[projects]] - digest = "1:ca3369c0fc8d471d8698f85a37a4f8c98a847402681a31431fb87a84fa2e5373" + digest = "1:08a552769d7194c0268ba31010f5da91878212603d11b4b551583a7586a76632" name = "github.com/godbus/dbus" packages = ["."] pruneopts = "NUT" - revision = "a389bdde4dd695d414e47b755e95e72b7826432c" - version = "v4.1.0" + revision = "2ff6f7ffd60f0f2410b3105864bdd12c7894f844" + version = "v5.0.1" [[projects]] digest = "1:0dfc35f448d29c2ff6a29fb3a6643f44175dc2a07925b1add2dea74e1dd6bf8d" @@ -350,7 +352,7 @@ revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02" [[projects]] - digest = "1:d470faddd8d4b027226f9a5ff9d88962c34bb1699dde2acd7e9bfb70a3703d45" + digest = "1:2d0636a8c490d2272dd725db26f74a537111b99b9dbdda0d8b98febe63702aa4" name = "github.com/golang/protobuf" packages = [ "proto", @@ -360,8 +362,8 @@ "ptypes/timestamp", ] pruneopts = "NUT" - revision = "925541529c1fa6821df4e44ce2723319eb2be768" - version = "v1.0.0" + revision = "c823c79ea1570fb5ff454033735a8e68575d1d0f" + version = "v1.3.0" [[projects]] branch = "master" @@ -373,11 +375,11 @@ [[projects]] branch = "master" - digest = "1:63e0b20cfa3fe456480edf93a7995f776afb610e49da8e3da04d8904472a44cc" + digest = "1:8deb0c5545c824dfeb0ac77ab8eb67a3d541eab76df5c85ce93064ef02d44cd0" name = "github.com/hashicorp/yamux" packages = ["."] pruneopts = "NUT" - revision = "3520598351bb3500a49ae9563f5539666ae0a27c" + revision = "2f1d1f20f75d5404f53b9edf6b53ed5505508675" [[projects]] digest = "1:270961b1d5e664d4939ffae00b990e256d92bb5039cae69208211a84c72fe5f5" @@ -414,7 +416,7 @@ "jwriter", ] pruneopts = "NUT" - revision = "60711f1a8329503b04e1c88535f419d0bb440bff" + revision = "6243d8e04c3f819e79757e8bc3faa15c3cb27003" [[projects]] digest = "1:0159dcdabe50788e5dcfb469521f8f8dcd362db3ab6b465b99a3d33a90726fc0" @@ -471,12 +473,12 @@ version = "v1.0.2" [[projects]] - digest = "1:5cf3f025cbee5951a4ee961de067c8a89fc95a5adabead774f82822efabab121" + digest = "1:14715f705ff5dfe0ffd6571d7d201dd8e921030f8070321a79380d8ca4ec1a24" name = "github.com/pkg/errors" packages = ["."] pruneopts = "NUT" - revision = "645ef00459ed84a119197bfb8d8205042c6df63d" - version = "v0.8.0" + revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4" + version = "v0.8.1" [[projects]] digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" @@ -519,7 +521,7 @@ revision = "890a5c3458b43e6104ff5da8dfa139d013d77544" [[projects]] - digest = "1:3d48c38e0eca8c66df62379c5ae7a83fb5cd839b94f241354c07ba077da7bc45" + digest = "1:87dee780f88f86f300bbd90116e933347cf4a3c65c1960072d412597a8896d50" name = "github.com/uber/jaeger-client-go" packages = [ ".", @@ -537,11 +539,12 @@ "thrift-gen/jaeger", "thrift-gen/sampling", "thrift-gen/zipkincore", + "transport", "utils", ] pruneopts = "NUT" - revision = "b043381d944715b469fd6b37addfd30145ca1758" - version = "v2.14.0" + revision = "1a782e2da844727691fef1757c72eb190c2909f0" + version = "v2.15.0" [[projects]] digest = "1:0f09db8429e19d57c8346ad76fbbc679341fa86073d3b8fb5ac919f0357d8f4c" @@ -577,11 +580,11 @@ [[projects]] branch = "master" - digest = "1:38cb27d3525635c34e84e2dbc2207c37d10832776997665bf0ddaeae2c861f1f" + digest = "1:058e9504b9a79bfe86092974d05bb3298d2aa0c312d266d43148de289a5065d9" name = "golang.org/x/crypto" packages = ["ssh/terminal"] pruneopts = "NUT" - revision = "91a49db82a88618983a78a06c1cbd4e00ab749ab" + revision = "8dd112bcdc25174059e45e07517d9fc663123347" [[projects]] digest = "1:b20c63a56900e442d5f435613fefc9392cbe8849467510fcc3869dbdad9441bb" @@ -634,11 +637,11 @@ [[projects]] branch = "master" - digest = "1:cd018653a358d4b743a9d3bee89e825521f2ab2f2ec0770164bf7632d8d73ab7" + digest = "1:c3076e7defee87de1236f1814beb588f40a75544c60121e6eb38b3b3721783e2" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] pruneopts = "NUT" - revision = "2c5e7ac708aaa719366570dd82bda44541ca2a63" + revision = "5fe7a883aa19554f42890211544aa549836af7b7" [[projects]] digest = "1:3d43152515ea791363eb0d1d21378fbf70e7df4a3954fd315898532cf5e64a8c" diff --git a/Gopkg.toml b/Gopkg.toml index 7cf7a7d68..3681b2322 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -74,6 +74,10 @@ name = "github.com/firecracker-microvm/firecracker-go-sdk" revision = "961461227bddf7e40a1d690634e866c343910f86" +[[constraint]] + name = "github.com/gogo/protobuf" + revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02" + [[override]] branch = "master" name = "github.com/hashicorp/yamux" diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in index 4be52f4a9..f655136bf 100644 --- a/cli/config/configuration-qemu.toml.in +++ b/cli/config/configuration-qemu.toml.in @@ -219,6 +219,30 @@ enable_iothreads = @DEFENABLEIOTHREADS@ # Default false #enable_template = true +# The number of caches of VMCache: +# unspecified or == 0 --> VMCache is disabled +# > 0 --> will be set to the specified number +# +# VMCache is a function that creates VMs as caches before using it. +# It helps speed up new container creation. +# The function consists of a server and some clients communicating +# through Unix socket. The protocol is gRPC in protocols/cache/cache.proto. +# The VMCache server will create some VMs and cache them by factory cache. +# It will convert the VM to gRPC format and transport it when gets +# requestion from clients. +# Factory grpccache is the VMCache client. It will request gRPC format +# VM and convert it back to a VM. If VMCache function is enabled, +# kata-runtime will request VM from factory grpccache when it creates +# a new sandbox. +# +# Default 0 +#vm_cache_number = 0 + +# Specify the address of the Unix socket that is used by VMCache. +# +# Default /var/run/kata-containers/cache.sock +#vm_cache_endpoint = "/var/run/kata-containers/cache.sock" + [proxy.@PROJECT_TYPE@] path = "@PROXYPATH@" diff --git a/cli/factory.go b/cli/factory.go index cab068010..da1793dd7 100644 --- a/cli/factory.go +++ b/cli/factory.go @@ -6,13 +6,23 @@ package main import ( - "errors" + "context" "fmt" + "net" + "os" + "os/signal" + "path/filepath" + "syscall" + "github.com/gogo/protobuf/types" + pb "github.com/kata-containers/runtime/protocols/cache" vc "github.com/kata-containers/runtime/virtcontainers" vf "github.com/kata-containers/runtime/virtcontainers/factory" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" + "github.com/pkg/errors" "github.com/urfave/cli" + "golang.org/x/sys/unix" + "google.golang.org/grpc" ) var factorySubCmds = []cli.Command{ @@ -30,6 +40,84 @@ var factoryCLICommand = cli.Command{ }, } +type cacheServer struct { + rpc *grpc.Server + factory vc.Factory +} + +var jsonVMConfig *pb.GrpcVMConfig + +// Config requests base factory config and convert it to gRPC protocol. +func (s *cacheServer) Config(ctx context.Context, empty *types.Empty) (*pb.GrpcVMConfig, error) { + if jsonVMConfig == nil { + config := s.factory.Config() + + var err error + jsonVMConfig, err = config.ToGrpc() + if err != nil { + return nil, err + } + } + + return jsonVMConfig, nil +} + +// GetBaseVM requests a paused VM and convert it to gRPC protocol. +func (s *cacheServer) GetBaseVM(ctx context.Context, empty *types.Empty) (*pb.GrpcVM, error) { + config := s.factory.Config() + + vm, err := s.factory.GetBaseVM(ctx, config) + if err != nil { + return nil, errors.Wrapf(err, "failed to GetBaseVM") + } + + return vm.ToGrpc(config) +} + +func getUnixListener(path string) (net.Listener, error) { + err := os.MkdirAll(filepath.Dir(path), 0755) + if err != nil { + return nil, err + } + if err = unix.Unlink(path); err != nil && !os.IsNotExist(err) { + return nil, err + } + l, err := net.Listen("unix", path) + if err != nil { + return nil, err + } + if err = os.Chmod(path, 0600); err != nil { + l.Close() + return nil, err + } + return l, nil +} + +var handledSignals = []os.Signal{ + syscall.SIGTERM, + syscall.SIGINT, + syscall.SIGPIPE, +} + +func handleSignals(s *cacheServer, signals chan os.Signal) chan struct{} { + done := make(chan struct{}, 1) + go func() { + for { + sig := <-signals + kataLog.WithField("signal", sig).Debug("received signal") + switch sig { + case unix.SIGPIPE: + continue + default: + s.rpc.GracefulStop() + close(done) + return + } + } + }() + return done +} + var initFactoryCommand = cli.Command{ Name: "init", Usage: "initialize a VM factory based on kata-runtime configuration", @@ -44,6 +132,51 @@ var initFactoryCommand = cli.Command{ return errors.New("invalid runtime config") } + if runtimeConfig.FactoryConfig.VMCacheNumber > 0 { + factoryConfig := vf.Config{ + Template: runtimeConfig.FactoryConfig.Template, + Cache: runtimeConfig.FactoryConfig.VMCacheNumber, + VMCache: true, + VMConfig: vc.VMConfig{ + HypervisorType: runtimeConfig.HypervisorType, + HypervisorConfig: runtimeConfig.HypervisorConfig, + AgentType: runtimeConfig.AgentType, + AgentConfig: runtimeConfig.AgentConfig, + ProxyType: runtimeConfig.ProxyType, + ProxyConfig: runtimeConfig.ProxyConfig, + }, + } + f, err := vf.NewFactory(ctx, factoryConfig, false) + if err != nil { + return err + } + defer f.CloseFactory(ctx) + + s := &cacheServer{ + rpc: grpc.NewServer(), + factory: f, + } + pb.RegisterCacheServiceServer(s.rpc, s) + + l, err := getUnixListener(runtimeConfig.FactoryConfig.VMCacheEndpoint) + if err != nil { + return err + } + defer l.Close() + + signals := make(chan os.Signal, 8) + done := handleSignals(s, signals) + signal.Notify(signals, handledSignals...) + + kataLog.WithField("endpoint", runtimeConfig.FactoryConfig.VMCacheEndpoint).Info("VM cache server start") + s.rpc.Serve(l) + + <-done + + kataLog.WithField("endpoint", runtimeConfig.FactoryConfig.VMCacheEndpoint).Info("VM cache server stop") + return nil + } + if runtimeConfig.FactoryConfig.Template { factoryConfig := vf.Config{ Template: true, diff --git a/hack/update-generated-runtime-proto.sh b/hack/update-generated-runtime-proto.sh new file mode 100755 index 000000000..d2881a48b --- /dev/null +++ b/hack/update-generated-runtime-proto.sh @@ -0,0 +1,14 @@ +# +# Copyright 2019 HyperHQ Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +protoc \ + -I=$GOPATH/src \ + -I=$GOPATH/src/github.com/gogo/protobuf/protobuf \ + --proto_path=protocols/cache \ + --gogofast_out=\ +Mgoogle/protobuf/empty.proto=github.com/gogo/protobuf/types,\ +plugins=grpc:protocols/cache \ + protocols/cache/cache.proto diff --git a/pkg/katautils/config-settings.go b/pkg/katautils/config-settings.go index b85efcb5f..da7ce7ba8 100644 --- a/pkg/katautils/config-settings.go +++ b/pkg/katautils/config-settings.go @@ -43,6 +43,8 @@ const defaultHotplugVFIOOnRootBus bool = false const defaultEntropySource = "/dev/urandom" const defaultGuestHookPath string = "" +const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock" + // Default config file used by stateless systems. var defaultRuntimeConfiguration = "/usr/share/defaults/kata-containers/configuration.toml" diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go index 10668baae..821381d94 100644 --- a/pkg/katautils/config.go +++ b/pkg/katautils/config.go @@ -77,7 +77,9 @@ type tomlConfig struct { } type factory struct { - Template bool `toml:"enable_template"` + Template bool `toml:"enable_template"` + VMCacheNumber uint `toml:"vm_cache_number"` + VMCacheEndpoint string `toml:"vm_cache_endpoint"` } type hypervisor struct { @@ -544,7 +546,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { } func newFactoryConfig(f factory) (oci.FactoryConfig, error) { - return oci.FactoryConfig{Template: f.Template}, nil + if f.VMCacheEndpoint == "" { + f.VMCacheEndpoint = defaultVMCacheEndpoint + } + return oci.FactoryConfig{ + Template: f.Template, + VMCacheNumber: f.VMCacheNumber, + VMCacheEndpoint: f.VMCacheEndpoint, + }, nil } func newShimConfig(s shim) (vc.ShimConfig, error) { @@ -910,6 +919,10 @@ func checkNetNsConfig(config oci.RuntimeConfig) error { // checkFactoryConfig ensures the VM factory configuration is valid. func checkFactoryConfig(config oci.RuntimeConfig) error { + if config.FactoryConfig.Template && config.FactoryConfig.VMCacheNumber > 0 { + return errors.New("VM factory cannot work together with VM cache") + } + if config.FactoryConfig.Template { if config.HypervisorConfig.InitrdPath == "" { return errors.New("Factory option enable_template requires an initrd image") @@ -920,6 +933,18 @@ func checkFactoryConfig(config oci.RuntimeConfig) error { } } + if config.FactoryConfig.VMCacheNumber > 0 { + if config.HypervisorType != vc.QemuHypervisor { + return errors.New("VM cache just support qemu") + } + if config.AgentType != vc.KataContainersAgent { + return errors.New("VM cache just support kata agent") + } + if config.HypervisorConfig.UseVSock { + return errors.New("config vsock conflicts with VM cache, please disable one of them") + } + } + return nil } diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go index 3c8efa06c..1a4f5f59e 100644 --- a/pkg/katautils/config_test.go +++ b/pkg/katautils/config_test.go @@ -184,6 +184,10 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf Enable: false, } + factoryConfig := oci.FactoryConfig{ + VMCacheEndpoint: defaultVMCacheEndpoint, + } + runtimeConfig := oci.RuntimeConfig{ HypervisorType: defaultHypervisor, HypervisorConfig: hypervisorConfig, @@ -199,6 +203,8 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf NetmonConfig: netmonConfig, DisableNewNetNs: disableNewNetNs, + + FactoryConfig: factoryConfig, } err = SetKernelParams(&runtimeConfig) @@ -626,6 +632,10 @@ func TestMinimalRuntimeConfig(t *testing.T) { Enable: false, } + expectedFactoryConfig := oci.FactoryConfig{ + VMCacheEndpoint: defaultVMCacheEndpoint, + } + expectedConfig := oci.RuntimeConfig{ HypervisorType: defaultHypervisor, HypervisorConfig: expectedHypervisorConfig, @@ -640,6 +650,8 @@ func TestMinimalRuntimeConfig(t *testing.T) { ShimConfig: expectedShimConfig, NetmonConfig: expectedNetmonConfig, + + FactoryConfig: expectedFactoryConfig, } err = SetKernelParams(&expectedConfig) if err != nil { @@ -1376,7 +1388,8 @@ func TestUpdateRuntimeConfigurationFactoryConfig(t *testing.T) { config := oci.RuntimeConfig{} expectedFactoryConfig := oci.FactoryConfig{ - Template: true, + Template: true, + VMCacheEndpoint: defaultVMCacheEndpoint, } tomlConf := tomlConfig{Factory: factory{Template: true}} diff --git a/pkg/katautils/create.go b/pkg/katautils/create.go index 0e07e8003..3b37393fd 100644 --- a/pkg/katautils/create.go +++ b/pkg/katautils/create.go @@ -53,12 +53,14 @@ func needSystemd(config vc.HypervisorConfig) bool { // HandleFactory set the factory func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeConfig) { - if !runtimeConfig.FactoryConfig.Template { + if !runtimeConfig.FactoryConfig.Template && runtimeConfig.FactoryConfig.VMCacheNumber == 0 { return } factoryConfig := vf.Config{ - Template: true, + Template: runtimeConfig.FactoryConfig.Template, + VMCache: runtimeConfig.FactoryConfig.VMCacheNumber > 0, + VMCacheEndpoint: runtimeConfig.FactoryConfig.VMCacheEndpoint, VMConfig: vc.VMConfig{ HypervisorType: runtimeConfig.HypervisorType, HypervisorConfig: runtimeConfig.HypervisorConfig, @@ -66,6 +68,10 @@ func HandleFactory(ctx context.Context, vci vc.VC, runtimeConfig *oci.RuntimeCon AgentConfig: runtimeConfig.AgentConfig, }, } + if runtimeConfig.FactoryConfig.VMCacheNumber > 0 { + factoryConfig.VMConfig.ProxyType = runtimeConfig.ProxyType + factoryConfig.VMConfig.ProxyConfig = runtimeConfig.ProxyConfig + } kataUtilsLogger.WithField("factory", factoryConfig).Info("load vm factory") diff --git a/protocols/cache/cache.pb.go b/protocols/cache/cache.pb.go new file mode 100644 index 000000000..052ee4db9 --- /dev/null +++ b/protocols/cache/cache.pb.go @@ -0,0 +1,844 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: cache.proto + +/* + Package cache is a generated protocol buffer package. + + It is generated from these files: + cache.proto + + It has these top-level messages: + GrpcVMConfig + GrpcVM +*/ +package cache + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import google_protobuf "github.com/gogo/protobuf/types" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type GrpcVMConfig struct { + Data []byte `protobuf:"bytes,1,opt,name=Data,proto3" json:"Data,omitempty"` + AgentConfig []byte `protobuf:"bytes,2,opt,name=AgentConfig,proto3" json:"AgentConfig,omitempty"` +} + +func (m *GrpcVMConfig) Reset() { *m = GrpcVMConfig{} } +func (m *GrpcVMConfig) String() string { return proto.CompactTextString(m) } +func (*GrpcVMConfig) ProtoMessage() {} +func (*GrpcVMConfig) Descriptor() ([]byte, []int) { return fileDescriptorCache, []int{0} } + +func (m *GrpcVMConfig) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +func (m *GrpcVMConfig) GetAgentConfig() []byte { + if m != nil { + return m.AgentConfig + } + return nil +} + +type GrpcVM struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Hypervisor []byte `protobuf:"bytes,2,opt,name=hypervisor,proto3" json:"hypervisor,omitempty"` + ProxyPid int64 `protobuf:"varint,3,opt,name=proxyPid,proto3" json:"proxyPid,omitempty"` + ProxyURL string `protobuf:"bytes,4,opt,name=proxyURL,proto3" json:"proxyURL,omitempty"` + Cpu uint32 `protobuf:"varint,5,opt,name=cpu,proto3" json:"cpu,omitempty"` + Memory uint32 `protobuf:"varint,6,opt,name=memory,proto3" json:"memory,omitempty"` + CpuDelta uint32 `protobuf:"varint,7,opt,name=cpuDelta,proto3" json:"cpuDelta,omitempty"` +} + +func (m *GrpcVM) Reset() { *m = GrpcVM{} } +func (m *GrpcVM) String() string { return proto.CompactTextString(m) } +func (*GrpcVM) ProtoMessage() {} +func (*GrpcVM) Descriptor() ([]byte, []int) { return fileDescriptorCache, []int{1} } + +func (m *GrpcVM) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *GrpcVM) GetHypervisor() []byte { + if m != nil { + return m.Hypervisor + } + return nil +} + +func (m *GrpcVM) GetProxyPid() int64 { + if m != nil { + return m.ProxyPid + } + return 0 +} + +func (m *GrpcVM) GetProxyURL() string { + if m != nil { + return m.ProxyURL + } + return "" +} + +func (m *GrpcVM) GetCpu() uint32 { + if m != nil { + return m.Cpu + } + return 0 +} + +func (m *GrpcVM) GetMemory() uint32 { + if m != nil { + return m.Memory + } + return 0 +} + +func (m *GrpcVM) GetCpuDelta() uint32 { + if m != nil { + return m.CpuDelta + } + return 0 +} + +func init() { + proto.RegisterType((*GrpcVMConfig)(nil), "cache.GrpcVMConfig") + proto.RegisterType((*GrpcVM)(nil), "cache.GrpcVM") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for CacheService service + +type CacheServiceClient interface { + Config(ctx context.Context, in *google_protobuf.Empty, opts ...grpc.CallOption) (*GrpcVMConfig, error) + GetBaseVM(ctx context.Context, in *google_protobuf.Empty, opts ...grpc.CallOption) (*GrpcVM, error) +} + +type cacheServiceClient struct { + cc *grpc.ClientConn +} + +func NewCacheServiceClient(cc *grpc.ClientConn) CacheServiceClient { + return &cacheServiceClient{cc} +} + +func (c *cacheServiceClient) Config(ctx context.Context, in *google_protobuf.Empty, opts ...grpc.CallOption) (*GrpcVMConfig, error) { + out := new(GrpcVMConfig) + err := grpc.Invoke(ctx, "/cache.CacheService/Config", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cacheServiceClient) GetBaseVM(ctx context.Context, in *google_protobuf.Empty, opts ...grpc.CallOption) (*GrpcVM, error) { + out := new(GrpcVM) + err := grpc.Invoke(ctx, "/cache.CacheService/GetBaseVM", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for CacheService service + +type CacheServiceServer interface { + Config(context.Context, *google_protobuf.Empty) (*GrpcVMConfig, error) + GetBaseVM(context.Context, *google_protobuf.Empty) (*GrpcVM, error) +} + +func RegisterCacheServiceServer(s *grpc.Server, srv CacheServiceServer) { + s.RegisterService(&_CacheService_serviceDesc, srv) +} + +func _CacheService_Config_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(google_protobuf.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CacheServiceServer).Config(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/cache.CacheService/Config", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CacheServiceServer).Config(ctx, req.(*google_protobuf.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _CacheService_GetBaseVM_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(google_protobuf.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CacheServiceServer).GetBaseVM(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/cache.CacheService/GetBaseVM", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CacheServiceServer).GetBaseVM(ctx, req.(*google_protobuf.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +var _CacheService_serviceDesc = grpc.ServiceDesc{ + ServiceName: "cache.CacheService", + HandlerType: (*CacheServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Config", + Handler: _CacheService_Config_Handler, + }, + { + MethodName: "GetBaseVM", + Handler: _CacheService_GetBaseVM_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "cache.proto", +} + +func (m *GrpcVMConfig) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GrpcVMConfig) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Data) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintCache(dAtA, i, uint64(len(m.Data))) + i += copy(dAtA[i:], m.Data) + } + if len(m.AgentConfig) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintCache(dAtA, i, uint64(len(m.AgentConfig))) + i += copy(dAtA[i:], m.AgentConfig) + } + return i, nil +} + +func (m *GrpcVM) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GrpcVM) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Id) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintCache(dAtA, i, uint64(len(m.Id))) + i += copy(dAtA[i:], m.Id) + } + if len(m.Hypervisor) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintCache(dAtA, i, uint64(len(m.Hypervisor))) + i += copy(dAtA[i:], m.Hypervisor) + } + if m.ProxyPid != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintCache(dAtA, i, uint64(m.ProxyPid)) + } + if len(m.ProxyURL) > 0 { + dAtA[i] = 0x22 + i++ + i = encodeVarintCache(dAtA, i, uint64(len(m.ProxyURL))) + i += copy(dAtA[i:], m.ProxyURL) + } + if m.Cpu != 0 { + dAtA[i] = 0x28 + i++ + i = encodeVarintCache(dAtA, i, uint64(m.Cpu)) + } + if m.Memory != 0 { + dAtA[i] = 0x30 + i++ + i = encodeVarintCache(dAtA, i, uint64(m.Memory)) + } + if m.CpuDelta != 0 { + dAtA[i] = 0x38 + i++ + i = encodeVarintCache(dAtA, i, uint64(m.CpuDelta)) + } + return i, nil +} + +func encodeVarintCache(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *GrpcVMConfig) Size() (n int) { + var l int + _ = l + l = len(m.Data) + if l > 0 { + n += 1 + l + sovCache(uint64(l)) + } + l = len(m.AgentConfig) + if l > 0 { + n += 1 + l + sovCache(uint64(l)) + } + return n +} + +func (m *GrpcVM) Size() (n int) { + var l int + _ = l + l = len(m.Id) + if l > 0 { + n += 1 + l + sovCache(uint64(l)) + } + l = len(m.Hypervisor) + if l > 0 { + n += 1 + l + sovCache(uint64(l)) + } + if m.ProxyPid != 0 { + n += 1 + sovCache(uint64(m.ProxyPid)) + } + l = len(m.ProxyURL) + if l > 0 { + n += 1 + l + sovCache(uint64(l)) + } + if m.Cpu != 0 { + n += 1 + sovCache(uint64(m.Cpu)) + } + if m.Memory != 0 { + n += 1 + sovCache(uint64(m.Memory)) + } + if m.CpuDelta != 0 { + n += 1 + sovCache(uint64(m.CpuDelta)) + } + return n +} + +func sovCache(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozCache(x uint64) (n int) { + return sovCache(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *GrpcVMConfig) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCache + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GrpcVMConfig: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GrpcVMConfig: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCache + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthCache + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field AgentConfig", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCache + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthCache + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.AgentConfig = append(m.AgentConfig[:0], dAtA[iNdEx:postIndex]...) + if m.AgentConfig == nil { + m.AgentConfig = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipCache(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthCache + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *GrpcVM) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCache + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GrpcVM: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GrpcVM: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Id", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCache + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCache + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Id = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Hypervisor", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCache + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthCache + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Hypervisor = append(m.Hypervisor[:0], dAtA[iNdEx:postIndex]...) + if m.Hypervisor == nil { + m.Hypervisor = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProxyPid", wireType) + } + m.ProxyPid = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCache + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProxyPid |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ProxyURL", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCache + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCache + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ProxyURL = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Cpu", wireType) + } + m.Cpu = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCache + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Cpu |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Memory", wireType) + } + m.Memory = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCache + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Memory |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CpuDelta", wireType) + } + m.CpuDelta = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCache + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CpuDelta |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipCache(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthCache + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipCache(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCache + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCache + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCache + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthCache + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCache + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipCache(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthCache = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowCache = fmt.Errorf("proto: integer overflow") +) + +func init() { proto.RegisterFile("cache.proto", fileDescriptorCache) } + +var fileDescriptorCache = []byte{ + // 299 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x90, 0xd1, 0x6a, 0xc2, 0x30, + 0x18, 0x85, 0x49, 0xab, 0xdd, 0xfc, 0xad, 0x63, 0x64, 0x20, 0xa1, 0x83, 0x52, 0xbc, 0xea, 0x55, + 0x05, 0xc7, 0x1e, 0x60, 0xda, 0xe1, 0xcd, 0x84, 0xd1, 0x31, 0xef, 0x6b, 0x1a, 0x6b, 0xc1, 0x9a, + 0x10, 0xd3, 0xb1, 0xbe, 0xd8, 0x9e, 0x61, 0x97, 0x7b, 0x84, 0xd1, 0x27, 0x19, 0x4d, 0xbb, 0xa2, + 0x17, 0xde, 0xe5, 0x9c, 0xff, 0x9c, 0x43, 0xf8, 0x60, 0x48, 0x63, 0xba, 0x63, 0x81, 0x90, 0x5c, + 0x71, 0xdc, 0xd7, 0xc2, 0xb9, 0x4f, 0x39, 0x4f, 0xf7, 0x6c, 0xaa, 0xcd, 0x4d, 0xb1, 0x9d, 0xb2, + 0x5c, 0xa8, 0xb2, 0xc9, 0x4c, 0x42, 0xb0, 0x97, 0x52, 0xd0, 0xf5, 0x6a, 0xc1, 0x0f, 0xdb, 0x2c, + 0xc5, 0x18, 0x7a, 0x61, 0xac, 0x62, 0x82, 0x3c, 0xe4, 0xdb, 0x91, 0x7e, 0x63, 0x0f, 0x86, 0x4f, + 0x29, 0x3b, 0xa8, 0x26, 0x42, 0x0c, 0x7d, 0x3a, 0xb5, 0x26, 0x5f, 0x08, 0xac, 0x66, 0x06, 0xdf, + 0x80, 0x91, 0x25, 0xba, 0x3e, 0x88, 0x8c, 0x2c, 0xc1, 0x2e, 0xc0, 0xae, 0x14, 0x4c, 0x7e, 0x64, + 0x47, 0x2e, 0xdb, 0xee, 0x89, 0x83, 0x1d, 0xb8, 0x16, 0x92, 0x7f, 0x96, 0xaf, 0x59, 0x42, 0x4c, + 0x0f, 0xf9, 0x66, 0xd4, 0xe9, 0xee, 0xf6, 0x1e, 0xbd, 0x90, 0x9e, 0x5e, 0xec, 0x34, 0xbe, 0x05, + 0x93, 0x8a, 0x82, 0xf4, 0x3d, 0xe4, 0x8f, 0xa2, 0xfa, 0x89, 0xc7, 0x60, 0xe5, 0x2c, 0xe7, 0xb2, + 0x24, 0x96, 0x36, 0x5b, 0x55, 0xaf, 0x50, 0x51, 0x84, 0x6c, 0xaf, 0x62, 0x72, 0xa5, 0x2f, 0x9d, + 0x9e, 0x95, 0x60, 0x2f, 0x6a, 0x48, 0x6f, 0xf5, 0x77, 0x28, 0xc3, 0x8f, 0x60, 0xb5, 0x20, 0xc6, + 0x41, 0x83, 0x2d, 0xf8, 0xc7, 0x16, 0x3c, 0xd7, 0xd8, 0x9c, 0xbb, 0xa0, 0x41, 0x7c, 0x46, 0x6d, + 0x06, 0x83, 0x25, 0x53, 0xf3, 0xf8, 0xc8, 0xd6, 0xab, 0x8b, 0xcd, 0xd1, 0x59, 0x73, 0x6e, 0x7f, + 0x57, 0x2e, 0xfa, 0xa9, 0x5c, 0xf4, 0x5b, 0xb9, 0x68, 0x63, 0xe9, 0xf0, 0xc3, 0x5f, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xb5, 0xef, 0x80, 0xc1, 0xc1, 0x01, 0x00, 0x00, +} diff --git a/protocols/cache/cache.proto b/protocols/cache/cache.proto new file mode 100644 index 000000000..6eada1eb0 --- /dev/null +++ b/protocols/cache/cache.proto @@ -0,0 +1,35 @@ +// +// Copyright 2019 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +syntax = "proto3"; + +package cache; + +import "google/protobuf/empty.proto"; + +service CacheService { + rpc Config(google.protobuf.Empty) returns (GrpcVMConfig); + rpc GetBaseVM(google.protobuf.Empty) returns (GrpcVM); +} + +message GrpcVMConfig { + bytes Data = 1; + bytes AgentConfig = 2; +} + +message GrpcVM { + string id = 1; + + bytes hypervisor = 2; + + int64 proxyPid = 3; + string proxyURL = 4; + + uint32 cpu = 5; + uint32 memory = 6; + + uint32 cpuDelta = 7; +} diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go index 9cfce0f34..804f47b78 100644 --- a/virtcontainers/agent.go +++ b/virtcontainers/agent.go @@ -151,6 +151,9 @@ type agent interface { // set to use an existing proxy setProxy(sandbox *Sandbox, proxy proxy, pid int, url string) error + // set to use an existing proxy from Grpc + setProxyFromGrpc(proxy proxy, pid int, url string) + // get agent url getAgentURL() (string, error) @@ -225,6 +228,9 @@ type agent interface { // configure will update agent settings based on provided arguments configure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error + // configureFromGrpc will update agent settings based on provided arguments which from Grpc + configureFromGrpc(id string, builtin bool, config interface{}) error + // getVMPath will return the agent vm socket's directory path getVMPath(id string) string diff --git a/virtcontainers/factory.go b/virtcontainers/factory.go index 45306a5f0..8579d8a97 100644 --- a/virtcontainers/factory.go +++ b/virtcontainers/factory.go @@ -9,9 +9,15 @@ import "context" // Factory controls how a new VM is created. type Factory interface { + // Config returns base factory config. + Config() VMConfig + // GetVM gets a new VM from the factory. GetVM(ctx context.Context, config VMConfig) (*VM, error) + // GetBaseVM returns a paused VM created by the base factory. + GetBaseVM(ctx context.Context, config VMConfig) (*VM, error) + // CloseFactory closes and cleans up the factory. CloseFactory(ctx context.Context) } diff --git a/virtcontainers/factory/factory.go b/virtcontainers/factory/factory.go index e11e1632c..4bbe19837 100644 --- a/virtcontainers/factory/factory.go +++ b/virtcontainers/factory/factory.go @@ -8,13 +8,14 @@ package factory import ( "context" "fmt" - "reflect" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/factory/base" "github.com/kata-containers/runtime/virtcontainers/factory/cache" "github.com/kata-containers/runtime/virtcontainers/factory/direct" + "github.com/kata-containers/runtime/virtcontainers/factory/grpccache" "github.com/kata-containers/runtime/virtcontainers/factory/template" + "github.com/kata-containers/runtime/virtcontainers/utils" opentracing "github.com/opentracing/opentracing-go" "github.com/sirupsen/logrus" ) @@ -24,7 +25,10 @@ var factoryLogger = logrus.FieldLogger(logrus.New()) // Config is a collection of VM factory configurations. type Config struct { Template bool - Cache uint + + VMCache bool + Cache uint + VMCacheEndpoint string VMConfig vc.VMConfig } @@ -65,6 +69,11 @@ func NewFactory(ctx context.Context, config Config, fetchOnly bool) (vc.Factory, } else { b = template.New(ctx, config.VMConfig) } + } else if config.VMCache && config.Cache == 0 { + b, err = grpccache.New(ctx, config.VMCacheEndpoint) + if err != nil { + return nil, err + } } else { b = direct.New(ctx, config.VMConfig) } @@ -100,71 +109,6 @@ func resetHypervisorConfig(config *vc.VMConfig) { config.ProxyConfig = vc.ProxyConfig{} } -func compareStruct(foo, bar reflect.Value) bool { - for i := 0; i < foo.NumField(); i++ { - if !deepCompareValue(foo.Field(i), bar.Field(i)) { - return false - } - } - - return true -} - -func compareMap(foo, bar reflect.Value) bool { - if foo.Len() != bar.Len() { - return false - } - - for _, k := range foo.MapKeys() { - if !deepCompareValue(foo.MapIndex(k), bar.MapIndex(k)) { - return false - } - } - - return true -} - -func compareSlice(foo, bar reflect.Value) bool { - if foo.Len() != bar.Len() { - return false - } - for j := 0; j < foo.Len(); j++ { - if !deepCompareValue(foo.Index(j), bar.Index(j)) { - return false - } - } - return true -} - -func deepCompareValue(foo, bar reflect.Value) bool { - if !foo.IsValid() || !bar.IsValid() { - return foo.IsValid() == bar.IsValid() - } - - if foo.Type() != bar.Type() { - return false - } - switch foo.Kind() { - case reflect.Map: - return compareMap(foo, bar) - case reflect.Array: - fallthrough - case reflect.Slice: - return compareSlice(foo, bar) - case reflect.Struct: - return compareStruct(foo, bar) - default: - return foo.Interface() == bar.Interface() - } -} - -func deepCompare(foo, bar interface{}) bool { - v1 := reflect.ValueOf(foo) - v2 := reflect.ValueOf(bar) - - return deepCompareValue(v1, v2) -} - // It's important that baseConfig and newConfig are passed by value! func checkVMConfig(config1, config2 vc.VMConfig) error { if config1.HypervisorType != config2.HypervisorType { @@ -179,7 +123,7 @@ func checkVMConfig(config1, config2 vc.VMConfig) error { resetHypervisorConfig(&config1) resetHypervisorConfig(&config2) - if !deepCompare(config1, config2) { + if !utils.DeepCompare(config1, config2) { return fmt.Errorf("hypervisor config does not match, base: %+v. new: %+v", config1, config2) } @@ -282,6 +226,16 @@ func (f *factory) GetVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error) return vm, nil } +// Config returns base factory config. +func (f *factory) Config() vc.VMConfig { + return f.base.Config() +} + +// GetBaseVM returns a paused VM created by the base factory. +func (f *factory) GetBaseVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error) { + return f.base.GetBaseVM(ctx, config) +} + // CloseFactory closes the factory. func (f *factory) CloseFactory(ctx context.Context) { f.base.CloseFactory(ctx) diff --git a/virtcontainers/factory/factory_test.go b/virtcontainers/factory/factory_test.go index deaa02c20..a8e08606c 100644 --- a/virtcontainers/factory/factory_test.go +++ b/virtcontainers/factory/factory_test.go @@ -10,11 +10,11 @@ import ( "io/ioutil" "testing" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/factory/base" + "github.com/kata-containers/runtime/virtcontainers/utils" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" ) func TestNewFactory(t *testing.T) { @@ -279,41 +279,41 @@ func TestDeepCompare(t *testing.T) { foo := vc.VMConfig{} bar := vc.VMConfig{} - assert.True(deepCompare(foo, bar)) + assert.True(utils.DeepCompare(foo, bar)) foo.HypervisorConfig.NumVCPUs = 1 - assert.False(deepCompare(foo, bar)) + assert.False(utils.DeepCompare(foo, bar)) bar.HypervisorConfig.NumVCPUs = 1 - assert.True(deepCompare(foo, bar)) + assert.True(utils.DeepCompare(foo, bar)) // slice foo.HypervisorConfig.KernelParams = []vc.Param{} - assert.True(deepCompare(foo, bar)) + assert.True(utils.DeepCompare(foo, bar)) foo.HypervisorConfig.KernelParams = append(foo.HypervisorConfig.KernelParams, vc.Param{Key: "key", Value: "value"}) - assert.False(deepCompare(foo, bar)) + assert.False(utils.DeepCompare(foo, bar)) bar.HypervisorConfig.KernelParams = append(bar.HypervisorConfig.KernelParams, vc.Param{Key: "key", Value: "value"}) - assert.True(deepCompare(foo, bar)) + assert.True(utils.DeepCompare(foo, bar)) // map var fooMap map[string]vc.VMConfig var barMap map[string]vc.VMConfig - assert.False(deepCompare(foo, fooMap)) - assert.True(deepCompare(fooMap, barMap)) + assert.False(utils.DeepCompare(foo, fooMap)) + assert.True(utils.DeepCompare(fooMap, barMap)) fooMap = make(map[string]vc.VMConfig) - assert.True(deepCompare(fooMap, barMap)) + assert.True(utils.DeepCompare(fooMap, barMap)) fooMap["foo"] = foo - assert.False(deepCompare(fooMap, barMap)) + assert.False(utils.DeepCompare(fooMap, barMap)) barMap = make(map[string]vc.VMConfig) - assert.False(deepCompare(fooMap, barMap)) + assert.False(utils.DeepCompare(fooMap, barMap)) barMap["foo"] = bar - assert.True(deepCompare(fooMap, barMap)) + assert.True(utils.DeepCompare(fooMap, barMap)) // invalid interface var f1 vc.Factory var f2 vc.Factory var f3 base.FactoryBase - assert.True(deepCompare(f1, f2)) - assert.True(deepCompare(f1, f3)) + assert.True(utils.DeepCompare(f1, f2)) + assert.True(utils.DeepCompare(f1, f3)) // valid interface var config Config @@ -331,8 +331,8 @@ func TestDeepCompare(t *testing.T) { } f1, err = NewFactory(ctx, config, false) assert.Nil(err) - assert.True(deepCompare(f1, f1)) + assert.True(utils.DeepCompare(f1, f1)) f2, err = NewFactory(ctx, config, false) assert.Nil(err) - assert.False(deepCompare(f1, f2)) + assert.False(utils.DeepCompare(f1, f2)) } diff --git a/virtcontainers/factory/grpccache/grpccache.go b/virtcontainers/factory/grpccache/grpccache.go new file mode 100644 index 000000000..8dfcb45eb --- /dev/null +++ b/virtcontainers/factory/grpccache/grpccache.go @@ -0,0 +1,62 @@ +// Copyright (c) 2019 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// grpccache implements base vm factory that get base vm from grpc + +package grpccache + +import ( + "context" + "fmt" + types "github.com/gogo/protobuf/types" + pb "github.com/kata-containers/runtime/protocols/cache" + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/factory/base" + "github.com/pkg/errors" + "google.golang.org/grpc" +) + +type grpccache struct { + conn *grpc.ClientConn + config *vc.VMConfig +} + +// New returns a new direct vm factory. +func New(ctx context.Context, endpoint string) (base.FactoryBase, error) { + conn, err := grpc.Dial(fmt.Sprintf("unix://%s", endpoint), grpc.WithInsecure()) + if err != nil { + return nil, errors.Wrapf(err, "failed to connect %q", endpoint) + } + + jConfig, err := pb.NewCacheServiceClient(conn).Config(ctx, &types.Empty{}) + if err != nil { + return nil, errors.Wrapf(err, "failed to Config") + } + + config, err := vc.GrpcToVMConfig(jConfig) + if err != nil { + return nil, errors.Wrapf(err, "failed to convert JSON to VMConfig") + } + + return &grpccache{conn: conn, config: config}, nil +} + +// Config returns the direct factory's configuration. +func (g *grpccache) Config() vc.VMConfig { + return *g.config +} + +// GetBaseVM create a new VM directly. +func (g *grpccache) GetBaseVM(ctx context.Context, config vc.VMConfig) (*vc.VM, error) { + defer g.conn.Close() + gVM, err := pb.NewCacheServiceClient(g.conn).GetBaseVM(ctx, &types.Empty{}) + if err != nil { + return nil, errors.Wrapf(err, "failed to GetBaseVM") + } + return vc.NewVMFromGrpc(ctx, gVM, *g.config) +} + +// CloseFactory closes the direct vm factory. +func (g *grpccache) CloseFactory(ctx context.Context) { +} diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index d8bcad9e4..c197e80a1 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -7,6 +7,7 @@ package virtcontainers import ( "context" + "errors" "fmt" "net/url" "os/exec" @@ -706,3 +707,11 @@ func (fc *firecracker) cleanup() error { func (fc *firecracker) pid() int { return fc.info.PID } + +func (fc *firecracker) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, store *store.VCStore, j []byte) error { + return errors.New("firecracker is not supported by VM cache") +} + +func (fc *firecracker) toGrpc() ([]byte, error) { + return nil, errors.New("firecracker is not supported by VM cache") +} diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index da6b9ecb6..dc9f3aed4 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -327,6 +327,10 @@ func (h *hyper) configure(hv hypervisor, id, sharePath string, builtin bool, con return hv.addDevice(sharedVolume, fsDev) } +func (h *hyper) configureFromGrpc(id string, builtin bool, config interface{}) error { + return nil +} + func (h *hyper) createSandbox(sandbox *Sandbox) (err error) { return h.configure(sandbox.hypervisor, "", h.getSharePath(sandbox.id), false, nil) } @@ -999,6 +1003,12 @@ func (h *hyper) setProxy(sandbox *Sandbox, proxy proxy, pid int, url string) err return nil } +func (h *hyper) setProxyFromGrpc(proxy proxy, pid int, url string) { + h.proxy = proxy + h.state.ProxyPid = pid + h.state.URL = url +} + func (h *hyper) getGuestDetails(*grpc.GuestDetailsRequest) (*grpc.GuestDetailsResponse, error) { // hyperstart-agent does not support getGuestDetails return nil, nil diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 6966b9e78..49885081a 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -610,4 +610,6 @@ type hypervisor interface { getThreadIDs() (*threadIDs, error) cleanup() error pid() int + fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, store *store.VCStore, j []byte) error + toGrpc() ([]byte, error) } diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index f07b89106..d26c512ef 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -221,7 +221,7 @@ func (k *kataAgent) capabilities() types.Capabilities { return caps } -func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error { +func (k *kataAgent) internalConfigure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error { if config != nil { switch c := config.(type) { case KataAgentConfig: @@ -234,20 +234,32 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, } } + if builtin { + k.proxyBuiltIn = true + } + + return nil +} + +func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, config interface{}) error { + err := k.internalConfigure(h, id, sharePath, builtin, config) + if err != nil { + return err + } + switch s := k.vmSocket.(type) { case types.Socket: - err := h.addDevice(s, serialPortDev) + err = h.addDevice(s, serialPortDev) if err != nil { return err } case kataVSOCK: - var err error s.vhostFd, s.contextID, err = utils.FindContextID() if err != nil { return err } s.port = uint32(vSockPort) - if err := h.addDevice(s, vSockPCIDev); err != nil { + if err = h.addDevice(s, vSockPCIDev); err != nil { return err } k.vmSocket = s @@ -255,10 +267,6 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, return fmt.Errorf("Invalid config type") } - if builtin { - k.proxyBuiltIn = true - } - // Neither create shared directory nor add 9p device if hypervisor // doesn't support filesystem sharing. caps := h.capabilities() @@ -273,13 +281,17 @@ func (k *kataAgent) configure(h hypervisor, id, sharePath string, builtin bool, HostPath: sharePath, } - if err := os.MkdirAll(sharedVolume.HostPath, store.DirMode); err != nil { + if err = os.MkdirAll(sharedVolume.HostPath, store.DirMode); err != nil { return err } return h.addDevice(sharedVolume, fsDev) } +func (k *kataAgent) configureFromGrpc(id string, builtin bool, config interface{}) error { + return k.internalConfigure(nil, id, "", builtin, config) +} + func (k *kataAgent) createSandbox(sandbox *Sandbox) error { span, _ := k.trace("createSandbox") defer span.Finish() @@ -587,6 +599,12 @@ func (k *kataAgent) setProxy(sandbox *Sandbox, proxy proxy, pid int, url string) return nil } +func (k *kataAgent) setProxyFromGrpc(proxy proxy, pid int, url string) { + k.proxy = proxy + k.state.ProxyPid = pid + k.state.URL = url +} + func (k *kataAgent) startSandbox(sandbox *Sandbox) error { span, _ := k.trace("startSandbox") defer span.Finish() diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go index fbf1c379b..b1efa6f67 100644 --- a/virtcontainers/mock_hypervisor.go +++ b/virtcontainers/mock_hypervisor.go @@ -7,6 +7,7 @@ package virtcontainers import ( "context" + "errors" "os" "github.com/kata-containers/runtime/virtcontainers/store" @@ -105,3 +106,11 @@ func (m *mockHypervisor) cleanup() error { func (m *mockHypervisor) pid() int { return m.mockPid } + +func (m *mockHypervisor) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, store *store.VCStore, j []byte) error { + return errors.New("mockHypervisor is not supported by VM cache") +} + +func (m *mockHypervisor) toGrpc() ([]byte, error) { + return nil, errors.New("firecracker is not supported by VM cache") +} diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go index 6ba15d4b3..db9a3dcbd 100644 --- a/virtcontainers/noop_agent.go +++ b/virtcontainers/noop_agent.go @@ -171,6 +171,10 @@ func (n *noopAgent) configure(h hypervisor, id, sharePath string, builtin bool, return nil } +func (n *noopAgent) configureFromGrpc(id string, builtin bool, config interface{}) error { + return nil +} + // getVMPath is the Noop agent vm path getter. It does nothing. func (n *noopAgent) getVMPath(id string) string { return "" @@ -201,6 +205,9 @@ func (n *noopAgent) setProxy(sandbox *Sandbox, proxy proxy, pid int, url string) return nil } +func (n *noopAgent) setProxyFromGrpc(proxy proxy, pid int, url string) { +} + // getGuestDetails is the Noop agent GuestDetails queryer. It does nothing. func (n *noopAgent) getGuestDetails(*grpc.GuestDetailsRequest) (*grpc.GuestDetailsResponse, error) { return nil, nil diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index 4ffb4ec95..269edc962 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -96,6 +96,12 @@ type CompatOCISpec struct { type FactoryConfig struct { // Template enables VM templating support in VM factory. Template bool + + // VMCacheNumber specifies the the number of caches of VMCache. + VMCacheNumber uint + + // VMCacheEndpoint specifies the endpoint of transport VM from the VM cache server to runtime. + VMCacheEndpoint string } // RuntimeConfig aggregates all runtime specific settings diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index 25e78ea06..7ee216433 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -7,6 +7,7 @@ package virtcontainers import ( "context" + "encoding/json" "fmt" "io/ioutil" "math" @@ -1600,3 +1601,55 @@ func (q *qemu) pid() int { return pid } + +type qemuGrpc struct { + ID string + QmpChannelpath string + State QemuState + NvdimmCount int + + // Most members of q.qemuConfig are just to generate + // q.qemuConfig.qemuParams that is used by LaunchQemu except + // q.qemuConfig.SMP. + // So just transport q.qemuConfig.SMP from VM Cache server to runtime. + QemuSMP govmmQemu.SMP +} + +func (q *qemu) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, store *store.VCStore, j []byte) error { + var qp qemuGrpc + err := json.Unmarshal(j, &qp) + if err != nil { + return err + } + + q.id = qp.ID + q.store = store + q.config = *hypervisorConfig + q.qmpMonitorCh.ctx = ctx + q.qmpMonitorCh.path = qp.QmpChannelpath + q.qemuConfig.Ctx = ctx + q.state = qp.State + q.arch = newQemuArch(q.config) + q.ctx = ctx + q.nvdimmCount = qp.NvdimmCount + + q.qemuConfig.SMP = qp.QemuSMP + + return nil +} + +func (q *qemu) toGrpc() ([]byte, error) { + q.qmpShutdown() + + q.cleanup() + qp := qemuGrpc{ + ID: q.id, + QmpChannelpath: q.qmpMonitorCh.path, + State: q.state, + NvdimmCount: q.nvdimmCount, + + QemuSMP: q.qemuConfig.SMP, + } + + return json.Marshal(&qp) +} diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 6c79a9297..067a9dfca 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -424,3 +424,22 @@ func TestQemuCleanup(t *testing.T) { err := q.cleanup() assert.Nil(err) } + +func TestQemuGrpc(t *testing.T) { + assert := assert.New(t) + + config := newQemuConfig() + q := &qemu{ + id: "testqemu", + config: config, + } + + json, err := q.toGrpc() + assert.Nil(err) + + var q2 qemu + err = q2.fromGrpc(context.Background(), &config, nil, json) + assert.Nil(err) + + assert.True(q.id == q2.id) +} diff --git a/virtcontainers/utils/compare.go b/virtcontainers/utils/compare.go new file mode 100644 index 000000000..e67b6bc47 --- /dev/null +++ b/virtcontainers/utils/compare.go @@ -0,0 +1,74 @@ +// Copyright (c) 2019 HyperHQ Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// + +package utils + +import "reflect" + +func compareStruct(foo, bar reflect.Value) bool { + for i := 0; i < foo.NumField(); i++ { + if !deepCompareValue(foo.Field(i), bar.Field(i)) { + return false + } + } + + return true +} + +func compareMap(foo, bar reflect.Value) bool { + if foo.Len() != bar.Len() { + return false + } + + for _, k := range foo.MapKeys() { + if !deepCompareValue(foo.MapIndex(k), bar.MapIndex(k)) { + return false + } + } + + return true +} + +func compareSlice(foo, bar reflect.Value) bool { + if foo.Len() != bar.Len() { + return false + } + for j := 0; j < foo.Len(); j++ { + if !deepCompareValue(foo.Index(j), bar.Index(j)) { + return false + } + } + return true +} + +func deepCompareValue(foo, bar reflect.Value) bool { + if !foo.IsValid() || !bar.IsValid() { + return foo.IsValid() == bar.IsValid() + } + + if foo.Type() != bar.Type() { + return false + } + switch foo.Kind() { + case reflect.Map: + return compareMap(foo, bar) + case reflect.Array: + fallthrough + case reflect.Slice: + return compareSlice(foo, bar) + case reflect.Struct: + return compareStruct(foo, bar) + default: + return foo.Interface() == bar.Interface() + } +} + +// DeepCompare compare foo and bar. +func DeepCompare(foo, bar interface{}) bool { + v1 := reflect.ValueOf(foo) + v2 := reflect.ValueOf(bar) + + return deepCompareValue(v1, v2) +} diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go index ea0e6c304..dc9d6f097 100644 --- a/virtcontainers/vm.go +++ b/virtcontainers/vm.go @@ -7,10 +7,13 @@ package virtcontainers import ( "context" + "encoding/json" + "fmt" "os" "path/filepath" "time" + pb "github.com/kata-containers/runtime/protocols/cache" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" "github.com/kata-containers/runtime/virtcontainers/store" "github.com/sirupsen/logrus" @@ -52,6 +55,63 @@ func (c *VMConfig) Valid() error { return c.HypervisorConfig.valid() } +// ToGrpc convert VMConfig struct to grpc format pb.GrpcVMConfig. +func (c *VMConfig) ToGrpc() (*pb.GrpcVMConfig, error) { + data, err := json.Marshal(&c) + if err != nil { + return nil, err + } + + var agentConfig []byte + switch aconf := c.AgentConfig.(type) { + case HyperConfig: + agentConfig, err = json.Marshal(&aconf) + case KataAgentConfig: + agentConfig, err = json.Marshal(&aconf) + default: + err = fmt.Errorf("agent type %s is not supported by VM cache", c.AgentType) + } + if err != nil { + return nil, err + } + + return &pb.GrpcVMConfig{ + Data: data, + AgentConfig: agentConfig, + }, nil +} + +// GrpcToVMConfig convert grpc format pb.GrpcVMConfig to VMConfig struct. +func GrpcToVMConfig(j *pb.GrpcVMConfig) (*VMConfig, error) { + var config VMConfig + err := json.Unmarshal(j.Data, &config) + if err != nil { + return nil, err + } + + switch config.AgentType { + case HyperstartAgent: + var hyperConfig HyperConfig + err := json.Unmarshal(j.AgentConfig, &hyperConfig) + if err == nil { + config.AgentConfig = hyperConfig + } + case KataContainersAgent: + var kataConfig KataAgentConfig + err := json.Unmarshal(j.AgentConfig, &kataConfig) + if err == nil { + config.AgentConfig = kataConfig + } + default: + err = fmt.Errorf("agent type %s is not supported by VM cache", config.AgentType) + } + if err != nil { + return nil, err + } + + return &config, nil +} + func setupProxy(h hypervisor, agent agent, config VMConfig, id string) (int, string, proxy, error) { consoleURL, err := h.getSandboxConsole(id) if err != nil { @@ -187,6 +247,57 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { }, nil } +// NewVMFromGrpc creates a new VM based on provided pb.GrpcVM and VMConfig. +func NewVMFromGrpc(ctx context.Context, v *pb.GrpcVM, config VMConfig) (*VM, error) { + virtLog.WithField("GrpcVM", v).WithField("config", config).Info("create new vm from Grpc") + + hypervisor, err := newHypervisor(config.HypervisorType) + if err != nil { + return nil, err + } + + vcStore, err := store.NewVCStore(ctx, + store.SandboxConfigurationRoot(v.Id), + store.SandboxRuntimeRoot(v.Id)) + if err != nil { + return nil, err + } + + defer func() { + if err != nil { + virtLog.WithField("vm", v.Id).WithError(err).Error("failed to create new vm from Grpc") + virtLog.WithField("vm", v.Id).Errorf("Deleting store for %s", v.Id) + vcStore.Delete() + } + }() + + err = hypervisor.fromGrpc(ctx, &config.HypervisorConfig, vcStore, v.Hypervisor) + if err != nil { + return nil, err + } + + agent := newAgent(config.AgentType) + agent.configureFromGrpc(v.Id, isProxyBuiltIn(config.ProxyType), config.AgentConfig) + + proxy, err := newProxy(config.ProxyType) + if err != nil { + return nil, err + } + agent.setProxyFromGrpc(proxy, int(v.ProxyPid), v.ProxyURL) + + return &VM{ + id: v.Id, + hypervisor: hypervisor, + agent: agent, + proxy: proxy, + proxyPid: int(v.ProxyPid), + proxyURL: v.ProxyURL, + cpu: v.Cpu, + memory: v.Memory, + cpuDelta: v.CpuDelta, + }, nil +} + func buildVMSharePath(id string) string { return filepath.Join(store.RunVMStoragePath, id, "shared") } @@ -353,3 +464,23 @@ func (v *VM) assignSandbox(s *Sandbox) error { return nil } + +// ToGrpc convert VM struct to Grpc format pb.GrpcVM. +func (v *VM) ToGrpc(config VMConfig) (*pb.GrpcVM, error) { + hJSON, err := v.hypervisor.toGrpc() + if err != nil { + return nil, err + } + + return &pb.GrpcVM{ + Id: v.id, + Hypervisor: hJSON, + + ProxyPid: int64(v.proxyPid), + ProxyURL: v.proxyURL, + + Cpu: v.cpu, + Memory: v.memory, + CpuDelta: v.cpuDelta, + }, nil +} diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go index 5fda0f98a..90025d238 100644 --- a/virtcontainers/vm_test.go +++ b/virtcontainers/vm_test.go @@ -10,6 +10,7 @@ import ( "io/ioutil" "testing" + "github.com/kata-containers/runtime/virtcontainers/utils" "github.com/stretchr/testify/assert" ) @@ -110,3 +111,22 @@ func TestSetupProxy(t *testing.T) { _, _, _, err = setupProxy(hypervisor, agent, config, "foobar") assert.Nil(err) } + +func TestVMConfigGrpc(t *testing.T) { + assert := assert.New(t) + config := VMConfig{ + HypervisorType: QemuHypervisor, + HypervisorConfig: newQemuConfig(), + AgentType: KataContainersAgent, + AgentConfig: KataAgentConfig{false, true}, + ProxyType: NoopProxyType, + } + + p, err := config.ToGrpc() + assert.Nil(err) + + config2, err := GrpcToVMConfig(p) + assert.Nil(err) + + assert.True(utils.DeepCompare(config, *config2)) +}