From f503755e5369ba191ee49e506636d09f51db2841 Mon Sep 17 00:00:00 2001 From: Madhan Raj Mookkandy Date: Tue, 1 Aug 2017 11:36:50 -0700 Subject: [PATCH 1/4] Add Windows Kernel Proxy support Windows Kernel now exposes "Internal Load Balancing" using VFP (Virtual Filtering Platform) part of Virtual Switch. An inbuild windows service HNS (Host Networking Service) acts as interface to program the VFP. VFP is synonymous to iptables in functionality. HNS uses json based data as input. With the help of the interface available in github.com/Microsoft/hcsshim, these APIs are exposed to the world in github to program HNS and use the feature. *** More info about the changes in this PR *** (1) For every endpoint available in the system, an HNS Endpoint is added (1.a) for local endpoints, a local HNS Endpoint would already exist, as part of container creation. (1.b) For all remote endpoints, a remote HNS Endpoint is created via HNS (2) For every Service, a HNS ILB LoadBalancer is added referring the endpoints created in (1) Sample Input to HNS: { "Policies": [ { "ExternalPort": 80, "InternalPort": 80, "Protocol": 6, "Type": "ELB", "VIPs": [ "11.0.98.129" ] } ], "References": [ "/endpoints/ca8b877b-ab90-499a-bc0e-7d736c425632", "/endpoints/ee0ef08b-8434-4f8b-b748-393884e77465" ] } (2-a) This is done for Cluster IP, LoadBalancer Ingress IP, NodePort, External IP Following the regular service and endpoint updates, the HNS is notified of the updates and the system is kept in sync. --- cmd/kube-proxy/app/server.go | 307 +--- cmd/kube-proxy/app/server_linux.go | 298 ++++ cmd/kube-proxy/app/server_windows.go | 187 ++ pkg/proxy/winkernel/metrics.go | 50 + pkg/proxy/winkernel/proxier.go | 1129 ++++++++++++ pkg/proxy/winkernel/proxier_test.go | 2474 ++++++++++++++++++++++++++ 6 files changed, 4150 insertions(+), 295 deletions(-) create mode 100644 cmd/kube-proxy/app/server_linux.go create mode 100644 cmd/kube-proxy/app/server_windows.go create mode 100644 pkg/proxy/winkernel/metrics.go create mode 100644 pkg/proxy/winkernel/proxier.go create mode 100644 pkg/proxy/winkernel/proxier_test.go diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 044c3268d2a..2b66fbea958 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -35,8 +35,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/runtime/serializer/json" - "k8s.io/apimachinery/pkg/types" - utilnet "k8s.io/apimachinery/pkg/util/net" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server/healthz" @@ -60,17 +58,13 @@ import ( "k8s.io/kubernetes/pkg/proxy/iptables" "k8s.io/kubernetes/pkg/proxy/ipvs" "k8s.io/kubernetes/pkg/proxy/userspace" - "k8s.io/kubernetes/pkg/proxy/winuserspace" "k8s.io/kubernetes/pkg/util/configz" - utildbus "k8s.io/kubernetes/pkg/util/dbus" utiliptables "k8s.io/kubernetes/pkg/util/iptables" utilipvs "k8s.io/kubernetes/pkg/util/ipvs" - utilnetsh "k8s.io/kubernetes/pkg/util/netsh" utilnode "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/util/oom" utilpointer "k8s.io/kubernetes/pkg/util/pointer" "k8s.io/kubernetes/pkg/util/resourcecontainer" - utilsysctl "k8s.io/kubernetes/pkg/util/sysctl" "k8s.io/kubernetes/pkg/version/verflag" "k8s.io/utils/exec" @@ -78,19 +72,19 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/spf13/cobra" "github.com/spf13/pflag" - "k8s.io/kubernetes/pkg/features" ) const ( - proxyModeUserspace = "userspace" - proxyModeIPTables = "iptables" - proxyModeIPVS = "ipvs" + proxyModeUserspace = "userspace" + proxyModeIPTables = "iptables" + proxyModeIPVS = "ipvs" + proxyModeKernelspace = "kernelspace" ) // checkKnownProxyMode returns true if proxyMode is valid. func checkKnownProxyMode(proxyMode string) bool { switch proxyMode { - case "", proxyModeUserspace, proxyModeIPTables, proxyModeIPVS: + case "", proxyModeUserspace, proxyModeIPTables, proxyModeIPVS, proxyModeKernelspace: return true } return false @@ -142,7 +136,7 @@ func AddFlags(options *Options, fs *pflag.FlagSet) { fs.StringVar(&options.config.ClientConnection.KubeConfigFile, "kubeconfig", options.config.ClientConnection.KubeConfigFile, "Path to kubeconfig file with authorization information (the master location is set by the master flag).") fs.Var(componentconfig.PortRangeVar{Val: &options.config.PortRange}, "proxy-port-range", "Range of host ports (beginPort-endPort, inclusive) that may be consumed in order to proxy service traffic. If unspecified (0-0) then ports will be randomly chosen.") fs.StringVar(&options.config.HostnameOverride, "hostname-override", options.config.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.") - fs.Var(&options.config.Mode, "proxy-mode", "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster) or 'ipvs'(experimental). If blank, use the best-available proxy (currently iptables). If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.") + fs.Var(&options.config.Mode, "proxy-mode", "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster) or 'ipvs'(experimental)'. If blank, use the best-available proxy (currently iptables). If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.") fs.Int32Var(options.config.IPTables.MasqueradeBit, "iptables-masquerade-bit", utilpointer.Int32PtrDerefOr(options.config.IPTables.MasqueradeBit, 14), "If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with. Must be within the range [0, 31].") fs.DurationVar(&options.config.IPTables.SyncPeriod.Duration, "iptables-sync-period", options.config.IPTables.SyncPeriod.Duration, "The maximum interval of how often iptables rules are refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.") fs.DurationVar(&options.config.IPTables.MinSyncPeriod.Duration, "iptables-min-sync-period", options.config.IPTables.MinSyncPeriod.Duration, "The minimum interval of how often the iptables rules can be refreshed as endpoints and services change (e.g. '5s', '1m', '2h22m').") @@ -425,229 +419,6 @@ func createClients(config componentconfig.ClientConnectionConfiguration, masterO return client, eventClient.CoreV1(), nil } -// NewProxyServer returns a new ProxyServer. -func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndExit bool, scheme *runtime.Scheme, master string) (*ProxyServer, error) { - if config == nil { - return nil, errors.New("config is required") - } - - if c, err := configz.New("componentconfig"); err == nil { - c.Set(config) - } else { - return nil, fmt.Errorf("unable to register configz: %s", err) - } - - protocol := utiliptables.ProtocolIpv4 - if net.ParseIP(config.BindAddress).To4() == nil { - protocol = utiliptables.ProtocolIpv6 - } - - var netshInterface utilnetsh.Interface - var iptInterface utiliptables.Interface - var ipvsInterface utilipvs.Interface - var dbus utildbus.Interface - - // Create a iptables utils. - execer := exec.New() - - if goruntime.GOOS == "windows" { - netshInterface = utilnetsh.New(execer) - } else { - dbus = utildbus.New() - iptInterface = utiliptables.New(execer, dbus, protocol) - ipvsInterface = utilipvs.New(execer) - } - - // We omit creation of pretty much everything if we run in cleanup mode - if cleanupAndExit { - return &ProxyServer{IptInterface: iptInterface, IpvsInterface: ipvsInterface, CleanupAndExit: cleanupAndExit}, nil - } - - client, eventClient, err := createClients(config.ClientConnection, master) - if err != nil { - return nil, err - } - - // Create event recorder - hostname := utilnode.GetHostname(config.HostnameOverride) - eventBroadcaster := record.NewBroadcaster() - recorder := eventBroadcaster.NewRecorder(scheme, v1.EventSource{Component: "kube-proxy", Host: hostname}) - - nodeRef := &v1.ObjectReference{ - Kind: "Node", - Name: hostname, - UID: types.UID(hostname), - Namespace: "", - } - - var healthzServer *healthcheck.HealthzServer - var healthzUpdater healthcheck.HealthzUpdater - if len(config.HealthzBindAddress) > 0 { - healthzServer = healthcheck.NewDefaultHealthzServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration, recorder, nodeRef) - healthzUpdater = healthzServer - } - - var proxier proxy.ProxyProvider - var serviceEventHandler proxyconfig.ServiceHandler - var endpointsEventHandler proxyconfig.EndpointsHandler - - proxyMode := getProxyMode(string(config.Mode), iptInterface, iptables.LinuxKernelCompatTester{}) - if proxyMode == proxyModeIPTables { - glog.V(0).Info("Using iptables Proxier.") - var nodeIP net.IP - if config.BindAddress != "0.0.0.0" { - nodeIP = net.ParseIP(config.BindAddress) - } else { - nodeIP = getNodeIP(client, hostname) - } - if config.IPTables.MasqueradeBit == nil { - // MasqueradeBit must be specified or defaulted. - return nil, fmt.Errorf("unable to read IPTables MasqueradeBit from config") - } - - // TODO this has side effects that should only happen when Run() is invoked. - proxierIPTables, err := iptables.NewProxier( - iptInterface, - utilsysctl.New(), - execer, - config.IPTables.SyncPeriod.Duration, - config.IPTables.MinSyncPeriod.Duration, - config.IPTables.MasqueradeAll, - int(*config.IPTables.MasqueradeBit), - config.ClusterCIDR, - hostname, - nodeIP, - recorder, - healthzUpdater, - ) - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) - } - iptables.RegisterMetrics() - proxier = proxierIPTables - serviceEventHandler = proxierIPTables - endpointsEventHandler = proxierIPTables - // No turning back. Remove artifacts that might still exist from the userspace Proxier. - glog.V(0).Info("Tearing down inactive rules.") - // TODO this has side effects that should only happen when Run() is invoked. - userspace.CleanupLeftovers(iptInterface) - // IPVS Proxier will generate some iptables rules, - // need to clean them before switching to other proxy mode. - ipvs.CleanupLeftovers(execer, ipvsInterface, iptInterface) - } else if proxyMode == proxyModeIPVS { - glog.V(0).Info("Using ipvs Proxier.") - proxierIPVS, err := ipvs.NewProxier( - iptInterface, - ipvsInterface, - utilsysctl.New(), - execer, - config.IPVS.SyncPeriod.Duration, - config.IPVS.MinSyncPeriod.Duration, - config.IPTables.MasqueradeAll, - int(*config.IPTables.MasqueradeBit), - config.ClusterCIDR, - hostname, - getNodeIP(client, hostname), - recorder, - healthzServer, - config.IPVS.Scheduler, - ) - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) - } - proxier = proxierIPVS - serviceEventHandler = proxierIPVS - endpointsEventHandler = proxierIPVS - glog.V(0).Info("Tearing down inactive rules.") - // TODO this has side effects that should only happen when Run() is invoked. - userspace.CleanupLeftovers(iptInterface) - iptables.CleanupLeftovers(iptInterface) - } else { - glog.V(0).Info("Using userspace Proxier.") - if goruntime.GOOS == "windows" { - // This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for - // our config.EndpointsConfigHandler. - loadBalancer := winuserspace.NewLoadBalancerRR() - // set EndpointsHandler to our loadBalancer - endpointsEventHandler = loadBalancer - proxierUserspace, err := winuserspace.NewProxier( - loadBalancer, - net.ParseIP(config.BindAddress), - netshInterface, - *utilnet.ParsePortRangeOrDie(config.PortRange), - // TODO @pires replace below with default values, if applicable - config.IPTables.SyncPeriod.Duration, - config.UDPIdleTimeout.Duration, - ) - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) - } - serviceEventHandler = proxierUserspace - proxier = proxierUserspace - } else { - // This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for - // our config.EndpointsConfigHandler. - loadBalancer := userspace.NewLoadBalancerRR() - // set EndpointsConfigHandler to our loadBalancer - endpointsEventHandler = loadBalancer - - // TODO this has side effects that should only happen when Run() is invoked. - proxierUserspace, err := userspace.NewProxier( - loadBalancer, - net.ParseIP(config.BindAddress), - iptInterface, - execer, - *utilnet.ParsePortRangeOrDie(config.PortRange), - config.IPTables.SyncPeriod.Duration, - config.IPTables.MinSyncPeriod.Duration, - config.UDPIdleTimeout.Duration, - ) - if err != nil { - return nil, fmt.Errorf("unable to create proxier: %v", err) - } - serviceEventHandler = proxierUserspace - proxier = proxierUserspace - } - // Remove artifacts from the iptables and ipvs Proxier, if not on Windows. - if goruntime.GOOS != "windows" { - glog.V(0).Info("Tearing down inactive rules.") - // TODO this has side effects that should only happen when Run() is invoked. - iptables.CleanupLeftovers(iptInterface) - // IPVS Proxier will generate some iptables rules, - // need to clean them before switching to other proxy mode. - ipvs.CleanupLeftovers(execer, ipvsInterface, iptInterface) - } - } - - // Add iptables reload function, if not on Windows. - if goruntime.GOOS != "windows" { - iptInterface.AddReloadFunc(proxier.Sync) - } - - return &ProxyServer{ - Client: client, - EventClient: eventClient, - IptInterface: iptInterface, - IpvsInterface: ipvsInterface, - execer: execer, - Proxier: proxier, - Broadcaster: eventBroadcaster, - Recorder: recorder, - ConntrackConfiguration: config.Conntrack, - Conntracker: &realConntracker{}, - ProxyMode: proxyMode, - NodeRef: nodeRef, - MetricsBindAddress: config.MetricsBindAddress, - EnableProfiling: config.EnableProfiling, - OOMScoreAdj: config.OOMScoreAdj, - ResourceContainer: config.ResourceContainer, - ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, - ServiceEventHandler: serviceEventHandler, - EndpointsEventHandler: endpointsEventHandler, - HealthzServer: healthzServer, - }, nil -} - // Run runs the specified ProxyServer. This should never exit (unless CleanupAndExit is set). func (s *ProxyServer) Run() error { // remove iptables rules and exit @@ -712,7 +483,8 @@ func (s *ProxyServer) Run() error { } // Tune conntrack, if requested - if s.Conntracker != nil && goruntime.GOOS != "windows" { + // Conntracker is always nil for windows + if s.Conntracker != nil { max, err := getConntrackMax(s.ConntrackConfiguration) if err != nil { return err @@ -776,6 +548,10 @@ func (s *ProxyServer) Run() error { return nil } +func (s *ProxyServer) birthCry() { + s.Recorder.Eventf(s.NodeRef, api.EventTypeNormal, "Starting", "Starting kube-proxy.") +} + func getConntrackMax(config componentconfig.KubeProxyConntrackConfiguration) (int, error) { if config.Max > 0 { if config.MaxPerCore > 0 { @@ -797,65 +573,6 @@ func getConntrackMax(config componentconfig.KubeProxyConntrackConfiguration) (in return 0, nil } -func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, kcompat iptables.KernelCompatTester) string { - if proxyMode == proxyModeUserspace { - return proxyModeUserspace - } - - if len(proxyMode) > 0 && proxyMode == proxyModeIPTables { - return tryIPTablesProxy(iptver, kcompat) - } - - if utilfeature.DefaultFeatureGate.Enabled(features.SupportIPVSProxyMode) { - if proxyMode == proxyModeIPVS { - return tryIPVSProxy(iptver, kcompat) - } else { - glog.Warningf("Can't use ipvs proxier, trying iptables proxier") - return tryIPTablesProxy(iptver, kcompat) - } - } - glog.Warningf("Flag proxy-mode=%q unknown, assuming iptables proxy", proxyMode) - return tryIPTablesProxy(iptver, kcompat) -} - -func tryIPVSProxy(iptver iptables.IPTablesVersioner, kcompat iptables.KernelCompatTester) string { - // guaranteed false on error, error only necessary for debugging - // IPVS Proxier relies on iptables - useIPVSProxy, err := ipvs.CanUseIPVSProxier() - if err != nil { - utilruntime.HandleError(fmt.Errorf("can't determine whether to use ipvs proxy, using userspace proxier: %v", err)) - return proxyModeUserspace - } - if useIPVSProxy { - return proxyModeIPVS - } - - // TODO: Check ipvs version - - // Try to fallback to iptables before falling back to userspace - glog.V(1).Infof("Can't use ipvs proxier, trying iptables proxier") - return tryIPTablesProxy(iptver, kcompat) -} - -func tryIPTablesProxy(iptver iptables.IPTablesVersioner, kcompat iptables.KernelCompatTester) string { - // guaranteed false on error, error only necessary for debugging - useIPTablesProxy, err := iptables.CanUseIPTablesProxier(iptver, kcompat) - if err != nil { - utilruntime.HandleError(fmt.Errorf("can't determine whether to use iptables proxy, using userspace proxier: %v", err)) - return proxyModeUserspace - } - if useIPTablesProxy { - return proxyModeIPTables - } - // Fallback. - glog.V(1).Infof("Can't use iptables proxy, using userspace proxier") - return proxyModeUserspace -} - -func (s *ProxyServer) birthCry() { - s.Recorder.Eventf(s.NodeRef, api.EventTypeNormal, "Starting", "Starting kube-proxy.") -} - func getNodeIP(client clientset.Interface, hostname string) net.IP { var nodeIP net.IP node, err := client.Core().Nodes().Get(hostname, metav1.GetOptions{}) diff --git a/cmd/kube-proxy/app/server_linux.go b/cmd/kube-proxy/app/server_linux.go new file mode 100644 index 00000000000..c92e2f1feeb --- /dev/null +++ b/cmd/kube-proxy/app/server_linux.go @@ -0,0 +1,298 @@ +// +build !windows + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package app does all of the work necessary to configure and run a +// Kubernetes app process. +package app + +import ( + "errors" + "fmt" + "net" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilnet "k8s.io/apimachinery/pkg/util/net" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/tools/record" + "k8s.io/kubernetes/pkg/apis/componentconfig" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/proxy" + proxyconfig "k8s.io/kubernetes/pkg/proxy/config" + "k8s.io/kubernetes/pkg/proxy/healthcheck" + "k8s.io/kubernetes/pkg/proxy/iptables" + "k8s.io/kubernetes/pkg/proxy/ipvs" + "k8s.io/kubernetes/pkg/proxy/userspace" + "k8s.io/kubernetes/pkg/util/configz" + utildbus "k8s.io/kubernetes/pkg/util/dbus" + utiliptables "k8s.io/kubernetes/pkg/util/iptables" + utilipvs "k8s.io/kubernetes/pkg/util/ipvs" + utilnode "k8s.io/kubernetes/pkg/util/node" + utilsysctl "k8s.io/kubernetes/pkg/util/sysctl" + "k8s.io/utils/exec" + + "github.com/golang/glog" +) + +func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndExit bool, scheme *runtime.Scheme, master string) (*ProxyServer, error) { + if config == nil { + return nil, errors.New("config is required") + } + + if c, err := configz.New("componentconfig"); err == nil { + c.Set(config) + } else { + return nil, fmt.Errorf("unable to register configz: %s", err) + } + + protocol := utiliptables.ProtocolIpv4 + if net.ParseIP(config.BindAddress).To4() == nil { + protocol = utiliptables.ProtocolIpv6 + } + + var iptInterface utiliptables.Interface + var ipvsInterface utilipvs.Interface + var dbus utildbus.Interface + + // Create a iptables utils. + execer := exec.New() + + dbus = utildbus.New() + iptInterface = utiliptables.New(execer, dbus, protocol) + ipvsInterface = utilipvs.New(execer) + + // We omit creation of pretty much everything if we run in cleanup mode + if cleanupAndExit { + return &ProxyServer{IptInterface: iptInterface, IpvsInterface: ipvsInterface, CleanupAndExit: cleanupAndExit}, nil + } + + client, eventClient, err := createClients(config.ClientConnection, master) + if err != nil { + return nil, err + } + + // Create event recorder + hostname := utilnode.GetHostname(config.HostnameOverride) + eventBroadcaster := record.NewBroadcaster() + recorder := eventBroadcaster.NewRecorder(scheme, v1.EventSource{Component: "kube-proxy", Host: hostname}) + + nodeRef := &v1.ObjectReference{ + Kind: "Node", + Name: hostname, + UID: types.UID(hostname), + Namespace: "", + } + + var healthzServer *healthcheck.HealthzServer + var healthzUpdater healthcheck.HealthzUpdater + if len(config.HealthzBindAddress) > 0 { + healthzServer = healthcheck.NewDefaultHealthzServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration, recorder, nodeRef) + healthzUpdater = healthzServer + } + + var proxier proxy.ProxyProvider + var serviceEventHandler proxyconfig.ServiceHandler + var endpointsEventHandler proxyconfig.EndpointsHandler + + proxyMode := getProxyMode(string(config.Mode), iptInterface, iptables.LinuxKernelCompatTester{}) + if proxyMode == proxyModeIPTables { + glog.V(0).Info("Using iptables Proxier.") + var nodeIP net.IP + if config.BindAddress != "0.0.0.0" { + nodeIP = net.ParseIP(config.BindAddress) + } else { + nodeIP = getNodeIP(client, hostname) + } + if config.IPTables.MasqueradeBit == nil { + // MasqueradeBit must be specified or defaulted. + return nil, fmt.Errorf("unable to read IPTables MasqueradeBit from config") + } + + // TODO this has side effects that should only happen when Run() is invoked. + proxierIPTables, err := iptables.NewProxier( + iptInterface, + utilsysctl.New(), + execer, + config.IPTables.SyncPeriod.Duration, + config.IPTables.MinSyncPeriod.Duration, + config.IPTables.MasqueradeAll, + int(*config.IPTables.MasqueradeBit), + config.ClusterCIDR, + hostname, + nodeIP, + recorder, + healthzUpdater, + ) + if err != nil { + return nil, fmt.Errorf("unable to create proxier: %v", err) + } + iptables.RegisterMetrics() + proxier = proxierIPTables + serviceEventHandler = proxierIPTables + endpointsEventHandler = proxierIPTables + // No turning back. Remove artifacts that might still exist from the userspace Proxier. + glog.V(0).Info("Tearing down inactive rules.") + // TODO this has side effects that should only happen when Run() is invoked. + userspace.CleanupLeftovers(iptInterface) + // IPVS Proxier will generate some iptables rules, + // need to clean them before switching to other proxy mode. + ipvs.CleanupLeftovers(execer, ipvsInterface, iptInterface) + } else if proxyMode == proxyModeIPVS { + glog.V(0).Info("Using ipvs Proxier.") + proxierIPVS, err := ipvs.NewProxier( + iptInterface, + ipvsInterface, + utilsysctl.New(), + execer, + config.IPVS.SyncPeriod.Duration, + config.IPVS.MinSyncPeriod.Duration, + config.IPTables.MasqueradeAll, + int(*config.IPTables.MasqueradeBit), + config.ClusterCIDR, + hostname, + getNodeIP(client, hostname), + recorder, + healthzServer, + config.IPVS.Scheduler, + ) + if err != nil { + return nil, fmt.Errorf("unable to create proxier: %v", err) + } + proxier = proxierIPVS + serviceEventHandler = proxierIPVS + endpointsEventHandler = proxierIPVS + glog.V(0).Info("Tearing down inactive rules.") + // TODO this has side effects that should only happen when Run() is invoked. + userspace.CleanupLeftovers(iptInterface) + iptables.CleanupLeftovers(iptInterface) + } else { + glog.V(0).Info("Using userspace Proxier.") + // This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for + // our config.EndpointsConfigHandler. + loadBalancer := userspace.NewLoadBalancerRR() + // set EndpointsConfigHandler to our loadBalancer + endpointsEventHandler = loadBalancer + + // TODO this has side effects that should only happen when Run() is invoked. + proxierUserspace, err := userspace.NewProxier( + loadBalancer, + net.ParseIP(config.BindAddress), + iptInterface, + execer, + *utilnet.ParsePortRangeOrDie(config.PortRange), + config.IPTables.SyncPeriod.Duration, + config.IPTables.MinSyncPeriod.Duration, + config.UDPIdleTimeout.Duration, + ) + if err != nil { + return nil, fmt.Errorf("unable to create proxier: %v", err) + } + serviceEventHandler = proxierUserspace + proxier = proxierUserspace + + // Remove artifacts from the iptables and ipvs Proxier, if not on Windows. + glog.V(0).Info("Tearing down inactive rules.") + // TODO this has side effects that should only happen when Run() is invoked. + iptables.CleanupLeftovers(iptInterface) + // IPVS Proxier will generate some iptables rules, + // need to clean them before switching to other proxy mode. + ipvs.CleanupLeftovers(execer, ipvsInterface, iptInterface) + } + + iptInterface.AddReloadFunc(proxier.Sync) + + return &ProxyServer{ + Client: client, + EventClient: eventClient, + IptInterface: iptInterface, + IpvsInterface: ipvsInterface, + execer: execer, + Proxier: proxier, + Broadcaster: eventBroadcaster, + Recorder: recorder, + ConntrackConfiguration: config.Conntrack, + Conntracker: &realConntracker{}, + ProxyMode: proxyMode, + NodeRef: nodeRef, + MetricsBindAddress: config.MetricsBindAddress, + EnableProfiling: config.EnableProfiling, + OOMScoreAdj: config.OOMScoreAdj, + ResourceContainer: config.ResourceContainer, + ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, + ServiceEventHandler: serviceEventHandler, + EndpointsEventHandler: endpointsEventHandler, + HealthzServer: healthzServer, + }, nil +} + +func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, kcompat iptables.KernelCompatTester) string { + if proxyMode == proxyModeUserspace { + return proxyModeUserspace + } + + if len(proxyMode) > 0 && proxyMode == proxyModeIPTables { + return tryIPTablesProxy(iptver, kcompat) + } + + if utilfeature.DefaultFeatureGate.Enabled(features.SupportIPVSProxyMode) { + if proxyMode == proxyModeIPVS { + return tryIPVSProxy(iptver, kcompat) + } else { + glog.Warningf("Can't use ipvs proxier, trying iptables proxier") + return tryIPTablesProxy(iptver, kcompat) + } + } + glog.Warningf("Flag proxy-mode=%q unknown, assuming iptables proxy", proxyMode) + return tryIPTablesProxy(iptver, kcompat) +} + +func tryIPVSProxy(iptver iptables.IPTablesVersioner, kcompat iptables.KernelCompatTester) string { + // guaranteed false on error, error only necessary for debugging + // IPVS Proxier relies on iptables + useIPVSProxy, err := ipvs.CanUseIPVSProxier() + if err != nil { + utilruntime.HandleError(fmt.Errorf("can't determine whether to use ipvs proxy, using userspace proxier: %v", err)) + return proxyModeUserspace + } + if useIPVSProxy { + return proxyModeIPVS + } + + // TODO: Check ipvs version + + // Try to fallback to iptables before falling back to userspace + glog.V(1).Infof("Can't use ipvs proxier, trying iptables proxier") + return tryIPTablesProxy(iptver, kcompat) +} + +func tryIPTablesProxy(iptver iptables.IPTablesVersioner, kcompat iptables.KernelCompatTester) string { + // guaranteed false on error, error only necessary for debugging + useIPTablesProxy, err := iptables.CanUseIPTablesProxier(iptver, kcompat) + if err != nil { + utilruntime.HandleError(fmt.Errorf("can't determine whether to use iptables proxy, using userspace proxier: %v", err)) + return proxyModeUserspace + } + if useIPTablesProxy { + return proxyModeIPTables + } + // Fallback. + glog.V(1).Infof("Can't use iptables proxy, using userspace proxier") + return proxyModeUserspace +} diff --git a/cmd/kube-proxy/app/server_windows.go b/cmd/kube-proxy/app/server_windows.go new file mode 100644 index 00000000000..f4f46386a74 --- /dev/null +++ b/cmd/kube-proxy/app/server_windows.go @@ -0,0 +1,187 @@ +// +build windows + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package app does all of the work necessary to configure and run a +// Kubernetes app process. +package app + +import ( + "errors" + "fmt" + "net" + _ "net/http/pprof" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/client-go/tools/record" + "k8s.io/kubernetes/pkg/apis/componentconfig" + "k8s.io/kubernetes/pkg/proxy" + proxyconfig "k8s.io/kubernetes/pkg/proxy/config" + "k8s.io/kubernetes/pkg/proxy/healthcheck" + "k8s.io/kubernetes/pkg/proxy/winkernel" + "k8s.io/kubernetes/pkg/proxy/winuserspace" + "k8s.io/kubernetes/pkg/util/configz" + utilnetsh "k8s.io/kubernetes/pkg/util/netsh" + utilnode "k8s.io/kubernetes/pkg/util/node" + "k8s.io/utils/exec" + + "github.com/golang/glog" +) + +// NewProxyServer returns a new ProxyServer. +func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndExit bool, scheme *runtime.Scheme, master string) (*ProxyServer, error) { + if config == nil { + return nil, errors.New("config is required") + } + + if c, err := configz.New("componentconfig"); err == nil { + c.Set(config) + } else { + return nil, fmt.Errorf("unable to register configz: %s", err) + } + + // We omit creation of pretty much everything if we run in cleanup mode + if cleanupAndExit { + return &ProxyServer{CleanupAndExit: cleanupAndExit}, nil + } + + client, eventClient, err := createClients(config.ClientConnection, master) + if err != nil { + return nil, err + } + + // Create event recorder + hostname := utilnode.GetHostname(config.HostnameOverride) + eventBroadcaster := record.NewBroadcaster() + recorder := eventBroadcaster.NewRecorder(scheme, v1.EventSource{Component: "kube-proxy", Host: hostname}) + + nodeRef := &v1.ObjectReference{ + Kind: "Node", + Name: hostname, + UID: types.UID(hostname), + Namespace: "", + } + + var healthzServer *healthcheck.HealthzServer + var healthzUpdater healthcheck.HealthzUpdater + if len(config.HealthzBindAddress) > 0 { + healthzServer = healthcheck.NewDefaultHealthzServer(config.HealthzBindAddress, 2*config.IPTables.SyncPeriod.Duration, recorder, nodeRef) + healthzUpdater = healthzServer + } + + var proxier proxy.ProxyProvider + var serviceEventHandler proxyconfig.ServiceHandler + var endpointsEventHandler proxyconfig.EndpointsHandler + + proxyMode := getProxyMode(string(config.Mode), winkernel.WindowsKernelCompatTester{}) + if proxyMode == proxyModeKernelspace { + glog.V(0).Info("Using Kernelspace Proxier.") + proxierKernelspace, err := winkernel.NewProxier( + config.IPTables.SyncPeriod.Duration, + config.IPTables.MinSyncPeriod.Duration, + config.IPTables.MasqueradeAll, + int(*config.IPTables.MasqueradeBit), + config.ClusterCIDR, + hostname, + getNodeIP(client, hostname), + recorder, + healthzUpdater, + ) + if err != nil { + return nil, fmt.Errorf("unable to create proxier: %v", err) + } + proxier = proxierKernelspace + endpointsEventHandler = proxierKernelspace + serviceEventHandler = proxierKernelspace + } else { + glog.V(0).Info("Using userspace Proxier.") + execer := exec.New() + var netshInterface utilnetsh.Interface + netshInterface = utilnetsh.New(execer) + + // This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for + // our config.EndpointsConfigHandler. + loadBalancer := winuserspace.NewLoadBalancerRR() + + // set EndpointsConfigHandler to our loadBalancer + endpointsEventHandler = loadBalancer + proxierUserspace, err := winuserspace.NewProxier( + loadBalancer, + net.ParseIP(config.BindAddress), + netshInterface, + *utilnet.ParsePortRangeOrDie(config.PortRange), + // TODO @pires replace below with default values, if applicable + config.IPTables.SyncPeriod.Duration, + config.UDPIdleTimeout.Duration, + ) + if err != nil { + return nil, fmt.Errorf("unable to create proxier: %v", err) + } + proxier = proxierUserspace + serviceEventHandler = proxierUserspace + glog.V(0).Info("Tearing down pure-winkernel proxy rules.") + winkernel.CleanupLeftovers() + } + + return &ProxyServer{ + Client: client, + EventClient: eventClient, + Proxier: proxier, + Broadcaster: eventBroadcaster, + Recorder: recorder, + ProxyMode: proxyMode, + NodeRef: nodeRef, + MetricsBindAddress: config.MetricsBindAddress, + EnableProfiling: config.EnableProfiling, + OOMScoreAdj: config.OOMScoreAdj, + ResourceContainer: config.ResourceContainer, + ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, + ServiceEventHandler: serviceEventHandler, + EndpointsEventHandler: endpointsEventHandler, + HealthzServer: healthzServer, + }, nil +} + +func getProxyMode(proxyMode string, kcompat winkernel.KernelCompatTester) string { + if proxyMode == proxyModeUserspace { + return proxyModeUserspace + } else if proxyMode == proxyModeKernelspace { + return tryWinKernelSpaceProxy(kcompat) + } + return proxyModeUserspace +} + +func tryWinKernelSpaceProxy(kcompat winkernel.KernelCompatTester) string { + // Check for Windows Kernel Version if we can support Kernel Space proxy + // Check for Windows Version + + // guaranteed false on error, error only necessary for debugging + useWinKerelProxy, err := winkernel.CanUseWinKernelProxier(kcompat) + if err != nil { + glog.Errorf("Can't determine whether to use windows kernel proxy, using userspace proxier: %v", err) + return proxyModeUserspace + } + if useWinKerelProxy { + return proxyModeKernelspace + } + // Fallback. + glog.V(1).Infof("Can't use winkernel proxy, using userspace proxier") + return proxyModeUserspace +} diff --git a/pkg/proxy/winkernel/metrics.go b/pkg/proxy/winkernel/metrics.go new file mode 100644 index 00000000000..100b6abba87 --- /dev/null +++ b/pkg/proxy/winkernel/metrics.go @@ -0,0 +1,50 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package winkernel + +import ( + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const kubeProxySubsystem = "kubeproxy" + +var ( + SyncProxyRulesLatency = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: kubeProxySubsystem, + Name: "sync_proxy_rules_latency_microseconds", + Help: "SyncProxyRules latency", + Buckets: prometheus.ExponentialBuckets(1000, 2, 15), + }, + ) +) + +var registerMetricsOnce sync.Once + +func RegisterMetrics() { + registerMetricsOnce.Do(func() { + prometheus.MustRegister(SyncProxyRulesLatency) + }) +} + +// Gets the time since the specified start in microseconds. +func sinceInMicroseconds(start time.Time) float64 { + return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds()) +} diff --git a/pkg/proxy/winkernel/proxier.go b/pkg/proxy/winkernel/proxier.go new file mode 100644 index 00000000000..d440dc410b1 --- /dev/null +++ b/pkg/proxy/winkernel/proxier.go @@ -0,0 +1,1129 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package winkernel + +import ( + "encoding/json" + "fmt" + "net" + "os" + "reflect" + "sync" + "sync/atomic" + "time" + + "github.com/Microsoft/hcsshim" + "github.com/davecgh/go-spew/spew" + "github.com/golang/glog" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/tools/record" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/helper" + apiservice "k8s.io/kubernetes/pkg/api/service" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/proxy" + "k8s.io/kubernetes/pkg/proxy/healthcheck" + "k8s.io/kubernetes/pkg/util/async" +) + +// KernelCompatTester tests whether the required kernel capabilities are +// present to run the windows kernel proxier. +type KernelCompatTester interface { + IsCompatible() error +} + +// CanUseWinKernelProxier returns true if we should use the Kernel Proxier +// instead of the "classic" userspace Proxier. This is determined by checking +// the windows kernel version and for the existence of kernel features. +func CanUseWinKernelProxier(kcompat KernelCompatTester) (bool, error) { + // Check that the kernel supports what we need. + if err := kcompat.IsCompatible(); err != nil { + return false, err + } + return true, nil +} + +type WindowsKernelCompatTester struct{} + +// TODO : Fix the below API to query the OS version +func (lkct WindowsKernelCompatTester) IsCompatible() error { + return nil +} + +type externalIPInfo struct { + ip string + hnsID string +} + +type loadBalancerIngressInfo struct { + ip string + hnsID string +} + +// internal struct for string service information +type serviceInfo struct { + clusterIP net.IP + port int + protocol api.Protocol + nodePort int + targetPort int + loadBalancerStatus api.LoadBalancerStatus + sessionAffinityType api.ServiceAffinity + stickyMaxAgeMinutes int + externalIPs []*externalIPInfo + loadBalancerIngressIPs []*loadBalancerIngressInfo + loadBalancerSourceRanges []string + onlyNodeLocalEndpoints bool + healthCheckNodePort int + hnsID string + nodePorthnsID string + policyApplied bool +} + +type hnsNetworkInfo struct { + name string + id string +} + +func Log(v interface{}, message string, level glog.Level) { + glog.V(level).Infof("%s, %s", message, spew.Sdump(v)) +} + +func LogJson(v interface{}, message string, level glog.Level) { + jsonString, err := json.Marshal(v) + if err == nil { + glog.V(level).Infof("%s, %s", message, string(jsonString)) + } +} + +// internal struct for endpoints information +type endpointsInfo struct { + ip string + port uint16 + isLocal bool + macAddress string + hnsID string + refCount uint16 +} + +func newEndpointInfo(ip string, port uint16, isLocal bool) *endpointsInfo { + info := &endpointsInfo{ + ip: ip, + port: port, + isLocal: isLocal, + macAddress: "00:11:22:33:44:55", // Hardcoding to some Random Mac + refCount: 0, + hnsID: "", + } + + return info +} + +func (ep *endpointsInfo) Cleanup() { + Log(ep, "Endpoint Cleanup", 3) + ep.refCount-- + // Remove the remote hns endpoint, if no service is referring it + // Never delete a Local Endpoint. Local Endpoints are already created by other entities. + // Remove only remote endpoints created by this service + if ep.refCount <= 0 && !ep.isLocal { + glog.V(4).Infof("Removing endpoints for %v, since no one is referencing it", ep) + deleteHnsEndpoint(ep.hnsID) + ep.hnsID = "" + } + +} + +// returns a new serviceInfo struct +func newServiceInfo(svcPortName proxy.ServicePortName, port *api.ServicePort, service *api.Service) *serviceInfo { + onlyNodeLocalEndpoints := false + if utilfeature.DefaultFeatureGate.Enabled(features.ExternalTrafficLocalOnly) && + apiservice.RequestsOnlyLocalTraffic(service) { + onlyNodeLocalEndpoints = true + } + + info := &serviceInfo{ + clusterIP: net.ParseIP(service.Spec.ClusterIP), + port: int(port.Port), + protocol: port.Protocol, + nodePort: int(port.NodePort), + targetPort: port.TargetPort.IntValue(), + // Deep-copy in case the service instance changes + loadBalancerStatus: *helper.LoadBalancerStatusDeepCopy(&service.Status.LoadBalancer), + sessionAffinityType: service.Spec.SessionAffinity, + stickyMaxAgeMinutes: 180, // TODO: paramaterize this in the API. + loadBalancerSourceRanges: make([]string, len(service.Spec.LoadBalancerSourceRanges)), + onlyNodeLocalEndpoints: onlyNodeLocalEndpoints, + } + + copy(info.loadBalancerSourceRanges, service.Spec.LoadBalancerSourceRanges) + for _, eip := range service.Spec.ExternalIPs { + info.externalIPs = append(info.externalIPs, &externalIPInfo{ip: eip}) + } + for _, ingress := range service.Status.LoadBalancer.Ingress { + info.loadBalancerIngressIPs = append(info.loadBalancerIngressIPs, &loadBalancerIngressInfo{ip: ingress.IP}) + } + + if apiservice.NeedsHealthCheck(service) { + p := service.Spec.HealthCheckNodePort + if p == 0 { + glog.Errorf("Service %q has no healthcheck nodeport", svcPortName.NamespacedName.String()) + } else { + info.healthCheckNodePort = int(p) + } + } + + return info +} + +type endpointsChange struct { + previous proxyEndpointsMap + current proxyEndpointsMap +} + +type endpointsChangeMap struct { + lock sync.Mutex + hostname string + items map[types.NamespacedName]*endpointsChange +} + +type serviceChange struct { + previous proxyServiceMap + current proxyServiceMap +} + +type serviceChangeMap struct { + lock sync.Mutex + items map[types.NamespacedName]*serviceChange +} + +type updateEndpointMapResult struct { + hcEndpoints map[types.NamespacedName]int + staleEndpoints map[endpointServicePair]bool + staleServiceNames map[proxy.ServicePortName]bool +} + +type updateServiceMapResult struct { + hcServices map[types.NamespacedName]uint16 + staleServices sets.String +} +type proxyServiceMap map[proxy.ServicePortName]*serviceInfo +type proxyEndpointsMap map[proxy.ServicePortName][]*endpointsInfo + +func newEndpointsChangeMap(hostname string) endpointsChangeMap { + return endpointsChangeMap{ + hostname: hostname, + items: make(map[types.NamespacedName]*endpointsChange), + } +} + +func (ecm *endpointsChangeMap) update(namespacedName *types.NamespacedName, previous, current *api.Endpoints) bool { + ecm.lock.Lock() + defer ecm.lock.Unlock() + + change, exists := ecm.items[*namespacedName] + if !exists { + change = &endpointsChange{} + change.previous = endpointsToEndpointsMap(previous, ecm.hostname) + ecm.items[*namespacedName] = change + } + change.current = endpointsToEndpointsMap(current, ecm.hostname) + if reflect.DeepEqual(change.previous, change.current) { + delete(ecm.items, *namespacedName) + } + return len(ecm.items) > 0 +} + +func newServiceChangeMap() serviceChangeMap { + return serviceChangeMap{ + items: make(map[types.NamespacedName]*serviceChange), + } +} + +func (scm *serviceChangeMap) update(namespacedName *types.NamespacedName, previous, current *api.Service) bool { + scm.lock.Lock() + defer scm.lock.Unlock() + + change, exists := scm.items[*namespacedName] + if !exists { + // Service is Added + change = &serviceChange{} + change.previous = serviceToServiceMap(previous) + scm.items[*namespacedName] = change + } + change.current = serviceToServiceMap(current) + if reflect.DeepEqual(change.previous, change.current) { + delete(scm.items, *namespacedName) + } + return len(scm.items) > 0 +} + +func (sm *proxyServiceMap) merge(other proxyServiceMap, curEndpoints proxyEndpointsMap) sets.String { + existingPorts := sets.NewString() + for svcPortName, info := range other { + existingPorts.Insert(svcPortName.Port) + svcInfo, exists := (*sm)[svcPortName] + if !exists { + glog.V(1).Infof("Adding new service port %q at %s:%d/%s", svcPortName, info.clusterIP, info.port, info.protocol) + } else { + glog.V(1).Infof("Updating existing service port %q at %s:%d/%s", svcPortName, info.clusterIP, info.port, info.protocol) + svcInfo.cleanupAllPolicies(curEndpoints[svcPortName]) + delete(*sm, svcPortName) + } + (*sm)[svcPortName] = info + } + return existingPorts +} + +func (sm *proxyServiceMap) unmerge(other proxyServiceMap, existingPorts, staleServices sets.String, curEndpoints proxyEndpointsMap) { + for svcPortName := range other { + if existingPorts.Has(svcPortName.Port) { + continue + } + info, exists := (*sm)[svcPortName] + if exists { + glog.V(1).Infof("Removing service port %q", svcPortName) + if info.protocol == api.ProtocolUDP { + staleServices.Insert(info.clusterIP.String()) + } + info.cleanupAllPolicies(curEndpoints[svcPortName]) + delete(*sm, svcPortName) + } else { + glog.Errorf("Service port %q removed, but doesn't exists", svcPortName) + } + } +} + +func (em proxyEndpointsMap) merge(other proxyEndpointsMap, curServices proxyServiceMap) { + // Endpoint Update/Add + for svcPortName := range other { + epInfos, exists := em[svcPortName] + if exists { + // + info, exists := curServices[svcPortName] + glog.V(1).Infof("Updating existing service port %q at %s:%d/%s", svcPortName, info.clusterIP, info.port, info.protocol) + if exists { + glog.V(2).Infof("Endpoints are modified. Service [%v] is stale", svcPortName) + info.cleanupAllPolicies(epInfos) + } else { + // If no service exists, just cleanup the remote endpoints + glog.V(2).Infof("Endpoints are orphaned. Cleaning up") + // Cleanup Endpoints references + for _, ep := range epInfos { + ep.Cleanup() + } + + } + + delete(em, svcPortName) + } + em[svcPortName] = other[svcPortName] + } +} + +func (em proxyEndpointsMap) unmerge(other proxyEndpointsMap, curServices proxyServiceMap) { + // Endpoint Update/Removal + for svcPortName := range other { + info, exists := curServices[svcPortName] + if exists { + glog.V(2).Infof("Service [%v] is stale", info) + info.cleanupAllPolicies(em[svcPortName]) + } else { + // If no service exists, just cleanup the remote endpoints + glog.V(2).Infof("Endpoints are orphaned. Cleaning up") + // Cleanup Endpoints references + epInfos, exists := em[svcPortName] + if exists { + for _, ep := range epInfos { + ep.Cleanup() + } + } + } + + delete(em, svcPortName) + } +} + +// Proxier is an hns based proxy for connections between a localhost:lport +// and services that provide the actual backends. +type Proxier struct { + // endpointsChanges and serviceChanges contains all changes to endpoints and + // services that happened since policies were synced. For a single object, + // changes are accumulated, i.e. previous is state from before all of them, + // current is state after applying all of those. + endpointsChanges endpointsChangeMap + serviceChanges serviceChangeMap + + mu sync.Mutex // protects the following fields + serviceMap proxyServiceMap + endpointsMap proxyEndpointsMap + portsMap map[localPort]closeable + // endpointsSynced and servicesSynced are set to true when corresponding + // objects are synced after startup. This is used to avoid updating hns policies + // with some partial data after kube-proxy restart. + endpointsSynced bool + servicesSynced bool + initialized int32 + syncRunner *async.BoundedFrequencyRunner // governs calls to syncProxyRules + + // These are effectively const and do not need the mutex to be held. + masqueradeAll bool + masqueradeMark string + clusterCIDR string + hostname string + nodeIP net.IP + recorder record.EventRecorder + healthChecker healthcheck.Server + healthzServer healthcheck.HealthzUpdater + + // Since converting probabilities (floats) to strings is expensive + // and we are using only probabilities in the format of 1/n, we are + // precomputing some number of those and cache for future reuse. + precomputedProbabilities []string + + network hnsNetworkInfo +} + +type localPort struct { + desc string + ip string + port int + protocol string +} + +func (lp *localPort) String() string { + return fmt.Sprintf("%q (%s:%d/%s)", lp.desc, lp.ip, lp.port, lp.protocol) +} + +func Enum(p api.Protocol) uint16 { + if p == api.ProtocolTCP { + return 6 + } + if p == api.ProtocolUDP { + return 17 + } + return 0 +} + +type closeable interface { + Close() error +} + +// Proxier implements ProxyProvider +var _ proxy.ProxyProvider = &Proxier{} + +// NewProxier returns a new Proxier +func NewProxier( + syncPeriod time.Duration, + minSyncPeriod time.Duration, + masqueradeAll bool, + masqueradeBit int, + clusterCIDR string, + hostname string, + nodeIP net.IP, + recorder record.EventRecorder, + healthzServer healthcheck.HealthzUpdater, +) (*Proxier, error) { + // check valid user input + if minSyncPeriod > syncPeriod { + return nil, fmt.Errorf("min-sync (%v) must be < sync(%v)", minSyncPeriod, syncPeriod) + } + + // Generate the masquerade mark to use for SNAT rules. + if masqueradeBit < 0 || masqueradeBit > 31 { + return nil, fmt.Errorf("invalid iptables-masquerade-bit %v not in [0, 31]", masqueradeBit) + } + masqueradeValue := 1 << uint(masqueradeBit) + masqueradeMark := fmt.Sprintf("%#08x/%#08x", masqueradeValue, masqueradeValue) + + if nodeIP == nil { + glog.Warningf("invalid nodeIP, initializing kube-proxy with 127.0.0.1 as nodeIP") + nodeIP = net.ParseIP("127.0.0.1") + } + + if len(clusterCIDR) == 0 { + glog.Warningf("clusterCIDR not specified, unable to distinguish between internal and external traffic") + } + + healthChecker := healthcheck.NewServer(hostname, recorder, nil, nil) // use default implementations of deps + + // TODO : Make this a param + hnsNetworkName := os.Getenv("KUBE_NETWORK") + if len(hnsNetworkName) == 0 { + return nil, fmt.Errorf("Environment variable KUBE_NETWORK not initialized") + } + hnsNetwork, err := getHnsNetworkInfo(hnsNetworkName) + if err != nil { + glog.Fatalf("Unable to find Hns Network speficied by %s. Please check environment variable KUBE_NETWORK", hnsNetworkName) + return nil, err + } + + glog.V(1).Infof("Hns Network loaded with info = %v", hnsNetwork) + + proxier := &Proxier{ + portsMap: make(map[localPort]closeable), + serviceMap: make(proxyServiceMap), + serviceChanges: newServiceChangeMap(), + endpointsMap: make(proxyEndpointsMap), + endpointsChanges: newEndpointsChangeMap(hostname), + masqueradeAll: masqueradeAll, + masqueradeMark: masqueradeMark, + clusterCIDR: clusterCIDR, + hostname: hostname, + nodeIP: nodeIP, + recorder: recorder, + healthChecker: healthChecker, + healthzServer: healthzServer, + network: *hnsNetwork, + } + + burstSyncs := 2 + glog.V(3).Infof("minSyncPeriod: %v, syncPeriod: %v, burstSyncs: %d", minSyncPeriod, syncPeriod, burstSyncs) + proxier.syncRunner = async.NewBoundedFrequencyRunner("sync-runner", proxier.syncProxyRules, minSyncPeriod, syncPeriod, burstSyncs) + return proxier, nil + +} + +// CleanupLeftovers removes all hns rules created by the Proxier +// It returns true if an error was encountered. Errors are logged. +func CleanupLeftovers() (encounteredError bool) { + // Delete all Hns Load Balancer Policies + deleteAllHnsLoadBalancerPolicy() + // TODO + // Delete all Hns Remote endpoints + + return encounteredError +} + +func (svcInfo *serviceInfo) cleanupAllPolicies(endpoints []*endpointsInfo) { + Log(svcInfo, "Service Cleanup", 3) + if svcInfo.policyApplied { + svcInfo.deleteAllHnsLoadBalancerPolicy() + // Cleanup Endpoints references + for _, ep := range endpoints { + ep.Cleanup() + } + + svcInfo.policyApplied = false + } +} + +func (svcInfo *serviceInfo) deleteAllHnsLoadBalancerPolicy() { + // Remove the Hns Policy corresponding to this service + deleteHnsLoadBalancerPolicy(svcInfo.hnsID) + svcInfo.hnsID = "" + for _, externalIp := range svcInfo.externalIPs { + deleteHnsLoadBalancerPolicy(externalIp.hnsID) + externalIp.hnsID = "" + } + for _, lbIngressIp := range svcInfo.loadBalancerIngressIPs { + deleteHnsLoadBalancerPolicy(lbIngressIp.hnsID) + lbIngressIp.hnsID = "" + } + +} + +func deleteAllHnsLoadBalancerPolicy() { + plists, err := hcsshim.HNSListPolicyListRequest() + if err != nil { + return + } + for _, plist := range plists { + LogJson(plist, "Remove Policy", 3) + _, err = plist.Delete() + if err != nil { + glog.Errorf("%v", err) + } + } + +} + +// getHnsLoadBalancer returns the LoadBalancer policy resource, if already found. +// If not, it would create one and return +func getHnsLoadBalancer(endpoints []hcsshim.HNSEndpoint, isILB bool, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*hcsshim.PolicyList, error) { + plists, err := hcsshim.HNSListPolicyListRequest() + if err != nil { + return nil, err + } + + for _, plist := range plists { + if len(plist.EndpointReferences) != len(endpoints) { + continue + } + // Validate if input meets any of the policy lists + elbPolicy := hcsshim.ELBPolicy{} + if err = json.Unmarshal(plist.Policies[0], &elbPolicy); err != nil { + continue + } + if elbPolicy.Protocol == protocol && elbPolicy.InternalPort == internalPort && elbPolicy.ExternalPort == externalPort && elbPolicy.ILB == isILB { + if len(vip) > 0 { + if len(elbPolicy.VIPs) > 0 && elbPolicy.VIPs[0] != vip { + continue + } + } + LogJson(plist, "Found existing Hns loadbalancer policy resource", 1) + return &plist, nil + + } + } + //TODO: sourceVip is not used. If required, expose this as a param + var sourceVip string + lb, err := hcsshim.AddLoadBalancer( + endpoints, + isILB, + sourceVip, + vip, + protocol, + internalPort, + externalPort, + ) + + if err == nil { + LogJson(lb, "Hns loadbalancer policy resource", 1) + } + return lb, err +} + +func deleteHnsLoadBalancerPolicy(hnsID string) { + if len(hnsID) == 0 { + // Return silently + return + } + + // Cleanup HNS policies + hnsloadBalancer, err := hcsshim.GetPolicyListByID(hnsID) + if err != nil { + glog.Errorf("%v", err) + return + } + LogJson(hnsloadBalancer, "Removing Policy", 2) + + _, err = hnsloadBalancer.Delete() + if err != nil { + glog.Errorf("%v", err) + } +} + +func deleteHnsEndpoint(hnsID string) { + hnsendpoint, err := hcsshim.GetHNSEndpointByID(hnsID) + if err != nil { + glog.Errorf("%v", err) + return + } + + _, err = hnsendpoint.Delete() + if err != nil { + glog.Errorf("%v", err) + } + + glog.V(3).Infof("Remote endpoint resource deleted id %s", hnsID) +} + +func getHnsNetworkInfo(hnsNetworkName string) (*hnsNetworkInfo, error) { + hnsnetwork, err := hcsshim.GetHNSNetworkByName(hnsNetworkName) + if err != nil { + glog.Errorf("%v", err) + return nil, err + } + + return &hnsNetworkInfo{ + id: hnsnetwork.Id, + name: hnsnetwork.Name, + }, nil +} + +func getHnsEndpointByIpAddress(ip net.IP, networkName string) (*hcsshim.HNSEndpoint, error) { + hnsnetwork, err := hcsshim.GetHNSNetworkByName(networkName) + if err != nil { + glog.Errorf("%v", err) + return nil, err + } + + endpoints, err := hcsshim.HNSListEndpointRequest() + for _, endpoint := range endpoints { + equal := reflect.DeepEqual(endpoint.IPAddress, ip) + if equal && endpoint.VirtualNetwork == hnsnetwork.Id { + return &endpoint, nil + } + } + + return nil, fmt.Errorf("Endpoint %v not found on network %s", ip, networkName) +} + +// Sync is called to synchronize the proxier state to hns as soon as possible. +func (proxier *Proxier) Sync() { + proxier.syncRunner.Run() +} + +// SyncLoop runs periodic work. This is expected to run as a goroutine or as the main loop of the app. It does not return. +func (proxier *Proxier) SyncLoop() { + // Update healthz timestamp at beginning in case Sync() never succeeds. + if proxier.healthzServer != nil { + proxier.healthzServer.UpdateTimestamp() + } + proxier.syncRunner.Loop(wait.NeverStop) +} + +func (proxier *Proxier) setInitialized(value bool) { + var initialized int32 + if value { + initialized = 1 + } + atomic.StoreInt32(&proxier.initialized, initialized) +} + +func (proxier *Proxier) isInitialized() bool { + return atomic.LoadInt32(&proxier.initialized) > 0 +} + +func (proxier *Proxier) OnServiceAdd(service *api.Service) { + namespacedName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} + if proxier.serviceChanges.update(&namespacedName, nil, service) && proxier.isInitialized() { + proxier.syncRunner.Run() + } +} + +func (proxier *Proxier) OnServiceUpdate(oldService, service *api.Service) { + namespacedName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} + if proxier.serviceChanges.update(&namespacedName, oldService, service) && proxier.isInitialized() { + proxier.syncRunner.Run() + } +} + +func (proxier *Proxier) OnServiceDelete(service *api.Service) { + namespacedName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} + if proxier.serviceChanges.update(&namespacedName, service, nil) && proxier.isInitialized() { + proxier.syncRunner.Run() + } +} + +func (proxier *Proxier) OnServiceSynced() { + proxier.mu.Lock() + proxier.servicesSynced = true + proxier.setInitialized(proxier.servicesSynced && proxier.endpointsSynced) + proxier.mu.Unlock() + + // Sync unconditionally - this is called once per lifetime. + proxier.syncProxyRules() +} + +func shouldSkipService(svcName types.NamespacedName, service *api.Service) bool { + // if ClusterIP is "None" or empty, skip proxying + if !helper.IsServiceIPSet(service) { + glog.V(3).Infof("Skipping service %s due to clusterIP = %q", svcName, service.Spec.ClusterIP) + return true + } + // Even if ClusterIP is set, ServiceTypeExternalName services don't get proxied + if service.Spec.Type == api.ServiceTypeExternalName { + glog.V(3).Infof("Skipping service %s due to Type=ExternalName", svcName) + return true + } + return false +} + +// is updated by this function (based on the given changes). +// map is cleared after applying them. +func (proxier *Proxier) updateServiceMap() (result updateServiceMapResult) { + result.staleServices = sets.NewString() + + var serviceMap proxyServiceMap = proxier.serviceMap + var changes *serviceChangeMap = &proxier.serviceChanges + + func() { + changes.lock.Lock() + defer changes.lock.Unlock() + for _, change := range changes.items { + existingPorts := serviceMap.merge(change.current, proxier.endpointsMap) + serviceMap.unmerge(change.previous, existingPorts, result.staleServices, proxier.endpointsMap) + } + changes.items = make(map[types.NamespacedName]*serviceChange) + }() + + // TODO: If this will appear to be computationally expensive, consider + // computing this incrementally similarly to serviceMap. + result.hcServices = make(map[types.NamespacedName]uint16) + for svcPortName, info := range serviceMap { + if info.healthCheckNodePort != 0 { + result.hcServices[svcPortName.NamespacedName] = uint16(info.healthCheckNodePort) + } + } + + return result +} + +func (proxier *Proxier) OnEndpointsAdd(endpoints *api.Endpoints) { + namespacedName := types.NamespacedName{Namespace: endpoints.Namespace, Name: endpoints.Name} + if proxier.endpointsChanges.update(&namespacedName, nil, endpoints) && proxier.isInitialized() { + proxier.syncRunner.Run() + } +} + +func (proxier *Proxier) OnEndpointsUpdate(oldEndpoints, endpoints *api.Endpoints) { + namespacedName := types.NamespacedName{Namespace: endpoints.Namespace, Name: endpoints.Name} + if proxier.endpointsChanges.update(&namespacedName, oldEndpoints, endpoints) && proxier.isInitialized() { + proxier.syncRunner.Run() + } +} + +func (proxier *Proxier) OnEndpointsDelete(endpoints *api.Endpoints) { + namespacedName := types.NamespacedName{Namespace: endpoints.Namespace, Name: endpoints.Name} + if proxier.endpointsChanges.update(&namespacedName, endpoints, nil) && proxier.isInitialized() { + proxier.syncRunner.Run() + } +} + +func (proxier *Proxier) OnEndpointsSynced() { + proxier.mu.Lock() + proxier.endpointsSynced = true + proxier.setInitialized(proxier.servicesSynced && proxier.endpointsSynced) + proxier.mu.Unlock() + + // Sync unconditionally - this is called once per lifetime. + proxier.syncProxyRules() +} + +// is updated by this function (based on the given changes). +// map is cleared after applying them. +func (proxier *Proxier) updateEndpointsMap() (result updateEndpointMapResult) { + result.staleEndpoints = make(map[endpointServicePair]bool) + result.staleServiceNames = make(map[proxy.ServicePortName]bool) + + var endpointsMap proxyEndpointsMap = proxier.endpointsMap + var changes *endpointsChangeMap = &proxier.endpointsChanges + + func() { + changes.lock.Lock() + defer changes.lock.Unlock() + for _, change := range changes.items { + endpointsMap.unmerge(change.previous, proxier.serviceMap) + endpointsMap.merge(change.current, proxier.serviceMap) + } + changes.items = make(map[types.NamespacedName]*endpointsChange) + }() + + if !utilfeature.DefaultFeatureGate.Enabled(features.ExternalTrafficLocalOnly) { + return + } + + // TODO: If this will appear to be computationally expensive, consider + // computing this incrementally similarly to endpointsMap. + result.hcEndpoints = make(map[types.NamespacedName]int) + localIPs := getLocalIPs(endpointsMap) + for nsn, ips := range localIPs { + result.hcEndpoints[nsn] = len(ips) + } + + return result +} +func getLocalIPs(endpointsMap proxyEndpointsMap) map[types.NamespacedName]sets.String { + localIPs := make(map[types.NamespacedName]sets.String) + for svcPortName := range endpointsMap { + for _, ep := range endpointsMap[svcPortName] { + if ep.isLocal { + nsn := svcPortName.NamespacedName + if localIPs[nsn] == nil { + localIPs[nsn] = sets.NewString() + } + localIPs[nsn].Insert(ep.ip) // just the IP part + } + } + } + return localIPs +} + +// Translates single Endpoints object to proxyEndpointsMap. +// This function is used for incremental updated of endpointsMap. +// +// NOTE: endpoints object should NOT be modified. +func endpointsToEndpointsMap(endpoints *api.Endpoints, hostname string) proxyEndpointsMap { + if endpoints == nil { + return nil + } + + endpointsMap := make(proxyEndpointsMap) + // We need to build a map of portname -> all ip:ports for that + // portname. Explode Endpoints.Subsets[*] into this structure. + for i := range endpoints.Subsets { + ss := &endpoints.Subsets[i] + for i := range ss.Ports { + port := &ss.Ports[i] + if port.Port == 0 { + glog.Warningf("ignoring invalid endpoint port %s", port.Name) + continue + } + svcPortName := proxy.ServicePortName{ + NamespacedName: types.NamespacedName{Namespace: endpoints.Namespace, Name: endpoints.Name}, + Port: port.Name, + } + for i := range ss.Addresses { + addr := &ss.Addresses[i] + if addr.IP == "" { + glog.Warningf("ignoring invalid endpoint port %s with empty host", port.Name) + continue + } + isLocal := addr.NodeName != nil && *addr.NodeName == hostname + epInfo := newEndpointInfo(addr.IP, uint16(port.Port), isLocal) + endpointsMap[svcPortName] = append(endpointsMap[svcPortName], epInfo) + } + if glog.V(3) { + newEPList := []*endpointsInfo{} + for _, ep := range endpointsMap[svcPortName] { + newEPList = append(newEPList, ep) + } + glog.Infof("Setting endpoints for %q to %+v", svcPortName, newEPList) + } + } + } + return endpointsMap +} + +// Translates single Service object to proxyServiceMap. +// +// NOTE: service object should NOT be modified. +func serviceToServiceMap(service *api.Service) proxyServiceMap { + if service == nil { + return nil + } + svcName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name} + if shouldSkipService(svcName, service) { + return nil + } + + serviceMap := make(proxyServiceMap) + for i := range service.Spec.Ports { + servicePort := &service.Spec.Ports[i] + svcPortName := proxy.ServicePortName{NamespacedName: svcName, Port: servicePort.Name} + serviceMap[svcPortName] = newServiceInfo(svcPortName, servicePort, service) + } + return serviceMap +} + +// This is where all of the hns -save/restore calls happen. +// The only other hns rules are those that are setup in iptablesInit() +// assumes proxier.mu is held +func (proxier *Proxier) syncProxyRules() { + proxier.mu.Lock() + defer proxier.mu.Unlock() + + start := time.Now() + defer func() { + SyncProxyRulesLatency.Observe(sinceInMicroseconds(start)) + glog.V(4).Infof("syncProxyRules took %v", time.Since(start)) + }() + // don't sync rules till we've received services and endpoints + if !proxier.endpointsSynced || !proxier.servicesSynced { + glog.V(2).Info("Not syncing hns until Services and Endpoints have been received from master") + return + } + + // We assume that if this was called, we really want to sync them, + // even if nothing changed in the meantime. In other words, callers are + // responsible for detecting no-op changes and not calling this function. + serviceUpdateResult := proxier.updateServiceMap() + endpointUpdateResult := proxier.updateEndpointsMap() + + staleServices := serviceUpdateResult.staleServices + // merge stale services gathered from updateEndpointsMap + for svcPortName := range endpointUpdateResult.staleServiceNames { + if svcInfo, ok := proxier.serviceMap[svcPortName]; ok && svcInfo != nil && svcInfo.protocol == api.ProtocolUDP { + glog.V(2).Infof("Stale udp service %v -> %s", svcPortName, svcInfo.clusterIP.String()) + staleServices.Insert(svcInfo.clusterIP.String()) + } + } + + glog.V(3).Infof("Syncing Policies") + + // Program HNS by adding corresponding policies for each service. + for svcName, svcInfo := range proxier.serviceMap { + if svcInfo.policyApplied { + glog.V(4).Infof("Policy already applied for %s", spew.Sdump(svcInfo)) + continue + } + + var hnsEndpoints []hcsshim.HNSEndpoint + glog.V(4).Infof("====Applying Policy for %s====", svcName) + // Create Remote endpoints for every endpoint, corresponding to the service + if len(proxier.endpointsMap[svcName]) > 0 { + for _, ep := range proxier.endpointsMap[svcName] { + var newHnsEndpoint *hcsshim.HNSEndpoint + hnsNetworkName := proxier.network.name + var err error + if len(ep.hnsID) > 0 { + newHnsEndpoint, err = hcsshim.GetHNSEndpointByID(ep.hnsID) + } + + if newHnsEndpoint == nil { + // First check if an endpoint resource exists for this IP, on the current host + // A Local endpoint could exist here already + // A remote endpoint was already created and proxy was restarted + newHnsEndpoint, err = getHnsEndpointByIpAddress(net.ParseIP(ep.ip), hnsNetworkName) + } + + if newHnsEndpoint == nil { + if ep.isLocal { + glog.Errorf("Local endpoint not found for %v: err : %v on network %s", ep.ip, err, hnsNetworkName) + continue + } + // hns Endpoint resource was not found, create one + hnsnetwork, err := hcsshim.GetHNSNetworkByName(hnsNetworkName) + if err != nil { + glog.Errorf("%v", err) + continue + } + + hnsEndpoint := &hcsshim.HNSEndpoint{ + MacAddress: ep.macAddress, + IPAddress: net.ParseIP(ep.ip), + } + + newHnsEndpoint, err = hnsnetwork.CreateRemoteEndpoint(hnsEndpoint) + if err != nil { + glog.Errorf("Remote endpoint creation failed: %v", err) + continue + } + } + + // Save the hnsId for reference + LogJson(newHnsEndpoint, "Hns Endpoint resource", 1) + hnsEndpoints = append(hnsEndpoints, *newHnsEndpoint) + ep.hnsID = newHnsEndpoint.Id + ep.refCount++ + Log(ep, "Endpoint resource found", 3) + } + } + + glog.V(3).Infof("Associated endpoints [%s] for service [%s]", spew.Sdump(hnsEndpoints), svcName) + + if len(svcInfo.hnsID) > 0 { + // This should not happen + glog.Warningf("Load Balancer already exists %s -- Debug ", svcInfo.hnsID) + } + + if len(hnsEndpoints) == 0 { + glog.Errorf("Endpoint information not available for service %s. Not applying any policy", svcName) + continue + } + + glog.V(4).Infof("Trying to Apply Policies for service %s", spew.Sdump(svcInfo)) + var hnsLoadBalancer *hcsshim.PolicyList + + hnsLoadBalancer, err := getHnsLoadBalancer( + hnsEndpoints, + false, + svcInfo.clusterIP.String(), + Enum(svcInfo.protocol), + uint16(svcInfo.port), + uint16(svcInfo.targetPort), + ) + if err != nil { + glog.Errorf("Policy creation failed: %v", err) + continue + } + + svcInfo.hnsID = hnsLoadBalancer.ID + glog.V(3).Infof("Hns LoadBalancer resource created for cluster ip resources %v, Id [%s]", svcInfo.clusterIP, hnsLoadBalancer.ID) + + // If nodePort is speficied, user should be able to use nodeIP:nodePort to reach the backend endpoints + if svcInfo.nodePort > 0 { + hnsLoadBalancer, err := getHnsLoadBalancer( + hnsEndpoints, + false, + "", // VIP has to be empty to automatically select the nodeIP + Enum(svcInfo.protocol), + uint16(svcInfo.port), + uint16(svcInfo.nodePort), + ) + if err != nil { + glog.Errorf("Policy creation failed: %v", err) + continue + } + + svcInfo.nodePorthnsID = hnsLoadBalancer.ID + glog.V(3).Infof("Hns LoadBalancer resource created for cluster ip resources %v, Id [%s]", svcInfo.clusterIP, hnsLoadBalancer.ID) + } + + // Create a Load Balancer Policy for each external IP + for _, externalIp := range svcInfo.externalIPs { + // Try loading existing policies, if already available + hnsLoadBalancer, err := getHnsLoadBalancer( + hnsEndpoints, + false, + externalIp.ip, + Enum(svcInfo.protocol), + uint16(svcInfo.port), + uint16(svcInfo.targetPort), + ) + if err != nil { + glog.Errorf("Policy creation failed: %v", err) + continue + } + externalIp.hnsID = hnsLoadBalancer.ID + glog.V(3).Infof("Hns LoadBalancer resource created for externalIp resources %v, Id[%s]", externalIp, hnsLoadBalancer.ID) + } + // Create a Load Balancer Policy for each loadbalancer ingress + for _, lbIngressIp := range svcInfo.loadBalancerIngressIPs { + // Try loading existing policies, if already available + hnsLoadBalancer, err := getHnsLoadBalancer( + hnsEndpoints, + false, + lbIngressIp.ip, + Enum(svcInfo.protocol), + uint16(svcInfo.port), + uint16(svcInfo.targetPort), + ) + if err != nil { + glog.Errorf("Policy creation failed: %v", err) + continue + } + lbIngressIp.hnsID = hnsLoadBalancer.ID + glog.V(3).Infof("Hns LoadBalancer resource created for loadBalancer Ingress resources %v", lbIngressIp) + } + svcInfo.policyApplied = true + Log(svcInfo, "+++Policy Successfully applied for service +++", 2) + } + + // Update healthz timestamp. + if proxier.healthzServer != nil { + proxier.healthzServer.UpdateTimestamp() + } + + // Update healthchecks. The endpoints list might include services that are + // not "OnlyLocal", but the services list will not, and the healthChecker + // will just drop those endpoints. + if err := proxier.healthChecker.SyncServices(serviceUpdateResult.hcServices); err != nil { + glog.Errorf("Error syncing healtcheck services: %v", err) + } + if err := proxier.healthChecker.SyncEndpoints(endpointUpdateResult.hcEndpoints); err != nil { + glog.Errorf("Error syncing healthcheck endoints: %v", err) + } + + // Finish housekeeping. + // TODO: these could be made more consistent. + for _, svcIP := range staleServices.List() { + // TODO : Check if this is required to cleanup stale services here + glog.V(5).Infof("Pending delete stale service IP %s connections", svcIP) + } + +} + +type endpointServicePair struct { + endpoint string + servicePortName proxy.ServicePortName +} diff --git a/pkg/proxy/winkernel/proxier_test.go b/pkg/proxy/winkernel/proxier_test.go new file mode 100644 index 00000000000..7ccce600225 --- /dev/null +++ b/pkg/proxy/winkernel/proxier_test.go @@ -0,0 +1,2474 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package winkernel + +import ( + "bytes" + "reflect" + "strconv" + "testing" + "time" + + "github.com/davecgh/go-spew/spew" + + "fmt" + "net" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/proxy" + "k8s.io/kubernetes/pkg/util/async" + utiliptables "k8s.io/kubernetes/pkg/util/iptables" + iptablestest "k8s.io/kubernetes/pkg/util/iptables/testing" + "k8s.io/utils/exec" + fakeexec "k8s.io/utils/exec/testing" +) + +func checkAllLines(t *testing.T, table utiliptables.Table, save []byte, expectedLines map[utiliptables.Chain]string) { + chainLines := utiliptables.GetChainLines(table, save) + for chain, line := range chainLines { + if expected, exists := expectedLines[chain]; exists { + if expected != line { + t.Errorf("getChainLines expected chain line not present. For chain: %s Expected: %s Got: %s", chain, expected, line) + } + } else { + t.Errorf("getChainLines expected chain not present: %s", chain) + } + } +} + +func TestReadLinesFromByteBuffer(t *testing.T) { + testFn := func(byteArray []byte, expected []string) { + index := 0 + readIndex := 0 + for ; readIndex < len(byteArray); index++ { + line, n := utiliptables.ReadLine(readIndex, byteArray) + readIndex = n + if expected[index] != line { + t.Errorf("expected:%q, actual:%q", expected[index], line) + } + } // for + if readIndex < len(byteArray) { + t.Errorf("Byte buffer was only partially read. Buffer length is:%d, readIndex is:%d", len(byteArray), readIndex) + } + if index < len(expected) { + t.Errorf("All expected strings were not compared. expected arr length:%d, matched count:%d", len(expected), index-1) + } + } + + byteArray1 := []byte("\n Line 1 \n\n\n L ine4 \nLine 5 \n \n") + expected1 := []string{"", "Line 1", "", "", "L ine4", "Line 5", ""} + testFn(byteArray1, expected1) + + byteArray1 = []byte("") + expected1 = []string{} + testFn(byteArray1, expected1) + + byteArray1 = []byte("\n\n") + expected1 = []string{"", ""} + testFn(byteArray1, expected1) +} + +func TestGetChainLines(t *testing.T) { + iptables_save := `# Generated by iptables-save v1.4.7 on Wed Oct 29 14:56:01 2014 + *nat + :PREROUTING ACCEPT [2136997:197881818] + :POSTROUTING ACCEPT [4284525:258542680] + :OUTPUT ACCEPT [5901660:357267963] + -A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER + COMMIT + # Completed on Wed Oct 29 14:56:01 2014` + expected := map[utiliptables.Chain]string{ + utiliptables.ChainPrerouting: ":PREROUTING ACCEPT [2136997:197881818]", + utiliptables.ChainPostrouting: ":POSTROUTING ACCEPT [4284525:258542680]", + utiliptables.ChainOutput: ":OUTPUT ACCEPT [5901660:357267963]", + } + checkAllLines(t, utiliptables.TableNAT, []byte(iptables_save), expected) +} + +func TestGetChainLinesMultipleTables(t *testing.T) { + iptables_save := `# Generated by iptables-save v1.4.21 on Fri Aug 7 14:47:37 2015 + *nat + :PREROUTING ACCEPT [2:138] + :INPUT ACCEPT [0:0] + :OUTPUT ACCEPT [0:0] + :POSTROUTING ACCEPT [0:0] + :DOCKER - [0:0] + :KUBE-NODEPORT-CONTAINER - [0:0] + :KUBE-NODEPORT-HOST - [0:0] + :KUBE-PORTALS-CONTAINER - [0:0] + :KUBE-PORTALS-HOST - [0:0] + :KUBE-SVC-1111111111111111 - [0:0] + :KUBE-SVC-2222222222222222 - [0:0] + :KUBE-SVC-3333333333333333 - [0:0] + :KUBE-SVC-4444444444444444 - [0:0] + :KUBE-SVC-5555555555555555 - [0:0] + :KUBE-SVC-6666666666666666 - [0:0] + -A PREROUTING -m comment --comment "handle ClusterIPs; NOTE: this must be before the NodePort rules" -j KUBE-PORTALS-CONTAINER + -A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER + -A PREROUTING -m addrtype --dst-type LOCAL -m comment --comment "handle service NodePorts; NOTE: this must be the last rule in the chain" -j KUBE-NODEPORT-CONTAINER + -A OUTPUT -m comment --comment "handle ClusterIPs; NOTE: this must be before the NodePort rules" -j KUBE-PORTALS-HOST + -A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER + -A OUTPUT -m addrtype --dst-type LOCAL -m comment --comment "handle service NodePorts; NOTE: this must be the last rule in the chain" -j KUBE-NODEPORT-HOST + -A POSTROUTING -s 10.246.1.0/24 ! -o cbr0 -j MASQUERADE + -A POSTROUTING -s 10.0.2.15/32 -d 10.0.2.15/32 -m comment --comment "handle pod connecting to self" -j MASQUERADE + -A KUBE-PORTALS-CONTAINER -d 10.247.0.1/32 -p tcp -m comment --comment "portal for default/kubernetes:" -m state --state NEW -m tcp --dport 443 -j KUBE-SVC-5555555555555555 + -A KUBE-PORTALS-CONTAINER -d 10.247.0.10/32 -p udp -m comment --comment "portal for kube-system/kube-dns:dns" -m state --state NEW -m udp --dport 53 -j KUBE-SVC-6666666666666666 + -A KUBE-PORTALS-CONTAINER -d 10.247.0.10/32 -p tcp -m comment --comment "portal for kube-system/kube-dns:dns-tcp" -m state --state NEW -m tcp --dport 53 -j KUBE-SVC-2222222222222222 + -A KUBE-PORTALS-HOST -d 10.247.0.1/32 -p tcp -m comment --comment "portal for default/kubernetes:" -m state --state NEW -m tcp --dport 443 -j KUBE-SVC-5555555555555555 + -A KUBE-PORTALS-HOST -d 10.247.0.10/32 -p udp -m comment --comment "portal for kube-system/kube-dns:dns" -m state --state NEW -m udp --dport 53 -j KUBE-SVC-6666666666666666 + -A KUBE-PORTALS-HOST -d 10.247.0.10/32 -p tcp -m comment --comment "portal for kube-system/kube-dns:dns-tcp" -m state --state NEW -m tcp --dport 53 -j KUBE-SVC-2222222222222222 + -A KUBE-SVC-1111111111111111 -p udp -m comment --comment "kube-system/kube-dns:dns" -m recent --set --name KUBE-SVC-1111111111111111 --mask 255.255.255.255 --rsource -j DNAT --to-destination 10.246.1.2:53 + -A KUBE-SVC-2222222222222222 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-SVC-3333333333333333 + -A KUBE-SVC-3333333333333333 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m recent --set --name KUBE-SVC-3333333333333333 --mask 255.255.255.255 --rsource -j DNAT --to-destination 10.246.1.2:53 + -A KUBE-SVC-4444444444444444 -p tcp -m comment --comment "default/kubernetes:" -m recent --set --name KUBE-SVC-4444444444444444 --mask 255.255.255.255 --rsource -j DNAT --to-destination 10.245.1.2:443 + -A KUBE-SVC-5555555555555555 -m comment --comment "default/kubernetes:" -j KUBE-SVC-4444444444444444 + -A KUBE-SVC-6666666666666666 -m comment --comment "kube-system/kube-dns:dns" -j KUBE-SVC-1111111111111111 + COMMIT + # Completed on Fri Aug 7 14:47:37 2015 + # Generated by iptables-save v1.4.21 on Fri Aug 7 14:47:37 2015 + *filter + :INPUT ACCEPT [17514:83115836] + :FORWARD ACCEPT [0:0] + :OUTPUT ACCEPT [8909:688225] + :DOCKER - [0:0] + -A FORWARD -o cbr0 -j DOCKER + -A FORWARD -o cbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + -A FORWARD -i cbr0 ! -o cbr0 -j ACCEPT + -A FORWARD -i cbr0 -o cbr0 -j ACCEPT + COMMIT + ` + expected := map[utiliptables.Chain]string{ + utiliptables.ChainPrerouting: ":PREROUTING ACCEPT [2:138]", + utiliptables.Chain("INPUT"): ":INPUT ACCEPT [0:0]", + utiliptables.Chain("OUTPUT"): ":OUTPUT ACCEPT [0:0]", + utiliptables.ChainPostrouting: ":POSTROUTING ACCEPT [0:0]", + utiliptables.Chain("DOCKER"): ":DOCKER - [0:0]", + utiliptables.Chain("KUBE-NODEPORT-CONTAINER"): ":KUBE-NODEPORT-CONTAINER - [0:0]", + utiliptables.Chain("KUBE-NODEPORT-HOST"): ":KUBE-NODEPORT-HOST - [0:0]", + utiliptables.Chain("KUBE-PORTALS-CONTAINER"): ":KUBE-PORTALS-CONTAINER - [0:0]", + utiliptables.Chain("KUBE-PORTALS-HOST"): ":KUBE-PORTALS-HOST - [0:0]", + utiliptables.Chain("KUBE-SVC-1111111111111111"): ":KUBE-SVC-1111111111111111 - [0:0]", + utiliptables.Chain("KUBE-SVC-2222222222222222"): ":KUBE-SVC-2222222222222222 - [0:0]", + utiliptables.Chain("KUBE-SVC-3333333333333333"): ":KUBE-SVC-3333333333333333 - [0:0]", + utiliptables.Chain("KUBE-SVC-4444444444444444"): ":KUBE-SVC-4444444444444444 - [0:0]", + utiliptables.Chain("KUBE-SVC-5555555555555555"): ":KUBE-SVC-5555555555555555 - [0:0]", + utiliptables.Chain("KUBE-SVC-6666666666666666"): ":KUBE-SVC-6666666666666666 - [0:0]", + } + checkAllLines(t, utiliptables.TableNAT, []byte(iptables_save), expected) +} + +func newFakeServiceInfo(service proxy.ServicePortName, ip net.IP, port int, protocol api.Protocol, onlyNodeLocalEndpoints bool) *serviceInfo { + return &serviceInfo{ + sessionAffinityType: api.ServiceAffinityNone, // default + stickyMaxAgeMinutes: 180, // TODO: paramaterize this in the API. + clusterIP: ip, + port: port, + protocol: protocol, + onlyNodeLocalEndpoints: onlyNodeLocalEndpoints, + } +} + +func TestDeleteEndpointConnections(t *testing.T) { + fcmd := fakeexec.FakeCmd{ + CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{ + func() ([]byte, error) { return []byte("1 flow entries have been deleted"), nil }, + func() ([]byte, error) { + return []byte(""), fmt.Errorf("conntrack v1.4.2 (conntrack-tools): 0 flow entries have been deleted.") + }, + }, + } + fexec := fakeexec.FakeExec{ + CommandScript: []fakeexec.FakeCommandAction{ + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, + }, + LookPathFunc: func(cmd string) (string, error) { return cmd, nil }, + } + + serviceMap := make(map[proxy.ServicePortName]*serviceInfo) + svc1 := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: "ns1", Name: "svc1"}, Port: "p80"} + svc2 := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: "ns1", Name: "svc2"}, Port: "p80"} + serviceMap[svc1] = newFakeServiceInfo(svc1, net.IPv4(10, 20, 30, 40), 80, api.ProtocolUDP, false) + serviceMap[svc2] = newFakeServiceInfo(svc1, net.IPv4(10, 20, 30, 41), 80, api.ProtocolTCP, false) + + fakeProxier := Proxier{exec: &fexec, serviceMap: serviceMap} + + testCases := []endpointServicePair{ + { + endpoint: "10.240.0.3:80", + servicePortName: svc1, + }, + { + endpoint: "10.240.0.4:80", + servicePortName: svc1, + }, + { + endpoint: "10.240.0.5:80", + servicePortName: svc2, + }, + } + + expectCommandExecCount := 0 + for i := range testCases { + input := map[endpointServicePair]bool{testCases[i]: true} + fakeProxier.deleteEndpointConnections(input) + svcInfo := fakeProxier.serviceMap[testCases[i].servicePortName] + if svcInfo.protocol == api.ProtocolUDP { + svcIp := svcInfo.clusterIP.String() + endpointIp := strings.Split(testCases[i].endpoint, ":")[0] + expectCommand := fmt.Sprintf("conntrack -D --orig-dst %s --dst-nat %s -p udp", svcIp, endpointIp) + execCommand := strings.Join(fcmd.CombinedOutputLog[expectCommandExecCount], " ") + if expectCommand != execCommand { + t.Errorf("Exepect comand: %s, but executed %s", expectCommand, execCommand) + } + expectCommandExecCount += 1 + } + + if expectCommandExecCount != fexec.CommandCalls { + t.Errorf("Exepect comand executed %d times, but got %d", expectCommandExecCount, fexec.CommandCalls) + } + } +} + +type fakeClosable struct { + closed bool +} + +func (c *fakeClosable) Close() error { + c.closed = true + return nil +} + +func TestRevertPorts(t *testing.T) { + testCases := []struct { + replacementPorts []localPort + existingPorts []localPort + expectToBeClose []bool + }{ + { + replacementPorts: []localPort{ + {port: 5001}, + {port: 5002}, + {port: 5003}, + }, + existingPorts: []localPort{}, + expectToBeClose: []bool{true, true, true}, + }, + { + replacementPorts: []localPort{}, + existingPorts: []localPort{ + {port: 5001}, + {port: 5002}, + {port: 5003}, + }, + expectToBeClose: []bool{}, + }, + { + replacementPorts: []localPort{ + {port: 5001}, + {port: 5002}, + {port: 5003}, + }, + existingPorts: []localPort{ + {port: 5001}, + {port: 5002}, + {port: 5003}, + }, + expectToBeClose: []bool{false, false, false}, + }, + { + replacementPorts: []localPort{ + {port: 5001}, + {port: 5002}, + {port: 5003}, + }, + existingPorts: []localPort{ + {port: 5001}, + {port: 5003}, + }, + expectToBeClose: []bool{false, true, false}, + }, + { + replacementPorts: []localPort{ + {port: 5001}, + {port: 5002}, + {port: 5003}, + }, + existingPorts: []localPort{ + {port: 5001}, + {port: 5002}, + {port: 5003}, + {port: 5004}, + }, + expectToBeClose: []bool{false, false, false}, + }, + } + + for i, tc := range testCases { + replacementPortsMap := make(map[localPort]closeable) + for _, lp := range tc.replacementPorts { + replacementPortsMap[lp] = &fakeClosable{} + } + existingPortsMap := make(map[localPort]closeable) + for _, lp := range tc.existingPorts { + existingPortsMap[lp] = &fakeClosable{} + } + revertPorts(replacementPortsMap, existingPortsMap) + for j, expectation := range tc.expectToBeClose { + if replacementPortsMap[tc.replacementPorts[j]].(*fakeClosable).closed != expectation { + t.Errorf("Expect replacement localport %v to be %v in test case %v", tc.replacementPorts[j], expectation, i) + } + } + for _, lp := range tc.existingPorts { + if existingPortsMap[lp].(*fakeClosable).closed == true { + t.Errorf("Expect existing localport %v to be false in test case %v", lp, i) + } + } + } + +} + +// fakePortOpener implements portOpener. +type fakePortOpener struct { + openPorts []*localPort +} + +// OpenLocalPort fakes out the listen() and bind() used by syncProxyRules +// to lock a local port. +func (f *fakePortOpener) OpenLocalPort(lp *localPort) (closeable, error) { + f.openPorts = append(f.openPorts, lp) + return nil, nil +} + +type fakeHealthChecker struct { + services map[types.NamespacedName]uint16 + endpoints map[types.NamespacedName]int +} + +func newFakeHealthChecker() *fakeHealthChecker { + return &fakeHealthChecker{ + services: map[types.NamespacedName]uint16{}, + endpoints: map[types.NamespacedName]int{}, + } +} + +func (fake *fakeHealthChecker) SyncServices(newServices map[types.NamespacedName]uint16) error { + fake.services = newServices + return nil +} + +func (fake *fakeHealthChecker) SyncEndpoints(newEndpoints map[types.NamespacedName]int) error { + fake.endpoints = newEndpoints + return nil +} + +const testHostname = "test-hostname" + +func NewFakeProxier(ipt utiliptables.Interface) *Proxier { + // TODO: Call NewProxier after refactoring out the goroutine + // invocation into a Run() method. + p := &Proxier{ + exec: &fakeexec.FakeExec{}, + serviceMap: make(proxyServiceMap), + serviceChanges: newServiceChangeMap(), + endpointsMap: make(proxyEndpointsMap), + endpointsChanges: newEndpointsChangeMap(testHostname), + iptables: ipt, + clusterCIDR: "10.0.0.0/24", + hostname: testHostname, + portsMap: make(map[localPort]closeable), + portMapper: &fakePortOpener{[]*localPort{}}, + healthChecker: newFakeHealthChecker(), + precomputedProbabilities: make([]string, 0, 1001), + iptablesData: bytes.NewBuffer(nil), + filterChains: bytes.NewBuffer(nil), + filterRules: bytes.NewBuffer(nil), + natChains: bytes.NewBuffer(nil), + natRules: bytes.NewBuffer(nil), + } + p.syncRunner = async.NewBoundedFrequencyRunner("test-sync-runner", p.syncProxyRules, 0, time.Minute, 1) + return p +} + +func hasJump(rules []iptablestest.Rule, destChain, destIP string, destPort int) bool { + destPortStr := strconv.Itoa(destPort) + match := false + for _, r := range rules { + if r[iptablestest.Jump] == destChain { + match = true + if destIP != "" { + if strings.Contains(r[iptablestest.Destination], destIP) && (strings.Contains(r[iptablestest.DPort], destPortStr) || r[iptablestest.DPort] == "") { + return true + } + match = false + } + if destPort != 0 { + if strings.Contains(r[iptablestest.DPort], destPortStr) && (strings.Contains(r[iptablestest.Destination], destIP) || r[iptablestest.Destination] == "") { + return true + } + match = false + } + } + } + return match +} + +func TestHasJump(t *testing.T) { + testCases := map[string]struct { + rules []iptablestest.Rule + destChain string + destIP string + destPort int + expected bool + }{ + "case 1": { + // Match the 1st rule(both dest IP and dest Port) + rules: []iptablestest.Rule{ + {"-d ": "10.20.30.41/32", "--dport ": "80", "-p ": "tcp", "-j ": "REJECT"}, + {"--dport ": "3001", "-p ": "tcp", "-j ": "KUBE-MARK-MASQ"}, + }, + destChain: "REJECT", + destIP: "10.20.30.41", + destPort: 80, + expected: true, + }, + "case 2": { + // Match the 2nd rule(dest Port) + rules: []iptablestest.Rule{ + {"-d ": "10.20.30.41/32", "-p ": "tcp", "-j ": "REJECT"}, + {"--dport ": "3001", "-p ": "tcp", "-j ": "REJECT"}, + }, + destChain: "REJECT", + destIP: "", + destPort: 3001, + expected: true, + }, + "case 3": { + // Match both dest IP and dest Port + rules: []iptablestest.Rule{ + {"-d ": "1.2.3.4/32", "--dport ": "80", "-p ": "tcp", "-j ": "KUBE-XLB-GF53O3C2HZEXL2XN"}, + }, + destChain: "KUBE-XLB-GF53O3C2HZEXL2XN", + destIP: "1.2.3.4", + destPort: 80, + expected: true, + }, + "case 4": { + // Match dest IP but doesn't match dest Port + rules: []iptablestest.Rule{ + {"-d ": "1.2.3.4/32", "--dport ": "80", "-p ": "tcp", "-j ": "KUBE-XLB-GF53O3C2HZEXL2XN"}, + }, + destChain: "KUBE-XLB-GF53O3C2HZEXL2XN", + destIP: "1.2.3.4", + destPort: 8080, + expected: false, + }, + "case 5": { + // Match dest Port but doesn't match dest IP + rules: []iptablestest.Rule{ + {"-d ": "1.2.3.4/32", "--dport ": "80", "-p ": "tcp", "-j ": "KUBE-XLB-GF53O3C2HZEXL2XN"}, + }, + destChain: "KUBE-XLB-GF53O3C2HZEXL2XN", + destIP: "10.20.30.40", + destPort: 80, + expected: false, + }, + "case 6": { + // Match the 2nd rule(dest IP) + rules: []iptablestest.Rule{ + {"-d ": "10.20.30.41/32", "-p ": "tcp", "-j ": "REJECT"}, + {"-d ": "1.2.3.4/32", "-p ": "tcp", "-j ": "REJECT"}, + {"--dport ": "3001", "-p ": "tcp", "-j ": "REJECT"}, + }, + destChain: "REJECT", + destIP: "1.2.3.4", + destPort: 8080, + expected: true, + }, + "case 7": { + // Match the 2nd rule(dest Port) + rules: []iptablestest.Rule{ + {"-d ": "10.20.30.41/32", "-p ": "tcp", "-j ": "REJECT"}, + {"--dport ": "3001", "-p ": "tcp", "-j ": "REJECT"}, + }, + destChain: "REJECT", + destIP: "1.2.3.4", + destPort: 3001, + expected: true, + }, + "case 8": { + // Match the 1st rule(dest IP) + rules: []iptablestest.Rule{ + {"-d ": "10.20.30.41/32", "-p ": "tcp", "-j ": "REJECT"}, + {"--dport ": "3001", "-p ": "tcp", "-j ": "REJECT"}, + }, + destChain: "REJECT", + destIP: "10.20.30.41", + destPort: 8080, + expected: true, + }, + "case 9": { + rules: []iptablestest.Rule{ + {"-j ": "KUBE-SEP-LWSOSDSHMKPJHHJV"}, + }, + destChain: "KUBE-SEP-LWSOSDSHMKPJHHJV", + destIP: "", + destPort: 0, + expected: true, + }, + "case 10": { + rules: []iptablestest.Rule{ + {"-j ": "KUBE-SEP-FOO"}, + }, + destChain: "KUBE-SEP-BAR", + destIP: "", + destPort: 0, + expected: false, + }, + } + + for k, tc := range testCases { + if got := hasJump(tc.rules, tc.destChain, tc.destIP, tc.destPort); got != tc.expected { + t.Errorf("%v: expected %v, got %v", k, tc.expected, got) + } + } +} + +func hasDNAT(rules []iptablestest.Rule, endpoint string) bool { + for _, r := range rules { + if r[iptablestest.ToDest] == endpoint { + return true + } + } + return false +} + +func errorf(msg string, rules []iptablestest.Rule, t *testing.T) { + for _, r := range rules { + t.Logf("%q", r) + } + t.Errorf("%v", msg) +} + +func TestClusterIPReject(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + svcIP := "10.20.30.41" + svcPort := 80 + svcPortName := proxy.ServicePortName{ + NamespacedName: makeNSN("ns1", "svc1"), + Port: "p80", + } + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Namespace, func(svc *api.Service) { + svc.Spec.ClusterIP = svcIP + svc.Spec.Ports = []api.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: api.ProtocolTCP, + }} + }), + ) + makeEndpointsMap(fp) + fp.syncProxyRules() + + svcChain := string(servicePortChainName(svcPortName.String(), strings.ToLower(string(api.ProtocolTCP)))) + svcRules := ipt.GetRules(svcChain) + if len(svcRules) != 0 { + errorf(fmt.Sprintf("Unexpected rule for chain %v service %v without endpoints", svcChain, svcPortName), svcRules, t) + } + kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) + if !hasJump(kubeSvcRules, iptablestest.Reject, svcIP, svcPort) { + errorf(fmt.Sprintf("Failed to find a %v rule for service %v with no endpoints", iptablestest.Reject, svcPortName), kubeSvcRules, t) + } +} + +func TestClusterIPEndpointsJump(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + svcIP := "10.20.30.41" + svcPort := 80 + svcPortName := proxy.ServicePortName{ + NamespacedName: makeNSN("ns1", "svc1"), + Port: "p80", + } + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) { + svc.Spec.ClusterIP = svcIP + svc.Spec.Ports = []api.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: api.ProtocolTCP, + }} + }), + ) + + epIP := "10.180.0.1" + makeEndpointsMap(fp, + makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: epIP, + }}, + Ports: []api.EndpointPort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + }}, + }} + }), + ) + + fp.syncProxyRules() + + epStr := fmt.Sprintf("%s:%d", epIP, svcPort) + svcChain := string(servicePortChainName(svcPortName.String(), strings.ToLower(string(api.ProtocolTCP)))) + epChain := string(servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(api.ProtocolTCP)), epStr)) + + kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) + if !hasJump(kubeSvcRules, svcChain, svcIP, svcPort) { + errorf(fmt.Sprintf("Failed to find jump from KUBE-SERVICES to %v chain", svcChain), kubeSvcRules, t) + } + + svcRules := ipt.GetRules(svcChain) + if !hasJump(svcRules, epChain, "", 0) { + errorf(fmt.Sprintf("Failed to jump to ep chain %v", epChain), svcRules, t) + } + epRules := ipt.GetRules(epChain) + if !hasDNAT(epRules, epStr) { + errorf(fmt.Sprintf("Endpoint chain %v lacks DNAT to %v", epChain, epStr), epRules, t) + } +} + +func TestLoadBalancer(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + svcIP := "10.20.30.41" + svcPort := 80 + svcNodePort := 3001 + svcLBIP := "1.2.3.4" + svcPortName := proxy.ServicePortName{ + NamespacedName: makeNSN("ns1", "svc1"), + Port: "p80", + } + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) { + svc.Spec.Type = "LoadBalancer" + svc.Spec.ClusterIP = svcIP + svc.Spec.Ports = []api.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: api.ProtocolTCP, + NodePort: int32(svcNodePort), + }} + svc.Status.LoadBalancer.Ingress = []api.LoadBalancerIngress{{ + IP: svcLBIP, + }} + }), + ) + + epIP := "10.180.0.1" + makeEndpointsMap(fp, + makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: epIP, + }}, + Ports: []api.EndpointPort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + }}, + }} + }), + ) + + fp.syncProxyRules() + + proto := strings.ToLower(string(api.ProtocolTCP)) + fwChain := string(serviceFirewallChainName(svcPortName.String(), proto)) + svcChain := string(servicePortChainName(svcPortName.String(), proto)) + //lbChain := string(serviceLBChainName(svcPortName.String(), proto)) + + kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) + if !hasJump(kubeSvcRules, fwChain, svcLBIP, svcPort) { + errorf(fmt.Sprintf("Failed to find jump to firewall chain %v", fwChain), kubeSvcRules, t) + } + + fwRules := ipt.GetRules(fwChain) + if !hasJump(fwRules, svcChain, "", 0) || !hasJump(fwRules, string(KubeMarkMasqChain), "", 0) { + errorf(fmt.Sprintf("Failed to find jump from firewall chain %v to svc chain %v", fwChain, svcChain), fwRules, t) + } +} + +func TestNodePort(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + svcIP := "10.20.30.41" + svcPort := 80 + svcNodePort := 3001 + svcPortName := proxy.ServicePortName{ + NamespacedName: makeNSN("ns1", "svc1"), + Port: "p80", + } + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) { + svc.Spec.Type = "NodePort" + svc.Spec.ClusterIP = svcIP + svc.Spec.Ports = []api.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: api.ProtocolTCP, + NodePort: int32(svcNodePort), + }} + }), + ) + + epIP := "10.180.0.1" + makeEndpointsMap(fp, + makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: epIP, + }}, + Ports: []api.EndpointPort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + }}, + }} + }), + ) + + fp.syncProxyRules() + + proto := strings.ToLower(string(api.ProtocolTCP)) + svcChain := string(servicePortChainName(svcPortName.String(), proto)) + + kubeNodePortRules := ipt.GetRules(string(kubeNodePortsChain)) + if !hasJump(kubeNodePortRules, svcChain, "", svcNodePort) { + errorf(fmt.Sprintf("Failed to find jump to svc chain %v", svcChain), kubeNodePortRules, t) + } +} + +func TestExternalIPsReject(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + svcIP := "10.20.30.41" + svcPort := 80 + svcExternalIPs := "50.60.70.81" + svcPortName := proxy.ServicePortName{ + NamespacedName: makeNSN("ns1", "svc1"), + Port: "p80", + } + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) { + svc.Spec.Type = "ClusterIP" + svc.Spec.ClusterIP = svcIP + svc.Spec.ExternalIPs = []string{svcExternalIPs} + svc.Spec.Ports = []api.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: api.ProtocolTCP, + TargetPort: intstr.FromInt(svcPort), + }} + }), + ) + makeEndpointsMap(fp) + + fp.syncProxyRules() + + kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) + if !hasJump(kubeSvcRules, iptablestest.Reject, svcExternalIPs, svcPort) { + errorf(fmt.Sprintf("Failed to a %v rule for externalIP %v with no endpoints", iptablestest.Reject, svcPortName), kubeSvcRules, t) + } +} + +func TestNodePortReject(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + svcIP := "10.20.30.41" + svcPort := 80 + svcNodePort := 3001 + svcPortName := proxy.ServicePortName{ + NamespacedName: makeNSN("ns1", "svc1"), + Port: "p80", + } + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) { + svc.Spec.Type = "NodePort" + svc.Spec.ClusterIP = svcIP + svc.Spec.Ports = []api.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: api.ProtocolTCP, + NodePort: int32(svcNodePort), + }} + }), + ) + makeEndpointsMap(fp) + + fp.syncProxyRules() + + kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) + if !hasJump(kubeSvcRules, iptablestest.Reject, svcIP, svcNodePort) { + errorf(fmt.Sprintf("Failed to find a %v rule for service %v with no endpoints", iptablestest.Reject, svcPortName), kubeSvcRules, t) + } +} + +func strPtr(s string) *string { + return &s +} + +func TestOnlyLocalLoadBalancing(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + svcIP := "10.20.30.41" + svcPort := 80 + svcNodePort := 3001 + svcLBIP := "1.2.3.4" + svcPortName := proxy.ServicePortName{ + NamespacedName: makeNSN("ns1", "svc1"), + Port: "p80", + } + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) { + svc.Spec.Type = "LoadBalancer" + svc.Spec.ClusterIP = svcIP + svc.Spec.Ports = []api.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: api.ProtocolTCP, + NodePort: int32(svcNodePort), + }} + svc.Status.LoadBalancer.Ingress = []api.LoadBalancerIngress{{ + IP: svcLBIP, + }} + svc.Annotations[api.BetaAnnotationExternalTraffic] = api.AnnotationValueExternalTrafficLocal + }), + ) + + epIP1 := "10.180.0.1" + epIP2 := "10.180.2.1" + epStrLocal := fmt.Sprintf("%s:%d", epIP1, svcPort) + epStrNonLocal := fmt.Sprintf("%s:%d", epIP2, svcPort) + makeEndpointsMap(fp, + makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: epIP1, + NodeName: nil, + }, { + IP: epIP2, + NodeName: strPtr(testHostname), + }}, + Ports: []api.EndpointPort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + }}, + }} + }), + ) + + fp.syncProxyRules() + + proto := strings.ToLower(string(api.ProtocolTCP)) + fwChain := string(serviceFirewallChainName(svcPortName.String(), proto)) + lbChain := string(serviceLBChainName(svcPortName.String(), proto)) + + nonLocalEpChain := string(servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(api.ProtocolTCP)), epStrLocal)) + localEpChain := string(servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(api.ProtocolTCP)), epStrNonLocal)) + + kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) + if !hasJump(kubeSvcRules, fwChain, svcLBIP, svcPort) { + errorf(fmt.Sprintf("Failed to find jump to firewall chain %v", fwChain), kubeSvcRules, t) + } + + fwRules := ipt.GetRules(fwChain) + if !hasJump(fwRules, lbChain, "", 0) { + errorf(fmt.Sprintf("Failed to find jump from firewall chain %v to svc chain %v", fwChain, lbChain), fwRules, t) + } + if hasJump(fwRules, string(KubeMarkMasqChain), "", 0) { + errorf(fmt.Sprintf("Found jump from fw chain %v to MASQUERADE", fwChain), fwRules, t) + } + + lbRules := ipt.GetRules(lbChain) + if hasJump(lbRules, nonLocalEpChain, "", 0) { + errorf(fmt.Sprintf("Found jump from lb chain %v to non-local ep %v", lbChain, epStrLocal), lbRules, t) + } + if !hasJump(lbRules, localEpChain, "", 0) { + errorf(fmt.Sprintf("Didn't find jump from lb chain %v to local ep %v", lbChain, epStrNonLocal), lbRules, t) + } +} + +func TestOnlyLocalNodePortsNoClusterCIDR(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + // set cluster CIDR to empty before test + fp.clusterCIDR = "" + onlyLocalNodePorts(t, fp, ipt) +} + +func TestOnlyLocalNodePorts(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + onlyLocalNodePorts(t, fp, ipt) +} + +func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTables) { + shouldLBTOSVCRuleExist := len(fp.clusterCIDR) > 0 + svcIP := "10.20.30.41" + svcPort := 80 + svcNodePort := 3001 + svcPortName := proxy.ServicePortName{ + NamespacedName: makeNSN("ns1", "svc1"), + Port: "p80", + } + + makeServiceMap(fp, + makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) { + svc.Spec.Type = "NodePort" + svc.Spec.ClusterIP = svcIP + svc.Spec.Ports = []api.ServicePort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + Protocol: api.ProtocolTCP, + NodePort: int32(svcNodePort), + }} + svc.Annotations[api.BetaAnnotationExternalTraffic] = api.AnnotationValueExternalTrafficLocal + }), + ) + + epIP1 := "10.180.0.1" + epIP2 := "10.180.2.1" + epStrLocal := fmt.Sprintf("%s:%d", epIP1, svcPort) + epStrNonLocal := fmt.Sprintf("%s:%d", epIP2, svcPort) + makeEndpointsMap(fp, + makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: epIP1, + NodeName: nil, + }, { + IP: epIP2, + NodeName: strPtr(testHostname), + }}, + Ports: []api.EndpointPort{{ + Name: svcPortName.Port, + Port: int32(svcPort), + }}, + }} + }), + ) + + fp.syncProxyRules() + + proto := strings.ToLower(string(api.ProtocolTCP)) + lbChain := string(serviceLBChainName(svcPortName.String(), proto)) + + nonLocalEpChain := string(servicePortEndpointChainName(svcPortName.String(), proto, epStrLocal)) + localEpChain := string(servicePortEndpointChainName(svcPortName.String(), proto, epStrNonLocal)) + + kubeNodePortRules := ipt.GetRules(string(kubeNodePortsChain)) + if !hasJump(kubeNodePortRules, lbChain, "", svcNodePort) { + errorf(fmt.Sprintf("Failed to find jump to lb chain %v", lbChain), kubeNodePortRules, t) + } + + svcChain := string(servicePortChainName(svcPortName.String(), proto)) + lbRules := ipt.GetRules(lbChain) + if hasJump(lbRules, nonLocalEpChain, "", 0) { + errorf(fmt.Sprintf("Found jump from lb chain %v to non-local ep %v", lbChain, epStrLocal), lbRules, t) + } + if hasJump(lbRules, svcChain, "", 0) != shouldLBTOSVCRuleExist { + prefix := "Did not find " + if !shouldLBTOSVCRuleExist { + prefix = "Found " + } + errorf(fmt.Sprintf("%s jump from lb chain %v to svc %v", prefix, lbChain, svcChain), lbRules, t) + } + if !hasJump(lbRules, localEpChain, "", 0) { + errorf(fmt.Sprintf("Didn't find jump from lb chain %v to local ep %v", lbChain, epStrLocal), lbRules, t) + } +} + +func makeTestService(namespace, name string, svcFunc func(*api.Service)) *api.Service { + svc := &api.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{}, + }, + Spec: api.ServiceSpec{}, + Status: api.ServiceStatus{}, + } + svcFunc(svc) + return svc +} + +func addTestPort(array []api.ServicePort, name string, protocol api.Protocol, port, nodeport int32, targetPort int) []api.ServicePort { + svcPort := api.ServicePort{ + Name: name, + Protocol: protocol, + Port: port, + NodePort: nodeport, + TargetPort: intstr.FromInt(targetPort), + } + return append(array, svcPort) +} + +func TestBuildServiceMapAddRemove(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + + services := []*api.Service{ + makeTestService("somewhere-else", "cluster-ip", func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeClusterIP + svc.Spec.ClusterIP = "172.16.55.4" + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "something", "UDP", 1234, 4321, 0) + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "UDP", 1235, 5321, 0) + }), + makeTestService("somewhere-else", "node-port", func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeNodePort + svc.Spec.ClusterIP = "172.16.55.10" + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "blahblah", "UDP", 345, 678, 0) + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "moreblahblah", "TCP", 344, 677, 0) + }), + makeTestService("somewhere", "load-balancer", func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Spec.ClusterIP = "172.16.55.11" + svc.Spec.LoadBalancerIP = "5.6.7.8" + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "foobar", "UDP", 8675, 30061, 7000) + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "baz", "UDP", 8676, 30062, 7001) + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{ + {IP: "10.1.2.4"}, + }, + } + }), + makeTestService("somewhere", "only-local-load-balancer", func(svc *api.Service) { + svc.ObjectMeta.Annotations = map[string]string{ + api.BetaAnnotationExternalTraffic: api.AnnotationValueExternalTrafficLocal, + api.BetaAnnotationHealthCheckNodePort: "345", + } + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Spec.ClusterIP = "172.16.55.12" + svc.Spec.LoadBalancerIP = "5.6.7.8" + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "foobar2", "UDP", 8677, 30063, 7002) + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "baz", "UDP", 8678, 30064, 7003) + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{ + {IP: "10.1.2.3"}, + }, + } + }), + } + + for i := range services { + fp.OnServiceAdd(services[i]) + } + result := updateServiceMap(fp.serviceMap, &fp.serviceChanges) + if len(fp.serviceMap) != 8 { + t.Errorf("expected service map length 8, got %v", fp.serviceMap) + } + + // The only-local-loadbalancer ones get added + if len(result.hcServices) != 1 { + t.Errorf("expected 1 healthcheck port, got %v", result.hcServices) + } else { + nsn := makeNSN("somewhere", "only-local-load-balancer") + if port, found := result.hcServices[nsn]; !found || port != 345 { + t.Errorf("expected healthcheck port [%q]=345: got %v", nsn, result.hcServices) + } + } + + if len(result.staleServices) != 0 { + // Services only added, so nothing stale yet + t.Errorf("expected stale UDP services length 0, got %d", len(result.staleServices)) + } + + // Remove some stuff + // oneService is a modification of services[0] with removed first port. + oneService := makeTestService("somewhere-else", "cluster-ip", func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeClusterIP + svc.Spec.ClusterIP = "172.16.55.4" + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "UDP", 1235, 5321, 0) + }) + + fp.OnServiceUpdate(services[0], oneService) + fp.OnServiceDelete(services[1]) + fp.OnServiceDelete(services[2]) + fp.OnServiceDelete(services[3]) + + result = updateServiceMap(fp.serviceMap, &fp.serviceChanges) + if len(fp.serviceMap) != 1 { + t.Errorf("expected service map length 1, got %v", fp.serviceMap) + } + + if len(result.hcServices) != 0 { + t.Errorf("expected 0 healthcheck ports, got %v", result.hcServices) + } + + // All services but one were deleted. While you'd expect only the ClusterIPs + // from the three deleted services here, we still have the ClusterIP for + // the not-deleted service, because one of it's ServicePorts was deleted. + expectedStaleUDPServices := []string{"172.16.55.10", "172.16.55.4", "172.16.55.11", "172.16.55.12"} + if len(result.staleServices) != len(expectedStaleUDPServices) { + t.Errorf("expected stale UDP services length %d, got %v", len(expectedStaleUDPServices), result.staleServices.List()) + } + for _, ip := range expectedStaleUDPServices { + if !result.staleServices.Has(ip) { + t.Errorf("expected stale UDP service service %s", ip) + } + } +} + +func TestBuildServiceMapServiceHeadless(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + + makeServiceMap(fp, + makeTestService("somewhere-else", "headless", func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeClusterIP + svc.Spec.ClusterIP = api.ClusterIPNone + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "rpc", "UDP", 1234, 0, 0) + }), + makeTestService("somewhere-else", "headless-without-port", func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeClusterIP + svc.Spec.ClusterIP = api.ClusterIPNone + }), + ) + + // Headless service should be ignored + result := updateServiceMap(fp.serviceMap, &fp.serviceChanges) + if len(fp.serviceMap) != 0 { + t.Errorf("expected service map length 0, got %d", len(fp.serviceMap)) + } + + // No proxied services, so no healthchecks + if len(result.hcServices) != 0 { + t.Errorf("expected healthcheck ports length 0, got %d", len(result.hcServices)) + } + + if len(result.staleServices) != 0 { + t.Errorf("expected stale UDP services length 0, got %d", len(result.staleServices)) + } +} + +func TestBuildServiceMapServiceTypeExternalName(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + + makeServiceMap(fp, + makeTestService("somewhere-else", "external-name", func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeExternalName + svc.Spec.ClusterIP = "172.16.55.4" // Should be ignored + svc.Spec.ExternalName = "foo2.bar.com" + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "blah", "UDP", 1235, 5321, 0) + }), + ) + + result := updateServiceMap(fp.serviceMap, &fp.serviceChanges) + if len(fp.serviceMap) != 0 { + t.Errorf("expected service map length 0, got %v", fp.serviceMap) + } + // No proxied services, so no healthchecks + if len(result.hcServices) != 0 { + t.Errorf("expected healthcheck ports length 0, got %v", result.hcServices) + } + if len(result.staleServices) != 0 { + t.Errorf("expected stale UDP services length 0, got %v", result.staleServices) + } +} + +func TestBuildServiceMapServiceUpdate(t *testing.T) { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + + servicev1 := makeTestService("somewhere", "some-service", func(svc *api.Service) { + svc.Spec.Type = api.ServiceTypeClusterIP + svc.Spec.ClusterIP = "172.16.55.4" + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "something", "UDP", 1234, 4321, 0) + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "TCP", 1235, 5321, 0) + }) + servicev2 := makeTestService("somewhere", "some-service", func(svc *api.Service) { + svc.ObjectMeta.Annotations = map[string]string{ + api.BetaAnnotationExternalTraffic: api.AnnotationValueExternalTrafficLocal, + api.BetaAnnotationHealthCheckNodePort: "345", + } + svc.Spec.Type = api.ServiceTypeLoadBalancer + svc.Spec.ClusterIP = "172.16.55.4" + svc.Spec.LoadBalancerIP = "5.6.7.8" + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "something", "UDP", 1234, 4321, 7002) + svc.Spec.Ports = addTestPort(svc.Spec.Ports, "somethingelse", "TCP", 1235, 5321, 7003) + svc.Status.LoadBalancer = api.LoadBalancerStatus{ + Ingress: []api.LoadBalancerIngress{ + {IP: "10.1.2.3"}, + }, + } + }) + + fp.OnServiceAdd(servicev1) + + result := updateServiceMap(fp.serviceMap, &fp.serviceChanges) + if len(fp.serviceMap) != 2 { + t.Errorf("expected service map length 2, got %v", fp.serviceMap) + } + if len(result.hcServices) != 0 { + t.Errorf("expected healthcheck ports length 0, got %v", result.hcServices) + } + if len(result.staleServices) != 0 { + // Services only added, so nothing stale yet + t.Errorf("expected stale UDP services length 0, got %d", len(result.staleServices)) + } + + // Change service to load-balancer + fp.OnServiceUpdate(servicev1, servicev2) + result = updateServiceMap(fp.serviceMap, &fp.serviceChanges) + if len(fp.serviceMap) != 2 { + t.Errorf("expected service map length 2, got %v", fp.serviceMap) + } + if len(result.hcServices) != 1 { + t.Errorf("expected healthcheck ports length 1, got %v", result.hcServices) + } + if len(result.staleServices) != 0 { + t.Errorf("expected stale UDP services length 0, got %v", result.staleServices.List()) + } + + // No change; make sure the service map stays the same and there are + // no health-check changes + fp.OnServiceUpdate(servicev2, servicev2) + result = updateServiceMap(fp.serviceMap, &fp.serviceChanges) + if len(fp.serviceMap) != 2 { + t.Errorf("expected service map length 2, got %v", fp.serviceMap) + } + if len(result.hcServices) != 1 { + t.Errorf("expected healthcheck ports length 1, got %v", result.hcServices) + } + if len(result.staleServices) != 0 { + t.Errorf("expected stale UDP services length 0, got %v", result.staleServices.List()) + } + + // And back to ClusterIP + fp.OnServiceUpdate(servicev2, servicev1) + result = updateServiceMap(fp.serviceMap, &fp.serviceChanges) + if len(fp.serviceMap) != 2 { + t.Errorf("expected service map length 2, got %v", fp.serviceMap) + } + if len(result.hcServices) != 0 { + t.Errorf("expected healthcheck ports length 0, got %v", result.hcServices) + } + if len(result.staleServices) != 0 { + // Services only added, so nothing stale yet + t.Errorf("expected stale UDP services length 0, got %d", len(result.staleServices)) + } +} + +func Test_getLocalIPs(t *testing.T) { + testCases := []struct { + endpointsMap map[proxy.ServicePortName][]*endpointsInfo + expected map[types.NamespacedName]sets.String + }{{ + // Case[0]: nothing + endpointsMap: map[proxy.ServicePortName][]*endpointsInfo{}, + expected: map[types.NamespacedName]sets.String{}, + }, { + // Case[1]: unnamed port + endpointsMap: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", ""): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + expected: map[types.NamespacedName]sets.String{}, + }, { + // Case[2]: unnamed port local + endpointsMap: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", ""): { + {endpoint: "1.1.1.1:11", isLocal: true}, + }, + }, + expected: map[types.NamespacedName]sets.String{ + {Namespace: "ns1", Name: "ep1"}: sets.NewString("1.1.1.1"), + }, + }, { + // Case[3]: named local and non-local ports for the same IP. + endpointsMap: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + {endpoint: "1.1.1.2:11", isLocal: true}, + }, + makeServicePortName("ns1", "ep1", "p12"): { + {endpoint: "1.1.1.1:12", isLocal: false}, + {endpoint: "1.1.1.2:12", isLocal: true}, + }, + }, + expected: map[types.NamespacedName]sets.String{ + {Namespace: "ns1", Name: "ep1"}: sets.NewString("1.1.1.2"), + }, + }, { + // Case[4]: named local and non-local ports for different IPs. + endpointsMap: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + makeServicePortName("ns2", "ep2", "p22"): { + {endpoint: "2.2.2.2:22", isLocal: true}, + {endpoint: "2.2.2.22:22", isLocal: true}, + }, + makeServicePortName("ns2", "ep2", "p23"): { + {endpoint: "2.2.2.3:23", isLocal: true}, + }, + makeServicePortName("ns4", "ep4", "p44"): { + {endpoint: "4.4.4.4:44", isLocal: true}, + {endpoint: "4.4.4.5:44", isLocal: false}, + }, + makeServicePortName("ns4", "ep4", "p45"): { + {endpoint: "4.4.4.6:45", isLocal: true}, + }, + }, + expected: map[types.NamespacedName]sets.String{ + {Namespace: "ns2", Name: "ep2"}: sets.NewString("2.2.2.2", "2.2.2.22", "2.2.2.3"), + {Namespace: "ns4", Name: "ep4"}: sets.NewString("4.4.4.4", "4.4.4.6"), + }, + }} + + for tci, tc := range testCases { + // outputs + localIPs := getLocalIPs(tc.endpointsMap) + + if !reflect.DeepEqual(localIPs, tc.expected) { + t.Errorf("[%d] expected %#v, got %#v", tci, tc.expected, localIPs) + } + } +} + +// This is a coarse test, but it offers some modicum of confidence as the code is evolved. +func Test_endpointsToEndpointsMap(t *testing.T) { + testCases := []struct { + newEndpoints *api.Endpoints + expected map[proxy.ServicePortName][]*endpointsInfo + }{{ + // Case[0]: nothing + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {}), + expected: map[proxy.ServicePortName][]*endpointsInfo{}, + }, { + // Case[1]: no changes, unnamed port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "", + Port: 11, + }}, + }, + } + }), + expected: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", ""): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + }, { + // Case[2]: no changes, named port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "port", + Port: 11, + }}, + }, + } + }), + expected: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "port"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + }, { + // Case[3]: new port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Port: 11, + }}, + }, + } + }), + expected: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", ""): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + }, { + // Case[4]: remove port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) {}), + expected: map[proxy.ServicePortName][]*endpointsInfo{}, + }, { + // Case[5]: new IP and port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }, { + IP: "2.2.2.2", + }}, + Ports: []api.EndpointPort{{ + Name: "p1", + Port: 11, + }, { + Name: "p2", + Port: 22, + }}, + }, + } + }), + expected: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p1"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + {endpoint: "2.2.2.2:11", isLocal: false}, + }, + makeServicePortName("ns1", "ep1", "p2"): { + {endpoint: "1.1.1.1:22", isLocal: false}, + {endpoint: "2.2.2.2:22", isLocal: false}, + }, + }, + }, { + // Case[6]: remove IP and port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "p1", + Port: 11, + }}, + }, + } + }), + expected: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p1"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + }, { + // Case[7]: rename port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "p2", + Port: 11, + }}, + }, + } + }), + expected: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p2"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + }, { + // Case[8]: renumber port + newEndpoints: makeTestEndpoints("ns1", "ep1", func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{ + { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "p1", + Port: 22, + }}, + }, + } + }), + expected: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p1"): { + {endpoint: "1.1.1.1:22", isLocal: false}, + }, + }, + }} + + for tci, tc := range testCases { + // outputs + newEndpoints := endpointsToEndpointsMap(tc.newEndpoints, "host") + + if len(newEndpoints) != len(tc.expected) { + t.Errorf("[%d] expected %d new, got %d: %v", tci, len(tc.expected), len(newEndpoints), spew.Sdump(newEndpoints)) + } + for x := range tc.expected { + if len(newEndpoints[x]) != len(tc.expected[x]) { + t.Errorf("[%d] expected %d endpoints for %v, got %d", tci, len(tc.expected[x]), x, len(newEndpoints[x])) + } else { + for i := range newEndpoints[x] { + if *(newEndpoints[x][i]) != *(tc.expected[x][i]) { + t.Errorf("[%d] expected new[%v][%d] to be %v, got %v", tci, x, i, tc.expected[x][i], *(newEndpoints[x][i])) + } + } + } + } + } +} + +func makeTestEndpoints(namespace, name string, eptFunc func(*api.Endpoints)) *api.Endpoints { + ept := &api.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + eptFunc(ept) + return ept +} + +func makeEndpointsMap(proxier *Proxier, allEndpoints ...*api.Endpoints) { + for i := range allEndpoints { + proxier.OnEndpointsAdd(allEndpoints[i]) + } + + proxier.mu.Lock() + defer proxier.mu.Unlock() + proxier.endpointsSynced = true +} + +func makeNSN(namespace, name string) types.NamespacedName { + return types.NamespacedName{Namespace: namespace, Name: name} +} + +func makeServicePortName(ns, name, port string) proxy.ServicePortName { + return proxy.ServicePortName{ + NamespacedName: makeNSN(ns, name), + Port: port, + } +} + +func makeServiceMap(proxier *Proxier, allServices ...*api.Service) { + for i := range allServices { + proxier.OnServiceAdd(allServices[i]) + } + + proxier.mu.Lock() + defer proxier.mu.Unlock() + proxier.servicesSynced = true +} + +func compareEndpointsMaps(t *testing.T, tci int, newMap, expected map[proxy.ServicePortName][]*endpointsInfo) { + if len(newMap) != len(expected) { + t.Errorf("[%d] expected %d results, got %d: %v", tci, len(expected), len(newMap), newMap) + } + for x := range expected { + if len(newMap[x]) != len(expected[x]) { + t.Errorf("[%d] expected %d endpoints for %v, got %d", tci, len(expected[x]), x, len(newMap[x])) + } else { + for i := range expected[x] { + if *(newMap[x][i]) != *(expected[x][i]) { + t.Errorf("[%d] expected new[%v][%d] to be %v, got %v", tci, x, i, expected[x][i], newMap[x][i]) + } + } + } + } +} + +func Test_updateEndpointsMap(t *testing.T) { + var nodeName = testHostname + + emptyEndpoint := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{} + } + unnamedPort := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Port: 11, + }}, + }} + } + unnamedPortLocal := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + NodeName: &nodeName, + }}, + Ports: []api.EndpointPort{{ + Port: 11, + }}, + }} + } + namedPortLocal := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + NodeName: &nodeName, + }}, + Ports: []api.EndpointPort{{ + Name: "p11", + Port: 11, + }}, + }} + } + namedPort := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "p11", + Port: 11, + }}, + }} + } + namedPortRenamed := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "p11-2", + Port: 11, + }}, + }} + } + namedPortRenumbered := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "p11", + Port: 22, + }}, + }} + } + namedPortsLocalNoLocal := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }, { + IP: "1.1.1.2", + NodeName: &nodeName, + }}, + Ports: []api.EndpointPort{{ + Name: "p11", + Port: 11, + }, { + Name: "p12", + Port: 12, + }}, + }} + } + multipleSubsets := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "p11", + Port: 11, + }}, + }, { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.2", + }}, + Ports: []api.EndpointPort{{ + Name: "p12", + Port: 12, + }}, + }} + } + multipleSubsetsWithLocal := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "p11", + Port: 11, + }}, + }, { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.2", + NodeName: &nodeName, + }}, + Ports: []api.EndpointPort{{ + Name: "p12", + Port: 12, + }}, + }} + } + multipleSubsetsMultiplePortsLocal := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + NodeName: &nodeName, + }}, + Ports: []api.EndpointPort{{ + Name: "p11", + Port: 11, + }, { + Name: "p12", + Port: 12, + }}, + }, { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.3", + }}, + Ports: []api.EndpointPort{{ + Name: "p13", + Port: 13, + }}, + }} + } + multipleSubsetsIPsPorts1 := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }, { + IP: "1.1.1.2", + NodeName: &nodeName, + }}, + Ports: []api.EndpointPort{{ + Name: "p11", + Port: 11, + }, { + Name: "p12", + Port: 12, + }}, + }, { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.3", + }, { + IP: "1.1.1.4", + NodeName: &nodeName, + }}, + Ports: []api.EndpointPort{{ + Name: "p13", + Port: 13, + }, { + Name: "p14", + Port: 14, + }}, + }} + } + multipleSubsetsIPsPorts2 := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "2.2.2.1", + }, { + IP: "2.2.2.2", + NodeName: &nodeName, + }}, + Ports: []api.EndpointPort{{ + Name: "p21", + Port: 21, + }, { + Name: "p22", + Port: 22, + }}, + }} + } + complexBefore1 := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }}, + Ports: []api.EndpointPort{{ + Name: "p11", + Port: 11, + }}, + }} + } + complexBefore2 := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "2.2.2.2", + NodeName: &nodeName, + }, { + IP: "2.2.2.22", + NodeName: &nodeName, + }}, + Ports: []api.EndpointPort{{ + Name: "p22", + Port: 22, + }}, + }, { + Addresses: []api.EndpointAddress{{ + IP: "2.2.2.3", + NodeName: &nodeName, + }}, + Ports: []api.EndpointPort{{ + Name: "p23", + Port: 23, + }}, + }} + } + complexBefore4 := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "4.4.4.4", + NodeName: &nodeName, + }, { + IP: "4.4.4.5", + NodeName: &nodeName, + }}, + Ports: []api.EndpointPort{{ + Name: "p44", + Port: 44, + }}, + }, { + Addresses: []api.EndpointAddress{{ + IP: "4.4.4.6", + NodeName: &nodeName, + }}, + Ports: []api.EndpointPort{{ + Name: "p45", + Port: 45, + }}, + }} + } + complexAfter1 := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.1", + }, { + IP: "1.1.1.11", + }}, + Ports: []api.EndpointPort{{ + Name: "p11", + Port: 11, + }}, + }, { + Addresses: []api.EndpointAddress{{ + IP: "1.1.1.2", + }}, + Ports: []api.EndpointPort{{ + Name: "p12", + Port: 12, + }, { + Name: "p122", + Port: 122, + }}, + }} + } + complexAfter3 := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "3.3.3.3", + }}, + Ports: []api.EndpointPort{{ + Name: "p33", + Port: 33, + }}, + }} + } + complexAfter4 := func(ept *api.Endpoints) { + ept.Subsets = []api.EndpointSubset{{ + Addresses: []api.EndpointAddress{{ + IP: "4.4.4.4", + NodeName: &nodeName, + }}, + Ports: []api.EndpointPort{{ + Name: "p44", + Port: 44, + }}, + }} + } + + testCases := []struct { + // previousEndpoints and currentEndpoints are used to call appropriate + // handlers OnEndpoints* (based on whether corresponding values are nil + // or non-nil) and must be of equal length. + previousEndpoints []*api.Endpoints + currentEndpoints []*api.Endpoints + oldEndpoints map[proxy.ServicePortName][]*endpointsInfo + expectedResult map[proxy.ServicePortName][]*endpointsInfo + expectedStaleEndpoints []endpointServicePair + expectedStaleServiceNames map[proxy.ServicePortName]bool + expectedHealthchecks map[types.NamespacedName]int + }{{ + // Case[0]: nothing + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{}, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{}, + expectedStaleEndpoints: []endpointServicePair{}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{}, + expectedHealthchecks: map[types.NamespacedName]int{}, + }, { + // Case[1]: no change, unnamed port + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", unnamedPort), + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", unnamedPort), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", ""): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", ""): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{}, + expectedHealthchecks: map[types.NamespacedName]int{}, + }, { + // Case[2]: no change, named port, local + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", namedPortLocal), + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", namedPortLocal), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: true}, + }, + }, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: true}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{}, + expectedHealthchecks: map[types.NamespacedName]int{ + makeNSN("ns1", "ep1"): 1, + }, + }, { + // Case[3]: no change, multiple subsets + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", multipleSubsets), + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", multipleSubsets), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + makeServicePortName("ns1", "ep1", "p12"): { + {endpoint: "1.1.1.2:12", isLocal: false}, + }, + }, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + makeServicePortName("ns1", "ep1", "p12"): { + {endpoint: "1.1.1.2:12", isLocal: false}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{}, + expectedHealthchecks: map[types.NamespacedName]int{}, + }, { + // Case[4]: no change, multiple subsets, multiple ports, local + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", multipleSubsetsMultiplePortsLocal), + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", multipleSubsetsMultiplePortsLocal), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: true}, + }, + makeServicePortName("ns1", "ep1", "p12"): { + {endpoint: "1.1.1.1:12", isLocal: true}, + }, + makeServicePortName("ns1", "ep1", "p13"): { + {endpoint: "1.1.1.3:13", isLocal: false}, + }, + }, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: true}, + }, + makeServicePortName("ns1", "ep1", "p12"): { + {endpoint: "1.1.1.1:12", isLocal: true}, + }, + makeServicePortName("ns1", "ep1", "p13"): { + {endpoint: "1.1.1.3:13", isLocal: false}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{}, + expectedHealthchecks: map[types.NamespacedName]int{ + makeNSN("ns1", "ep1"): 1, + }, + }, { + // Case[5]: no change, multiple endpoints, subsets, IPs, and ports + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", multipleSubsetsIPsPorts1), + makeTestEndpoints("ns2", "ep2", multipleSubsetsIPsPorts2), + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", multipleSubsetsIPsPorts1), + makeTestEndpoints("ns2", "ep2", multipleSubsetsIPsPorts2), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + {endpoint: "1.1.1.2:11", isLocal: true}, + }, + makeServicePortName("ns1", "ep1", "p12"): { + {endpoint: "1.1.1.1:12", isLocal: false}, + {endpoint: "1.1.1.2:12", isLocal: true}, + }, + makeServicePortName("ns1", "ep1", "p13"): { + {endpoint: "1.1.1.3:13", isLocal: false}, + {endpoint: "1.1.1.4:13", isLocal: true}, + }, + makeServicePortName("ns1", "ep1", "p14"): { + {endpoint: "1.1.1.3:14", isLocal: false}, + {endpoint: "1.1.1.4:14", isLocal: true}, + }, + makeServicePortName("ns2", "ep2", "p21"): { + {endpoint: "2.2.2.1:21", isLocal: false}, + {endpoint: "2.2.2.2:21", isLocal: true}, + }, + makeServicePortName("ns2", "ep2", "p22"): { + {endpoint: "2.2.2.1:22", isLocal: false}, + {endpoint: "2.2.2.2:22", isLocal: true}, + }, + }, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + {endpoint: "1.1.1.2:11", isLocal: true}, + }, + makeServicePortName("ns1", "ep1", "p12"): { + {endpoint: "1.1.1.1:12", isLocal: false}, + {endpoint: "1.1.1.2:12", isLocal: true}, + }, + makeServicePortName("ns1", "ep1", "p13"): { + {endpoint: "1.1.1.3:13", isLocal: false}, + {endpoint: "1.1.1.4:13", isLocal: true}, + }, + makeServicePortName("ns1", "ep1", "p14"): { + {endpoint: "1.1.1.3:14", isLocal: false}, + {endpoint: "1.1.1.4:14", isLocal: true}, + }, + makeServicePortName("ns2", "ep2", "p21"): { + {endpoint: "2.2.2.1:21", isLocal: false}, + {endpoint: "2.2.2.2:21", isLocal: true}, + }, + makeServicePortName("ns2", "ep2", "p22"): { + {endpoint: "2.2.2.1:22", isLocal: false}, + {endpoint: "2.2.2.2:22", isLocal: true}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{}, + expectedHealthchecks: map[types.NamespacedName]int{ + makeNSN("ns1", "ep1"): 2, + makeNSN("ns2", "ep2"): 1, + }, + }, { + // Case[6]: add an Endpoints + previousEndpoints: []*api.Endpoints{ + nil, + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", unnamedPortLocal), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{}, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", ""): { + {endpoint: "1.1.1.1:11", isLocal: true}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{ + makeServicePortName("ns1", "ep1", ""): true, + }, + expectedHealthchecks: map[types.NamespacedName]int{ + makeNSN("ns1", "ep1"): 1, + }, + }, { + // Case[7]: remove an Endpoints + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", unnamedPortLocal), + }, + currentEndpoints: []*api.Endpoints{ + nil, + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", ""): { + {endpoint: "1.1.1.1:11", isLocal: true}, + }, + }, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{}, + expectedStaleEndpoints: []endpointServicePair{{ + endpoint: "1.1.1.1:11", + servicePortName: makeServicePortName("ns1", "ep1", ""), + }}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{}, + expectedHealthchecks: map[types.NamespacedName]int{}, + }, { + // Case[8]: add an IP and port + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", namedPort), + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", namedPortsLocalNoLocal), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + {endpoint: "1.1.1.2:11", isLocal: true}, + }, + makeServicePortName("ns1", "ep1", "p12"): { + {endpoint: "1.1.1.1:12", isLocal: false}, + {endpoint: "1.1.1.2:12", isLocal: true}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{ + makeServicePortName("ns1", "ep1", "p12"): true, + }, + expectedHealthchecks: map[types.NamespacedName]int{ + makeNSN("ns1", "ep1"): 1, + }, + }, { + // Case[9]: remove an IP and port + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", namedPortsLocalNoLocal), + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", namedPort), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + {endpoint: "1.1.1.2:11", isLocal: true}, + }, + makeServicePortName("ns1", "ep1", "p12"): { + {endpoint: "1.1.1.1:12", isLocal: false}, + {endpoint: "1.1.1.2:12", isLocal: true}, + }, + }, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{{ + endpoint: "1.1.1.2:11", + servicePortName: makeServicePortName("ns1", "ep1", "p11"), + }, { + endpoint: "1.1.1.1:12", + servicePortName: makeServicePortName("ns1", "ep1", "p12"), + }, { + endpoint: "1.1.1.2:12", + servicePortName: makeServicePortName("ns1", "ep1", "p12"), + }}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{}, + expectedHealthchecks: map[types.NamespacedName]int{}, + }, { + // Case[10]: add a subset + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", namedPort), + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", multipleSubsetsWithLocal), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + makeServicePortName("ns1", "ep1", "p12"): { + {endpoint: "1.1.1.2:12", isLocal: true}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{ + makeServicePortName("ns1", "ep1", "p12"): true, + }, + expectedHealthchecks: map[types.NamespacedName]int{ + makeNSN("ns1", "ep1"): 1, + }, + }, { + // Case[11]: remove a subset + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", multipleSubsets), + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", namedPort), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + makeServicePortName("ns1", "ep1", "p12"): { + {endpoint: "1.1.1.2:12", isLocal: false}, + }, + }, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{{ + endpoint: "1.1.1.2:12", + servicePortName: makeServicePortName("ns1", "ep1", "p12"), + }}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{}, + expectedHealthchecks: map[types.NamespacedName]int{}, + }, { + // Case[12]: rename a port + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", namedPort), + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", namedPortRenamed), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11-2"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{{ + endpoint: "1.1.1.1:11", + servicePortName: makeServicePortName("ns1", "ep1", "p11"), + }}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{ + makeServicePortName("ns1", "ep1", "p11-2"): true, + }, + expectedHealthchecks: map[types.NamespacedName]int{}, + }, { + // Case[13]: renumber a port + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", namedPort), + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", namedPortRenumbered), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:22", isLocal: false}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{{ + endpoint: "1.1.1.1:11", + servicePortName: makeServicePortName("ns1", "ep1", "p11"), + }}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{}, + expectedHealthchecks: map[types.NamespacedName]int{}, + }, { + // Case[14]: complex add and remove + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", complexBefore1), + makeTestEndpoints("ns2", "ep2", complexBefore2), + nil, + makeTestEndpoints("ns4", "ep4", complexBefore4), + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", complexAfter1), + nil, + makeTestEndpoints("ns3", "ep3", complexAfter3), + makeTestEndpoints("ns4", "ep4", complexAfter4), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + makeServicePortName("ns2", "ep2", "p22"): { + {endpoint: "2.2.2.2:22", isLocal: true}, + {endpoint: "2.2.2.22:22", isLocal: true}, + }, + makeServicePortName("ns2", "ep2", "p23"): { + {endpoint: "2.2.2.3:23", isLocal: true}, + }, + makeServicePortName("ns4", "ep4", "p44"): { + {endpoint: "4.4.4.4:44", isLocal: true}, + {endpoint: "4.4.4.5:44", isLocal: true}, + }, + makeServicePortName("ns4", "ep4", "p45"): { + {endpoint: "4.4.4.6:45", isLocal: true}, + }, + }, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", "p11"): { + {endpoint: "1.1.1.1:11", isLocal: false}, + {endpoint: "1.1.1.11:11", isLocal: false}, + }, + makeServicePortName("ns1", "ep1", "p12"): { + {endpoint: "1.1.1.2:12", isLocal: false}, + }, + makeServicePortName("ns1", "ep1", "p122"): { + {endpoint: "1.1.1.2:122", isLocal: false}, + }, + makeServicePortName("ns3", "ep3", "p33"): { + {endpoint: "3.3.3.3:33", isLocal: false}, + }, + makeServicePortName("ns4", "ep4", "p44"): { + {endpoint: "4.4.4.4:44", isLocal: true}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{{ + endpoint: "2.2.2.2:22", + servicePortName: makeServicePortName("ns2", "ep2", "p22"), + }, { + endpoint: "2.2.2.22:22", + servicePortName: makeServicePortName("ns2", "ep2", "p22"), + }, { + endpoint: "2.2.2.3:23", + servicePortName: makeServicePortName("ns2", "ep2", "p23"), + }, { + endpoint: "4.4.4.5:44", + servicePortName: makeServicePortName("ns4", "ep4", "p44"), + }, { + endpoint: "4.4.4.6:45", + servicePortName: makeServicePortName("ns4", "ep4", "p45"), + }}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{ + makeServicePortName("ns1", "ep1", "p12"): true, + makeServicePortName("ns1", "ep1", "p122"): true, + makeServicePortName("ns3", "ep3", "p33"): true, + }, + expectedHealthchecks: map[types.NamespacedName]int{ + makeNSN("ns4", "ep4"): 1, + }, + }, { + // Case[15]: change from 0 endpoint address to 1 unnamed port + previousEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", emptyEndpoint), + }, + currentEndpoints: []*api.Endpoints{ + makeTestEndpoints("ns1", "ep1", unnamedPort), + }, + oldEndpoints: map[proxy.ServicePortName][]*endpointsInfo{}, + expectedResult: map[proxy.ServicePortName][]*endpointsInfo{ + makeServicePortName("ns1", "ep1", ""): { + {endpoint: "1.1.1.1:11", isLocal: false}, + }, + }, + expectedStaleEndpoints: []endpointServicePair{}, + expectedStaleServiceNames: map[proxy.ServicePortName]bool{ + makeServicePortName("ns1", "ep1", ""): true, + }, + expectedHealthchecks: map[types.NamespacedName]int{}, + }, + } + + for tci, tc := range testCases { + ipt := iptablestest.NewFake() + fp := NewFakeProxier(ipt) + fp.hostname = nodeName + + // First check that after adding all previous versions of endpoints, + // the fp.oldEndpoints is as we expect. + for i := range tc.previousEndpoints { + if tc.previousEndpoints[i] != nil { + fp.OnEndpointsAdd(tc.previousEndpoints[i]) + } + } + updateEndpointsMap(fp.endpointsMap, &fp.endpointsChanges, fp.hostname) + compareEndpointsMaps(t, tci, fp.endpointsMap, tc.oldEndpoints) + + // Now let's call appropriate handlers to get to state we want to be. + if len(tc.previousEndpoints) != len(tc.currentEndpoints) { + t.Fatalf("[%d] different lengths of previous and current endpoints", tci) + continue + } + + for i := range tc.previousEndpoints { + prev, curr := tc.previousEndpoints[i], tc.currentEndpoints[i] + switch { + case prev == nil: + fp.OnEndpointsAdd(curr) + case curr == nil: + fp.OnEndpointsDelete(prev) + default: + fp.OnEndpointsUpdate(prev, curr) + } + } + result := updateEndpointsMap(fp.endpointsMap, &fp.endpointsChanges, fp.hostname) + newMap := fp.endpointsMap + compareEndpointsMaps(t, tci, newMap, tc.expectedResult) + if len(result.staleEndpoints) != len(tc.expectedStaleEndpoints) { + t.Errorf("[%d] expected %d staleEndpoints, got %d: %v", tci, len(tc.expectedStaleEndpoints), len(result.staleEndpoints), result.staleEndpoints) + } + for _, x := range tc.expectedStaleEndpoints { + if result.staleEndpoints[x] != true { + t.Errorf("[%d] expected staleEndpoints[%v], but didn't find it: %v", tci, x, result.staleEndpoints) + } + } + if len(result.staleServiceNames) != len(tc.expectedStaleServiceNames) { + t.Errorf("[%d] expected %d staleServiceNames, got %d: %v", tci, len(tc.expectedStaleServiceNames), len(result.staleServiceNames), result.staleServiceNames) + } + for svcName := range tc.expectedStaleServiceNames { + if result.staleServiceNames[svcName] != true { + t.Errorf("[%d] expected staleServiceNames[%v], but didn't find it: %v", tci, svcName, result.staleServiceNames) + } + } + if !reflect.DeepEqual(result.hcEndpoints, tc.expectedHealthchecks) { + t.Errorf("[%d] expected healthchecks %v, got %v", tci, tc.expectedHealthchecks, result.hcEndpoints) + } + } +} + +// TODO(thockin): add *more* tests for syncProxyRules() or break it down further and test the pieces. From 5b875139721950fa0ce4641ac2b63f43a4eaa9ed Mon Sep 17 00:00:00 2001 From: Madhan Raj Mookkandy Date: Thu, 24 Aug 2017 18:15:54 -0700 Subject: [PATCH 2/4] Fix Bazel build --- cmd/kube-proxy/app/BUILD | 3 +- pkg/proxy/BUILD | 1 + pkg/proxy/winkernel/BUILD | 70 +++++++++++++++++++++++++++++++++++++++ vendor/BUILD | 1 + 4 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 pkg/proxy/winkernel/BUILD diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index 86565782341..2beec9ddc12 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -11,6 +11,7 @@ go_library( srcs = [ "conntrack.go", "server.go", + "server_linux.go", ], deps = [ "//pkg/api:go_default_library", @@ -28,13 +29,11 @@ go_library( "//pkg/proxy/iptables:go_default_library", "//pkg/proxy/ipvs:go_default_library", "//pkg/proxy/userspace:go_default_library", - "//pkg/proxy/winuserspace:go_default_library", "//pkg/util/configz:go_default_library", "//pkg/util/dbus:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/ipvs:go_default_library", "//pkg/util/mount:go_default_library", - "//pkg/util/netsh:go_default_library", "//pkg/util/node:go_default_library", "//pkg/util/oom:go_default_library", "//pkg/util/pointer:go_default_library", diff --git a/pkg/proxy/BUILD b/pkg/proxy/BUILD index 3e40d13ba27..03f7f583ac2 100644 --- a/pkg/proxy/BUILD +++ b/pkg/proxy/BUILD @@ -31,6 +31,7 @@ filegroup( "//pkg/proxy/ipvs:all-srcs", "//pkg/proxy/userspace:all-srcs", "//pkg/proxy/util:all-srcs", + "//pkg/proxy/winkernel:all-srcs", "//pkg/proxy/winuserspace:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/proxy/winkernel/BUILD b/pkg/proxy/winkernel/BUILD new file mode 100644 index 00000000000..96b2652cbb9 --- /dev/null +++ b/pkg/proxy/winkernel/BUILD @@ -0,0 +1,70 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + srcs = ["proxier_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/proxy:go_default_library", + "//pkg/util/async:go_default_library", + "//pkg/util/iptables:go_default_library", + "//pkg/util/iptables/testing:go_default_library", + "//vendor/github.com/davecgh/go-spew/spew:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + "//vendor/k8s.io/utils/exec/testing:go_default_library", + ], +) + +go_library( + name = "go_default_library", + srcs = [ + "metrics.go", + "proxier.go", + ], + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/api/helper:go_default_library", + "//pkg/api/service:go_default_library", + "//pkg/features:go_default_library", + "//pkg/proxy:go_default_library", + "//pkg/proxy/healthcheck:go_default_library", + "//pkg/util/async:go_default_library", + "//vendor/github.com/Microsoft/hcsshim:go_default_library", + "//vendor/github.com/davecgh/go-spew/spew:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/vendor/BUILD b/vendor/BUILD index 80286c1ff93..0b25450951f 100644 --- a/vendor/BUILD +++ b/vendor/BUILD @@ -23,6 +23,7 @@ filegroup( "//vendor/github.com/Azure/go-autorest/autorest:all-srcs", "//vendor/github.com/MakeNowJust/heredoc:all-srcs", "//vendor/github.com/Microsoft/go-winio:all-srcs", + "//vendor/github.com/Microsoft/hcsshim:all-srcs", "//vendor/github.com/NYTimes/gziphandler:all-srcs", "//vendor/github.com/Nvveen/Gotty:all-srcs", "//vendor/github.com/PuerkitoBio/purell:all-srcs", From 63020d5f72b9192788d2b38bc1edfc20ecacedde Mon Sep 17 00:00:00 2001 From: Madhan Raj Mookkandy Date: Fri, 25 Aug 2017 12:41:16 -0700 Subject: [PATCH 3/4] Vendor changes Vendoring (NEW) in github.com/Microsoft/hcsshim --- Godeps/Godeps.json | 9 +- Godeps/LICENSES | 57 + cmd/kube-proxy/app/BUILD | 33 +- pkg/proxy/winkernel/BUILD | 104 +- pkg/proxy/winkernel/proxier.go | 4 +- pkg/proxy/winkernel/proxier_test.go | 545 +-------- vendor/github.com/Microsoft/go-winio/BUILD | 1 + .../github.com/Microsoft/go-winio/backup.go | 14 +- vendor/github.com/Microsoft/go-winio/ea.go | 137 +++ vendor/github.com/Microsoft/go-winio/file.go | 29 +- vendor/github.com/Microsoft/go-winio/pipe.go | 4 +- vendor/github.com/Microsoft/hcsshim/BUILD | 63 + vendor/github.com/Microsoft/hcsshim/LICENSE | 21 + vendor/github.com/Microsoft/hcsshim/README.md | 12 + .../Microsoft/hcsshim/activatelayer.go | 28 + .../github.com/Microsoft/hcsshim/baselayer.go | 183 +++ .../github.com/Microsoft/hcsshim/callback.go | 79 ++ vendor/github.com/Microsoft/hcsshim/cgo.go | 7 + .../github.com/Microsoft/hcsshim/container.go | 794 +++++++++++++ .../Microsoft/hcsshim/createlayer.go | 27 + .../Microsoft/hcsshim/createsandboxlayer.go | 35 + .../Microsoft/hcsshim/deactivatelayer.go | 26 + .../Microsoft/hcsshim/destroylayer.go | 27 + vendor/github.com/Microsoft/hcsshim/errors.go | 239 ++++ .../Microsoft/hcsshim/expandsandboxsize.go | 26 + .../Microsoft/hcsshim/exportlayer.go | 156 +++ .../Microsoft/hcsshim/getlayermountpath.go | 55 + .../Microsoft/hcsshim/getsharedbaseimages.go | 22 + vendor/github.com/Microsoft/hcsshim/guid.go | 19 + .../github.com/Microsoft/hcsshim/hcsshim.go | 166 +++ .../Microsoft/hcsshim/hnsendpoint.go | 318 +++++ .../github.com/Microsoft/hcsshim/hnsfuncs.go | 40 + .../Microsoft/hcsshim/hnsnetwork.go | 142 +++ .../github.com/Microsoft/hcsshim/hnspolicy.go | 95 ++ .../Microsoft/hcsshim/hnspolicylist.go | 200 ++++ .../Microsoft/hcsshim/importlayer.go | 212 ++++ .../github.com/Microsoft/hcsshim/interface.go | 187 +++ .../Microsoft/hcsshim/layerexists.go | 30 + .../Microsoft/hcsshim/layerutils.go | 111 ++ vendor/github.com/Microsoft/hcsshim/legacy.go | 741 ++++++++++++ .../Microsoft/hcsshim/mksyscall_windows.go | 934 +++++++++++++++ .../Microsoft/hcsshim/nametoguid.go | 20 + .../Microsoft/hcsshim/preparelayer.go | 46 + .../github.com/Microsoft/hcsshim/process.go | 384 ++++++ .../Microsoft/hcsshim/processimage.go | 23 + .../Microsoft/hcsshim/unpreparelayer.go | 27 + vendor/github.com/Microsoft/hcsshim/utils.go | 33 + .../github.com/Microsoft/hcsshim/version.go | 7 + .../Microsoft/hcsshim/waithelper.go | 63 + .../github.com/Microsoft/hcsshim/zhcsshim.go | 1042 +++++++++++++++++ 50 files changed, 7013 insertions(+), 564 deletions(-) create mode 100644 vendor/github.com/Microsoft/go-winio/ea.go create mode 100644 vendor/github.com/Microsoft/hcsshim/BUILD create mode 100644 vendor/github.com/Microsoft/hcsshim/LICENSE create mode 100644 vendor/github.com/Microsoft/hcsshim/README.md create mode 100644 vendor/github.com/Microsoft/hcsshim/activatelayer.go create mode 100644 vendor/github.com/Microsoft/hcsshim/baselayer.go create mode 100644 vendor/github.com/Microsoft/hcsshim/callback.go create mode 100644 vendor/github.com/Microsoft/hcsshim/cgo.go create mode 100644 vendor/github.com/Microsoft/hcsshim/container.go create mode 100644 vendor/github.com/Microsoft/hcsshim/createlayer.go create mode 100644 vendor/github.com/Microsoft/hcsshim/createsandboxlayer.go create mode 100644 vendor/github.com/Microsoft/hcsshim/deactivatelayer.go create mode 100644 vendor/github.com/Microsoft/hcsshim/destroylayer.go create mode 100644 vendor/github.com/Microsoft/hcsshim/errors.go create mode 100644 vendor/github.com/Microsoft/hcsshim/expandsandboxsize.go create mode 100644 vendor/github.com/Microsoft/hcsshim/exportlayer.go create mode 100644 vendor/github.com/Microsoft/hcsshim/getlayermountpath.go create mode 100644 vendor/github.com/Microsoft/hcsshim/getsharedbaseimages.go create mode 100644 vendor/github.com/Microsoft/hcsshim/guid.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hcsshim.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hnsendpoint.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hnsfuncs.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hnsnetwork.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hnspolicy.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hnspolicylist.go create mode 100644 vendor/github.com/Microsoft/hcsshim/importlayer.go create mode 100644 vendor/github.com/Microsoft/hcsshim/interface.go create mode 100644 vendor/github.com/Microsoft/hcsshim/layerexists.go create mode 100644 vendor/github.com/Microsoft/hcsshim/layerutils.go create mode 100644 vendor/github.com/Microsoft/hcsshim/legacy.go create mode 100644 vendor/github.com/Microsoft/hcsshim/mksyscall_windows.go create mode 100644 vendor/github.com/Microsoft/hcsshim/nametoguid.go create mode 100644 vendor/github.com/Microsoft/hcsshim/preparelayer.go create mode 100644 vendor/github.com/Microsoft/hcsshim/process.go create mode 100644 vendor/github.com/Microsoft/hcsshim/processimage.go create mode 100644 vendor/github.com/Microsoft/hcsshim/unpreparelayer.go create mode 100644 vendor/github.com/Microsoft/hcsshim/utils.go create mode 100644 vendor/github.com/Microsoft/hcsshim/version.go create mode 100644 vendor/github.com/Microsoft/hcsshim/waithelper.go create mode 100644 vendor/github.com/Microsoft/hcsshim/zhcsshim.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 9e0290c78f7..1fa0d8a3de3 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -101,8 +101,13 @@ }, { "ImportPath": "github.com/Microsoft/go-winio", - "Comment": "v0.4.2", - "Rev": "f533f7a102197536779ea3a8cb881d639e21ec5a" + "Comment": "v0.4.5", + "Rev": "78439966b38d69bf38227fbf57ac8a6fee70f69a" + }, + { + "ImportPath": "github.com/Microsoft/hcsshim", + "Comment": "V0.6.3", + "Rev": "6ea7fe54f719d95721e7d9b26ac0add224c9b923" }, { "ImportPath": "github.com/NYTimes/gziphandler", diff --git a/Godeps/LICENSES b/Godeps/LICENSES index a1d6c00de78..9696e91425e 100644 --- a/Godeps/LICENSES +++ b/Godeps/LICENSES @@ -68207,6 +68207,34 @@ SOFTWARE. ================================================================================ +================================================================================ += vendor/github.com/Microsoft/hcsshim licensed under: = + +The MIT License (MIT) + +Copyright (c) 2015 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. += vendor/github.com/Microsoft/hcsshim/LICENSE d4c2cbbea5ee1e7c86dff68a7073718e - +================================================================================ + + ================================================================================ = vendor/github.com/miekg/coredns/middleware/etcd/msg licensed under: = @@ -80979,6 +81007,35 @@ THE SOFTWARE. ================================================================================ +================================================================================ += vendor/github.com/sirupsen/logrus licensed under: = + +The MIT License (MIT) + +Copyright (c) 2014 Simon Eskildsen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + += vendor/github.com/sirupsen/logrus/LICENSE 8dadfef729c08ec4e631c4f6fc5d43a0 - +================================================================================ + + ================================================================================ = vendor/github.com/spf13/afero licensed under: = diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index 2beec9ddc12..06540166fc9 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -11,8 +11,15 @@ go_library( srcs = [ "conntrack.go", "server.go", - "server_linux.go", - ], + ] + select({ + "@io_bazel_rules_go//go/platform:linux_amd64": [ + "server_linux.go", + ], + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "server_windows.go", + ], + "//conditions:default": [], + }), deps = [ "//pkg/api:go_default_library", "//pkg/apis/componentconfig:go_default_library", @@ -30,7 +37,6 @@ go_library( "//pkg/proxy/ipvs:go_default_library", "//pkg/proxy/userspace:go_default_library", "//pkg/util/configz:go_default_library", - "//pkg/util/dbus:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/ipvs:go_default_library", "//pkg/util/mount:go_default_library", @@ -49,8 +55,6 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library", @@ -60,8 +64,23 @@ go_library( "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - ], + ] + select({ + "@io_bazel_rules_go//go/platform:linux_amd64": [ + "//pkg/util/dbus:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + ], + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "//pkg/proxy/winkernel:go_default_library", + "//pkg/proxy/winuserspace:go_default_library", + "//pkg/util/netsh:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + ], + "//conditions:default": [], + }), ) go_test( diff --git a/pkg/proxy/winkernel/BUILD b/pkg/proxy/winkernel/BUILD index 96b2652cbb9..f60ceb4efbb 100644 --- a/pkg/proxy/winkernel/BUILD +++ b/pkg/proxy/winkernel/BUILD @@ -1,59 +1,64 @@ -package(default_visibility = ["//visibility:public"]) - -licenses(["notice"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_test( - name = "go_default_test", - srcs = ["proxier_test.go"], - library = ":go_default_library", - tags = ["automanaged"], - deps = [ - "//pkg/api:go_default_library", - "//pkg/proxy:go_default_library", - "//pkg/util/async:go_default_library", - "//pkg/util/iptables:go_default_library", - "//pkg/util/iptables/testing:go_default_library", - "//vendor/github.com/davecgh/go-spew/spew:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/exec/testing:go_default_library", - ], -) +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ "metrics.go", - "proxier.go", - ], - tags = ["automanaged"], + ] + select({ + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "proxier.go", + ], + "//conditions:default": [], + }), + visibility = ["//visibility:public"], deps = [ - "//pkg/api:go_default_library", - "//pkg/api/helper:go_default_library", - "//pkg/api/service:go_default_library", - "//pkg/features:go_default_library", - "//pkg/proxy:go_default_library", - "//pkg/proxy/healthcheck:go_default_library", - "//pkg/util/async:go_default_library", - "//vendor/github.com/Microsoft/hcsshim:go_default_library", - "//vendor/github.com/davecgh/go-spew/spew:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/k8s.io/client-go/tools/record:go_default_library", - ], + ] + select({ + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "//pkg/api:go_default_library", + "//pkg/api/helper:go_default_library", + "//pkg/api/service:go_default_library", + "//pkg/features:go_default_library", + "//pkg/proxy:go_default_library", + "//pkg/proxy/healthcheck:go_default_library", + "//pkg/util/async:go_default_library", + "//vendor/github.com/Microsoft/hcsshim:go_default_library", + "//vendor/github.com/davecgh/go-spew/spew:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", + ], + "//conditions:default": [], + }), +) + +go_test( + name = "go_default_test", + srcs = select({ + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "proxier_test.go", + ], + "//conditions:default": [], + }), + library = ":go_default_library", + deps = select({ + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "//pkg/api:go_default_library", + "//pkg/proxy:go_default_library", + "//pkg/util/async:go_default_library", + "//vendor/github.com/davecgh/go-spew/spew:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", + "//vendor/k8s.io/utils/exec/testing:go_default_library", + ], + "//conditions:default": [], + }), ) filegroup( @@ -67,4 +72,5 @@ filegroup( name = "all-srcs", srcs = [":package-srcs"], tags = ["automanaged"], + visibility = ["//visibility:public"], ) diff --git a/pkg/proxy/winkernel/proxier.go b/pkg/proxy/winkernel/proxier.go index d440dc410b1..dc36003ce29 100644 --- a/pkg/proxy/winkernel/proxier.go +++ b/pkg/proxy/winkernel/proxier.go @@ -1,3 +1,5 @@ +// +build windows + /* Copyright 2015 The Kubernetes Authors. @@ -63,7 +65,7 @@ func CanUseWinKernelProxier(kcompat KernelCompatTester) (bool, error) { type WindowsKernelCompatTester struct{} -// TODO : Fix the below API to query the OS version +// Todo : Fix the below API to query the OS version func (lkct WindowsKernelCompatTester) IsCompatible() error { return nil } diff --git a/pkg/proxy/winkernel/proxier_test.go b/pkg/proxy/winkernel/proxier_test.go index 7ccce600225..24987bc7b5d 100644 --- a/pkg/proxy/winkernel/proxier_test.go +++ b/pkg/proxy/winkernel/proxier_test.go @@ -1,3 +1,5 @@ +// +build windows + /* Copyright 2015 The Kubernetes Authors. @@ -17,9 +19,7 @@ limitations under the License. package winkernel import ( - "bytes" "reflect" - "strconv" "testing" "time" @@ -36,150 +36,14 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/proxy" "k8s.io/kubernetes/pkg/util/async" - utiliptables "k8s.io/kubernetes/pkg/util/iptables" - iptablestest "k8s.io/kubernetes/pkg/util/iptables/testing" "k8s.io/utils/exec" fakeexec "k8s.io/utils/exec/testing" ) -func checkAllLines(t *testing.T, table utiliptables.Table, save []byte, expectedLines map[utiliptables.Chain]string) { - chainLines := utiliptables.GetChainLines(table, save) - for chain, line := range chainLines { - if expected, exists := expectedLines[chain]; exists { - if expected != line { - t.Errorf("getChainLines expected chain line not present. For chain: %s Expected: %s Got: %s", chain, expected, line) - } - } else { - t.Errorf("getChainLines expected chain not present: %s", chain) - } - } -} - -func TestReadLinesFromByteBuffer(t *testing.T) { - testFn := func(byteArray []byte, expected []string) { - index := 0 - readIndex := 0 - for ; readIndex < len(byteArray); index++ { - line, n := utiliptables.ReadLine(readIndex, byteArray) - readIndex = n - if expected[index] != line { - t.Errorf("expected:%q, actual:%q", expected[index], line) - } - } // for - if readIndex < len(byteArray) { - t.Errorf("Byte buffer was only partially read. Buffer length is:%d, readIndex is:%d", len(byteArray), readIndex) - } - if index < len(expected) { - t.Errorf("All expected strings were not compared. expected arr length:%d, matched count:%d", len(expected), index-1) - } - } - - byteArray1 := []byte("\n Line 1 \n\n\n L ine4 \nLine 5 \n \n") - expected1 := []string{"", "Line 1", "", "", "L ine4", "Line 5", ""} - testFn(byteArray1, expected1) - - byteArray1 = []byte("") - expected1 = []string{} - testFn(byteArray1, expected1) - - byteArray1 = []byte("\n\n") - expected1 = []string{"", ""} - testFn(byteArray1, expected1) -} - -func TestGetChainLines(t *testing.T) { - iptables_save := `# Generated by iptables-save v1.4.7 on Wed Oct 29 14:56:01 2014 - *nat - :PREROUTING ACCEPT [2136997:197881818] - :POSTROUTING ACCEPT [4284525:258542680] - :OUTPUT ACCEPT [5901660:357267963] - -A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER - COMMIT - # Completed on Wed Oct 29 14:56:01 2014` - expected := map[utiliptables.Chain]string{ - utiliptables.ChainPrerouting: ":PREROUTING ACCEPT [2136997:197881818]", - utiliptables.ChainPostrouting: ":POSTROUTING ACCEPT [4284525:258542680]", - utiliptables.ChainOutput: ":OUTPUT ACCEPT [5901660:357267963]", - } - checkAllLines(t, utiliptables.TableNAT, []byte(iptables_save), expected) -} - -func TestGetChainLinesMultipleTables(t *testing.T) { - iptables_save := `# Generated by iptables-save v1.4.21 on Fri Aug 7 14:47:37 2015 - *nat - :PREROUTING ACCEPT [2:138] - :INPUT ACCEPT [0:0] - :OUTPUT ACCEPT [0:0] - :POSTROUTING ACCEPT [0:0] - :DOCKER - [0:0] - :KUBE-NODEPORT-CONTAINER - [0:0] - :KUBE-NODEPORT-HOST - [0:0] - :KUBE-PORTALS-CONTAINER - [0:0] - :KUBE-PORTALS-HOST - [0:0] - :KUBE-SVC-1111111111111111 - [0:0] - :KUBE-SVC-2222222222222222 - [0:0] - :KUBE-SVC-3333333333333333 - [0:0] - :KUBE-SVC-4444444444444444 - [0:0] - :KUBE-SVC-5555555555555555 - [0:0] - :KUBE-SVC-6666666666666666 - [0:0] - -A PREROUTING -m comment --comment "handle ClusterIPs; NOTE: this must be before the NodePort rules" -j KUBE-PORTALS-CONTAINER - -A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER - -A PREROUTING -m addrtype --dst-type LOCAL -m comment --comment "handle service NodePorts; NOTE: this must be the last rule in the chain" -j KUBE-NODEPORT-CONTAINER - -A OUTPUT -m comment --comment "handle ClusterIPs; NOTE: this must be before the NodePort rules" -j KUBE-PORTALS-HOST - -A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER - -A OUTPUT -m addrtype --dst-type LOCAL -m comment --comment "handle service NodePorts; NOTE: this must be the last rule in the chain" -j KUBE-NODEPORT-HOST - -A POSTROUTING -s 10.246.1.0/24 ! -o cbr0 -j MASQUERADE - -A POSTROUTING -s 10.0.2.15/32 -d 10.0.2.15/32 -m comment --comment "handle pod connecting to self" -j MASQUERADE - -A KUBE-PORTALS-CONTAINER -d 10.247.0.1/32 -p tcp -m comment --comment "portal for default/kubernetes:" -m state --state NEW -m tcp --dport 443 -j KUBE-SVC-5555555555555555 - -A KUBE-PORTALS-CONTAINER -d 10.247.0.10/32 -p udp -m comment --comment "portal for kube-system/kube-dns:dns" -m state --state NEW -m udp --dport 53 -j KUBE-SVC-6666666666666666 - -A KUBE-PORTALS-CONTAINER -d 10.247.0.10/32 -p tcp -m comment --comment "portal for kube-system/kube-dns:dns-tcp" -m state --state NEW -m tcp --dport 53 -j KUBE-SVC-2222222222222222 - -A KUBE-PORTALS-HOST -d 10.247.0.1/32 -p tcp -m comment --comment "portal for default/kubernetes:" -m state --state NEW -m tcp --dport 443 -j KUBE-SVC-5555555555555555 - -A KUBE-PORTALS-HOST -d 10.247.0.10/32 -p udp -m comment --comment "portal for kube-system/kube-dns:dns" -m state --state NEW -m udp --dport 53 -j KUBE-SVC-6666666666666666 - -A KUBE-PORTALS-HOST -d 10.247.0.10/32 -p tcp -m comment --comment "portal for kube-system/kube-dns:dns-tcp" -m state --state NEW -m tcp --dport 53 -j KUBE-SVC-2222222222222222 - -A KUBE-SVC-1111111111111111 -p udp -m comment --comment "kube-system/kube-dns:dns" -m recent --set --name KUBE-SVC-1111111111111111 --mask 255.255.255.255 --rsource -j DNAT --to-destination 10.246.1.2:53 - -A KUBE-SVC-2222222222222222 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-SVC-3333333333333333 - -A KUBE-SVC-3333333333333333 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m recent --set --name KUBE-SVC-3333333333333333 --mask 255.255.255.255 --rsource -j DNAT --to-destination 10.246.1.2:53 - -A KUBE-SVC-4444444444444444 -p tcp -m comment --comment "default/kubernetes:" -m recent --set --name KUBE-SVC-4444444444444444 --mask 255.255.255.255 --rsource -j DNAT --to-destination 10.245.1.2:443 - -A KUBE-SVC-5555555555555555 -m comment --comment "default/kubernetes:" -j KUBE-SVC-4444444444444444 - -A KUBE-SVC-6666666666666666 -m comment --comment "kube-system/kube-dns:dns" -j KUBE-SVC-1111111111111111 - COMMIT - # Completed on Fri Aug 7 14:47:37 2015 - # Generated by iptables-save v1.4.21 on Fri Aug 7 14:47:37 2015 - *filter - :INPUT ACCEPT [17514:83115836] - :FORWARD ACCEPT [0:0] - :OUTPUT ACCEPT [8909:688225] - :DOCKER - [0:0] - -A FORWARD -o cbr0 -j DOCKER - -A FORWARD -o cbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - -A FORWARD -i cbr0 ! -o cbr0 -j ACCEPT - -A FORWARD -i cbr0 -o cbr0 -j ACCEPT - COMMIT - ` - expected := map[utiliptables.Chain]string{ - utiliptables.ChainPrerouting: ":PREROUTING ACCEPT [2:138]", - utiliptables.Chain("INPUT"): ":INPUT ACCEPT [0:0]", - utiliptables.Chain("OUTPUT"): ":OUTPUT ACCEPT [0:0]", - utiliptables.ChainPostrouting: ":POSTROUTING ACCEPT [0:0]", - utiliptables.Chain("DOCKER"): ":DOCKER - [0:0]", - utiliptables.Chain("KUBE-NODEPORT-CONTAINER"): ":KUBE-NODEPORT-CONTAINER - [0:0]", - utiliptables.Chain("KUBE-NODEPORT-HOST"): ":KUBE-NODEPORT-HOST - [0:0]", - utiliptables.Chain("KUBE-PORTALS-CONTAINER"): ":KUBE-PORTALS-CONTAINER - [0:0]", - utiliptables.Chain("KUBE-PORTALS-HOST"): ":KUBE-PORTALS-HOST - [0:0]", - utiliptables.Chain("KUBE-SVC-1111111111111111"): ":KUBE-SVC-1111111111111111 - [0:0]", - utiliptables.Chain("KUBE-SVC-2222222222222222"): ":KUBE-SVC-2222222222222222 - [0:0]", - utiliptables.Chain("KUBE-SVC-3333333333333333"): ":KUBE-SVC-3333333333333333 - [0:0]", - utiliptables.Chain("KUBE-SVC-4444444444444444"): ":KUBE-SVC-4444444444444444 - [0:0]", - utiliptables.Chain("KUBE-SVC-5555555555555555"): ":KUBE-SVC-5555555555555555 - [0:0]", - utiliptables.Chain("KUBE-SVC-6666666666666666"): ":KUBE-SVC-6666666666666666 - [0:0]", - } - checkAllLines(t, utiliptables.TableNAT, []byte(iptables_save), expected) -} - func newFakeServiceInfo(service proxy.ServicePortName, ip net.IP, port int, protocol api.Protocol, onlyNodeLocalEndpoints bool) *serviceInfo { return &serviceInfo{ sessionAffinityType: api.ServiceAffinityNone, // default - stickyMaxAgeMinutes: 180, // TODO: paramaterize this in the API. + stickyMaxAgeMinutes: 180, clusterIP: ip, port: port, protocol: protocol, @@ -381,187 +245,34 @@ func (fake *fakeHealthChecker) SyncEndpoints(newEndpoints map[types.NamespacedNa return nil } +func getFakeHnsNetwork() *hnsNetworkInfo { + return &hnsNetworkInfo{ + id: "00000000-0000-0000-0000-000000000001", + name: "fakeNetwork", + }, nil +} + const testHostname = "test-hostname" -func NewFakeProxier(ipt utiliptables.Interface) *Proxier { +func NewFakeProxier() *Proxier { + fakeHnsNetwork := getFakeHnsNetwork() // TODO: Call NewProxier after refactoring out the goroutine // invocation into a Run() method. p := &Proxier{ - exec: &fakeexec.FakeExec{}, - serviceMap: make(proxyServiceMap), - serviceChanges: newServiceChangeMap(), - endpointsMap: make(proxyEndpointsMap), - endpointsChanges: newEndpointsChangeMap(testHostname), - iptables: ipt, - clusterCIDR: "10.0.0.0/24", - hostname: testHostname, - portsMap: make(map[localPort]closeable), - portMapper: &fakePortOpener{[]*localPort{}}, - healthChecker: newFakeHealthChecker(), - precomputedProbabilities: make([]string, 0, 1001), - iptablesData: bytes.NewBuffer(nil), - filterChains: bytes.NewBuffer(nil), - filterRules: bytes.NewBuffer(nil), - natChains: bytes.NewBuffer(nil), - natRules: bytes.NewBuffer(nil), + serviceMap: make(proxyServiceMap), + serviceChanges: newServiceChangeMap(), + endpointsMap: make(proxyEndpointsMap), + endpointsChanges: newEndpointsChangeMap(testHostname), + clusterCIDR: "10.0.0.0/24", + hostname: testHostname, + portsMap: make(map[localPort]closeable), + healthChecker: newFakeHealthChecker(), + network: fakeHnsNetwork, } p.syncRunner = async.NewBoundedFrequencyRunner("test-sync-runner", p.syncProxyRules, 0, time.Minute, 1) return p } -func hasJump(rules []iptablestest.Rule, destChain, destIP string, destPort int) bool { - destPortStr := strconv.Itoa(destPort) - match := false - for _, r := range rules { - if r[iptablestest.Jump] == destChain { - match = true - if destIP != "" { - if strings.Contains(r[iptablestest.Destination], destIP) && (strings.Contains(r[iptablestest.DPort], destPortStr) || r[iptablestest.DPort] == "") { - return true - } - match = false - } - if destPort != 0 { - if strings.Contains(r[iptablestest.DPort], destPortStr) && (strings.Contains(r[iptablestest.Destination], destIP) || r[iptablestest.Destination] == "") { - return true - } - match = false - } - } - } - return match -} - -func TestHasJump(t *testing.T) { - testCases := map[string]struct { - rules []iptablestest.Rule - destChain string - destIP string - destPort int - expected bool - }{ - "case 1": { - // Match the 1st rule(both dest IP and dest Port) - rules: []iptablestest.Rule{ - {"-d ": "10.20.30.41/32", "--dport ": "80", "-p ": "tcp", "-j ": "REJECT"}, - {"--dport ": "3001", "-p ": "tcp", "-j ": "KUBE-MARK-MASQ"}, - }, - destChain: "REJECT", - destIP: "10.20.30.41", - destPort: 80, - expected: true, - }, - "case 2": { - // Match the 2nd rule(dest Port) - rules: []iptablestest.Rule{ - {"-d ": "10.20.30.41/32", "-p ": "tcp", "-j ": "REJECT"}, - {"--dport ": "3001", "-p ": "tcp", "-j ": "REJECT"}, - }, - destChain: "REJECT", - destIP: "", - destPort: 3001, - expected: true, - }, - "case 3": { - // Match both dest IP and dest Port - rules: []iptablestest.Rule{ - {"-d ": "1.2.3.4/32", "--dport ": "80", "-p ": "tcp", "-j ": "KUBE-XLB-GF53O3C2HZEXL2XN"}, - }, - destChain: "KUBE-XLB-GF53O3C2HZEXL2XN", - destIP: "1.2.3.4", - destPort: 80, - expected: true, - }, - "case 4": { - // Match dest IP but doesn't match dest Port - rules: []iptablestest.Rule{ - {"-d ": "1.2.3.4/32", "--dport ": "80", "-p ": "tcp", "-j ": "KUBE-XLB-GF53O3C2HZEXL2XN"}, - }, - destChain: "KUBE-XLB-GF53O3C2HZEXL2XN", - destIP: "1.2.3.4", - destPort: 8080, - expected: false, - }, - "case 5": { - // Match dest Port but doesn't match dest IP - rules: []iptablestest.Rule{ - {"-d ": "1.2.3.4/32", "--dport ": "80", "-p ": "tcp", "-j ": "KUBE-XLB-GF53O3C2HZEXL2XN"}, - }, - destChain: "KUBE-XLB-GF53O3C2HZEXL2XN", - destIP: "10.20.30.40", - destPort: 80, - expected: false, - }, - "case 6": { - // Match the 2nd rule(dest IP) - rules: []iptablestest.Rule{ - {"-d ": "10.20.30.41/32", "-p ": "tcp", "-j ": "REJECT"}, - {"-d ": "1.2.3.4/32", "-p ": "tcp", "-j ": "REJECT"}, - {"--dport ": "3001", "-p ": "tcp", "-j ": "REJECT"}, - }, - destChain: "REJECT", - destIP: "1.2.3.4", - destPort: 8080, - expected: true, - }, - "case 7": { - // Match the 2nd rule(dest Port) - rules: []iptablestest.Rule{ - {"-d ": "10.20.30.41/32", "-p ": "tcp", "-j ": "REJECT"}, - {"--dport ": "3001", "-p ": "tcp", "-j ": "REJECT"}, - }, - destChain: "REJECT", - destIP: "1.2.3.4", - destPort: 3001, - expected: true, - }, - "case 8": { - // Match the 1st rule(dest IP) - rules: []iptablestest.Rule{ - {"-d ": "10.20.30.41/32", "-p ": "tcp", "-j ": "REJECT"}, - {"--dport ": "3001", "-p ": "tcp", "-j ": "REJECT"}, - }, - destChain: "REJECT", - destIP: "10.20.30.41", - destPort: 8080, - expected: true, - }, - "case 9": { - rules: []iptablestest.Rule{ - {"-j ": "KUBE-SEP-LWSOSDSHMKPJHHJV"}, - }, - destChain: "KUBE-SEP-LWSOSDSHMKPJHHJV", - destIP: "", - destPort: 0, - expected: true, - }, - "case 10": { - rules: []iptablestest.Rule{ - {"-j ": "KUBE-SEP-FOO"}, - }, - destChain: "KUBE-SEP-BAR", - destIP: "", - destPort: 0, - expected: false, - }, - } - - for k, tc := range testCases { - if got := hasJump(tc.rules, tc.destChain, tc.destIP, tc.destPort); got != tc.expected { - t.Errorf("%v: expected %v, got %v", k, tc.expected, got) - } - } -} - -func hasDNAT(rules []iptablestest.Rule, endpoint string) bool { - for _, r := range rules { - if r[iptablestest.ToDest] == endpoint { - return true - } - } - return false -} - func errorf(msg string, rules []iptablestest.Rule, t *testing.T) { for _, r := range rules { t.Logf("%q", r) @@ -569,100 +280,8 @@ func errorf(msg string, rules []iptablestest.Rule, t *testing.T) { t.Errorf("%v", msg) } -func TestClusterIPReject(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - svcIP := "10.20.30.41" - svcPort := 80 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Namespace, func(svc *api.Service) { - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []api.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: api.ProtocolTCP, - }} - }), - ) - makeEndpointsMap(fp) - fp.syncProxyRules() - - svcChain := string(servicePortChainName(svcPortName.String(), strings.ToLower(string(api.ProtocolTCP)))) - svcRules := ipt.GetRules(svcChain) - if len(svcRules) != 0 { - errorf(fmt.Sprintf("Unexpected rule for chain %v service %v without endpoints", svcChain, svcPortName), svcRules, t) - } - kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) - if !hasJump(kubeSvcRules, iptablestest.Reject, svcIP, svcPort) { - errorf(fmt.Sprintf("Failed to find a %v rule for service %v with no endpoints", iptablestest.Reject, svcPortName), kubeSvcRules, t) - } -} - -func TestClusterIPEndpointsJump(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - svcIP := "10.20.30.41" - svcPort := 80 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - } - - makeServiceMap(fp, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *api.Service) { - svc.Spec.ClusterIP = svcIP - svc.Spec.Ports = []api.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: api.ProtocolTCP, - }} - }), - ) - - epIP := "10.180.0.1" - makeEndpointsMap(fp, - makeTestEndpoints(svcPortName.Namespace, svcPortName.Name, func(ept *api.Endpoints) { - ept.Subsets = []api.EndpointSubset{{ - Addresses: []api.EndpointAddress{{ - IP: epIP, - }}, - Ports: []api.EndpointPort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - }}, - }} - }), - ) - - fp.syncProxyRules() - - epStr := fmt.Sprintf("%s:%d", epIP, svcPort) - svcChain := string(servicePortChainName(svcPortName.String(), strings.ToLower(string(api.ProtocolTCP)))) - epChain := string(servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(api.ProtocolTCP)), epStr)) - - kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) - if !hasJump(kubeSvcRules, svcChain, svcIP, svcPort) { - errorf(fmt.Sprintf("Failed to find jump from KUBE-SERVICES to %v chain", svcChain), kubeSvcRules, t) - } - - svcRules := ipt.GetRules(svcChain) - if !hasJump(svcRules, epChain, "", 0) { - errorf(fmt.Sprintf("Failed to jump to ep chain %v", epChain), svcRules, t) - } - epRules := ipt.GetRules(epChain) - if !hasDNAT(epRules, epStr) { - errorf(fmt.Sprintf("Endpoint chain %v lacks DNAT to %v", epChain, epStr), epRules, t) - } -} - func TestLoadBalancer(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) + fp := NewFakeProxier() svcIP := "10.20.30.41" svcPort := 80 svcNodePort := 3001 @@ -710,20 +329,13 @@ func TestLoadBalancer(t *testing.T) { svcChain := string(servicePortChainName(svcPortName.String(), proto)) //lbChain := string(serviceLBChainName(svcPortName.String(), proto)) - kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) - if !hasJump(kubeSvcRules, fwChain, svcLBIP, svcPort) { - errorf(fmt.Sprintf("Failed to find jump to firewall chain %v", fwChain), kubeSvcRules, t) - } + // TODO - fwRules := ipt.GetRules(fwChain) - if !hasJump(fwRules, svcChain, "", 0) || !hasJump(fwRules, string(KubeMarkMasqChain), "", 0) { - errorf(fmt.Sprintf("Failed to find jump from firewall chain %v to svc chain %v", fwChain, svcChain), fwRules, t) - } } func TestNodePort(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) + + fp := NewFakeProxier() svcIP := "10.20.30.41" svcPort := 80 svcNodePort := 3001 @@ -765,15 +377,12 @@ func TestNodePort(t *testing.T) { proto := strings.ToLower(string(api.ProtocolTCP)) svcChain := string(servicePortChainName(svcPortName.String(), proto)) - kubeNodePortRules := ipt.GetRules(string(kubeNodePortsChain)) - if !hasJump(kubeNodePortRules, svcChain, "", svcNodePort) { - errorf(fmt.Sprintf("Failed to find jump to svc chain %v", svcChain), kubeNodePortRules, t) - } + // TODO } func TestExternalIPsReject(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) + + fp := NewFakeProxier() svcIP := "10.20.30.41" svcPort := 80 svcExternalIPs := "50.60.70.81" @@ -799,15 +408,11 @@ func TestExternalIPsReject(t *testing.T) { fp.syncProxyRules() - kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) - if !hasJump(kubeSvcRules, iptablestest.Reject, svcExternalIPs, svcPort) { - errorf(fmt.Sprintf("Failed to a %v rule for externalIP %v with no endpoints", iptablestest.Reject, svcPortName), kubeSvcRules, t) - } } func TestNodePortReject(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) + + fp := NewFakeProxier() svcIP := "10.20.30.41" svcPort := 80 svcNodePort := 3001 @@ -832,10 +437,7 @@ func TestNodePortReject(t *testing.T) { fp.syncProxyRules() - kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) - if !hasJump(kubeSvcRules, iptablestest.Reject, svcIP, svcNodePort) { - errorf(fmt.Sprintf("Failed to find a %v rule for service %v with no endpoints", iptablestest.Reject, svcPortName), kubeSvcRules, t) - } + // TODO } func strPtr(s string) *string { @@ -843,8 +445,8 @@ func strPtr(s string) *string { } func TestOnlyLocalLoadBalancing(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) + + fp := NewFakeProxier() svcIP := "10.20.30.41" svcPort := 80 svcNodePort := 3001 @@ -902,43 +504,24 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { nonLocalEpChain := string(servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(api.ProtocolTCP)), epStrLocal)) localEpChain := string(servicePortEndpointChainName(svcPortName.String(), strings.ToLower(string(api.ProtocolTCP)), epStrNonLocal)) - kubeSvcRules := ipt.GetRules(string(kubeServicesChain)) - if !hasJump(kubeSvcRules, fwChain, svcLBIP, svcPort) { - errorf(fmt.Sprintf("Failed to find jump to firewall chain %v", fwChain), kubeSvcRules, t) - } - - fwRules := ipt.GetRules(fwChain) - if !hasJump(fwRules, lbChain, "", 0) { - errorf(fmt.Sprintf("Failed to find jump from firewall chain %v to svc chain %v", fwChain, lbChain), fwRules, t) - } - if hasJump(fwRules, string(KubeMarkMasqChain), "", 0) { - errorf(fmt.Sprintf("Found jump from fw chain %v to MASQUERADE", fwChain), fwRules, t) - } - - lbRules := ipt.GetRules(lbChain) - if hasJump(lbRules, nonLocalEpChain, "", 0) { - errorf(fmt.Sprintf("Found jump from lb chain %v to non-local ep %v", lbChain, epStrLocal), lbRules, t) - } - if !hasJump(lbRules, localEpChain, "", 0) { - errorf(fmt.Sprintf("Didn't find jump from lb chain %v to local ep %v", lbChain, epStrNonLocal), lbRules, t) - } + // TODO } func TestOnlyLocalNodePortsNoClusterCIDR(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) + + fp := NewFakeProxier() // set cluster CIDR to empty before test fp.clusterCIDR = "" - onlyLocalNodePorts(t, fp, ipt) + onlyLocalNodePorts(t, fp) } func TestOnlyLocalNodePorts(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) - onlyLocalNodePorts(t, fp, ipt) + + fp := NewFakeProxier() + onlyLocalNodePorts(t, fp) } -func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTables) { +func onlyLocalNodePorts(t *testing.T, fp *Proxier) { shouldLBTOSVCRuleExist := len(fp.clusterCIDR) > 0 svcIP := "10.20.30.41" svcPort := 80 @@ -986,32 +569,7 @@ func onlyLocalNodePorts(t *testing.T, fp *Proxier, ipt *iptablestest.FakeIPTable fp.syncProxyRules() - proto := strings.ToLower(string(api.ProtocolTCP)) - lbChain := string(serviceLBChainName(svcPortName.String(), proto)) - - nonLocalEpChain := string(servicePortEndpointChainName(svcPortName.String(), proto, epStrLocal)) - localEpChain := string(servicePortEndpointChainName(svcPortName.String(), proto, epStrNonLocal)) - - kubeNodePortRules := ipt.GetRules(string(kubeNodePortsChain)) - if !hasJump(kubeNodePortRules, lbChain, "", svcNodePort) { - errorf(fmt.Sprintf("Failed to find jump to lb chain %v", lbChain), kubeNodePortRules, t) - } - - svcChain := string(servicePortChainName(svcPortName.String(), proto)) - lbRules := ipt.GetRules(lbChain) - if hasJump(lbRules, nonLocalEpChain, "", 0) { - errorf(fmt.Sprintf("Found jump from lb chain %v to non-local ep %v", lbChain, epStrLocal), lbRules, t) - } - if hasJump(lbRules, svcChain, "", 0) != shouldLBTOSVCRuleExist { - prefix := "Did not find " - if !shouldLBTOSVCRuleExist { - prefix = "Found " - } - errorf(fmt.Sprintf("%s jump from lb chain %v to svc %v", prefix, lbChain, svcChain), lbRules, t) - } - if !hasJump(lbRules, localEpChain, "", 0) { - errorf(fmt.Sprintf("Didn't find jump from lb chain %v to local ep %v", lbChain, epStrLocal), lbRules, t) - } + // TODO } func makeTestService(namespace, name string, svcFunc func(*api.Service)) *api.Service { @@ -1040,8 +598,8 @@ func addTestPort(array []api.ServicePort, name string, protocol api.Protocol, po } func TestBuildServiceMapAddRemove(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) + + fp := NewFakeProxier() services := []*api.Service{ makeTestService("somewhere-else", "cluster-ip", func(svc *api.Service) { @@ -1146,8 +704,8 @@ func TestBuildServiceMapAddRemove(t *testing.T) { } func TestBuildServiceMapServiceHeadless(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) + + fp := NewFakeProxier() makeServiceMap(fp, makeTestService("somewhere-else", "headless", func(svc *api.Service) { @@ -1178,8 +736,8 @@ func TestBuildServiceMapServiceHeadless(t *testing.T) { } func TestBuildServiceMapServiceTypeExternalName(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) + + fp := NewFakeProxier() makeServiceMap(fp, makeTestService("somewhere-else", "external-name", func(svc *api.Service) { @@ -1204,8 +762,7 @@ func TestBuildServiceMapServiceTypeExternalName(t *testing.T) { } func TestBuildServiceMapServiceUpdate(t *testing.T) { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) + fp := NewFakeProxier() servicev1 := makeTestService("somewhere", "some-service", func(svc *api.Service) { svc.Spec.Type = api.ServiceTypeClusterIP @@ -2415,8 +1972,8 @@ func Test_updateEndpointsMap(t *testing.T) { } for tci, tc := range testCases { - ipt := iptablestest.NewFake() - fp := NewFakeProxier(ipt) + + fp := NewFakeProxier() fp.hostname = nodeName // First check that after adding all previous versions of endpoints, diff --git a/vendor/github.com/Microsoft/go-winio/BUILD b/vendor/github.com/Microsoft/go-winio/BUILD index 1b01469a499..ba46acfa214 100644 --- a/vendor/github.com/Microsoft/go-winio/BUILD +++ b/vendor/github.com/Microsoft/go-winio/BUILD @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = [ + "ea.go", "reparse.go", "syscall.go", ] + select({ diff --git a/vendor/github.com/Microsoft/go-winio/backup.go b/vendor/github.com/Microsoft/go-winio/backup.go index 27d6ace0c91..2be34af4310 100644 --- a/vendor/github.com/Microsoft/go-winio/backup.go +++ b/vendor/github.com/Microsoft/go-winio/backup.go @@ -68,10 +68,20 @@ func NewBackupStreamReader(r io.Reader) *BackupStreamReader { return &BackupStreamReader{r, 0} } -// Next returns the next backup stream and prepares for calls to Write(). It skips the remainder of the current stream if +// Next returns the next backup stream and prepares for calls to Read(). It skips the remainder of the current stream if // it was not completely read. func (r *BackupStreamReader) Next() (*BackupHeader, error) { if r.bytesLeft > 0 { + if s, ok := r.r.(io.Seeker); ok { + // Make sure Seek on io.SeekCurrent sometimes succeeds + // before trying the actual seek. + if _, err := s.Seek(0, io.SeekCurrent); err == nil { + if _, err = s.Seek(r.bytesLeft, io.SeekCurrent); err != nil { + return nil, err + } + r.bytesLeft = 0 + } + } if _, err := io.Copy(ioutil.Discard, r); err != nil { return nil, err } @@ -220,7 +230,7 @@ type BackupFileWriter struct { ctx uintptr } -// NewBackupFileWrtier returns a new BackupFileWriter from a file handle. If includeSecurity is true, +// NewBackupFileWriter returns a new BackupFileWriter from a file handle. If includeSecurity is true, // Write() will attempt to restore the security descriptor from the stream. func NewBackupFileWriter(f *os.File, includeSecurity bool) *BackupFileWriter { w := &BackupFileWriter{f, includeSecurity, 0} diff --git a/vendor/github.com/Microsoft/go-winio/ea.go b/vendor/github.com/Microsoft/go-winio/ea.go new file mode 100644 index 00000000000..b37e930d6af --- /dev/null +++ b/vendor/github.com/Microsoft/go-winio/ea.go @@ -0,0 +1,137 @@ +package winio + +import ( + "bytes" + "encoding/binary" + "errors" +) + +type fileFullEaInformation struct { + NextEntryOffset uint32 + Flags uint8 + NameLength uint8 + ValueLength uint16 +} + +var ( + fileFullEaInformationSize = binary.Size(&fileFullEaInformation{}) + + errInvalidEaBuffer = errors.New("invalid extended attribute buffer") + errEaNameTooLarge = errors.New("extended attribute name too large") + errEaValueTooLarge = errors.New("extended attribute value too large") +) + +// ExtendedAttribute represents a single Windows EA. +type ExtendedAttribute struct { + Name string + Value []byte + Flags uint8 +} + +func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) { + var info fileFullEaInformation + err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info) + if err != nil { + err = errInvalidEaBuffer + return + } + + nameOffset := fileFullEaInformationSize + nameLen := int(info.NameLength) + valueOffset := nameOffset + int(info.NameLength) + 1 + valueLen := int(info.ValueLength) + nextOffset := int(info.NextEntryOffset) + if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) { + err = errInvalidEaBuffer + return + } + + ea.Name = string(b[nameOffset : nameOffset+nameLen]) + ea.Value = b[valueOffset : valueOffset+valueLen] + ea.Flags = info.Flags + if info.NextEntryOffset != 0 { + nb = b[info.NextEntryOffset:] + } + return +} + +// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION +// buffer retrieved from BackupRead, ZwQueryEaFile, etc. +func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) { + for len(b) != 0 { + ea, nb, err := parseEa(b) + if err != nil { + return nil, err + } + + eas = append(eas, ea) + b = nb + } + return +} + +func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error { + if int(uint8(len(ea.Name))) != len(ea.Name) { + return errEaNameTooLarge + } + if int(uint16(len(ea.Value))) != len(ea.Value) { + return errEaValueTooLarge + } + entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value)) + withPadding := (entrySize + 3) &^ 3 + nextOffset := uint32(0) + if !last { + nextOffset = withPadding + } + info := fileFullEaInformation{ + NextEntryOffset: nextOffset, + Flags: ea.Flags, + NameLength: uint8(len(ea.Name)), + ValueLength: uint16(len(ea.Value)), + } + + err := binary.Write(buf, binary.LittleEndian, &info) + if err != nil { + return err + } + + _, err = buf.Write([]byte(ea.Name)) + if err != nil { + return err + } + + err = buf.WriteByte(0) + if err != nil { + return err + } + + _, err = buf.Write(ea.Value) + if err != nil { + return err + } + + _, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize]) + if err != nil { + return err + } + + return nil +} + +// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION +// buffer for use with BackupWrite, ZwSetEaFile, etc. +func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) { + var buf bytes.Buffer + for i := range eas { + last := false + if i == len(eas)-1 { + last = true + } + + err := writeEa(&buf, &eas[i], last) + if err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} diff --git a/vendor/github.com/Microsoft/go-winio/file.go b/vendor/github.com/Microsoft/go-winio/file.go index 613f31b520e..57ac3696a91 100644 --- a/vendor/github.com/Microsoft/go-winio/file.go +++ b/vendor/github.com/Microsoft/go-winio/file.go @@ -23,6 +23,13 @@ type atomicBool int32 func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 } func (b *atomicBool) setFalse() { atomic.StoreInt32((*int32)(b), 0) } func (b *atomicBool) setTrue() { atomic.StoreInt32((*int32)(b), 1) } +func (b *atomicBool) swap(new bool) bool { + var newInt int32 + if new { + newInt = 1 + } + return atomic.SwapInt32((*int32)(b), newInt) == 1 +} const ( cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS = 1 @@ -71,7 +78,8 @@ func initIo() { type win32File struct { handle syscall.Handle wg sync.WaitGroup - closing bool + wgLock sync.RWMutex + closing atomicBool readDeadline deadlineHandler writeDeadline deadlineHandler } @@ -107,14 +115,18 @@ func MakeOpenFile(h syscall.Handle) (io.ReadWriteCloser, error) { // closeHandle closes the resources associated with a Win32 handle func (f *win32File) closeHandle() { - if !f.closing { + f.wgLock.Lock() + // Atomically set that we are closing, releasing the resources only once. + if !f.closing.swap(true) { + f.wgLock.Unlock() // cancel all IO and wait for it to complete - f.closing = true cancelIoEx(f.handle, nil) f.wg.Wait() // at this point, no new IO can start syscall.Close(f.handle) f.handle = 0 + } else { + f.wgLock.Unlock() } } @@ -127,10 +139,13 @@ func (f *win32File) Close() error { // prepareIo prepares for a new IO operation. // The caller must call f.wg.Done() when the IO is finished, prior to Close() returning. func (f *win32File) prepareIo() (*ioOperation, error) { - f.wg.Add(1) - if f.closing { + f.wgLock.RLock() + if f.closing.isSet() { + f.wgLock.RUnlock() return nil, ErrFileClosed } + f.wg.Add(1) + f.wgLock.RUnlock() c := &ioOperation{} c.ch = make(chan ioResult) return c, nil @@ -159,7 +174,7 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er return int(bytes), err } - if f.closing { + if f.closing.isSet() { cancelIoEx(f.handle, &c.o) } @@ -175,7 +190,7 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er case r = <-c.ch: err = r.err if err == syscall.ERROR_OPERATION_ABORTED { - if f.closing { + if f.closing.isSet() { err = ErrFileClosed } } diff --git a/vendor/github.com/Microsoft/go-winio/pipe.go b/vendor/github.com/Microsoft/go-winio/pipe.go index da706cc8a7d..44340b8167b 100644 --- a/vendor/github.com/Microsoft/go-winio/pipe.go +++ b/vendor/github.com/Microsoft/go-winio/pipe.go @@ -265,9 +265,9 @@ func (l *win32PipeListener) listenerRoutine() { if err == nil { // Wait for the client to connect. ch := make(chan error) - go func() { + go func(p *win32File) { ch <- connectPipe(p) - }() + }(p) select { case err = <-ch: if err != nil { diff --git a/vendor/github.com/Microsoft/hcsshim/BUILD b/vendor/github.com/Microsoft/hcsshim/BUILD new file mode 100644 index 00000000000..966f658b693 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/BUILD @@ -0,0 +1,63 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "activatelayer.go", + "baselayer.go", + "callback.go", + "cgo.go", + "container.go", + "createlayer.go", + "createsandboxlayer.go", + "deactivatelayer.go", + "destroylayer.go", + "errors.go", + "expandsandboxsize.go", + "exportlayer.go", + "getlayermountpath.go", + "getsharedbaseimages.go", + "guid.go", + "hcsshim.go", + "hnsendpoint.go", + "hnsfuncs.go", + "hnsnetwork.go", + "hnspolicy.go", + "hnspolicylist.go", + "importlayer.go", + "interface.go", + "layerexists.go", + "layerutils.go", + "legacy.go", + "nametoguid.go", + "preparelayer.go", + "process.go", + "processimage.go", + "unpreparelayer.go", + "utils.go", + "version.go", + "waithelper.go", + "zhcsshim.go", + ], + cgo = True, + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/Microsoft/go-winio:go_default_library", + "//vendor/github.com/sirupsen/logrus:go_default_library", + "//vendor/golang.org/x/sys/windows:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/vendor/github.com/Microsoft/hcsshim/LICENSE b/vendor/github.com/Microsoft/hcsshim/LICENSE new file mode 100644 index 00000000000..49d21669aee --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/Microsoft/hcsshim/README.md b/vendor/github.com/Microsoft/hcsshim/README.md new file mode 100644 index 00000000000..30991a12e4c --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/README.md @@ -0,0 +1,12 @@ +# hcsshim + +This package supports launching Windows Server containers from Go. It is +primarily used in the [Docker Engine](https://github.com/docker/docker) project, +but it can be freely used by other projects as well. + +This project has adopted the [Microsoft Open Source Code of +Conduct](https://opensource.microsoft.com/codeofconduct/). For more information +see the [Code of Conduct +FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact +[opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional +questions or comments. diff --git a/vendor/github.com/Microsoft/hcsshim/activatelayer.go b/vendor/github.com/Microsoft/hcsshim/activatelayer.go new file mode 100644 index 00000000000..6d824d7a79a --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/activatelayer.go @@ -0,0 +1,28 @@ +package hcsshim + +import "github.com/sirupsen/logrus" + +// ActivateLayer will find the layer with the given id and mount it's filesystem. +// For a read/write layer, the mounted filesystem will appear as a volume on the +// host, while a read-only layer is generally expected to be a no-op. +// An activated layer must later be deactivated via DeactivateLayer. +func ActivateLayer(info DriverInfo, id string) error { + title := "hcsshim::ActivateLayer " + logrus.Debugf(title+"Flavour %d ID %s", info.Flavour, id) + + infop, err := convertDriverInfo(info) + if err != nil { + logrus.Error(err) + return err + } + + err = activateLayer(&infop, id) + if err != nil { + err = makeErrorf(err, title, "id=%s flavour=%d", id, info.Flavour) + logrus.Error(err) + return err + } + + logrus.Debugf(title+" - succeeded id=%s flavour=%d", id, info.Flavour) + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/baselayer.go b/vendor/github.com/Microsoft/hcsshim/baselayer.go new file mode 100644 index 00000000000..9babd4e18ab --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/baselayer.go @@ -0,0 +1,183 @@ +package hcsshim + +import ( + "errors" + "os" + "path/filepath" + "syscall" + + "github.com/Microsoft/go-winio" +) + +type baseLayerWriter struct { + root string + f *os.File + bw *winio.BackupFileWriter + err error + hasUtilityVM bool + dirInfo []dirInfo +} + +type dirInfo struct { + path string + fileInfo winio.FileBasicInfo +} + +// reapplyDirectoryTimes reapplies directory modification, creation, etc. times +// after processing of the directory tree has completed. The times are expected +// to be ordered such that parent directories come before child directories. +func reapplyDirectoryTimes(dis []dirInfo) error { + for i := range dis { + di := &dis[len(dis)-i-1] // reverse order: process child directories first + f, err := winio.OpenForBackup(di.path, syscall.GENERIC_READ|syscall.GENERIC_WRITE, syscall.FILE_SHARE_READ, syscall.OPEN_EXISTING) + if err != nil { + return err + } + + err = winio.SetFileBasicInfo(f, &di.fileInfo) + f.Close() + if err != nil { + return err + } + } + return nil +} + +func (w *baseLayerWriter) closeCurrentFile() error { + if w.f != nil { + err := w.bw.Close() + err2 := w.f.Close() + w.f = nil + w.bw = nil + if err != nil { + return err + } + if err2 != nil { + return err2 + } + } + return nil +} + +func (w *baseLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) (err error) { + defer func() { + if err != nil { + w.err = err + } + }() + + err = w.closeCurrentFile() + if err != nil { + return err + } + + if filepath.ToSlash(name) == `UtilityVM/Files` { + w.hasUtilityVM = true + } + + path := filepath.Join(w.root, name) + path, err = makeLongAbsPath(path) + if err != nil { + return err + } + + var f *os.File + defer func() { + if f != nil { + f.Close() + } + }() + + createmode := uint32(syscall.CREATE_NEW) + if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { + err := os.Mkdir(path, 0) + if err != nil && !os.IsExist(err) { + return err + } + createmode = syscall.OPEN_EXISTING + if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT == 0 { + w.dirInfo = append(w.dirInfo, dirInfo{path, *fileInfo}) + } + } + + mode := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE | winio.WRITE_DAC | winio.WRITE_OWNER | winio.ACCESS_SYSTEM_SECURITY) + f, err = winio.OpenForBackup(path, mode, syscall.FILE_SHARE_READ, createmode) + if err != nil { + return makeError(err, "Failed to OpenForBackup", path) + } + + err = winio.SetFileBasicInfo(f, fileInfo) + if err != nil { + return makeError(err, "Failed to SetFileBasicInfo", path) + } + + w.f = f + w.bw = winio.NewBackupFileWriter(f, true) + f = nil + return nil +} + +func (w *baseLayerWriter) AddLink(name string, target string) (err error) { + defer func() { + if err != nil { + w.err = err + } + }() + + err = w.closeCurrentFile() + if err != nil { + return err + } + + linkpath, err := makeLongAbsPath(filepath.Join(w.root, name)) + if err != nil { + return err + } + + linktarget, err := makeLongAbsPath(filepath.Join(w.root, target)) + if err != nil { + return err + } + + return os.Link(linktarget, linkpath) +} + +func (w *baseLayerWriter) Remove(name string) error { + return errors.New("base layer cannot have tombstones") +} + +func (w *baseLayerWriter) Write(b []byte) (int, error) { + n, err := w.bw.Write(b) + if err != nil { + w.err = err + } + return n, err +} + +func (w *baseLayerWriter) Close() error { + err := w.closeCurrentFile() + if err != nil { + return err + } + if w.err == nil { + // Restore the file times of all the directories, since they may have + // been modified by creating child directories. + err = reapplyDirectoryTimes(w.dirInfo) + if err != nil { + return err + } + + err = ProcessBaseLayer(w.root) + if err != nil { + return err + } + + if w.hasUtilityVM { + err = ProcessUtilityVMImage(filepath.Join(w.root, "UtilityVM")) + if err != nil { + return err + } + } + } + return w.err +} diff --git a/vendor/github.com/Microsoft/hcsshim/callback.go b/vendor/github.com/Microsoft/hcsshim/callback.go new file mode 100644 index 00000000000..e8c2b00c8ad --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/callback.go @@ -0,0 +1,79 @@ +package hcsshim + +import ( + "sync" + "syscall" +) + +var ( + nextCallback uintptr + callbackMap = map[uintptr]*notifcationWatcherContext{} + callbackMapLock = sync.RWMutex{} + + notificationWatcherCallback = syscall.NewCallback(notificationWatcher) + + // Notifications for HCS_SYSTEM handles + hcsNotificationSystemExited hcsNotification = 0x00000001 + hcsNotificationSystemCreateCompleted hcsNotification = 0x00000002 + hcsNotificationSystemStartCompleted hcsNotification = 0x00000003 + hcsNotificationSystemPauseCompleted hcsNotification = 0x00000004 + hcsNotificationSystemResumeCompleted hcsNotification = 0x00000005 + + // Notifications for HCS_PROCESS handles + hcsNotificationProcessExited hcsNotification = 0x00010000 + + // Common notifications + hcsNotificationInvalid hcsNotification = 0x00000000 + hcsNotificationServiceDisconnect hcsNotification = 0x01000000 +) + +type hcsNotification uint32 +type notificationChannel chan error + +type notifcationWatcherContext struct { + channels notificationChannels + handle hcsCallback +} + +type notificationChannels map[hcsNotification]notificationChannel + +func newChannels() notificationChannels { + channels := make(notificationChannels) + + channels[hcsNotificationSystemExited] = make(notificationChannel, 1) + channels[hcsNotificationSystemCreateCompleted] = make(notificationChannel, 1) + channels[hcsNotificationSystemStartCompleted] = make(notificationChannel, 1) + channels[hcsNotificationSystemPauseCompleted] = make(notificationChannel, 1) + channels[hcsNotificationSystemResumeCompleted] = make(notificationChannel, 1) + channels[hcsNotificationProcessExited] = make(notificationChannel, 1) + channels[hcsNotificationServiceDisconnect] = make(notificationChannel, 1) + return channels +} +func closeChannels(channels notificationChannels) { + close(channels[hcsNotificationSystemExited]) + close(channels[hcsNotificationSystemCreateCompleted]) + close(channels[hcsNotificationSystemStartCompleted]) + close(channels[hcsNotificationSystemPauseCompleted]) + close(channels[hcsNotificationSystemResumeCompleted]) + close(channels[hcsNotificationProcessExited]) + close(channels[hcsNotificationServiceDisconnect]) +} + +func notificationWatcher(notificationType hcsNotification, callbackNumber uintptr, notificationStatus uintptr, notificationData *uint16) uintptr { + var result error + if int32(notificationStatus) < 0 { + result = syscall.Errno(win32FromHresult(notificationStatus)) + } + + callbackMapLock.RLock() + context := callbackMap[callbackNumber] + callbackMapLock.RUnlock() + + if context == nil { + return 0 + } + + context.channels[notificationType] <- result + + return 0 +} diff --git a/vendor/github.com/Microsoft/hcsshim/cgo.go b/vendor/github.com/Microsoft/hcsshim/cgo.go new file mode 100644 index 00000000000..2003332330a --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/cgo.go @@ -0,0 +1,7 @@ +package hcsshim + +import "C" + +// This import is needed to make the library compile as CGO because HCSSHIM +// only works with CGO due to callbacks from HCS comming back from a C thread +// which is not supported without CGO. See https://github.com/golang/go/issues/10973 diff --git a/vendor/github.com/Microsoft/hcsshim/container.go b/vendor/github.com/Microsoft/hcsshim/container.go new file mode 100644 index 00000000000..b924d39f46b --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/container.go @@ -0,0 +1,794 @@ +package hcsshim + +import ( + "encoding/json" + "fmt" + "os" + "sync" + "syscall" + "time" + + "github.com/sirupsen/logrus" +) + +var ( + defaultTimeout = time.Minute * 4 +) + +const ( + pendingUpdatesQuery = `{ "PropertyTypes" : ["PendingUpdates"]}` + statisticsQuery = `{ "PropertyTypes" : ["Statistics"]}` + processListQuery = `{ "PropertyTypes" : ["ProcessList"]}` + mappedVirtualDiskQuery = `{ "PropertyTypes" : ["MappedVirtualDisk"]}` +) + +type container struct { + handleLock sync.RWMutex + handle hcsSystem + id string + callbackNumber uintptr +} + +// ContainerProperties holds the properties for a container and the processes running in that container +type ContainerProperties struct { + ID string `json:"Id"` + Name string + SystemType string + Owner string + SiloGUID string `json:"SiloGuid,omitempty"` + RuntimeID string `json:"RuntimeId,omitempty"` + IsRuntimeTemplate bool `json:",omitempty"` + RuntimeImagePath string `json:",omitempty"` + Stopped bool `json:",omitempty"` + ExitType string `json:",omitempty"` + AreUpdatesPending bool `json:",omitempty"` + ObRoot string `json:",omitempty"` + Statistics Statistics `json:",omitempty"` + ProcessList []ProcessListItem `json:",omitempty"` + MappedVirtualDiskControllers map[int]MappedVirtualDiskController `json:",omitempty"` +} + +// MemoryStats holds the memory statistics for a container +type MemoryStats struct { + UsageCommitBytes uint64 `json:"MemoryUsageCommitBytes,omitempty"` + UsageCommitPeakBytes uint64 `json:"MemoryUsageCommitPeakBytes,omitempty"` + UsagePrivateWorkingSetBytes uint64 `json:"MemoryUsagePrivateWorkingSetBytes,omitempty"` +} + +// ProcessorStats holds the processor statistics for a container +type ProcessorStats struct { + TotalRuntime100ns uint64 `json:",omitempty"` + RuntimeUser100ns uint64 `json:",omitempty"` + RuntimeKernel100ns uint64 `json:",omitempty"` +} + +// StorageStats holds the storage statistics for a container +type StorageStats struct { + ReadCountNormalized uint64 `json:",omitempty"` + ReadSizeBytes uint64 `json:",omitempty"` + WriteCountNormalized uint64 `json:",omitempty"` + WriteSizeBytes uint64 `json:",omitempty"` +} + +// NetworkStats holds the network statistics for a container +type NetworkStats struct { + BytesReceived uint64 `json:",omitempty"` + BytesSent uint64 `json:",omitempty"` + PacketsReceived uint64 `json:",omitempty"` + PacketsSent uint64 `json:",omitempty"` + DroppedPacketsIncoming uint64 `json:",omitempty"` + DroppedPacketsOutgoing uint64 `json:",omitempty"` + EndpointId string `json:",omitempty"` + InstanceId string `json:",omitempty"` +} + +// Statistics is the structure returned by a statistics call on a container +type Statistics struct { + Timestamp time.Time `json:",omitempty"` + ContainerStartTime time.Time `json:",omitempty"` + Uptime100ns uint64 `json:",omitempty"` + Memory MemoryStats `json:",omitempty"` + Processor ProcessorStats `json:",omitempty"` + Storage StorageStats `json:",omitempty"` + Network []NetworkStats `json:",omitempty"` +} + +// ProcessList is the structure of an item returned by a ProcessList call on a container +type ProcessListItem struct { + CreateTimestamp time.Time `json:",omitempty"` + ImageName string `json:",omitempty"` + KernelTime100ns uint64 `json:",omitempty"` + MemoryCommitBytes uint64 `json:",omitempty"` + MemoryWorkingSetPrivateBytes uint64 `json:",omitempty"` + MemoryWorkingSetSharedBytes uint64 `json:",omitempty"` + ProcessId uint32 `json:",omitempty"` + UserTime100ns uint64 `json:",omitempty"` +} + +// MappedVirtualDiskController is the structure of an item returned by a MappedVirtualDiskList call on a container +type MappedVirtualDiskController struct { + MappedVirtualDisks map[int]MappedVirtualDisk `json:",omitempty"` +} + +// Type of Request Support in ModifySystem +type RequestType string + +// Type of Resource Support in ModifySystem +type ResourceType string + +// RequestType const +const ( + Add RequestType = "Add" + Remove RequestType = "Remove" + Network ResourceType = "Network" +) + +// ResourceModificationRequestResponse is the structure used to send request to the container to modify the system +// Supported resource types are Network and Request Types are Add/Remove +type ResourceModificationRequestResponse struct { + Resource ResourceType `json:"ResourceType"` + Data interface{} `json:"Settings"` + Request RequestType `json:"RequestType,omitempty"` +} + +// createContainerAdditionalJSON is read from the environment at initialisation +// time. It allows an environment variable to define additional JSON which +// is merged in the CreateContainer call to HCS. +var createContainerAdditionalJSON string + +func init() { + createContainerAdditionalJSON = os.Getenv("HCSSHIM_CREATECONTAINER_ADDITIONALJSON") +} + +// CreateContainer creates a new container with the given configuration but does not start it. +func CreateContainer(id string, c *ContainerConfig) (Container, error) { + return createContainerWithJSON(id, c, "") +} + +// CreateContainerWithJSON creates a new container with the given configuration but does not start it. +// It is identical to CreateContainer except that optional additional JSON can be merged before passing to HCS. +func CreateContainerWithJSON(id string, c *ContainerConfig, additionalJSON string) (Container, error) { + return createContainerWithJSON(id, c, additionalJSON) +} + +func createContainerWithJSON(id string, c *ContainerConfig, additionalJSON string) (Container, error) { + operation := "CreateContainer" + title := "HCSShim::" + operation + + container := &container{ + id: id, + } + + configurationb, err := json.Marshal(c) + if err != nil { + return nil, err + } + + configuration := string(configurationb) + logrus.Debugf(title+" id=%s config=%s", id, configuration) + + // Merge any additional JSON. Priority is given to what is passed in explicitly, + // falling back to what's set in the environment. + if additionalJSON == "" && createContainerAdditionalJSON != "" { + additionalJSON = createContainerAdditionalJSON + } + if additionalJSON != "" { + configurationMap := map[string]interface{}{} + if err := json.Unmarshal([]byte(configuration), &configurationMap); err != nil { + return nil, fmt.Errorf("failed to unmarshal %s: %s", configuration, err) + } + + additionalMap := map[string]interface{}{} + if err := json.Unmarshal([]byte(additionalJSON), &additionalMap); err != nil { + return nil, fmt.Errorf("failed to unmarshal %s: %s", additionalJSON, err) + } + + mergedMap := mergeMaps(additionalMap, configurationMap) + mergedJSON, err := json.Marshal(mergedMap) + if err != nil { + return nil, fmt.Errorf("failed to marshal merged configuration map %+v: %s", mergedMap, err) + } + + configuration = string(mergedJSON) + logrus.Debugf(title+" id=%s merged config=%s", id, configuration) + } + + var ( + resultp *uint16 + identity syscall.Handle + ) + createError := hcsCreateComputeSystem(id, configuration, identity, &container.handle, &resultp) + + if createError == nil || IsPending(createError) { + if err := container.registerCallback(); err != nil { + return nil, makeContainerError(container, operation, "", err) + } + } + + err = processAsyncHcsResult(createError, resultp, container.callbackNumber, hcsNotificationSystemCreateCompleted, &defaultTimeout) + if err != nil { + return nil, makeContainerError(container, operation, configuration, err) + } + + logrus.Debugf(title+" succeeded id=%s handle=%d", id, container.handle) + return container, nil +} + +// mergeMaps recursively merges map `fromMap` into map `ToMap`. Any pre-existing values +// in ToMap are overwritten. Values in fromMap are added to ToMap. +// From http://stackoverflow.com/questions/40491438/merging-two-json-strings-in-golang +func mergeMaps(fromMap, ToMap interface{}) interface{} { + switch fromMap := fromMap.(type) { + case map[string]interface{}: + ToMap, ok := ToMap.(map[string]interface{}) + if !ok { + return fromMap + } + for keyToMap, valueToMap := range ToMap { + if valueFromMap, ok := fromMap[keyToMap]; ok { + fromMap[keyToMap] = mergeMaps(valueFromMap, valueToMap) + } else { + fromMap[keyToMap] = valueToMap + } + } + case nil: + // merge(nil, map[string]interface{...}) -> map[string]interface{...} + ToMap, ok := ToMap.(map[string]interface{}) + if ok { + return ToMap + } + } + return fromMap +} + +// OpenContainer opens an existing container by ID. +func OpenContainer(id string) (Container, error) { + operation := "OpenContainer" + title := "HCSShim::" + operation + logrus.Debugf(title+" id=%s", id) + + container := &container{ + id: id, + } + + var ( + handle hcsSystem + resultp *uint16 + ) + err := hcsOpenComputeSystem(id, &handle, &resultp) + err = processHcsResult(err, resultp) + if err != nil { + return nil, makeContainerError(container, operation, "", err) + } + + container.handle = handle + + if err := container.registerCallback(); err != nil { + return nil, makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s handle=%d", id, handle) + return container, nil +} + +// GetContainers gets a list of the containers on the system that match the query +func GetContainers(q ComputeSystemQuery) ([]ContainerProperties, error) { + operation := "GetContainers" + title := "HCSShim::" + operation + + queryb, err := json.Marshal(q) + if err != nil { + return nil, err + } + + query := string(queryb) + logrus.Debugf(title+" query=%s", query) + + var ( + resultp *uint16 + computeSystemsp *uint16 + ) + err = hcsEnumerateComputeSystems(query, &computeSystemsp, &resultp) + err = processHcsResult(err, resultp) + if err != nil { + return nil, err + } + + if computeSystemsp == nil { + return nil, ErrUnexpectedValue + } + computeSystemsRaw := convertAndFreeCoTaskMemBytes(computeSystemsp) + computeSystems := []ContainerProperties{} + if err := json.Unmarshal(computeSystemsRaw, &computeSystems); err != nil { + return nil, err + } + + logrus.Debugf(title + " succeeded") + return computeSystems, nil +} + +// Start synchronously starts the container. +func (container *container) Start() error { + container.handleLock.RLock() + defer container.handleLock.RUnlock() + operation := "Start" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s", container.id) + + if container.handle == 0 { + return makeContainerError(container, operation, "", ErrAlreadyClosed) + } + + var resultp *uint16 + err := hcsStartComputeSystem(container.handle, "", &resultp) + err = processAsyncHcsResult(err, resultp, container.callbackNumber, hcsNotificationSystemStartCompleted, &defaultTimeout) + if err != nil { + return makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s", container.id) + return nil +} + +// Shutdown requests a container shutdown, if IsPending() on the error returned is true, +// it may not actually be shut down until Wait() succeeds. +func (container *container) Shutdown() error { + container.handleLock.RLock() + defer container.handleLock.RUnlock() + operation := "Shutdown" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s", container.id) + + if container.handle == 0 { + return makeContainerError(container, operation, "", ErrAlreadyClosed) + } + + var resultp *uint16 + err := hcsShutdownComputeSystem(container.handle, "", &resultp) + err = processHcsResult(err, resultp) + if err != nil { + return makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s", container.id) + return nil +} + +// Terminate requests a container terminate, if IsPending() on the error returned is true, +// it may not actually be shut down until Wait() succeeds. +func (container *container) Terminate() error { + container.handleLock.RLock() + defer container.handleLock.RUnlock() + operation := "Terminate" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s", container.id) + + if container.handle == 0 { + return makeContainerError(container, operation, "", ErrAlreadyClosed) + } + + var resultp *uint16 + err := hcsTerminateComputeSystem(container.handle, "", &resultp) + err = processHcsResult(err, resultp) + if err != nil { + return makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s", container.id) + return nil +} + +// Wait synchronously waits for the container to shutdown or terminate. +func (container *container) Wait() error { + operation := "Wait" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s", container.id) + + err := waitForNotification(container.callbackNumber, hcsNotificationSystemExited, nil) + if err != nil { + return makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s", container.id) + return nil +} + +// WaitTimeout synchronously waits for the container to terminate or the duration to elapse. +// If the timeout expires, IsTimeout(err) == true +func (container *container) WaitTimeout(timeout time.Duration) error { + operation := "WaitTimeout" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s", container.id) + + err := waitForNotification(container.callbackNumber, hcsNotificationSystemExited, &timeout) + if err != nil { + return makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s", container.id) + return nil +} + +func (container *container) properties(query string) (*ContainerProperties, error) { + var ( + resultp *uint16 + propertiesp *uint16 + ) + err := hcsGetComputeSystemProperties(container.handle, query, &propertiesp, &resultp) + err = processHcsResult(err, resultp) + if err != nil { + return nil, err + } + + if propertiesp == nil { + return nil, ErrUnexpectedValue + } + propertiesRaw := convertAndFreeCoTaskMemBytes(propertiesp) + properties := &ContainerProperties{} + if err := json.Unmarshal(propertiesRaw, properties); err != nil { + return nil, err + } + return properties, nil +} + +// HasPendingUpdates returns true if the container has updates pending to install +func (container *container) HasPendingUpdates() (bool, error) { + container.handleLock.RLock() + defer container.handleLock.RUnlock() + operation := "HasPendingUpdates" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s", container.id) + + if container.handle == 0 { + return false, makeContainerError(container, operation, "", ErrAlreadyClosed) + } + + properties, err := container.properties(pendingUpdatesQuery) + if err != nil { + return false, makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s", container.id) + return properties.AreUpdatesPending, nil +} + +// Statistics returns statistics for the container +func (container *container) Statistics() (Statistics, error) { + container.handleLock.RLock() + defer container.handleLock.RUnlock() + operation := "Statistics" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s", container.id) + + if container.handle == 0 { + return Statistics{}, makeContainerError(container, operation, "", ErrAlreadyClosed) + } + + properties, err := container.properties(statisticsQuery) + if err != nil { + return Statistics{}, makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s", container.id) + return properties.Statistics, nil +} + +// ProcessList returns an array of ProcessListItems for the container +func (container *container) ProcessList() ([]ProcessListItem, error) { + container.handleLock.RLock() + defer container.handleLock.RUnlock() + operation := "ProcessList" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s", container.id) + + if container.handle == 0 { + return nil, makeContainerError(container, operation, "", ErrAlreadyClosed) + } + + properties, err := container.properties(processListQuery) + if err != nil { + return nil, makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s", container.id) + return properties.ProcessList, nil +} + +// MappedVirtualDisks returns a map of the controllers and the disks mapped +// to a container. +// +// Example of JSON returned by the query. +//{ +// "Id":"1126e8d7d279c707a666972a15976371d365eaf622c02cea2c442b84f6f550a3_svm", +// "SystemType":"Container", +// "RuntimeOsType":"Linux", +// "RuntimeId":"00000000-0000-0000-0000-000000000000", +// "State":"Running", +// "MappedVirtualDiskControllers":{ +// "0":{ +// "MappedVirtualDisks":{ +// "2":{ +// "HostPath":"C:\\lcow\\lcow\\scratch\\1126e8d7d279c707a666972a15976371d365eaf622c02cea2c442b84f6f550a3.vhdx", +// "ContainerPath":"/mnt/gcs/LinuxServiceVM/scratch", +// "Lun":2, +// "CreateInUtilityVM":true +// }, +// "3":{ +// "HostPath":"C:\\lcow\\lcow\\1126e8d7d279c707a666972a15976371d365eaf622c02cea2c442b84f6f550a3\\sandbox.vhdx", +// "Lun":3, +// "CreateInUtilityVM":true, +// "AttachOnly":true +// } +// } +// } +// } +//} +func (container *container) MappedVirtualDisks() (map[int]MappedVirtualDiskController, error) { + container.handleLock.RLock() + defer container.handleLock.RUnlock() + operation := "MappedVirtualDiskList" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s", container.id) + + if container.handle == 0 { + return nil, makeContainerError(container, operation, "", ErrAlreadyClosed) + } + + properties, err := container.properties(mappedVirtualDiskQuery) + if err != nil { + return nil, makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s", container.id) + return properties.MappedVirtualDiskControllers, nil +} + +// Pause pauses the execution of the container. This feature is not enabled in TP5. +func (container *container) Pause() error { + container.handleLock.RLock() + defer container.handleLock.RUnlock() + operation := "Pause" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s", container.id) + + if container.handle == 0 { + return makeContainerError(container, operation, "", ErrAlreadyClosed) + } + + var resultp *uint16 + err := hcsPauseComputeSystem(container.handle, "", &resultp) + err = processAsyncHcsResult(err, resultp, container.callbackNumber, hcsNotificationSystemPauseCompleted, &defaultTimeout) + if err != nil { + return makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s", container.id) + return nil +} + +// Resume resumes the execution of the container. This feature is not enabled in TP5. +func (container *container) Resume() error { + container.handleLock.RLock() + defer container.handleLock.RUnlock() + operation := "Resume" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s", container.id) + + if container.handle == 0 { + return makeContainerError(container, operation, "", ErrAlreadyClosed) + } + + var resultp *uint16 + err := hcsResumeComputeSystem(container.handle, "", &resultp) + err = processAsyncHcsResult(err, resultp, container.callbackNumber, hcsNotificationSystemResumeCompleted, &defaultTimeout) + if err != nil { + return makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s", container.id) + return nil +} + +// CreateProcess launches a new process within the container. +func (container *container) CreateProcess(c *ProcessConfig) (Process, error) { + container.handleLock.RLock() + defer container.handleLock.RUnlock() + operation := "CreateProcess" + title := "HCSShim::Container::" + operation + var ( + processInfo hcsProcessInformation + processHandle hcsProcess + resultp *uint16 + ) + + if container.handle == 0 { + return nil, makeContainerError(container, operation, "", ErrAlreadyClosed) + } + + // If we are not emulating a console, ignore any console size passed to us + if !c.EmulateConsole { + c.ConsoleSize[0] = 0 + c.ConsoleSize[1] = 0 + } + + configurationb, err := json.Marshal(c) + if err != nil { + return nil, makeContainerError(container, operation, "", err) + } + + configuration := string(configurationb) + logrus.Debugf(title+" id=%s config=%s", container.id, configuration) + + err = hcsCreateProcess(container.handle, configuration, &processInfo, &processHandle, &resultp) + err = processHcsResult(err, resultp) + if err != nil { + return nil, makeContainerError(container, operation, configuration, err) + } + + process := &process{ + handle: processHandle, + processID: int(processInfo.ProcessId), + container: container, + cachedPipes: &cachedPipes{ + stdIn: processInfo.StdInput, + stdOut: processInfo.StdOutput, + stdErr: processInfo.StdError, + }, + } + + if err := process.registerCallback(); err != nil { + return nil, makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s processid=%d", container.id, process.processID) + return process, nil +} + +// OpenProcess gets an interface to an existing process within the container. +func (container *container) OpenProcess(pid int) (Process, error) { + container.handleLock.RLock() + defer container.handleLock.RUnlock() + operation := "OpenProcess" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s, processid=%d", container.id, pid) + var ( + processHandle hcsProcess + resultp *uint16 + ) + + if container.handle == 0 { + return nil, makeContainerError(container, operation, "", ErrAlreadyClosed) + } + + err := hcsOpenProcess(container.handle, uint32(pid), &processHandle, &resultp) + err = processHcsResult(err, resultp) + if err != nil { + return nil, makeContainerError(container, operation, "", err) + } + + process := &process{ + handle: processHandle, + processID: pid, + container: container, + } + + if err := process.registerCallback(); err != nil { + return nil, makeContainerError(container, operation, "", err) + } + + logrus.Debugf(title+" succeeded id=%s processid=%s", container.id, process.processID) + return process, nil +} + +// Close cleans up any state associated with the container but does not terminate or wait for it. +func (container *container) Close() error { + container.handleLock.Lock() + defer container.handleLock.Unlock() + operation := "Close" + title := "HCSShim::Container::" + operation + logrus.Debugf(title+" id=%s", container.id) + + // Don't double free this + if container.handle == 0 { + return nil + } + + if err := container.unregisterCallback(); err != nil { + return makeContainerError(container, operation, "", err) + } + + if err := hcsCloseComputeSystem(container.handle); err != nil { + return makeContainerError(container, operation, "", err) + } + + container.handle = 0 + + logrus.Debugf(title+" succeeded id=%s", container.id) + return nil +} + +func (container *container) registerCallback() error { + context := ¬ifcationWatcherContext{ + channels: newChannels(), + } + + callbackMapLock.Lock() + callbackNumber := nextCallback + nextCallback++ + callbackMap[callbackNumber] = context + callbackMapLock.Unlock() + + var callbackHandle hcsCallback + err := hcsRegisterComputeSystemCallback(container.handle, notificationWatcherCallback, callbackNumber, &callbackHandle) + if err != nil { + return err + } + context.handle = callbackHandle + container.callbackNumber = callbackNumber + + return nil +} + +func (container *container) unregisterCallback() error { + callbackNumber := container.callbackNumber + + callbackMapLock.RLock() + context := callbackMap[callbackNumber] + callbackMapLock.RUnlock() + + if context == nil { + return nil + } + + handle := context.handle + + if handle == 0 { + return nil + } + + // hcsUnregisterComputeSystemCallback has its own syncronization + // to wait for all callbacks to complete. We must NOT hold the callbackMapLock. + err := hcsUnregisterComputeSystemCallback(handle) + if err != nil { + return err + } + + closeChannels(context.channels) + + callbackMapLock.Lock() + callbackMap[callbackNumber] = nil + callbackMapLock.Unlock() + + handle = 0 + + return nil +} + +// Modifies the System by sending a request to HCS +func (container *container) Modify(config *ResourceModificationRequestResponse) error { + container.handleLock.RLock() + defer container.handleLock.RUnlock() + operation := "Modify" + title := "HCSShim::Container::" + operation + + if container.handle == 0 { + return makeContainerError(container, operation, "", ErrAlreadyClosed) + } + + requestJSON, err := json.Marshal(config) + if err != nil { + return err + } + + requestString := string(requestJSON) + logrus.Debugf(title+" id=%s request=%s", container.id, requestString) + + var resultp *uint16 + err = hcsModifyComputeSystem(container.handle, requestString, &resultp) + err = processHcsResult(err, resultp) + if err != nil { + return makeContainerError(container, operation, "", err) + } + logrus.Debugf(title+" succeeded id=%s", container.id) + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/createlayer.go b/vendor/github.com/Microsoft/hcsshim/createlayer.go new file mode 100644 index 00000000000..035d9c3947c --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/createlayer.go @@ -0,0 +1,27 @@ +package hcsshim + +import "github.com/sirupsen/logrus" + +// CreateLayer creates a new, empty, read-only layer on the filesystem based on +// the parent layer provided. +func CreateLayer(info DriverInfo, id, parent string) error { + title := "hcsshim::CreateLayer " + logrus.Debugf(title+"Flavour %d ID %s parent %s", info.Flavour, id, parent) + + // Convert info to API calling convention + infop, err := convertDriverInfo(info) + if err != nil { + logrus.Error(err) + return err + } + + err = createLayer(&infop, id, parent) + if err != nil { + err = makeErrorf(err, title, "id=%s parent=%s flavour=%d", id, parent, info.Flavour) + logrus.Error(err) + return err + } + + logrus.Debugf(title+" - succeeded id=%s parent=%s flavour=%d", id, parent, info.Flavour) + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/createsandboxlayer.go b/vendor/github.com/Microsoft/hcsshim/createsandboxlayer.go new file mode 100644 index 00000000000..7a6a8854cf9 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/createsandboxlayer.go @@ -0,0 +1,35 @@ +package hcsshim + +import "github.com/sirupsen/logrus" + +// CreateSandboxLayer creates and populates new read-write layer for use by a container. +// This requires both the id of the direct parent layer, as well as the full list +// of paths to all parent layers up to the base (and including the direct parent +// whose id was provided). +func CreateSandboxLayer(info DriverInfo, layerId, parentId string, parentLayerPaths []string) error { + title := "hcsshim::CreateSandboxLayer " + logrus.Debugf(title+"layerId %s parentId %s", layerId, parentId) + + // Generate layer descriptors + layers, err := layerPathsToDescriptors(parentLayerPaths) + if err != nil { + return err + } + + // Convert info to API calling convention + infop, err := convertDriverInfo(info) + if err != nil { + logrus.Error(err) + return err + } + + err = createSandboxLayer(&infop, layerId, parentId, layers) + if err != nil { + err = makeErrorf(err, title, "layerId=%s parentId=%s", layerId, parentId) + logrus.Error(err) + return err + } + + logrus.Debugf(title+"- succeeded layerId=%s parentId=%s", layerId, parentId) + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/deactivatelayer.go b/vendor/github.com/Microsoft/hcsshim/deactivatelayer.go new file mode 100644 index 00000000000..fd785030fba --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/deactivatelayer.go @@ -0,0 +1,26 @@ +package hcsshim + +import "github.com/sirupsen/logrus" + +// DeactivateLayer will dismount a layer that was mounted via ActivateLayer. +func DeactivateLayer(info DriverInfo, id string) error { + title := "hcsshim::DeactivateLayer " + logrus.Debugf(title+"Flavour %d ID %s", info.Flavour, id) + + // Convert info to API calling convention + infop, err := convertDriverInfo(info) + if err != nil { + logrus.Error(err) + return err + } + + err = deactivateLayer(&infop, id) + if err != nil { + err = makeErrorf(err, title, "id=%s flavour=%d", id, info.Flavour) + logrus.Error(err) + return err + } + + logrus.Debugf(title+"succeeded flavour=%d id=%s", info.Flavour, id) + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/destroylayer.go b/vendor/github.com/Microsoft/hcsshim/destroylayer.go new file mode 100644 index 00000000000..b1e3b89fc70 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/destroylayer.go @@ -0,0 +1,27 @@ +package hcsshim + +import "github.com/sirupsen/logrus" + +// DestroyLayer will remove the on-disk files representing the layer with the given +// id, including that layer's containing folder, if any. +func DestroyLayer(info DriverInfo, id string) error { + title := "hcsshim::DestroyLayer " + logrus.Debugf(title+"Flavour %d ID %s", info.Flavour, id) + + // Convert info to API calling convention + infop, err := convertDriverInfo(info) + if err != nil { + logrus.Error(err) + return err + } + + err = destroyLayer(&infop, id) + if err != nil { + err = makeErrorf(err, title, "id=%s flavour=%d", id, info.Flavour) + logrus.Error(err) + return err + } + + logrus.Debugf(title+"succeeded flavour=%d id=%s", info.Flavour, id) + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/errors.go b/vendor/github.com/Microsoft/hcsshim/errors.go new file mode 100644 index 00000000000..d2f9cc8bd20 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/errors.go @@ -0,0 +1,239 @@ +package hcsshim + +import ( + "errors" + "fmt" + "syscall" +) + +var ( + // ErrComputeSystemDoesNotExist is an error encountered when the container being operated on no longer exists + ErrComputeSystemDoesNotExist = syscall.Errno(0xc037010e) + + // ErrElementNotFound is an error encountered when the object being referenced does not exist + ErrElementNotFound = syscall.Errno(0x490) + + // ErrElementNotFound is an error encountered when the object being referenced does not exist + ErrNotSupported = syscall.Errno(0x32) + + // ErrInvalidData is an error encountered when the request being sent to hcs is invalid/unsupported + // decimal -2147024883 / hex 0x8007000d + ErrInvalidData = syscall.Errno(0xd) + + // ErrHandleClose is an error encountered when the handle generating the notification being waited on has been closed + ErrHandleClose = errors.New("hcsshim: the handle generating this notification has been closed") + + // ErrAlreadyClosed is an error encountered when using a handle that has been closed by the Close method + ErrAlreadyClosed = errors.New("hcsshim: the handle has already been closed") + + // ErrInvalidNotificationType is an error encountered when an invalid notification type is used + ErrInvalidNotificationType = errors.New("hcsshim: invalid notification type") + + // ErrInvalidProcessState is an error encountered when the process is not in a valid state for the requested operation + ErrInvalidProcessState = errors.New("the process is in an invalid state for the attempted operation") + + // ErrTimeout is an error encountered when waiting on a notification times out + ErrTimeout = errors.New("hcsshim: timeout waiting for notification") + + // ErrUnexpectedContainerExit is the error encountered when a container exits while waiting for + // a different expected notification + ErrUnexpectedContainerExit = errors.New("unexpected container exit") + + // ErrUnexpectedProcessAbort is the error encountered when communication with the compute service + // is lost while waiting for a notification + ErrUnexpectedProcessAbort = errors.New("lost communication with compute service") + + // ErrUnexpectedValue is an error encountered when hcs returns an invalid value + ErrUnexpectedValue = errors.New("unexpected value returned from hcs") + + // ErrVmcomputeAlreadyStopped is an error encountered when a shutdown or terminate request is made on a stopped container + ErrVmcomputeAlreadyStopped = syscall.Errno(0xc0370110) + + // ErrVmcomputeOperationPending is an error encountered when the operation is being completed asynchronously + ErrVmcomputeOperationPending = syscall.Errno(0xC0370103) + + // ErrVmcomputeOperationInvalidState is an error encountered when the compute system is not in a valid state for the requested operation + ErrVmcomputeOperationInvalidState = syscall.Errno(0xc0370105) + + // ErrProcNotFound is an error encountered when the the process cannot be found + ErrProcNotFound = syscall.Errno(0x7f) + + // ErrVmcomputeOperationAccessIsDenied is an error which can be encountered when enumerating compute systems in RS1/RS2 + // builds when the underlying silo might be in the process of terminating. HCS was fixed in RS3. + ErrVmcomputeOperationAccessIsDenied = syscall.Errno(0x5) + + // ErrVmcomputeInvalidJSON is an error encountered when the compute system does not support/understand the messages sent by management + ErrVmcomputeInvalidJSON = syscall.Errno(0xc037010d) + + // ErrVmcomputeUnknownMessage is an error encountered guest compute system doesn't support the message + ErrVmcomputeUnknownMessage = syscall.Errno(0xc037010b) + + // ErrNotSupported is an error encountered when hcs doesn't support the request + ErrPlatformNotSupported = errors.New("unsupported platform request") +) + +// ProcessError is an error encountered in HCS during an operation on a Process object +type ProcessError struct { + Process *process + Operation string + ExtraInfo string + Err error +} + +// ContainerError is an error encountered in HCS during an operation on a Container object +type ContainerError struct { + Container *container + Operation string + ExtraInfo string + Err error +} + +func (e *ContainerError) Error() string { + if e == nil { + return "" + } + + if e.Container == nil { + return "unexpected nil container for error: " + e.Err.Error() + } + + s := "container " + e.Container.id + + if e.Operation != "" { + s += " encountered an error during " + e.Operation + } + + switch e.Err.(type) { + case nil: + break + case syscall.Errno: + s += fmt.Sprintf(": failure in a Windows system call: %s (0x%x)", e.Err, win32FromError(e.Err)) + default: + s += fmt.Sprintf(": %s", e.Err.Error()) + } + + if e.ExtraInfo != "" { + s += " extra info: " + e.ExtraInfo + } + + return s +} + +func makeContainerError(container *container, operation string, extraInfo string, err error) error { + // Don't double wrap errors + if _, ok := err.(*ContainerError); ok { + return err + } + containerError := &ContainerError{Container: container, Operation: operation, ExtraInfo: extraInfo, Err: err} + return containerError +} + +func (e *ProcessError) Error() string { + if e == nil { + return "" + } + + if e.Process == nil { + return "Unexpected nil process for error: " + e.Err.Error() + } + + s := fmt.Sprintf("process %d", e.Process.processID) + + if e.Process.container != nil { + s += " in container " + e.Process.container.id + } + + if e.Operation != "" { + s += " encountered an error during " + e.Operation + } + + switch e.Err.(type) { + case nil: + break + case syscall.Errno: + s += fmt.Sprintf(": failure in a Windows system call: %s (0x%x)", e.Err, win32FromError(e.Err)) + default: + s += fmt.Sprintf(": %s", e.Err.Error()) + } + + return s +} + +func makeProcessError(process *process, operation string, extraInfo string, err error) error { + // Don't double wrap errors + if _, ok := err.(*ProcessError); ok { + return err + } + processError := &ProcessError{Process: process, Operation: operation, ExtraInfo: extraInfo, Err: err} + return processError +} + +// IsNotExist checks if an error is caused by the Container or Process not existing. +// Note: Currently, ErrElementNotFound can mean that a Process has either +// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist +// will currently return true when the error is ErrElementNotFound or ErrProcNotFound. +func IsNotExist(err error) bool { + err = getInnerError(err) + return err == ErrComputeSystemDoesNotExist || + err == ErrElementNotFound || + err == ErrProcNotFound +} + +// IsAlreadyClosed checks if an error is caused by the Container or Process having been +// already closed by a call to the Close() method. +func IsAlreadyClosed(err error) bool { + err = getInnerError(err) + return err == ErrAlreadyClosed +} + +// IsPending returns a boolean indicating whether the error is that +// the requested operation is being completed in the background. +func IsPending(err error) bool { + err = getInnerError(err) + return err == ErrVmcomputeOperationPending +} + +// IsTimeout returns a boolean indicating whether the error is caused by +// a timeout waiting for the operation to complete. +func IsTimeout(err error) bool { + err = getInnerError(err) + return err == ErrTimeout +} + +// IsAlreadyStopped returns a boolean indicating whether the error is caused by +// a Container or Process being already stopped. +// Note: Currently, ErrElementNotFound can mean that a Process has either +// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist +// will currently return true when the error is ErrElementNotFound or ErrProcNotFound. +func IsAlreadyStopped(err error) bool { + err = getInnerError(err) + return err == ErrVmcomputeAlreadyStopped || + err == ErrElementNotFound || + err == ErrProcNotFound +} + +// IsNotSupported returns a boolean indicating whether the error is caused by +// unsupported platform requests +// Note: Currently Unsupported platform requests can be mean either +// ErrVmcomputeInvalidJSON, ErrInvalidData, ErrNotSupported or ErrVmcomputeUnknownMessage +// is thrown from the Platform +func IsNotSupported(err error) bool { + err = getInnerError(err) + // If Platform doesn't recognize or support the request sent, below errors are seen + return err == ErrVmcomputeInvalidJSON || + err == ErrInvalidData || + err == ErrNotSupported || + err == ErrVmcomputeUnknownMessage +} + +func getInnerError(err error) error { + switch pe := err.(type) { + case nil: + return nil + case *ContainerError: + err = pe.Err + case *ProcessError: + err = pe.Err + } + return err +} diff --git a/vendor/github.com/Microsoft/hcsshim/expandsandboxsize.go b/vendor/github.com/Microsoft/hcsshim/expandsandboxsize.go new file mode 100644 index 00000000000..6946c6a84f1 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/expandsandboxsize.go @@ -0,0 +1,26 @@ +package hcsshim + +import "github.com/sirupsen/logrus" + +// ExpandSandboxSize expands the size of a layer to at least size bytes. +func ExpandSandboxSize(info DriverInfo, layerId string, size uint64) error { + title := "hcsshim::ExpandSandboxSize " + logrus.Debugf(title+"layerId=%s size=%d", layerId, size) + + // Convert info to API calling convention + infop, err := convertDriverInfo(info) + if err != nil { + logrus.Error(err) + return err + } + + err = expandSandboxSize(&infop, layerId, size) + if err != nil { + err = makeErrorf(err, title, "layerId=%s size=%d", layerId, size) + logrus.Error(err) + return err + } + + logrus.Debugf(title+"- succeeded layerId=%s size=%d", layerId, size) + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/exportlayer.go b/vendor/github.com/Microsoft/hcsshim/exportlayer.go new file mode 100644 index 00000000000..d7025f20bab --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/exportlayer.go @@ -0,0 +1,156 @@ +package hcsshim + +import ( + "io" + "io/ioutil" + "os" + "syscall" + + "github.com/Microsoft/go-winio" + "github.com/sirupsen/logrus" +) + +// ExportLayer will create a folder at exportFolderPath and fill that folder with +// the transport format version of the layer identified by layerId. This transport +// format includes any metadata required for later importing the layer (using +// ImportLayer), and requires the full list of parent layer paths in order to +// perform the export. +func ExportLayer(info DriverInfo, layerId string, exportFolderPath string, parentLayerPaths []string) error { + title := "hcsshim::ExportLayer " + logrus.Debugf(title+"flavour %d layerId %s folder %s", info.Flavour, layerId, exportFolderPath) + + // Generate layer descriptors + layers, err := layerPathsToDescriptors(parentLayerPaths) + if err != nil { + return err + } + + // Convert info to API calling convention + infop, err := convertDriverInfo(info) + if err != nil { + logrus.Error(err) + return err + } + + err = exportLayer(&infop, layerId, exportFolderPath, layers) + if err != nil { + err = makeErrorf(err, title, "layerId=%s flavour=%d folder=%s", layerId, info.Flavour, exportFolderPath) + logrus.Error(err) + return err + } + + logrus.Debugf(title+"succeeded flavour=%d layerId=%s folder=%s", info.Flavour, layerId, exportFolderPath) + return nil +} + +type LayerReader interface { + Next() (string, int64, *winio.FileBasicInfo, error) + Read(b []byte) (int, error) + Close() error +} + +// FilterLayerReader provides an interface for extracting the contents of an on-disk layer. +type FilterLayerReader struct { + context uintptr +} + +// Next reads the next available file from a layer, ensuring that parent directories are always read +// before child files and directories. +// +// Next returns the file's relative path, size, and basic file metadata. Read() should be used to +// extract a Win32 backup stream with the remainder of the metadata and the data. +func (r *FilterLayerReader) Next() (string, int64, *winio.FileBasicInfo, error) { + var fileNamep *uint16 + fileInfo := &winio.FileBasicInfo{} + var deleted uint32 + var fileSize int64 + err := exportLayerNext(r.context, &fileNamep, fileInfo, &fileSize, &deleted) + if err != nil { + if err == syscall.ERROR_NO_MORE_FILES { + err = io.EOF + } else { + err = makeError(err, "ExportLayerNext", "") + } + return "", 0, nil, err + } + fileName := convertAndFreeCoTaskMemString(fileNamep) + if deleted != 0 { + fileInfo = nil + } + if fileName[0] == '\\' { + fileName = fileName[1:] + } + return fileName, fileSize, fileInfo, nil +} + +// Read reads from the current file's Win32 backup stream. +func (r *FilterLayerReader) Read(b []byte) (int, error) { + var bytesRead uint32 + err := exportLayerRead(r.context, b, &bytesRead) + if err != nil { + return 0, makeError(err, "ExportLayerRead", "") + } + if bytesRead == 0 { + return 0, io.EOF + } + return int(bytesRead), nil +} + +// Close frees resources associated with the layer reader. It will return an +// error if there was an error while reading the layer or of the layer was not +// completely read. +func (r *FilterLayerReader) Close() (err error) { + if r.context != 0 { + err = exportLayerEnd(r.context) + if err != nil { + err = makeError(err, "ExportLayerEnd", "") + } + r.context = 0 + } + return +} + +// NewLayerReader returns a new layer reader for reading the contents of an on-disk layer. +// The caller must have taken the SeBackupPrivilege privilege +// to call this and any methods on the resulting LayerReader. +func NewLayerReader(info DriverInfo, layerID string, parentLayerPaths []string) (LayerReader, error) { + if procExportLayerBegin.Find() != nil { + // The new layer reader is not available on this Windows build. Fall back to the + // legacy export code path. + path, err := ioutil.TempDir("", "hcs") + if err != nil { + return nil, err + } + err = ExportLayer(info, layerID, path, parentLayerPaths) + if err != nil { + os.RemoveAll(path) + return nil, err + } + return &legacyLayerReaderWrapper{newLegacyLayerReader(path)}, nil + } + + layers, err := layerPathsToDescriptors(parentLayerPaths) + if err != nil { + return nil, err + } + infop, err := convertDriverInfo(info) + if err != nil { + return nil, err + } + r := &FilterLayerReader{} + err = exportLayerBegin(&infop, layerID, layers, &r.context) + if err != nil { + return nil, makeError(err, "ExportLayerBegin", "") + } + return r, err +} + +type legacyLayerReaderWrapper struct { + *legacyLayerReader +} + +func (r *legacyLayerReaderWrapper) Close() error { + err := r.legacyLayerReader.Close() + os.RemoveAll(r.root) + return err +} diff --git a/vendor/github.com/Microsoft/hcsshim/getlayermountpath.go b/vendor/github.com/Microsoft/hcsshim/getlayermountpath.go new file mode 100644 index 00000000000..89f8079d0f1 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/getlayermountpath.go @@ -0,0 +1,55 @@ +package hcsshim + +import ( + "syscall" + + "github.com/sirupsen/logrus" +) + +// GetLayerMountPath will look for a mounted layer with the given id and return +// the path at which that layer can be accessed. This path may be a volume path +// if the layer is a mounted read-write layer, otherwise it is expected to be the +// folder path at which the layer is stored. +func GetLayerMountPath(info DriverInfo, id string) (string, error) { + title := "hcsshim::GetLayerMountPath " + logrus.Debugf(title+"Flavour %d ID %s", info.Flavour, id) + + // Convert info to API calling convention + infop, err := convertDriverInfo(info) + if err != nil { + logrus.Error(err) + return "", err + } + + var mountPathLength uintptr + mountPathLength = 0 + + // Call the procedure itself. + logrus.Debugf("Calling proc (1)") + err = getLayerMountPath(&infop, id, &mountPathLength, nil) + if err != nil { + err = makeErrorf(err, title, "(first call) id=%s flavour=%d", id, info.Flavour) + logrus.Error(err) + return "", err + } + + // Allocate a mount path of the returned length. + if mountPathLength == 0 { + return "", nil + } + mountPathp := make([]uint16, mountPathLength) + mountPathp[0] = 0 + + // Call the procedure again + logrus.Debugf("Calling proc (2)") + err = getLayerMountPath(&infop, id, &mountPathLength, &mountPathp[0]) + if err != nil { + err = makeErrorf(err, title, "(second call) id=%s flavour=%d", id, info.Flavour) + logrus.Error(err) + return "", err + } + + path := syscall.UTF16ToString(mountPathp[0:]) + logrus.Debugf(title+"succeeded flavour=%d id=%s path=%s", info.Flavour, id, path) + return path, nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/getsharedbaseimages.go b/vendor/github.com/Microsoft/hcsshim/getsharedbaseimages.go new file mode 100644 index 00000000000..05d3d9532ae --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/getsharedbaseimages.go @@ -0,0 +1,22 @@ +package hcsshim + +import "github.com/sirupsen/logrus" + +// GetSharedBaseImages will enumerate the images stored in the common central +// image store and return descriptive info about those images for the purpose +// of registering them with the graphdriver, graph, and tagstore. +func GetSharedBaseImages() (imageData string, err error) { + title := "hcsshim::GetSharedBaseImages " + + logrus.Debugf("Calling proc") + var buffer *uint16 + err = getBaseImages(&buffer) + if err != nil { + err = makeError(err, title, "") + logrus.Error(err) + return + } + imageData = convertAndFreeCoTaskMemString(buffer) + logrus.Debugf(title+" - succeeded output=%s", imageData) + return +} diff --git a/vendor/github.com/Microsoft/hcsshim/guid.go b/vendor/github.com/Microsoft/hcsshim/guid.go new file mode 100644 index 00000000000..620aba123cb --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/guid.go @@ -0,0 +1,19 @@ +package hcsshim + +import ( + "crypto/sha1" + "fmt" +) + +type GUID [16]byte + +func NewGUID(source string) *GUID { + h := sha1.Sum([]byte(source)) + var g GUID + copy(g[0:], h[0:16]) + return &g +} + +func (g *GUID) ToString() string { + return fmt.Sprintf("%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x-%02x", g[3], g[2], g[1], g[0], g[5], g[4], g[7], g[6], g[8:10], g[10:]) +} diff --git a/vendor/github.com/Microsoft/hcsshim/hcsshim.go b/vendor/github.com/Microsoft/hcsshim/hcsshim.go new file mode 100644 index 00000000000..236ba1fa30d --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hcsshim.go @@ -0,0 +1,166 @@ +// Shim for the Host Compute Service (HCS) to manage Windows Server +// containers and Hyper-V containers. + +package hcsshim + +import ( + "fmt" + "syscall" + "unsafe" + + "github.com/sirupsen/logrus" +) + +//go:generate go run mksyscall_windows.go -output zhcsshim.go hcsshim.go + +//sys coTaskMemFree(buffer unsafe.Pointer) = ole32.CoTaskMemFree +//sys SetCurrentThreadCompartmentId(compartmentId uint32) (hr error) = iphlpapi.SetCurrentThreadCompartmentId + +//sys activateLayer(info *driverInfo, id string) (hr error) = vmcompute.ActivateLayer? +//sys copyLayer(info *driverInfo, srcId string, dstId string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.CopyLayer? +//sys createLayer(info *driverInfo, id string, parent string) (hr error) = vmcompute.CreateLayer? +//sys createSandboxLayer(info *driverInfo, id string, parent string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.CreateSandboxLayer? +//sys expandSandboxSize(info *driverInfo, id string, size uint64) (hr error) = vmcompute.ExpandSandboxSize? +//sys deactivateLayer(info *driverInfo, id string) (hr error) = vmcompute.DeactivateLayer? +//sys destroyLayer(info *driverInfo, id string) (hr error) = vmcompute.DestroyLayer? +//sys exportLayer(info *driverInfo, id string, path string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.ExportLayer? +//sys getLayerMountPath(info *driverInfo, id string, length *uintptr, buffer *uint16) (hr error) = vmcompute.GetLayerMountPath? +//sys getBaseImages(buffer **uint16) (hr error) = vmcompute.GetBaseImages? +//sys importLayer(info *driverInfo, id string, path string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.ImportLayer? +//sys layerExists(info *driverInfo, id string, exists *uint32) (hr error) = vmcompute.LayerExists? +//sys nameToGuid(name string, guid *GUID) (hr error) = vmcompute.NameToGuid? +//sys prepareLayer(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.PrepareLayer? +//sys unprepareLayer(info *driverInfo, id string) (hr error) = vmcompute.UnprepareLayer? +//sys processBaseImage(path string) (hr error) = vmcompute.ProcessBaseImage? +//sys processUtilityImage(path string) (hr error) = vmcompute.ProcessUtilityImage? + +//sys importLayerBegin(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) = vmcompute.ImportLayerBegin? +//sys importLayerNext(context uintptr, fileName string, fileInfo *winio.FileBasicInfo) (hr error) = vmcompute.ImportLayerNext? +//sys importLayerWrite(context uintptr, buffer []byte) (hr error) = vmcompute.ImportLayerWrite? +//sys importLayerEnd(context uintptr) (hr error) = vmcompute.ImportLayerEnd? + +//sys exportLayerBegin(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) = vmcompute.ExportLayerBegin? +//sys exportLayerNext(context uintptr, fileName **uint16, fileInfo *winio.FileBasicInfo, fileSize *int64, deleted *uint32) (hr error) = vmcompute.ExportLayerNext? +//sys exportLayerRead(context uintptr, buffer []byte, bytesRead *uint32) (hr error) = vmcompute.ExportLayerRead? +//sys exportLayerEnd(context uintptr) (hr error) = vmcompute.ExportLayerEnd? + +//sys hcsEnumerateComputeSystems(query string, computeSystems **uint16, result **uint16) (hr error) = vmcompute.HcsEnumerateComputeSystems? +//sys hcsCreateComputeSystem(id string, configuration string, identity syscall.Handle, computeSystem *hcsSystem, result **uint16) (hr error) = vmcompute.HcsCreateComputeSystem? +//sys hcsOpenComputeSystem(id string, computeSystem *hcsSystem, result **uint16) (hr error) = vmcompute.HcsOpenComputeSystem? +//sys hcsCloseComputeSystem(computeSystem hcsSystem) (hr error) = vmcompute.HcsCloseComputeSystem? +//sys hcsStartComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsStartComputeSystem? +//sys hcsShutdownComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsShutdownComputeSystem? +//sys hcsTerminateComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsTerminateComputeSystem? +//sys hcsPauseComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsPauseComputeSystem? +//sys hcsResumeComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsResumeComputeSystem? +//sys hcsGetComputeSystemProperties(computeSystem hcsSystem, propertyQuery string, properties **uint16, result **uint16) (hr error) = vmcompute.HcsGetComputeSystemProperties? +//sys hcsModifyComputeSystem(computeSystem hcsSystem, configuration string, result **uint16) (hr error) = vmcompute.HcsModifyComputeSystem? +//sys hcsRegisterComputeSystemCallback(computeSystem hcsSystem, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) = vmcompute.HcsRegisterComputeSystemCallback? +//sys hcsUnregisterComputeSystemCallback(callbackHandle hcsCallback) (hr error) = vmcompute.HcsUnregisterComputeSystemCallback? + +//sys hcsCreateProcess(computeSystem hcsSystem, processParameters string, processInformation *hcsProcessInformation, process *hcsProcess, result **uint16) (hr error) = vmcompute.HcsCreateProcess? +//sys hcsOpenProcess(computeSystem hcsSystem, pid uint32, process *hcsProcess, result **uint16) (hr error) = vmcompute.HcsOpenProcess? +//sys hcsCloseProcess(process hcsProcess) (hr error) = vmcompute.HcsCloseProcess? +//sys hcsTerminateProcess(process hcsProcess, result **uint16) (hr error) = vmcompute.HcsTerminateProcess? +//sys hcsGetProcessInfo(process hcsProcess, processInformation *hcsProcessInformation, result **uint16) (hr error) = vmcompute.HcsGetProcessInfo? +//sys hcsGetProcessProperties(process hcsProcess, processProperties **uint16, result **uint16) (hr error) = vmcompute.HcsGetProcessProperties? +//sys hcsModifyProcess(process hcsProcess, settings string, result **uint16) (hr error) = vmcompute.HcsModifyProcess? +//sys hcsGetServiceProperties(propertyQuery string, properties **uint16, result **uint16) (hr error) = vmcompute.HcsGetServiceProperties? +//sys hcsRegisterProcessCallback(process hcsProcess, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) = vmcompute.HcsRegisterProcessCallback? +//sys hcsUnregisterProcessCallback(callbackHandle hcsCallback) (hr error) = vmcompute.HcsUnregisterProcessCallback? + +//sys hcsModifyServiceSettings(settings string, result **uint16) (hr error) = vmcompute.HcsModifyServiceSettings? + +//sys _hnsCall(method string, path string, object string, response **uint16) (hr error) = vmcompute.HNSCall? + +const ( + // Specific user-visible exit codes + WaitErrExecFailed = 32767 + + ERROR_GEN_FAILURE = syscall.Errno(31) + ERROR_SHUTDOWN_IN_PROGRESS = syscall.Errno(1115) + WSAEINVAL = syscall.Errno(10022) + + // Timeout on wait calls + TimeoutInfinite = 0xFFFFFFFF +) + +type HcsError struct { + title string + rest string + Err error +} + +type hcsSystem syscall.Handle +type hcsProcess syscall.Handle +type hcsCallback syscall.Handle + +type hcsProcessInformation struct { + ProcessId uint32 + Reserved uint32 + StdInput syscall.Handle + StdOutput syscall.Handle + StdError syscall.Handle +} + +func makeError(err error, title, rest string) error { + // Pass through DLL errors directly since they do not originate from HCS. + if _, ok := err.(*syscall.DLLError); ok { + return err + } + return &HcsError{title, rest, err} +} + +func makeErrorf(err error, title, format string, a ...interface{}) error { + return makeError(err, title, fmt.Sprintf(format, a...)) +} + +func win32FromError(err error) uint32 { + if herr, ok := err.(*HcsError); ok { + return win32FromError(herr.Err) + } + if code, ok := err.(syscall.Errno); ok { + return uint32(code) + } + return uint32(ERROR_GEN_FAILURE) +} + +func win32FromHresult(hr uintptr) uintptr { + if hr&0x1fff0000 == 0x00070000 { + return hr & 0xffff + } + return hr +} + +func (e *HcsError) Error() string { + s := e.title + if len(s) > 0 && s[len(s)-1] != ' ' { + s += " " + } + s += fmt.Sprintf("failed in Win32: %s (0x%x)", e.Err, win32FromError(e.Err)) + if e.rest != "" { + if e.rest[0] != ' ' { + s += " " + } + s += e.rest + } + return s +} + +func convertAndFreeCoTaskMemString(buffer *uint16) string { + str := syscall.UTF16ToString((*[1 << 30]uint16)(unsafe.Pointer(buffer))[:]) + coTaskMemFree(unsafe.Pointer(buffer)) + return str +} + +func convertAndFreeCoTaskMemBytes(buffer *uint16) []byte { + return []byte(convertAndFreeCoTaskMemString(buffer)) +} + +func processHcsResult(err error, resultp *uint16) error { + if resultp != nil { + result := convertAndFreeCoTaskMemString(resultp) + logrus.Debugf("Result: %s", result) + } + return err +} diff --git a/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go b/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go new file mode 100644 index 00000000000..92afc0c249c --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go @@ -0,0 +1,318 @@ +package hcsshim + +import ( + "encoding/json" + "fmt" + "net" + + "github.com/sirupsen/logrus" +) + +// HNSEndpoint represents a network endpoint in HNS +type HNSEndpoint struct { + Id string `json:"ID,omitempty"` + Name string `json:",omitempty"` + VirtualNetwork string `json:",omitempty"` + VirtualNetworkName string `json:",omitempty"` + Policies []json.RawMessage `json:",omitempty"` + MacAddress string `json:",omitempty"` + IPAddress net.IP `json:",omitempty"` + DNSSuffix string `json:",omitempty"` + DNSServerList string `json:",omitempty"` + GatewayAddress string `json:",omitempty"` + EnableInternalDNS bool `json:",omitempty"` + DisableICC bool `json:",omitempty"` + PrefixLength uint8 `json:",omitempty"` + IsRemoteEndpoint bool `json:",omitempty"` +} + +//SystemType represents the type of the system on which actions are done +type SystemType string + +// SystemType const +const ( + ContainerType SystemType = "Container" + VirtualMachineType SystemType = "VirtualMachine" + HostType SystemType = "Host" +) + +// EndpointAttachDetachRequest is the structure used to send request to the container to modify the system +// Supported resource types are Network and Request Types are Add/Remove +type EndpointAttachDetachRequest struct { + ContainerID string `json:"ContainerId,omitempty"` + SystemType SystemType `json:"SystemType"` + CompartmentID uint16 `json:"CompartmentId,omitempty"` + VirtualNICName string `json:"VirtualNicName,omitempty"` +} + +// EndpointResquestResponse is object to get the endpoint request response +type EndpointResquestResponse struct { + Success bool + Error string +} + +// HNSEndpointRequest makes a HNS call to modify/query a network endpoint +func HNSEndpointRequest(method, path, request string) (*HNSEndpoint, error) { + endpoint := &HNSEndpoint{} + err := hnsCall(method, "/endpoints/"+path, request, &endpoint) + if err != nil { + return nil, err + } + + return endpoint, nil +} + +// HNSListEndpointRequest makes a HNS call to query the list of available endpoints +func HNSListEndpointRequest() ([]HNSEndpoint, error) { + var endpoint []HNSEndpoint + err := hnsCall("GET", "/endpoints/", "", &endpoint) + if err != nil { + return nil, err + } + + return endpoint, nil +} + +// HotAttachEndpoint makes a HCS Call to attach the endpoint to the container +func HotAttachEndpoint(containerID string, endpointID string) error { + return modifyNetworkEndpoint(containerID, endpointID, Add) +} + +// HotDetachEndpoint makes a HCS Call to detach the endpoint from the container +func HotDetachEndpoint(containerID string, endpointID string) error { + return modifyNetworkEndpoint(containerID, endpointID, Remove) +} + +// ModifyContainer corresponding to the container id, by sending a request +func modifyContainer(id string, request *ResourceModificationRequestResponse) error { + container, err := OpenContainer(id) + if err != nil { + if IsNotExist(err) { + return ErrComputeSystemDoesNotExist + } + return getInnerError(err) + } + defer container.Close() + err = container.Modify(request) + if err != nil { + if IsNotSupported(err) { + return ErrPlatformNotSupported + } + return getInnerError(err) + } + + return nil +} + +func modifyNetworkEndpoint(containerID string, endpointID string, request RequestType) error { + requestMessage := &ResourceModificationRequestResponse{ + Resource: Network, + Request: request, + Data: endpointID, + } + err := modifyContainer(containerID, requestMessage) + + if err != nil { + return err + } + + return nil +} + +// GetHNSEndpointByID get the Endpoint by ID +func GetHNSEndpointByID(endpointID string) (*HNSEndpoint, error) { + return HNSEndpointRequest("GET", endpointID, "") +} + +// GetHNSEndpointByName gets the endpoint filtered by Name +func GetHNSEndpointByName(endpointName string) (*HNSEndpoint, error) { + hnsResponse, err := HNSListEndpointRequest() + if err != nil { + return nil, err + } + for _, hnsEndpoint := range hnsResponse { + if hnsEndpoint.Name == endpointName { + return &hnsEndpoint, nil + } + } + return nil, fmt.Errorf("Endpoint %v not found", endpointName) +} + +// Create Endpoint by sending EndpointRequest to HNS. TODO: Create a separate HNS interface to place all these methods +func (endpoint *HNSEndpoint) Create() (*HNSEndpoint, error) { + operation := "Create" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + + jsonString, err := json.Marshal(endpoint) + if err != nil { + return nil, err + } + return HNSEndpointRequest("POST", "", string(jsonString)) +} + +// Delete Endpoint by sending EndpointRequest to HNS +func (endpoint *HNSEndpoint) Delete() (*HNSEndpoint, error) { + operation := "Delete" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + + return HNSEndpointRequest("DELETE", endpoint.Id, "") +} + +// Update Endpoint +func (endpoint *HNSEndpoint) Update() (*HNSEndpoint, error) { + operation := "Update" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + jsonString, err := json.Marshal(endpoint) + if err != nil { + return nil, err + } + err = hnsCall("POST", "/endpoints/"+endpoint.Id, string(jsonString), &endpoint) + + return endpoint, err +} + +// ContainerHotAttach attaches an endpoint to a running container +func (endpoint *HNSEndpoint) ContainerHotAttach(containerID string) error { + operation := "ContainerHotAttach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s, containerId=%s", endpoint.Id, containerID) + + return modifyNetworkEndpoint(containerID, endpoint.Id, Add) +} + +// ContainerHotDetach detaches an endpoint from a running container +func (endpoint *HNSEndpoint) ContainerHotDetach(containerID string) error { + operation := "ContainerHotDetach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s, containerId=%s", endpoint.Id, containerID) + + return modifyNetworkEndpoint(containerID, endpoint.Id, Remove) +} + +// ApplyACLPolicy applies Acl Policy on the Endpoint +func (endpoint *HNSEndpoint) ApplyACLPolicy(policy *ACLPolicy) error { + operation := "ApplyACLPolicy" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + + jsonString, err := json.Marshal(policy) + if err != nil { + return err + } + endpoint.Policies[0] = jsonString + _, err = endpoint.Update() + return err +} + +// ContainerAttach attaches an endpoint to container +func (endpoint *HNSEndpoint) ContainerAttach(containerID string, compartmentID uint16) error { + operation := "ContainerAttach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + + requestMessage := &EndpointAttachDetachRequest{ + ContainerID: containerID, + CompartmentID: compartmentID, + SystemType: ContainerType, + } + response := &EndpointResquestResponse{} + jsonString, err := json.Marshal(requestMessage) + if err != nil { + return err + } + return hnsCall("POST", "/endpoints/"+endpoint.Id+"/attach", string(jsonString), &response) +} + +// ContainerDetach detaches an endpoint from container +func (endpoint *HNSEndpoint) ContainerDetach(containerID string) error { + operation := "ContainerDetach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + + requestMessage := &EndpointAttachDetachRequest{ + ContainerID: containerID, + SystemType: ContainerType, + } + response := &EndpointResquestResponse{} + + jsonString, err := json.Marshal(requestMessage) + if err != nil { + return err + } + return hnsCall("POST", "/endpoints/"+endpoint.Id+"/detach", string(jsonString), &response) +} + +// HostAttach attaches a nic on the host +func (endpoint *HNSEndpoint) HostAttach(compartmentID uint16) error { + operation := "HostAttach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + requestMessage := &EndpointAttachDetachRequest{ + CompartmentID: compartmentID, + SystemType: HostType, + } + response := &EndpointResquestResponse{} + + jsonString, err := json.Marshal(requestMessage) + if err != nil { + return err + } + return hnsCall("POST", "/endpoints/"+endpoint.Id+"/attach", string(jsonString), &response) + +} + +// HostDetach detaches a nic on the host +func (endpoint *HNSEndpoint) HostDetach() error { + operation := "HostDetach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + requestMessage := &EndpointAttachDetachRequest{ + SystemType: HostType, + } + response := &EndpointResquestResponse{} + + jsonString, err := json.Marshal(requestMessage) + if err != nil { + return err + } + return hnsCall("POST", "/endpoints/"+endpoint.Id+"/detach", string(jsonString), &response) +} + +// VirtualMachineNICAttach attaches a endpoint to a virtual machine +func (endpoint *HNSEndpoint) VirtualMachineNICAttach(virtualMachineNICName string) error { + operation := "VirtualMachineNicAttach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + requestMessage := &EndpointAttachDetachRequest{ + VirtualNICName: virtualMachineNICName, + SystemType: VirtualMachineType, + } + response := &EndpointResquestResponse{} + + jsonString, err := json.Marshal(requestMessage) + if err != nil { + return err + } + return hnsCall("POST", "/endpoints/"+endpoint.Id+"/attach", string(jsonString), &response) +} + +// VirtualMachineNICDetach detaches a endpoint from a virtual machine +func (endpoint *HNSEndpoint) VirtualMachineNICDetach() error { + operation := "VirtualMachineNicDetach" + title := "HCSShim::HNSEndpoint::" + operation + logrus.Debugf(title+" id=%s", endpoint.Id) + + requestMessage := &EndpointAttachDetachRequest{ + SystemType: VirtualMachineType, + } + response := &EndpointResquestResponse{} + + jsonString, err := json.Marshal(requestMessage) + if err != nil { + return err + } + return hnsCall("POST", "/endpoints/"+endpoint.Id+"/detach", string(jsonString), &response) +} diff --git a/vendor/github.com/Microsoft/hcsshim/hnsfuncs.go b/vendor/github.com/Microsoft/hcsshim/hnsfuncs.go new file mode 100644 index 00000000000..2c1b979ae84 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hnsfuncs.go @@ -0,0 +1,40 @@ +package hcsshim + +import ( + "encoding/json" + "fmt" + + "github.com/sirupsen/logrus" +) + +func hnsCall(method, path, request string, returnResponse interface{}) error { + var responseBuffer *uint16 + logrus.Debugf("[%s]=>[%s] Request : %s", method, path, request) + + err := _hnsCall(method, path, request, &responseBuffer) + if err != nil { + return makeError(err, "hnsCall ", "") + } + response := convertAndFreeCoTaskMemString(responseBuffer) + + hnsresponse := &hnsResponse{} + if err = json.Unmarshal([]byte(response), &hnsresponse); err != nil { + return err + } + + if !hnsresponse.Success { + return fmt.Errorf("HNS failed with error : %s", hnsresponse.Error) + } + + if len(hnsresponse.Output) == 0 { + return nil + } + + logrus.Debugf("Network Response : %s", hnsresponse.Output) + err = json.Unmarshal(hnsresponse.Output, returnResponse) + if err != nil { + return err + } + + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/hnsnetwork.go b/vendor/github.com/Microsoft/hcsshim/hnsnetwork.go new file mode 100644 index 00000000000..3345bfa3f27 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hnsnetwork.go @@ -0,0 +1,142 @@ +package hcsshim + +import ( + "encoding/json" + "fmt" + "net" + + "github.com/sirupsen/logrus" +) + +// Subnet is assoicated with a network and represents a list +// of subnets available to the network +type Subnet struct { + AddressPrefix string `json:",omitempty"` + GatewayAddress string `json:",omitempty"` + Policies []json.RawMessage `json:",omitempty"` +} + +// MacPool is assoicated with a network and represents a list +// of macaddresses available to the network +type MacPool struct { + StartMacAddress string `json:",omitempty"` + EndMacAddress string `json:",omitempty"` +} + +// HNSNetwork represents a network in HNS +type HNSNetwork struct { + Id string `json:"ID,omitempty"` + Name string `json:",omitempty"` + Type string `json:",omitempty"` + NetworkAdapterName string `json:",omitempty"` + SourceMac string `json:",omitempty"` + Policies []json.RawMessage `json:",omitempty"` + MacPools []MacPool `json:",omitempty"` + Subnets []Subnet `json:",omitempty"` + DNSSuffix string `json:",omitempty"` + DNSServerList string `json:",omitempty"` + DNSServerCompartment uint32 `json:",omitempty"` + ManagementIP string `json:",omitempty"` + AutomaticDNS bool `json:",omitempty"` +} + +type hnsNetworkResponse struct { + Success bool + Error string + Output HNSNetwork +} + +type hnsResponse struct { + Success bool + Error string + Output json.RawMessage +} + +// HNSNetworkRequest makes a call into HNS to update/query a single network +func HNSNetworkRequest(method, path, request string) (*HNSNetwork, error) { + var network HNSNetwork + err := hnsCall(method, "/networks/"+path, request, &network) + if err != nil { + return nil, err + } + + return &network, nil +} + +// HNSListNetworkRequest makes a HNS call to query the list of available networks +func HNSListNetworkRequest(method, path, request string) ([]HNSNetwork, error) { + var network []HNSNetwork + err := hnsCall(method, "/networks/"+path, request, &network) + if err != nil { + return nil, err + } + + return network, nil +} + +// GetHNSNetworkByID +func GetHNSNetworkByID(networkID string) (*HNSNetwork, error) { + return HNSNetworkRequest("GET", networkID, "") +} + +// GetHNSNetworkName filtered by Name +func GetHNSNetworkByName(networkName string) (*HNSNetwork, error) { + hsnnetworks, err := HNSListNetworkRequest("GET", "", "") + if err != nil { + return nil, err + } + for _, hnsnetwork := range hsnnetworks { + if hnsnetwork.Name == networkName { + return &hnsnetwork, nil + } + } + return nil, fmt.Errorf("Network %v not found", networkName) +} + +// Create Network by sending NetworkRequest to HNS. +func (network *HNSNetwork) Create() (*HNSNetwork, error) { + operation := "Create" + title := "HCSShim::HNSNetwork::" + operation + logrus.Debugf(title+" id=%s", network.Id) + + jsonString, err := json.Marshal(network) + if err != nil { + return nil, err + } + return HNSNetworkRequest("POST", "", string(jsonString)) +} + +// Delete Network by sending NetworkRequest to HNS +func (network *HNSNetwork) Delete() (*HNSNetwork, error) { + operation := "Delete" + title := "HCSShim::HNSNetwork::" + operation + logrus.Debugf(title+" id=%s", network.Id) + + return HNSNetworkRequest("DELETE", network.Id, "") +} + +// Creates an endpoint on the Network. +func (network *HNSNetwork) NewEndpoint(ipAddress net.IP, macAddress net.HardwareAddr) *HNSEndpoint { + return &HNSEndpoint{ + VirtualNetwork: network.Id, + IPAddress: ipAddress, + MacAddress: string(macAddress), + } +} + +func (network *HNSNetwork) CreateEndpoint(endpoint *HNSEndpoint) (*HNSEndpoint, error) { + operation := "CreateEndpoint" + title := "HCSShim::HNSNetwork::" + operation + logrus.Debugf(title+" id=%s, endpointId=%s", network.Id, endpoint.Id) + + endpoint.VirtualNetwork = network.Id + return endpoint.Create() +} + +func (network *HNSNetwork) CreateRemoteEndpoint(endpoint *HNSEndpoint) (*HNSEndpoint, error) { + operation := "CreateRemoteEndpoint" + title := "HCSShim::HNSNetwork::" + operation + logrus.Debugf(title+" id=%s", network.Id) + endpoint.IsRemoteEndpoint = true + return network.CreateEndpoint(endpoint) +} diff --git a/vendor/github.com/Microsoft/hcsshim/hnspolicy.go b/vendor/github.com/Microsoft/hcsshim/hnspolicy.go new file mode 100644 index 00000000000..ecfbf0eda37 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hnspolicy.go @@ -0,0 +1,95 @@ +package hcsshim + +// Type of Request Support in ModifySystem +type PolicyType string + +// RequestType const +const ( + Nat PolicyType = "NAT" + ACL PolicyType = "ACL" + PA PolicyType = "PA" + VLAN PolicyType = "VLAN" + VSID PolicyType = "VSID" + VNet PolicyType = "VNET" + L2Driver PolicyType = "L2Driver" + Isolation PolicyType = "Isolation" + QOS PolicyType = "QOS" + OutboundNat PolicyType = "OutBoundNAT" + ExternalLoadBalancer PolicyType = "ELB" + Route PolicyType = "ROUTE" +) + +type NatPolicy struct { + Type PolicyType `json:"Type"` + Protocol string + InternalPort uint16 + ExternalPort uint16 +} + +type QosPolicy struct { + Type PolicyType `json:"Type"` + MaximumOutgoingBandwidthInBytes uint64 +} + +type IsolationPolicy struct { + Type PolicyType `json:"Type"` + VLAN uint + VSID uint + InDefaultIsolation bool +} + +type VlanPolicy struct { + Type PolicyType `json:"Type"` + VLAN uint +} + +type VsidPolicy struct { + Type PolicyType `json:"Type"` + VSID uint +} + +type PaPolicy struct { + Type PolicyType `json:"Type"` + PA string `json:"PA"` +} + +type OutboundNatPolicy struct { + Policy + VIP string `json:"VIP,omitempty"` + Exceptions []string `json:"ExceptionList,omitempty"` +} + +type ActionType string +type DirectionType string +type RuleType string + +const ( + Allow ActionType = "Allow" + Block ActionType = "Block" + + In DirectionType = "In" + Out DirectionType = "Out" + + Host RuleType = "Host" + Switch RuleType = "Switch" +) + +type ACLPolicy struct { + Type PolicyType `json:"Type"` + Protocol uint16 + InternalPort uint16 + Action ActionType + Direction DirectionType + LocalAddress string + RemoteAddress string + LocalPort uint16 + RemotePort uint16 + RuleType RuleType `json:"RuleType,omitempty"` + + Priority uint16 + ServiceName string +} + +type Policy struct { + Type PolicyType `json:"Type"` +} diff --git a/vendor/github.com/Microsoft/hcsshim/hnspolicylist.go b/vendor/github.com/Microsoft/hcsshim/hnspolicylist.go new file mode 100644 index 00000000000..bbd7e1edb08 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hnspolicylist.go @@ -0,0 +1,200 @@ +package hcsshim + +import ( + "encoding/json" + + "github.com/sirupsen/logrus" +) + +// RoutePolicy is a structure defining schema for Route based Policy +type RoutePolicy struct { + Policy + DestinationPrefix string `json:"DestinationPrefix,omitempty"` + NextHop string `json:"NextHop,omitempty"` + EncapEnabled bool `json:"NeedEncap,omitempty"` +} + +// ELBPolicy is a structure defining schema for ELB LoadBalancing based Policy +type ELBPolicy struct { + LBPolicy + SourceVIP string `json:"SourceVIP,omitempty"` + VIPs []string `json:"VIPs,omitempty"` + ILB bool `json:"ILB,omitempty"` +} + +// LBPolicy is a structure defining schema for LoadBalancing based Policy +type LBPolicy struct { + Policy + Protocol uint16 `json:"Protocol,omitempty"` + InternalPort uint16 + ExternalPort uint16 +} + +// PolicyList is a structure defining schema for Policy list request +type PolicyList struct { + ID string `json:"ID,omitempty"` + EndpointReferences []string `json:"References,omitempty"` + Policies []json.RawMessage `json:"Policies,omitempty"` +} + +// HNSPolicyListRequest makes a call into HNS to update/query a single network +func HNSPolicyListRequest(method, path, request string) (*PolicyList, error) { + var policy PolicyList + err := hnsCall(method, "/policylists/"+path, request, &policy) + if err != nil { + return nil, err + } + + return &policy, nil +} + +// HNSListPolicyListRequest gets all the policy list +func HNSListPolicyListRequest() ([]PolicyList, error) { + var plist []PolicyList + err := hnsCall("GET", "/policylists/", "", &plist) + if err != nil { + return nil, err + } + + return plist, nil +} + +// PolicyListRequest makes a HNS call to modify/query a network policy list +func PolicyListRequest(method, path, request string) (*PolicyList, error) { + policylist := &PolicyList{} + err := hnsCall(method, "/policylists/"+path, request, &policylist) + if err != nil { + return nil, err + } + + return policylist, nil +} + +// GetPolicyListByID get the policy list by ID +func GetPolicyListByID(policyListID string) (*PolicyList, error) { + return PolicyListRequest("GET", policyListID, "") +} + +// Create PolicyList by sending PolicyListRequest to HNS. +func (policylist *PolicyList) Create() (*PolicyList, error) { + operation := "Create" + title := "HCSShim::PolicyList::" + operation + logrus.Debugf(title+" id=%s", policylist.ID) + jsonString, err := json.Marshal(policylist) + if err != nil { + return nil, err + } + return PolicyListRequest("POST", "", string(jsonString)) +} + +// Delete deletes PolicyList +func (policylist *PolicyList) Delete() (*PolicyList, error) { + operation := "Delete" + title := "HCSShim::PolicyList::" + operation + logrus.Debugf(title+" id=%s", policylist.ID) + + return PolicyListRequest("DELETE", policylist.ID, "") +} + +// AddEndpoint add an endpoint to a Policy List +func (policylist *PolicyList) AddEndpoint(endpoint *HNSEndpoint) (*PolicyList, error) { + operation := "AddEndpoint" + title := "HCSShim::PolicyList::" + operation + logrus.Debugf(title+" id=%s, endpointId:%s", policylist.ID, endpoint.Id) + + _, err := policylist.Delete() + if err != nil { + return nil, err + } + + // Add Endpoint to the Existing List + policylist.EndpointReferences = append(policylist.EndpointReferences, "/endpoints/"+endpoint.Id) + + return policylist.Create() +} + +// RemoveEndpoint removes an endpoint from the Policy List +func (policylist *PolicyList) RemoveEndpoint(endpoint *HNSEndpoint) (*PolicyList, error) { + operation := "RemoveEndpoint" + title := "HCSShim::PolicyList::" + operation + logrus.Debugf(title+" id=%s, endpointId:%s", policylist.ID, endpoint.Id) + + _, err := policylist.Delete() + if err != nil { + return nil, err + } + + elementToRemove := "/endpoints/" + endpoint.Id + + var references []string + + for _, endpointReference := range policylist.EndpointReferences { + if endpointReference == elementToRemove { + continue + } + references = append(references, endpointReference) + } + policylist.EndpointReferences = references + return policylist.Create() +} + +// AddLoadBalancer policy list for the specified endpoints +func AddLoadBalancer(endpoints []HNSEndpoint, isILB bool, sourceVIP, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*PolicyList, error) { + operation := "AddLoadBalancer" + title := "HCSShim::PolicyList::" + operation + logrus.Debugf(title+" endpointId=%v, isILB=%v, sourceVIP=%s, vip=%s, protocol=%v, internalPort=%v, externalPort=%v", endpoints, isILB, sourceVIP, vip, protocol, internalPort, externalPort) + + policylist := &PolicyList{} + + elbPolicy := &ELBPolicy{ + SourceVIP: sourceVIP, + ILB: isILB, + } + + if len(vip) > 0 { + elbPolicy.VIPs = []string{vip} + } + elbPolicy.Type = ExternalLoadBalancer + elbPolicy.Protocol = protocol + elbPolicy.InternalPort = internalPort + elbPolicy.ExternalPort = externalPort + + for _, endpoint := range endpoints { + policylist.EndpointReferences = append(policylist.EndpointReferences, "/endpoints/"+endpoint.Id) + } + + jsonString, err := json.Marshal(elbPolicy) + if err != nil { + return nil, err + } + policylist.Policies = append(policylist.Policies, jsonString) + return policylist.Create() +} + +// AddRoute adds route policy list for the specified endpoints +func AddRoute(endpoints []HNSEndpoint, destinationPrefix string, nextHop string, encapEnabled bool) (*PolicyList, error) { + operation := "AddRoute" + title := "HCSShim::PolicyList::" + operation + logrus.Debugf(title+" destinationPrefix:%s", destinationPrefix) + + policylist := &PolicyList{} + + rPolicy := &RoutePolicy{ + DestinationPrefix: destinationPrefix, + NextHop: nextHop, + EncapEnabled: encapEnabled, + } + rPolicy.Type = Route + + for _, endpoint := range endpoints { + policylist.EndpointReferences = append(policylist.EndpointReferences, "/endpoints/"+endpoint.Id) + } + + jsonString, err := json.Marshal(rPolicy) + if err != nil { + return nil, err + } + + policylist.Policies = append(policylist.Policies, jsonString) + return policylist.Create() +} diff --git a/vendor/github.com/Microsoft/hcsshim/importlayer.go b/vendor/github.com/Microsoft/hcsshim/importlayer.go new file mode 100644 index 00000000000..3aed14376a4 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/importlayer.go @@ -0,0 +1,212 @@ +package hcsshim + +import ( + "errors" + "io/ioutil" + "os" + "path/filepath" + + "github.com/Microsoft/go-winio" + "github.com/sirupsen/logrus" +) + +// ImportLayer will take the contents of the folder at importFolderPath and import +// that into a layer with the id layerId. Note that in order to correctly populate +// the layer and interperet the transport format, all parent layers must already +// be present on the system at the paths provided in parentLayerPaths. +func ImportLayer(info DriverInfo, layerID string, importFolderPath string, parentLayerPaths []string) error { + title := "hcsshim::ImportLayer " + logrus.Debugf(title+"flavour %d layerId %s folder %s", info.Flavour, layerID, importFolderPath) + + // Generate layer descriptors + layers, err := layerPathsToDescriptors(parentLayerPaths) + if err != nil { + return err + } + + // Convert info to API calling convention + infop, err := convertDriverInfo(info) + if err != nil { + logrus.Error(err) + return err + } + + err = importLayer(&infop, layerID, importFolderPath, layers) + if err != nil { + err = makeErrorf(err, title, "layerId=%s flavour=%d folder=%s", layerID, info.Flavour, importFolderPath) + logrus.Error(err) + return err + } + + logrus.Debugf(title+"succeeded flavour=%d layerId=%s folder=%s", info.Flavour, layerID, importFolderPath) + return nil +} + +// LayerWriter is an interface that supports writing a new container image layer. +type LayerWriter interface { + // Add adds a file to the layer with given metadata. + Add(name string, fileInfo *winio.FileBasicInfo) error + // AddLink adds a hard link to the layer. The target must already have been added. + AddLink(name string, target string) error + // Remove removes a file that was present in a parent layer from the layer. + Remove(name string) error + // Write writes data to the current file. The data must be in the format of a Win32 + // backup stream. + Write(b []byte) (int, error) + // Close finishes the layer writing process and releases any resources. + Close() error +} + +// FilterLayerWriter provides an interface to write the contents of a layer to the file system. +type FilterLayerWriter struct { + context uintptr +} + +// Add adds a file or directory to the layer. The file's parent directory must have already been added. +// +// name contains the file's relative path. fileInfo contains file times and file attributes; the rest +// of the file metadata and the file data must be written as a Win32 backup stream to the Write() method. +// winio.BackupStreamWriter can be used to facilitate this. +func (w *FilterLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) error { + if name[0] != '\\' { + name = `\` + name + } + err := importLayerNext(w.context, name, fileInfo) + if err != nil { + return makeError(err, "ImportLayerNext", "") + } + return nil +} + +// AddLink adds a hard link to the layer. The target of the link must have already been added. +func (w *FilterLayerWriter) AddLink(name string, target string) error { + return errors.New("hard links not yet supported") +} + +// Remove removes a file from the layer. The file must have been present in the parent layer. +// +// name contains the file's relative path. +func (w *FilterLayerWriter) Remove(name string) error { + if name[0] != '\\' { + name = `\` + name + } + err := importLayerNext(w.context, name, nil) + if err != nil { + return makeError(err, "ImportLayerNext", "") + } + return nil +} + +// Write writes more backup stream data to the current file. +func (w *FilterLayerWriter) Write(b []byte) (int, error) { + err := importLayerWrite(w.context, b) + if err != nil { + err = makeError(err, "ImportLayerWrite", "") + return 0, err + } + return len(b), err +} + +// Close completes the layer write operation. The error must be checked to ensure that the +// operation was successful. +func (w *FilterLayerWriter) Close() (err error) { + if w.context != 0 { + err = importLayerEnd(w.context) + if err != nil { + err = makeError(err, "ImportLayerEnd", "") + } + w.context = 0 + } + return +} + +type legacyLayerWriterWrapper struct { + *legacyLayerWriter + info DriverInfo + layerID string + path string + parentLayerPaths []string +} + +func (r *legacyLayerWriterWrapper) Close() error { + defer os.RemoveAll(r.root) + err := r.legacyLayerWriter.Close() + if err != nil { + return err + } + + // Use the original path here because ImportLayer does not support long paths for the source in TP5. + // But do use a long path for the destination to work around another bug with directories + // with MAX_PATH - 12 < length < MAX_PATH. + info := r.info + fullPath, err := makeLongAbsPath(filepath.Join(info.HomeDir, r.layerID)) + if err != nil { + return err + } + + info.HomeDir = "" + if err = ImportLayer(info, fullPath, r.path, r.parentLayerPaths); err != nil { + return err + } + // Add any hard links that were collected. + for _, lnk := range r.PendingLinks { + if err = os.Remove(lnk.Path); err != nil && !os.IsNotExist(err) { + return err + } + if err = os.Link(lnk.Target, lnk.Path); err != nil { + return err + } + } + // Prepare the utility VM for use if one is present in the layer. + if r.HasUtilityVM { + err = ProcessUtilityVMImage(filepath.Join(fullPath, "UtilityVM")) + if err != nil { + return err + } + } + return nil +} + +// NewLayerWriter returns a new layer writer for creating a layer on disk. +// The caller must have taken the SeBackupPrivilege and SeRestorePrivilege privileges +// to call this and any methods on the resulting LayerWriter. +func NewLayerWriter(info DriverInfo, layerID string, parentLayerPaths []string) (LayerWriter, error) { + if len(parentLayerPaths) == 0 { + // This is a base layer. It gets imported differently. + return &baseLayerWriter{ + root: filepath.Join(info.HomeDir, layerID), + }, nil + } + + if procImportLayerBegin.Find() != nil { + // The new layer reader is not available on this Windows build. Fall back to the + // legacy export code path. + path, err := ioutil.TempDir("", "hcs") + if err != nil { + return nil, err + } + return &legacyLayerWriterWrapper{ + legacyLayerWriter: newLegacyLayerWriter(path, parentLayerPaths, filepath.Join(info.HomeDir, layerID)), + info: info, + layerID: layerID, + path: path, + parentLayerPaths: parentLayerPaths, + }, nil + } + layers, err := layerPathsToDescriptors(parentLayerPaths) + if err != nil { + return nil, err + } + + infop, err := convertDriverInfo(info) + if err != nil { + return nil, err + } + + w := &FilterLayerWriter{} + err = importLayerBegin(&infop, layerID, layers, &w.context) + if err != nil { + return nil, makeError(err, "ImportLayerStart", "") + } + return w, nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/interface.go b/vendor/github.com/Microsoft/hcsshim/interface.go new file mode 100644 index 00000000000..9fc7852e41b --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/interface.go @@ -0,0 +1,187 @@ +package hcsshim + +import ( + "encoding/json" + "io" + "time" +) + +// ProcessConfig is used as both the input of Container.CreateProcess +// and to convert the parameters to JSON for passing onto the HCS +type ProcessConfig struct { + ApplicationName string `json:",omitempty"` + CommandLine string `json:",omitempty"` + CommandArgs []string `json:",omitempty"` // Used by Linux Containers on Windows + User string `json:",omitempty"` + WorkingDirectory string `json:",omitempty"` + Environment map[string]string `json:",omitempty"` + EmulateConsole bool `json:",omitempty"` + CreateStdInPipe bool `json:",omitempty"` + CreateStdOutPipe bool `json:",omitempty"` + CreateStdErrPipe bool `json:",omitempty"` + ConsoleSize [2]uint `json:",omitempty"` + CreateInUtilityVm bool `json:",omitempty"` // Used by Linux Containers on Windows + OCISpecification *json.RawMessage `json:",omitempty"` // Used by Linux Containers on Windows +} + +type Layer struct { + ID string + Path string +} + +type MappedDir struct { + HostPath string + ContainerPath string + ReadOnly bool + BandwidthMaximum uint64 + IOPSMaximum uint64 +} + +type MappedPipe struct { + HostPath string + ContainerPipeName string +} + +type HvRuntime struct { + ImagePath string `json:",omitempty"` + SkipTemplate bool `json:",omitempty"` + LinuxInitrdFile string `json:",omitempty"` // File under ImagePath on host containing an initrd image for starting a Linux utility VM + LinuxKernelFile string `json:",omitempty"` // File under ImagePath on host containing a kernel for starting a Linux utility VM + LinuxBootParameters string `json:",omitempty"` // Additional boot parameters for starting a Linux Utility VM in initrd mode + BootSource string `json:",omitempty"` // "Vhd" for Linux Utility VM booting from VHD + WritableBootSource bool `json:",omitempty"` // Linux Utility VM booting from VHD +} + +type MappedVirtualDisk struct { + HostPath string `json:",omitempty"` // Path to VHD on the host + ContainerPath string // Platform-specific mount point path in the container + CreateInUtilityVM bool `json:",omitempty"` + ReadOnly bool `json:",omitempty"` + Cache string `json:",omitempty"` // "" (Unspecified); "Disabled"; "Enabled"; "Private"; "PrivateAllowSharing" + AttachOnly bool `json:",omitempty:` +} + +// ContainerConfig is used as both the input of CreateContainer +// and to convert the parameters to JSON for passing onto the HCS +type ContainerConfig struct { + SystemType string // HCS requires this to be hard-coded to "Container" + Name string // Name of the container. We use the docker ID. + Owner string `json:",omitempty"` // The management platform that created this container + VolumePath string `json:",omitempty"` // Windows volume path for scratch space. Used by Windows Server Containers only. Format \\?\\Volume{GUID} + IgnoreFlushesDuringBoot bool `json:",omitempty"` // Optimization hint for container startup in Windows + LayerFolderPath string `json:",omitempty"` // Where the layer folders are located. Used by Windows Server Containers only. Format %root%\windowsfilter\containerID + Layers []Layer // List of storage layers. Required for Windows Server and Hyper-V Containers. Format ID=GUID;Path=%root%\windowsfilter\layerID + Credentials string `json:",omitempty"` // Credentials information + ProcessorCount uint32 `json:",omitempty"` // Number of processors to assign to the container. + ProcessorWeight uint64 `json:",omitempty"` // CPU shares (relative weight to other containers with cpu shares). Range is from 1 to 10000. A value of 0 results in default shares. + ProcessorMaximum int64 `json:",omitempty"` // Specifies the portion of processor cycles that this container can use as a percentage times 100. Range is from 1 to 10000. A value of 0 results in no limit. + StorageIOPSMaximum uint64 `json:",omitempty"` // Maximum Storage IOPS + StorageBandwidthMaximum uint64 `json:",omitempty"` // Maximum Storage Bandwidth in bytes per second + StorageSandboxSize uint64 `json:",omitempty"` // Size in bytes that the container system drive should be expanded to if smaller + MemoryMaximumInMB int64 `json:",omitempty"` // Maximum memory available to the container in Megabytes + HostName string `json:",omitempty"` // Hostname + MappedDirectories []MappedDir `json:",omitempty"` // List of mapped directories (volumes/mounts) + MappedPipes []MappedPipe `json:",omitempty"` // List of mapped Windows named pipes + HvPartition bool // True if it a Hyper-V Container + NetworkSharedContainerName string `json:",omitempty"` // Name (ID) of the container that we will share the network stack with. + EndpointList []string `json:",omitempty"` // List of networking endpoints to be attached to container + HvRuntime *HvRuntime `json:",omitempty"` // Hyper-V container settings. Used by Hyper-V containers only. Format ImagePath=%root%\BaseLayerID\UtilityVM + Servicing bool `json:",omitempty"` // True if this container is for servicing + AllowUnqualifiedDNSQuery bool `json:",omitempty"` // True to allow unqualified DNS name resolution + DNSSearchList string `json:",omitempty"` // Comma seperated list of DNS suffixes to use for name resolution + ContainerType string `json:",omitempty"` // "Linux" for Linux containers on Windows. Omitted otherwise. + TerminateOnLastHandleClosed bool `json:",omitempty"` // Should HCS terminate the container once all handles have been closed + MappedVirtualDisks []MappedVirtualDisk `json:",omitempty"` // Array of virtual disks to mount at start +} + +type ComputeSystemQuery struct { + IDs []string `json:"Ids,omitempty"` + Types []string `json:",omitempty"` + Names []string `json:",omitempty"` + Owners []string `json:",omitempty"` +} + +// Container represents a created (but not necessarily running) container. +type Container interface { + // Start synchronously starts the container. + Start() error + + // Shutdown requests a container shutdown, but it may not actually be shutdown until Wait() succeeds. + Shutdown() error + + // Terminate requests a container terminate, but it may not actually be terminated until Wait() succeeds. + Terminate() error + + // Waits synchronously waits for the container to shutdown or terminate. + Wait() error + + // WaitTimeout synchronously waits for the container to terminate or the duration to elapse. It + // returns false if timeout occurs. + WaitTimeout(time.Duration) error + + // Pause pauses the execution of a container. + Pause() error + + // Resume resumes the execution of a container. + Resume() error + + // HasPendingUpdates returns true if the container has updates pending to install. + HasPendingUpdates() (bool, error) + + // Statistics returns statistics for a container. + Statistics() (Statistics, error) + + // ProcessList returns details for the processes in a container. + ProcessList() ([]ProcessListItem, error) + + // MappedVirtualDisks returns virtual disks mapped to a utility VM, indexed by controller + MappedVirtualDisks() (map[int]MappedVirtualDiskController, error) + + // CreateProcess launches a new process within the container. + CreateProcess(c *ProcessConfig) (Process, error) + + // OpenProcess gets an interface to an existing process within the container. + OpenProcess(pid int) (Process, error) + + // Close cleans up any state associated with the container but does not terminate or wait for it. + Close() error + + // Modify the System + Modify(config *ResourceModificationRequestResponse) error +} + +// Process represents a running or exited process. +type Process interface { + // Pid returns the process ID of the process within the container. + Pid() int + + // Kill signals the process to terminate but does not wait for it to finish terminating. + Kill() error + + // Wait waits for the process to exit. + Wait() error + + // WaitTimeout waits for the process to exit or the duration to elapse. It returns + // false if timeout occurs. + WaitTimeout(time.Duration) error + + // ExitCode returns the exit code of the process. The process must have + // already terminated. + ExitCode() (int, error) + + // ResizeConsole resizes the console of the process. + ResizeConsole(width, height uint16) error + + // Stdio returns the stdin, stdout, and stderr pipes, respectively. Closing + // these pipes does not close the underlying pipes; it should be possible to + // call this multiple times to get multiple interfaces. + Stdio() (io.WriteCloser, io.ReadCloser, io.ReadCloser, error) + + // CloseStdin closes the write side of the stdin pipe so that the process is + // notified on the read side that there is no more data in stdin. + CloseStdin() error + + // Close cleans up any state associated with the process but does not kill + // or wait on it. + Close() error +} diff --git a/vendor/github.com/Microsoft/hcsshim/layerexists.go b/vendor/github.com/Microsoft/hcsshim/layerexists.go new file mode 100644 index 00000000000..fe46f404c38 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/layerexists.go @@ -0,0 +1,30 @@ +package hcsshim + +import "github.com/sirupsen/logrus" + +// LayerExists will return true if a layer with the given id exists and is known +// to the system. +func LayerExists(info DriverInfo, id string) (bool, error) { + title := "hcsshim::LayerExists " + logrus.Debugf(title+"Flavour %d ID %s", info.Flavour, id) + + // Convert info to API calling convention + infop, err := convertDriverInfo(info) + if err != nil { + logrus.Error(err) + return false, err + } + + // Call the procedure itself. + var exists uint32 + + err = layerExists(&infop, id, &exists) + if err != nil { + err = makeErrorf(err, title, "id=%s flavour=%d", id, info.Flavour) + logrus.Error(err) + return false, err + } + + logrus.Debugf(title+"succeeded flavour=%d id=%s exists=%d", info.Flavour, id, exists) + return exists != 0, nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/layerutils.go b/vendor/github.com/Microsoft/hcsshim/layerutils.go new file mode 100644 index 00000000000..c0e55037731 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/layerutils.go @@ -0,0 +1,111 @@ +package hcsshim + +// This file contains utility functions to support storage (graph) related +// functionality. + +import ( + "path/filepath" + "syscall" + + "github.com/sirupsen/logrus" +) + +/* To pass into syscall, we need a struct matching the following: +enum GraphDriverType +{ + DiffDriver, + FilterDriver +}; + +struct DriverInfo { + GraphDriverType Flavour; + LPCWSTR HomeDir; +}; +*/ +type DriverInfo struct { + Flavour int + HomeDir string +} + +type driverInfo struct { + Flavour int + HomeDirp *uint16 +} + +func convertDriverInfo(info DriverInfo) (driverInfo, error) { + homedirp, err := syscall.UTF16PtrFromString(info.HomeDir) + if err != nil { + logrus.Debugf("Failed conversion of home to pointer for driver info: %s", err.Error()) + return driverInfo{}, err + } + + return driverInfo{ + Flavour: info.Flavour, + HomeDirp: homedirp, + }, nil +} + +/* To pass into syscall, we need a struct matching the following: +typedef struct _WC_LAYER_DESCRIPTOR { + + // + // The ID of the layer + // + + GUID LayerId; + + // + // Additional flags + // + + union { + struct { + ULONG Reserved : 31; + ULONG Dirty : 1; // Created from sandbox as a result of snapshot + }; + ULONG Value; + } Flags; + + // + // Path to the layer root directory, null-terminated + // + + PCWSTR Path; + +} WC_LAYER_DESCRIPTOR, *PWC_LAYER_DESCRIPTOR; +*/ +type WC_LAYER_DESCRIPTOR struct { + LayerId GUID + Flags uint32 + Pathp *uint16 +} + +func layerPathsToDescriptors(parentLayerPaths []string) ([]WC_LAYER_DESCRIPTOR, error) { + // Array of descriptors that gets constructed. + var layers []WC_LAYER_DESCRIPTOR + + for i := 0; i < len(parentLayerPaths); i++ { + // Create a layer descriptor, using the folder name + // as the source for a GUID LayerId + _, folderName := filepath.Split(parentLayerPaths[i]) + g, err := NameToGuid(folderName) + if err != nil { + logrus.Debugf("Failed to convert name to guid %s", err) + return nil, err + } + + p, err := syscall.UTF16PtrFromString(parentLayerPaths[i]) + if err != nil { + logrus.Debugf("Failed conversion of parentLayerPath to pointer %s", err) + return nil, err + } + + layers = append(layers, WC_LAYER_DESCRIPTOR{ + LayerId: g, + Flags: 0, + Pathp: p, + }) + } + + return layers, nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/legacy.go b/vendor/github.com/Microsoft/hcsshim/legacy.go new file mode 100644 index 00000000000..c7f6073ac3c --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/legacy.go @@ -0,0 +1,741 @@ +package hcsshim + +import ( + "bufio" + "encoding/binary" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/Microsoft/go-winio" +) + +var errorIterationCanceled = errors.New("") + +var mutatedUtilityVMFiles = map[string]bool{ + `EFI\Microsoft\Boot\BCD`: true, + `EFI\Microsoft\Boot\BCD.LOG`: true, + `EFI\Microsoft\Boot\BCD.LOG1`: true, + `EFI\Microsoft\Boot\BCD.LOG2`: true, +} + +const ( + filesPath = `Files` + hivesPath = `Hives` + utilityVMPath = `UtilityVM` + utilityVMFilesPath = `UtilityVM\Files` +) + +func openFileOrDir(path string, mode uint32, createDisposition uint32) (file *os.File, err error) { + return winio.OpenForBackup(path, mode, syscall.FILE_SHARE_READ, createDisposition) +} + +func makeLongAbsPath(path string) (string, error) { + if strings.HasPrefix(path, `\\?\`) || strings.HasPrefix(path, `\\.\`) { + return path, nil + } + if !filepath.IsAbs(path) { + absPath, err := filepath.Abs(path) + if err != nil { + return "", err + } + path = absPath + } + if strings.HasPrefix(path, `\\`) { + return `\\?\UNC\` + path[2:], nil + } + return `\\?\` + path, nil +} + +func hasPathPrefix(p, prefix string) bool { + return strings.HasPrefix(p, prefix) && len(p) > len(prefix) && p[len(prefix)] == '\\' +} + +type fileEntry struct { + path string + fi os.FileInfo + err error +} + +type legacyLayerReader struct { + root string + result chan *fileEntry + proceed chan bool + currentFile *os.File + backupReader *winio.BackupFileReader +} + +// newLegacyLayerReader returns a new LayerReader that can read the Windows +// container layer transport format from disk. +func newLegacyLayerReader(root string) *legacyLayerReader { + r := &legacyLayerReader{ + root: root, + result: make(chan *fileEntry), + proceed: make(chan bool), + } + go r.walk() + return r +} + +func readTombstones(path string) (map[string]([]string), error) { + tf, err := os.Open(filepath.Join(path, "tombstones.txt")) + if err != nil { + return nil, err + } + defer tf.Close() + s := bufio.NewScanner(tf) + if !s.Scan() || s.Text() != "\xef\xbb\xbfVersion 1.0" { + return nil, errors.New("Invalid tombstones file") + } + + ts := make(map[string]([]string)) + for s.Scan() { + t := filepath.Join(filesPath, s.Text()[1:]) // skip leading `\` + dir := filepath.Dir(t) + ts[dir] = append(ts[dir], t) + } + if err = s.Err(); err != nil { + return nil, err + } + + return ts, nil +} + +func (r *legacyLayerReader) walkUntilCancelled() error { + root, err := makeLongAbsPath(r.root) + if err != nil { + return err + } + + r.root = root + ts, err := readTombstones(r.root) + if err != nil { + return err + } + + err = filepath.Walk(r.root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if path == r.root || path == filepath.Join(r.root, "tombstones.txt") || strings.HasSuffix(path, ".$wcidirs$") { + return nil + } + + r.result <- &fileEntry{path, info, nil} + if !<-r.proceed { + return errorIterationCanceled + } + + // List all the tombstones. + if info.IsDir() { + relPath, err := filepath.Rel(r.root, path) + if err != nil { + return err + } + if dts, ok := ts[relPath]; ok { + for _, t := range dts { + r.result <- &fileEntry{filepath.Join(r.root, t), nil, nil} + if !<-r.proceed { + return errorIterationCanceled + } + } + } + } + return nil + }) + if err == errorIterationCanceled { + return nil + } + if err == nil { + return io.EOF + } + return err +} + +func (r *legacyLayerReader) walk() { + defer close(r.result) + if !<-r.proceed { + return + } + + err := r.walkUntilCancelled() + if err != nil { + for { + r.result <- &fileEntry{err: err} + if !<-r.proceed { + return + } + } + } +} + +func (r *legacyLayerReader) reset() { + if r.backupReader != nil { + r.backupReader.Close() + r.backupReader = nil + } + if r.currentFile != nil { + r.currentFile.Close() + r.currentFile = nil + } +} + +func findBackupStreamSize(r io.Reader) (int64, error) { + br := winio.NewBackupStreamReader(r) + for { + hdr, err := br.Next() + if err != nil { + if err == io.EOF { + err = nil + } + return 0, err + } + if hdr.Id == winio.BackupData { + return hdr.Size, nil + } + } +} + +func (r *legacyLayerReader) Next() (path string, size int64, fileInfo *winio.FileBasicInfo, err error) { + r.reset() + r.proceed <- true + fe := <-r.result + if fe == nil { + err = errors.New("LegacyLayerReader closed") + return + } + if fe.err != nil { + err = fe.err + return + } + + path, err = filepath.Rel(r.root, fe.path) + if err != nil { + return + } + + if fe.fi == nil { + // This is a tombstone. Return a nil fileInfo. + return + } + + if fe.fi.IsDir() && hasPathPrefix(path, filesPath) { + fe.path += ".$wcidirs$" + } + + f, err := openFileOrDir(fe.path, syscall.GENERIC_READ, syscall.OPEN_EXISTING) + if err != nil { + return + } + defer func() { + if f != nil { + f.Close() + } + }() + + fileInfo, err = winio.GetFileBasicInfo(f) + if err != nil { + return + } + + if !hasPathPrefix(path, filesPath) { + size = fe.fi.Size() + r.backupReader = winio.NewBackupFileReader(f, false) + if path == hivesPath || path == filesPath { + // The Hives directory has a non-deterministic file time because of the + // nature of the import process. Use the times from System_Delta. + var g *os.File + g, err = os.Open(filepath.Join(r.root, hivesPath, `System_Delta`)) + if err != nil { + return + } + attr := fileInfo.FileAttributes + fileInfo, err = winio.GetFileBasicInfo(g) + g.Close() + if err != nil { + return + } + fileInfo.FileAttributes = attr + } + + // The creation time and access time get reset for files outside of the Files path. + fileInfo.CreationTime = fileInfo.LastWriteTime + fileInfo.LastAccessTime = fileInfo.LastWriteTime + + } else { + // The file attributes are written before the backup stream. + var attr uint32 + err = binary.Read(f, binary.LittleEndian, &attr) + if err != nil { + return + } + fileInfo.FileAttributes = uintptr(attr) + beginning := int64(4) + + // Find the accurate file size. + if !fe.fi.IsDir() { + size, err = findBackupStreamSize(f) + if err != nil { + err = &os.PathError{Op: "findBackupStreamSize", Path: fe.path, Err: err} + return + } + } + + // Return back to the beginning of the backup stream. + _, err = f.Seek(beginning, 0) + if err != nil { + return + } + } + + r.currentFile = f + f = nil + return +} + +func (r *legacyLayerReader) Read(b []byte) (int, error) { + if r.backupReader == nil { + if r.currentFile == nil { + return 0, io.EOF + } + return r.currentFile.Read(b) + } + return r.backupReader.Read(b) +} + +func (r *legacyLayerReader) Seek(offset int64, whence int) (int64, error) { + if r.backupReader == nil { + if r.currentFile == nil { + return 0, errors.New("no current file") + } + return r.currentFile.Seek(offset, whence) + } + return 0, errors.New("seek not supported on this stream") +} + +func (r *legacyLayerReader) Close() error { + r.proceed <- false + <-r.result + r.reset() + return nil +} + +type pendingLink struct { + Path, Target string +} + +type legacyLayerWriter struct { + root string + parentRoots []string + destRoot string + currentFile *os.File + backupWriter *winio.BackupFileWriter + tombstones []string + pathFixed bool + HasUtilityVM bool + uvmDi []dirInfo + addedFiles map[string]bool + PendingLinks []pendingLink +} + +// newLegacyLayerWriter returns a LayerWriter that can write the contaler layer +// transport format to disk. +func newLegacyLayerWriter(root string, parentRoots []string, destRoot string) *legacyLayerWriter { + return &legacyLayerWriter{ + root: root, + parentRoots: parentRoots, + destRoot: destRoot, + addedFiles: make(map[string]bool), + } +} + +func (w *legacyLayerWriter) init() error { + if !w.pathFixed { + path, err := makeLongAbsPath(w.root) + if err != nil { + return err + } + for i, p := range w.parentRoots { + w.parentRoots[i], err = makeLongAbsPath(p) + if err != nil { + return err + } + } + destPath, err := makeLongAbsPath(w.destRoot) + if err != nil { + return err + } + w.root = path + w.destRoot = destPath + w.pathFixed = true + } + return nil +} + +func (w *legacyLayerWriter) initUtilityVM() error { + if !w.HasUtilityVM { + err := os.Mkdir(filepath.Join(w.destRoot, utilityVMPath), 0) + if err != nil { + return err + } + // Server 2016 does not support multiple layers for the utility VM, so + // clone the utility VM from the parent layer into this layer. Use hard + // links to avoid unnecessary copying, since most of the files are + // immutable. + err = cloneTree(filepath.Join(w.parentRoots[0], utilityVMFilesPath), filepath.Join(w.destRoot, utilityVMFilesPath), mutatedUtilityVMFiles) + if err != nil { + return fmt.Errorf("cloning the parent utility VM image failed: %s", err) + } + w.HasUtilityVM = true + } + return nil +} + +func (w *legacyLayerWriter) reset() { + if w.backupWriter != nil { + w.backupWriter.Close() + w.backupWriter = nil + } + if w.currentFile != nil { + w.currentFile.Close() + w.currentFile = nil + } +} + +// copyFileWithMetadata copies a file using the backup/restore APIs in order to preserve metadata +func copyFileWithMetadata(srcPath, destPath string, isDir bool) (fileInfo *winio.FileBasicInfo, err error) { + createDisposition := uint32(syscall.CREATE_NEW) + if isDir { + err = os.Mkdir(destPath, 0) + if err != nil { + return nil, err + } + createDisposition = syscall.OPEN_EXISTING + } + + src, err := openFileOrDir(srcPath, syscall.GENERIC_READ|winio.ACCESS_SYSTEM_SECURITY, syscall.OPEN_EXISTING) + if err != nil { + return nil, err + } + defer src.Close() + srcr := winio.NewBackupFileReader(src, true) + defer srcr.Close() + + fileInfo, err = winio.GetFileBasicInfo(src) + if err != nil { + return nil, err + } + + dest, err := openFileOrDir(destPath, syscall.GENERIC_READ|syscall.GENERIC_WRITE|winio.WRITE_DAC|winio.WRITE_OWNER|winio.ACCESS_SYSTEM_SECURITY, createDisposition) + if err != nil { + return nil, err + } + defer dest.Close() + + err = winio.SetFileBasicInfo(dest, fileInfo) + if err != nil { + return nil, err + } + + destw := winio.NewBackupFileWriter(dest, true) + defer func() { + cerr := destw.Close() + if err == nil { + err = cerr + } + }() + + _, err = io.Copy(destw, srcr) + if err != nil { + return nil, err + } + + return fileInfo, nil +} + +// cloneTree clones a directory tree using hard links. It skips hard links for +// the file names in the provided map and just copies those files. +func cloneTree(srcPath, destPath string, mutatedFiles map[string]bool) error { + var di []dirInfo + err := filepath.Walk(srcPath, func(srcFilePath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(srcPath, srcFilePath) + if err != nil { + return err + } + destFilePath := filepath.Join(destPath, relPath) + + // Directories, reparse points, and files that will be mutated during + // utility VM import must be copied. All other files can be hard linked. + isReparsePoint := info.Sys().(*syscall.Win32FileAttributeData).FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0 + if info.IsDir() || isReparsePoint || mutatedFiles[relPath] { + fi, err := copyFileWithMetadata(srcFilePath, destFilePath, info.IsDir()) + if err != nil { + return err + } + if info.IsDir() && !isReparsePoint { + di = append(di, dirInfo{path: destFilePath, fileInfo: *fi}) + } + } else { + err = os.Link(srcFilePath, destFilePath) + if err != nil { + return err + } + } + + // Don't recurse on reparse points. + if info.IsDir() && isReparsePoint { + return filepath.SkipDir + } + + return nil + }) + if err != nil { + return err + } + + return reapplyDirectoryTimes(di) +} + +func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) error { + w.reset() + err := w.init() + if err != nil { + return err + } + + if name == utilityVMPath { + return w.initUtilityVM() + } + + if hasPathPrefix(name, utilityVMPath) { + if !w.HasUtilityVM { + return errors.New("missing UtilityVM directory") + } + if !hasPathPrefix(name, utilityVMFilesPath) && name != utilityVMFilesPath { + return errors.New("invalid UtilityVM layer") + } + path := filepath.Join(w.destRoot, name) + createDisposition := uint32(syscall.OPEN_EXISTING) + if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 { + st, err := os.Lstat(path) + if err != nil && !os.IsNotExist(err) { + return err + } + if st != nil { + // Delete the existing file/directory if it is not the same type as this directory. + existingAttr := st.Sys().(*syscall.Win32FileAttributeData).FileAttributes + if (uint32(fileInfo.FileAttributes)^existingAttr)&(syscall.FILE_ATTRIBUTE_DIRECTORY|syscall.FILE_ATTRIBUTE_REPARSE_POINT) != 0 { + if err = os.RemoveAll(path); err != nil { + return err + } + st = nil + } + } + if st == nil { + if err = os.Mkdir(path, 0); err != nil { + return err + } + } + if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT == 0 { + w.uvmDi = append(w.uvmDi, dirInfo{path: path, fileInfo: *fileInfo}) + } + } else { + // Overwrite any existing hard link. + err = os.Remove(path) + if err != nil && !os.IsNotExist(err) { + return err + } + createDisposition = syscall.CREATE_NEW + } + + f, err := openFileOrDir(path, syscall.GENERIC_READ|syscall.GENERIC_WRITE|winio.WRITE_DAC|winio.WRITE_OWNER|winio.ACCESS_SYSTEM_SECURITY, createDisposition) + if err != nil { + return err + } + defer func() { + if f != nil { + f.Close() + os.Remove(path) + } + }() + + err = winio.SetFileBasicInfo(f, fileInfo) + if err != nil { + return err + } + + w.backupWriter = winio.NewBackupFileWriter(f, true) + w.currentFile = f + w.addedFiles[name] = true + f = nil + return nil + } + + path := filepath.Join(w.root, name) + if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 { + err := os.Mkdir(path, 0) + if err != nil { + return err + } + path += ".$wcidirs$" + } + + f, err := openFileOrDir(path, syscall.GENERIC_READ|syscall.GENERIC_WRITE, syscall.CREATE_NEW) + if err != nil { + return err + } + defer func() { + if f != nil { + f.Close() + os.Remove(path) + } + }() + + strippedFi := *fileInfo + strippedFi.FileAttributes = 0 + err = winio.SetFileBasicInfo(f, &strippedFi) + if err != nil { + return err + } + + if hasPathPrefix(name, hivesPath) { + w.backupWriter = winio.NewBackupFileWriter(f, false) + } else { + // The file attributes are written before the stream. + err = binary.Write(f, binary.LittleEndian, uint32(fileInfo.FileAttributes)) + if err != nil { + return err + } + } + + w.currentFile = f + w.addedFiles[name] = true + f = nil + return nil +} + +func (w *legacyLayerWriter) AddLink(name string, target string) error { + w.reset() + err := w.init() + if err != nil { + return err + } + + var roots []string + if hasPathPrefix(target, filesPath) { + // Look for cross-layer hard link targets in the parent layers, since + // nothing is in the destination path yet. + roots = w.parentRoots + } else if hasPathPrefix(target, utilityVMFilesPath) { + // Since the utility VM is fully cloned into the destination path + // already, look for cross-layer hard link targets directly in the + // destination path. + roots = []string{w.destRoot} + } + + if roots == nil || (!hasPathPrefix(name, filesPath) && !hasPathPrefix(name, utilityVMFilesPath)) { + return errors.New("invalid hard link in layer") + } + + // Find to try the target of the link in a previously added file. If that + // fails, search in parent layers. + var selectedRoot string + if _, ok := w.addedFiles[target]; ok { + selectedRoot = w.destRoot + } else { + for _, r := range roots { + if _, err = os.Lstat(filepath.Join(r, target)); err != nil { + if !os.IsNotExist(err) { + return err + } + } else { + selectedRoot = r + break + } + } + if selectedRoot == "" { + return fmt.Errorf("failed to find link target for '%s' -> '%s'", name, target) + } + } + // The link can't be written until after the ImportLayer call. + w.PendingLinks = append(w.PendingLinks, pendingLink{ + Path: filepath.Join(w.destRoot, name), + Target: filepath.Join(selectedRoot, target), + }) + w.addedFiles[name] = true + return nil +} + +func (w *legacyLayerWriter) Remove(name string) error { + if hasPathPrefix(name, filesPath) { + w.tombstones = append(w.tombstones, name[len(filesPath)+1:]) + } else if hasPathPrefix(name, utilityVMFilesPath) { + err := w.initUtilityVM() + if err != nil { + return err + } + // Make sure the path exists; os.RemoveAll will not fail if the file is + // already gone, and this needs to be a fatal error for diagnostics + // purposes. + path := filepath.Join(w.destRoot, name) + if _, err := os.Lstat(path); err != nil { + return err + } + err = os.RemoveAll(path) + if err != nil { + return err + } + } else { + return fmt.Errorf("invalid tombstone %s", name) + } + + return nil +} + +func (w *legacyLayerWriter) Write(b []byte) (int, error) { + if w.backupWriter == nil { + if w.currentFile == nil { + return 0, errors.New("closed") + } + return w.currentFile.Write(b) + } + return w.backupWriter.Write(b) +} + +func (w *legacyLayerWriter) Close() error { + w.reset() + err := w.init() + if err != nil { + return err + } + tf, err := os.Create(filepath.Join(w.root, "tombstones.txt")) + if err != nil { + return err + } + defer tf.Close() + _, err = tf.Write([]byte("\xef\xbb\xbfVersion 1.0\n")) + if err != nil { + return err + } + for _, t := range w.tombstones { + _, err = tf.Write([]byte(filepath.Join(`\`, t) + "\n")) + if err != nil { + return err + } + } + if w.HasUtilityVM { + err = reapplyDirectoryTimes(w.uvmDi) + if err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/mksyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/mksyscall_windows.go new file mode 100644 index 00000000000..82393ca3670 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/mksyscall_windows.go @@ -0,0 +1,934 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +mksyscall_windows generates windows system call bodies + +It parses all files specified on command line containing function +prototypes (like syscall_windows.go) and prints system call bodies +to standard output. + +The prototypes are marked by lines beginning with "//sys" and read +like func declarations if //sys is replaced by func, but: + +* The parameter lists must give a name for each argument. This + includes return parameters. + +* The parameter lists must give a type for each argument: + the (x, y, z int) shorthand is not allowed. + +* If the return parameter is an error number, it must be named err. + +* If go func name needs to be different from it's winapi dll name, + the winapi name could be specified at the end, after "=" sign, like + //sys LoadLibrary(libname string) (handle uint32, err error) = LoadLibraryA + +* Each function that returns err needs to supply a condition, that + return value of winapi will be tested against to detect failure. + This would set err to windows "last-error", otherwise it will be nil. + The value can be provided at end of //sys declaration, like + //sys LoadLibrary(libname string) (handle uint32, err error) [failretval==-1] = LoadLibraryA + and is [failretval==0] by default. + +Usage: + mksyscall_windows [flags] [path ...] + +The flags are: + -output + Specify output file name (outputs to console if blank). + -trace + Generate print statement after every syscall. +*/ +package main + +import ( + "bufio" + "bytes" + "errors" + "flag" + "fmt" + "go/format" + "go/parser" + "go/token" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "text/template" +) + +var ( + filename = flag.String("output", "", "output file name (standard output if omitted)") + printTraceFlag = flag.Bool("trace", false, "generate print statement after every syscall") + systemDLL = flag.Bool("systemdll", true, "whether all DLLs should be loaded from the Windows system directory") +) + +func trim(s string) string { + return strings.Trim(s, " \t") +} + +var packageName string + +func packagename() string { + return packageName +} + +func syscalldot() string { + if packageName == "syscall" { + return "" + } + return "syscall." +} + +// Param is function parameter +type Param struct { + Name string + Type string + fn *Fn + tmpVarIdx int +} + +// tmpVar returns temp variable name that will be used to represent p during syscall. +func (p *Param) tmpVar() string { + if p.tmpVarIdx < 0 { + p.tmpVarIdx = p.fn.curTmpVarIdx + p.fn.curTmpVarIdx++ + } + return fmt.Sprintf("_p%d", p.tmpVarIdx) +} + +// BoolTmpVarCode returns source code for bool temp variable. +func (p *Param) BoolTmpVarCode() string { + const code = `var %s uint32 + if %s { + %s = 1 + } else { + %s = 0 + }` + tmp := p.tmpVar() + return fmt.Sprintf(code, tmp, p.Name, tmp, tmp) +} + +// SliceTmpVarCode returns source code for slice temp variable. +func (p *Param) SliceTmpVarCode() string { + const code = `var %s *%s + if len(%s) > 0 { + %s = &%s[0] + }` + tmp := p.tmpVar() + return fmt.Sprintf(code, tmp, p.Type[2:], p.Name, tmp, p.Name) +} + +// StringTmpVarCode returns source code for string temp variable. +func (p *Param) StringTmpVarCode() string { + errvar := p.fn.Rets.ErrorVarName() + if errvar == "" { + errvar = "_" + } + tmp := p.tmpVar() + const code = `var %s %s + %s, %s = %s(%s)` + s := fmt.Sprintf(code, tmp, p.fn.StrconvType(), tmp, errvar, p.fn.StrconvFunc(), p.Name) + if errvar == "-" { + return s + } + const morecode = ` + if %s != nil { + return + }` + return s + fmt.Sprintf(morecode, errvar) +} + +// TmpVarCode returns source code for temp variable. +func (p *Param) TmpVarCode() string { + switch { + case p.Type == "bool": + return p.BoolTmpVarCode() + case strings.HasPrefix(p.Type, "[]"): + return p.SliceTmpVarCode() + default: + return "" + } +} + +// TmpVarHelperCode returns source code for helper's temp variable. +func (p *Param) TmpVarHelperCode() string { + if p.Type != "string" { + return "" + } + return p.StringTmpVarCode() +} + +// SyscallArgList returns source code fragments representing p parameter +// in syscall. Slices are translated into 2 syscall parameters: pointer to +// the first element and length. +func (p *Param) SyscallArgList() []string { + t := p.HelperType() + var s string + switch { + case t[0] == '*': + s = fmt.Sprintf("unsafe.Pointer(%s)", p.Name) + case t == "bool": + s = p.tmpVar() + case strings.HasPrefix(t, "[]"): + return []string{ + fmt.Sprintf("uintptr(unsafe.Pointer(%s))", p.tmpVar()), + fmt.Sprintf("uintptr(len(%s))", p.Name), + } + default: + s = p.Name + } + return []string{fmt.Sprintf("uintptr(%s)", s)} +} + +// IsError determines if p parameter is used to return error. +func (p *Param) IsError() bool { + return p.Name == "err" && p.Type == "error" +} + +// HelperType returns type of parameter p used in helper function. +func (p *Param) HelperType() string { + if p.Type == "string" { + return p.fn.StrconvType() + } + return p.Type +} + +// join concatenates parameters ps into a string with sep separator. +// Each parameter is converted into string by applying fn to it +// before conversion. +func join(ps []*Param, fn func(*Param) string, sep string) string { + if len(ps) == 0 { + return "" + } + a := make([]string, 0) + for _, p := range ps { + a = append(a, fn(p)) + } + return strings.Join(a, sep) +} + +// Rets describes function return parameters. +type Rets struct { + Name string + Type string + ReturnsError bool + FailCond string +} + +// ErrorVarName returns error variable name for r. +func (r *Rets) ErrorVarName() string { + if r.ReturnsError { + return "err" + } + if r.Type == "error" { + return r.Name + } + return "" +} + +// ToParams converts r into slice of *Param. +func (r *Rets) ToParams() []*Param { + ps := make([]*Param, 0) + if len(r.Name) > 0 { + ps = append(ps, &Param{Name: r.Name, Type: r.Type}) + } + if r.ReturnsError { + ps = append(ps, &Param{Name: "err", Type: "error"}) + } + return ps +} + +// List returns source code of syscall return parameters. +func (r *Rets) List() string { + s := join(r.ToParams(), func(p *Param) string { return p.Name + " " + p.Type }, ", ") + if len(s) > 0 { + s = "(" + s + ")" + } + return s +} + +// PrintList returns source code of trace printing part correspondent +// to syscall return values. +func (r *Rets) PrintList() string { + return join(r.ToParams(), func(p *Param) string { return fmt.Sprintf(`"%s=", %s, `, p.Name, p.Name) }, `", ", `) +} + +// SetReturnValuesCode returns source code that accepts syscall return values. +func (r *Rets) SetReturnValuesCode() string { + if r.Name == "" && !r.ReturnsError { + return "" + } + retvar := "r0" + if r.Name == "" { + retvar = "r1" + } + errvar := "_" + if r.ReturnsError { + errvar = "e1" + } + return fmt.Sprintf("%s, _, %s := ", retvar, errvar) +} + +func (r *Rets) useLongHandleErrorCode(retvar string) string { + const code = `if %s { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = %sEINVAL + } + }` + cond := retvar + " == 0" + if r.FailCond != "" { + cond = strings.Replace(r.FailCond, "failretval", retvar, 1) + } + return fmt.Sprintf(code, cond, syscalldot()) +} + +// SetErrorCode returns source code that sets return parameters. +func (r *Rets) SetErrorCode() string { + const code = `if r0 != 0 { + %s = %sErrno(r0) + }` + const hrCode = `if int32(r0) < 0 { + %s = %sErrno(win32FromHresult(r0)) + }` + if r.Name == "" && !r.ReturnsError { + return "" + } + if r.Name == "" { + return r.useLongHandleErrorCode("r1") + } + if r.Type == "error" { + if r.Name == "hr" { + return fmt.Sprintf(hrCode, r.Name, syscalldot()) + } else { + return fmt.Sprintf(code, r.Name, syscalldot()) + } + } + s := "" + switch { + case r.Type[0] == '*': + s = fmt.Sprintf("%s = (%s)(unsafe.Pointer(r0))", r.Name, r.Type) + case r.Type == "bool": + s = fmt.Sprintf("%s = r0 != 0", r.Name) + default: + s = fmt.Sprintf("%s = %s(r0)", r.Name, r.Type) + } + if !r.ReturnsError { + return s + } + return s + "\n\t" + r.useLongHandleErrorCode(r.Name) +} + +// Fn describes syscall function. +type Fn struct { + Name string + Params []*Param + Rets *Rets + PrintTrace bool + confirmproc bool + dllname string + dllfuncname string + src string + // TODO: get rid of this field and just use parameter index instead + curTmpVarIdx int // insure tmp variables have uniq names +} + +// extractParams parses s to extract function parameters. +func extractParams(s string, f *Fn) ([]*Param, error) { + s = trim(s) + if s == "" { + return nil, nil + } + a := strings.Split(s, ",") + ps := make([]*Param, len(a)) + for i := range ps { + s2 := trim(a[i]) + b := strings.Split(s2, " ") + if len(b) != 2 { + b = strings.Split(s2, "\t") + if len(b) != 2 { + return nil, errors.New("Could not extract function parameter from \"" + s2 + "\"") + } + } + ps[i] = &Param{ + Name: trim(b[0]), + Type: trim(b[1]), + fn: f, + tmpVarIdx: -1, + } + } + return ps, nil +} + +// extractSection extracts text out of string s starting after start +// and ending just before end. found return value will indicate success, +// and prefix, body and suffix will contain correspondent parts of string s. +func extractSection(s string, start, end rune) (prefix, body, suffix string, found bool) { + s = trim(s) + if strings.HasPrefix(s, string(start)) { + // no prefix + body = s[1:] + } else { + a := strings.SplitN(s, string(start), 2) + if len(a) != 2 { + return "", "", s, false + } + prefix = a[0] + body = a[1] + } + a := strings.SplitN(body, string(end), 2) + if len(a) != 2 { + return "", "", "", false + } + return prefix, a[0], a[1], true +} + +// newFn parses string s and return created function Fn. +func newFn(s string) (*Fn, error) { + s = trim(s) + f := &Fn{ + Rets: &Rets{}, + src: s, + PrintTrace: *printTraceFlag, + } + // function name and args + prefix, body, s, found := extractSection(s, '(', ')') + if !found || prefix == "" { + return nil, errors.New("Could not extract function name and parameters from \"" + f.src + "\"") + } + f.Name = prefix + var err error + f.Params, err = extractParams(body, f) + if err != nil { + return nil, err + } + // return values + _, body, s, found = extractSection(s, '(', ')') + if found { + r, err := extractParams(body, f) + if err != nil { + return nil, err + } + switch len(r) { + case 0: + case 1: + if r[0].IsError() { + f.Rets.ReturnsError = true + } else { + f.Rets.Name = r[0].Name + f.Rets.Type = r[0].Type + } + case 2: + if !r[1].IsError() { + return nil, errors.New("Only last windows error is allowed as second return value in \"" + f.src + "\"") + } + f.Rets.ReturnsError = true + f.Rets.Name = r[0].Name + f.Rets.Type = r[0].Type + default: + return nil, errors.New("Too many return values in \"" + f.src + "\"") + } + } + // fail condition + _, body, s, found = extractSection(s, '[', ']') + if found { + f.Rets.FailCond = body + } + // dll and dll function names + s = trim(s) + if s == "" { + return f, nil + } + if !strings.HasPrefix(s, "=") { + return nil, errors.New("Could not extract dll name from \"" + f.src + "\"") + } + s = trim(s[1:]) + a := strings.Split(s, ".") + switch len(a) { + case 1: + f.dllfuncname = a[0] + case 2: + f.dllname = a[0] + f.dllfuncname = a[1] + default: + return nil, errors.New("Could not extract dll name from \"" + f.src + "\"") + } + if f.dllfuncname[len(f.dllfuncname)-1] == '?' { + f.confirmproc = true + f.dllfuncname = f.dllfuncname[0 : len(f.dllfuncname)-1] + } + return f, nil +} + +// DLLName returns DLL name for function f. +func (f *Fn) DLLName() string { + if f.dllname == "" { + return "kernel32" + } + return f.dllname +} + +// DLLName returns DLL function name for function f. +func (f *Fn) DLLFuncName() string { + if f.dllfuncname == "" { + return f.Name + } + return f.dllfuncname +} + +func (f *Fn) ConfirmProc() bool { + return f.confirmproc +} + +// ParamList returns source code for function f parameters. +func (f *Fn) ParamList() string { + return join(f.Params, func(p *Param) string { return p.Name + " " + p.Type }, ", ") +} + +// HelperParamList returns source code for helper function f parameters. +func (f *Fn) HelperParamList() string { + return join(f.Params, func(p *Param) string { return p.Name + " " + p.HelperType() }, ", ") +} + +// ParamPrintList returns source code of trace printing part correspondent +// to syscall input parameters. +func (f *Fn) ParamPrintList() string { + return join(f.Params, func(p *Param) string { return fmt.Sprintf(`"%s=", %s, `, p.Name, p.Name) }, `", ", `) +} + +// ParamCount return number of syscall parameters for function f. +func (f *Fn) ParamCount() int { + n := 0 + for _, p := range f.Params { + n += len(p.SyscallArgList()) + } + return n +} + +// SyscallParamCount determines which version of Syscall/Syscall6/Syscall9/... +// to use. It returns parameter count for correspondent SyscallX function. +func (f *Fn) SyscallParamCount() int { + n := f.ParamCount() + switch { + case n <= 3: + return 3 + case n <= 6: + return 6 + case n <= 9: + return 9 + case n <= 12: + return 12 + case n <= 15: + return 15 + default: + panic("too many arguments to system call") + } +} + +// Syscall determines which SyscallX function to use for function f. +func (f *Fn) Syscall() string { + c := f.SyscallParamCount() + if c == 3 { + return syscalldot() + "Syscall" + } + return syscalldot() + "Syscall" + strconv.Itoa(c) +} + +// SyscallParamList returns source code for SyscallX parameters for function f. +func (f *Fn) SyscallParamList() string { + a := make([]string, 0) + for _, p := range f.Params { + a = append(a, p.SyscallArgList()...) + } + for len(a) < f.SyscallParamCount() { + a = append(a, "0") + } + return strings.Join(a, ", ") +} + +// HelperCallParamList returns source code of call into function f helper. +func (f *Fn) HelperCallParamList() string { + a := make([]string, 0, len(f.Params)) + for _, p := range f.Params { + s := p.Name + if p.Type == "string" { + s = p.tmpVar() + } + a = append(a, s) + } + return strings.Join(a, ", ") +} + +// IsUTF16 is true, if f is W (utf16) function. It is false +// for all A (ascii) functions. +func (_ *Fn) IsUTF16() bool { + return true +} + +// StrconvFunc returns name of Go string to OS string function for f. +func (f *Fn) StrconvFunc() string { + if f.IsUTF16() { + return syscalldot() + "UTF16PtrFromString" + } + return syscalldot() + "BytePtrFromString" +} + +// StrconvType returns Go type name used for OS string for f. +func (f *Fn) StrconvType() string { + if f.IsUTF16() { + return "*uint16" + } + return "*byte" +} + +// HasStringParam is true, if f has at least one string parameter. +// Otherwise it is false. +func (f *Fn) HasStringParam() bool { + for _, p := range f.Params { + if p.Type == "string" { + return true + } + } + return false +} + +var uniqDllFuncName = make(map[string]bool) + +// IsNotDuplicate is true if f is not a duplicated function +func (f *Fn) IsNotDuplicate() bool { + funcName := f.DLLFuncName() + if uniqDllFuncName[funcName] == false { + uniqDllFuncName[funcName] = true + return true + } + return false +} + +// HelperName returns name of function f helper. +func (f *Fn) HelperName() string { + if !f.HasStringParam() { + return f.Name + } + return "_" + f.Name +} + +// Source files and functions. +type Source struct { + Funcs []*Fn + Files []string + StdLibImports []string + ExternalImports []string +} + +func (src *Source) Import(pkg string) { + src.StdLibImports = append(src.StdLibImports, pkg) + sort.Strings(src.StdLibImports) +} + +func (src *Source) ExternalImport(pkg string) { + src.ExternalImports = append(src.ExternalImports, pkg) + sort.Strings(src.ExternalImports) +} + +// ParseFiles parses files listed in fs and extracts all syscall +// functions listed in sys comments. It returns source files +// and functions collection *Source if successful. +func ParseFiles(fs []string) (*Source, error) { + src := &Source{ + Funcs: make([]*Fn, 0), + Files: make([]string, 0), + StdLibImports: []string{ + "unsafe", + }, + ExternalImports: make([]string, 0), + } + for _, file := range fs { + if err := src.ParseFile(file); err != nil { + return nil, err + } + } + return src, nil +} + +// DLLs return dll names for a source set src. +func (src *Source) DLLs() []string { + uniq := make(map[string]bool) + r := make([]string, 0) + for _, f := range src.Funcs { + name := f.DLLName() + if _, found := uniq[name]; !found { + uniq[name] = true + r = append(r, name) + } + } + return r +} + +// ParseFile adds additional file path to a source set src. +func (src *Source) ParseFile(path string) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + s := bufio.NewScanner(file) + for s.Scan() { + t := trim(s.Text()) + if len(t) < 7 { + continue + } + if !strings.HasPrefix(t, "//sys") { + continue + } + t = t[5:] + if !(t[0] == ' ' || t[0] == '\t') { + continue + } + f, err := newFn(t[1:]) + if err != nil { + return err + } + src.Funcs = append(src.Funcs, f) + } + if err := s.Err(); err != nil { + return err + } + src.Files = append(src.Files, path) + + // get package name + fset := token.NewFileSet() + _, err = file.Seek(0, 0) + if err != nil { + return err + } + pkg, err := parser.ParseFile(fset, "", file, parser.PackageClauseOnly) + if err != nil { + return err + } + packageName = pkg.Name.Name + + return nil +} + +// IsStdRepo returns true if src is part of standard library. +func (src *Source) IsStdRepo() (bool, error) { + if len(src.Files) == 0 { + return false, errors.New("no input files provided") + } + abspath, err := filepath.Abs(src.Files[0]) + if err != nil { + return false, err + } + goroot := runtime.GOROOT() + if runtime.GOOS == "windows" { + abspath = strings.ToLower(abspath) + goroot = strings.ToLower(goroot) + } + sep := string(os.PathSeparator) + if !strings.HasSuffix(goroot, sep) { + goroot += sep + } + return strings.HasPrefix(abspath, goroot), nil +} + +// Generate output source file from a source set src. +func (src *Source) Generate(w io.Writer) error { + const ( + pkgStd = iota // any package in std library + pkgXSysWindows // x/sys/windows package + pkgOther + ) + isStdRepo, err := src.IsStdRepo() + if err != nil { + return err + } + var pkgtype int + switch { + case isStdRepo: + pkgtype = pkgStd + case packageName == "windows": + // TODO: this needs better logic than just using package name + pkgtype = pkgXSysWindows + default: + pkgtype = pkgOther + } + if *systemDLL { + switch pkgtype { + case pkgStd: + src.Import("internal/syscall/windows/sysdll") + case pkgXSysWindows: + default: + src.ExternalImport("golang.org/x/sys/windows") + } + } + src.ExternalImport("github.com/Microsoft/go-winio") + if packageName != "syscall" { + src.Import("syscall") + } + funcMap := template.FuncMap{ + "packagename": packagename, + "syscalldot": syscalldot, + "newlazydll": func(dll string) string { + arg := "\"" + dll + ".dll\"" + if !*systemDLL { + return syscalldot() + "NewLazyDLL(" + arg + ")" + } + switch pkgtype { + case pkgStd: + return syscalldot() + "NewLazyDLL(sysdll.Add(" + arg + "))" + case pkgXSysWindows: + return "NewLazySystemDLL(" + arg + ")" + default: + return "windows.NewLazySystemDLL(" + arg + ")" + } + }, + } + t := template.Must(template.New("main").Funcs(funcMap).Parse(srcTemplate)) + err = t.Execute(w, src) + if err != nil { + return errors.New("Failed to execute template: " + err.Error()) + } + return nil +} + +func usage() { + fmt.Fprintf(os.Stderr, "usage: mksyscall_windows [flags] [path ...]\n") + flag.PrintDefaults() + os.Exit(1) +} + +func main() { + flag.Usage = usage + flag.Parse() + if len(flag.Args()) <= 0 { + fmt.Fprintf(os.Stderr, "no files to parse provided\n") + usage() + } + + src, err := ParseFiles(flag.Args()) + if err != nil { + log.Fatal(err) + } + + var buf bytes.Buffer + if err := src.Generate(&buf); err != nil { + log.Fatal(err) + } + + data, err := format.Source(buf.Bytes()) + if err != nil { + log.Fatal(err) + } + if *filename == "" { + _, err = os.Stdout.Write(data) + } else { + err = ioutil.WriteFile(*filename, data, 0644) + } + if err != nil { + log.Fatal(err) + } +} + +// TODO: use println instead to print in the following template +const srcTemplate = ` + +{{define "main"}}// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT + +package {{packagename}} + +import ( +{{range .StdLibImports}}"{{.}}" +{{end}} + +{{range .ExternalImports}}"{{.}}" +{{end}} +) + +var _ unsafe.Pointer + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = {{syscalldot}}Errno(errnoERROR_IO_PENDING) +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e {{syscalldot}}Errno) error { + switch e { + case 0: + return nil + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + +var ( +{{template "dlls" .}} +{{template "funcnames" .}}) +{{range .Funcs}}{{if .HasStringParam}}{{template "helperbody" .}}{{end}}{{template "funcbody" .}}{{end}} +{{end}} + +{{/* help functions */}} + +{{define "dlls"}}{{range .DLLs}} mod{{.}} = {{newlazydll .}} +{{end}}{{end}} + +{{define "funcnames"}}{{range .Funcs}}{{if .IsNotDuplicate}} proc{{.DLLFuncName}} = mod{{.DLLName}}.NewProc("{{.DLLFuncName}}"){{end}} +{{end}}{{end}} + +{{define "helperbody"}} +func {{.Name}}({{.ParamList}}) {{template "results" .}}{ +{{template "helpertmpvars" .}} return {{.HelperName}}({{.HelperCallParamList}}) +} +{{end}} + +{{define "funcbody"}} +func {{.HelperName}}({{.HelperParamList}}) {{template "results" .}}{ +{{template "tmpvars" .}} {{template "syscallcheck" .}}{{template "syscall" .}} +{{template "seterror" .}}{{template "printtrace" .}} return +} +{{end}} + +{{define "helpertmpvars"}}{{range .Params}}{{if .TmpVarHelperCode}} {{.TmpVarHelperCode}} +{{end}}{{end}}{{end}} + +{{define "tmpvars"}}{{range .Params}}{{if .TmpVarCode}} {{.TmpVarCode}} +{{end}}{{end}}{{end}} + +{{define "results"}}{{if .Rets.List}}{{.Rets.List}} {{end}}{{end}} + +{{define "syscall"}}{{.Rets.SetReturnValuesCode}}{{.Syscall}}(proc{{.DLLFuncName}}.Addr(), {{.ParamCount}}, {{.SyscallParamList}}){{end}} + +{{define "syscallcheck"}}{{if .ConfirmProc}}if {{.Rets.ErrorVarName}} = proc{{.DLLFuncName}}.Find(); {{.Rets.ErrorVarName}} != nil { + return +} +{{end}}{{end}} + + +{{define "seterror"}}{{if .Rets.SetErrorCode}} {{.Rets.SetErrorCode}} +{{end}}{{end}} + +{{define "printtrace"}}{{if .PrintTrace}} print("SYSCALL: {{.Name}}(", {{.ParamPrintList}}") (", {{.Rets.PrintList}}")\n") +{{end}}{{end}} + +` diff --git a/vendor/github.com/Microsoft/hcsshim/nametoguid.go b/vendor/github.com/Microsoft/hcsshim/nametoguid.go new file mode 100644 index 00000000000..b7c6d020c68 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/nametoguid.go @@ -0,0 +1,20 @@ +package hcsshim + +import "github.com/sirupsen/logrus" + +// NameToGuid converts the given string into a GUID using the algorithm in the +// Host Compute Service, ensuring GUIDs generated with the same string are common +// across all clients. +func NameToGuid(name string) (id GUID, err error) { + title := "hcsshim::NameToGuid " + logrus.Debugf(title+"Name %s", name) + + err = nameToGuid(name, &id) + if err != nil { + err = makeErrorf(err, title, "name=%s", name) + logrus.Error(err) + return + } + + return +} diff --git a/vendor/github.com/Microsoft/hcsshim/preparelayer.go b/vendor/github.com/Microsoft/hcsshim/preparelayer.go new file mode 100644 index 00000000000..5c5b618411c --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/preparelayer.go @@ -0,0 +1,46 @@ +package hcsshim + +import ( + "sync" + + "github.com/sirupsen/logrus" +) + +var prepareLayerLock sync.Mutex + +// PrepareLayer finds a mounted read-write layer matching layerId and enables the +// the filesystem filter for use on that layer. This requires the paths to all +// parent layers, and is necessary in order to view or interact with the layer +// as an actual filesystem (reading and writing files, creating directories, etc). +// Disabling the filter must be done via UnprepareLayer. +func PrepareLayer(info DriverInfo, layerId string, parentLayerPaths []string) error { + title := "hcsshim::PrepareLayer " + logrus.Debugf(title+"flavour %d layerId %s", info.Flavour, layerId) + + // Generate layer descriptors + layers, err := layerPathsToDescriptors(parentLayerPaths) + if err != nil { + return err + } + + // Convert info to API calling convention + infop, err := convertDriverInfo(info) + if err != nil { + logrus.Error(err) + return err + } + + // This lock is a temporary workaround for a Windows bug. Only allowing one + // call to prepareLayer at a time vastly reduces the chance of a timeout. + prepareLayerLock.Lock() + defer prepareLayerLock.Unlock() + err = prepareLayer(&infop, layerId, layers) + if err != nil { + err = makeErrorf(err, title, "layerId=%s flavour=%d", layerId, info.Flavour) + logrus.Error(err) + return err + } + + logrus.Debugf(title+"succeeded flavour=%d layerId=%s", info.Flavour, layerId) + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/process.go b/vendor/github.com/Microsoft/hcsshim/process.go new file mode 100644 index 00000000000..faee2cfeeb5 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/process.go @@ -0,0 +1,384 @@ +package hcsshim + +import ( + "encoding/json" + "io" + "sync" + "syscall" + "time" + + "github.com/sirupsen/logrus" +) + +// ContainerError is an error encountered in HCS +type process struct { + handleLock sync.RWMutex + handle hcsProcess + processID int + container *container + cachedPipes *cachedPipes + callbackNumber uintptr +} + +type cachedPipes struct { + stdIn syscall.Handle + stdOut syscall.Handle + stdErr syscall.Handle +} + +type processModifyRequest struct { + Operation string + ConsoleSize *consoleSize `json:",omitempty"` + CloseHandle *closeHandle `json:",omitempty"` +} + +type consoleSize struct { + Height uint16 + Width uint16 +} + +type closeHandle struct { + Handle string +} + +type processStatus struct { + ProcessID uint32 + Exited bool + ExitCode uint32 + LastWaitResult int32 +} + +const ( + stdIn string = "StdIn" + stdOut string = "StdOut" + stdErr string = "StdErr" +) + +const ( + modifyConsoleSize string = "ConsoleSize" + modifyCloseHandle string = "CloseHandle" +) + +// Pid returns the process ID of the process within the container. +func (process *process) Pid() int { + return process.processID +} + +// Kill signals the process to terminate but does not wait for it to finish terminating. +func (process *process) Kill() error { + process.handleLock.RLock() + defer process.handleLock.RUnlock() + operation := "Kill" + title := "HCSShim::Process::" + operation + logrus.Debugf(title+" processid=%d", process.processID) + + if process.handle == 0 { + return makeProcessError(process, operation, "", ErrAlreadyClosed) + } + + var resultp *uint16 + err := hcsTerminateProcess(process.handle, &resultp) + err = processHcsResult(err, resultp) + if err != nil { + return makeProcessError(process, operation, "", err) + } + + logrus.Debugf(title+" succeeded processid=%d", process.processID) + return nil +} + +// Wait waits for the process to exit. +func (process *process) Wait() error { + operation := "Wait" + title := "HCSShim::Process::" + operation + logrus.Debugf(title+" processid=%d", process.processID) + + err := waitForNotification(process.callbackNumber, hcsNotificationProcessExited, nil) + if err != nil { + return makeProcessError(process, operation, "", err) + } + + logrus.Debugf(title+" succeeded processid=%d", process.processID) + return nil +} + +// WaitTimeout waits for the process to exit or the duration to elapse. It returns +// false if timeout occurs. +func (process *process) WaitTimeout(timeout time.Duration) error { + operation := "WaitTimeout" + title := "HCSShim::Process::" + operation + logrus.Debugf(title+" processid=%d", process.processID) + + err := waitForNotification(process.callbackNumber, hcsNotificationProcessExited, &timeout) + if err != nil { + return makeProcessError(process, operation, "", err) + } + + logrus.Debugf(title+" succeeded processid=%d", process.processID) + return nil +} + +// ExitCode returns the exit code of the process. The process must have +// already terminated. +func (process *process) ExitCode() (int, error) { + process.handleLock.RLock() + defer process.handleLock.RUnlock() + operation := "ExitCode" + title := "HCSShim::Process::" + operation + logrus.Debugf(title+" processid=%d", process.processID) + + if process.handle == 0 { + return 0, makeProcessError(process, operation, "", ErrAlreadyClosed) + } + + properties, err := process.properties() + if err != nil { + return 0, makeProcessError(process, operation, "", err) + } + + if properties.Exited == false { + return 0, makeProcessError(process, operation, "", ErrInvalidProcessState) + } + + if properties.LastWaitResult != 0 { + return 0, makeProcessError(process, operation, "", syscall.Errno(properties.LastWaitResult)) + } + + logrus.Debugf(title+" succeeded processid=%d exitCode=%d", process.processID, properties.ExitCode) + return int(properties.ExitCode), nil +} + +// ResizeConsole resizes the console of the process. +func (process *process) ResizeConsole(width, height uint16) error { + process.handleLock.RLock() + defer process.handleLock.RUnlock() + operation := "ResizeConsole" + title := "HCSShim::Process::" + operation + logrus.Debugf(title+" processid=%d", process.processID) + + if process.handle == 0 { + return makeProcessError(process, operation, "", ErrAlreadyClosed) + } + + modifyRequest := processModifyRequest{ + Operation: modifyConsoleSize, + ConsoleSize: &consoleSize{ + Height: height, + Width: width, + }, + } + + modifyRequestb, err := json.Marshal(modifyRequest) + if err != nil { + return err + } + + modifyRequestStr := string(modifyRequestb) + + var resultp *uint16 + err = hcsModifyProcess(process.handle, modifyRequestStr, &resultp) + err = processHcsResult(err, resultp) + if err != nil { + return makeProcessError(process, operation, "", err) + } + + logrus.Debugf(title+" succeeded processid=%d", process.processID) + return nil +} + +func (process *process) properties() (*processStatus, error) { + operation := "properties" + title := "HCSShim::Process::" + operation + logrus.Debugf(title+" processid=%d", process.processID) + + var ( + resultp *uint16 + propertiesp *uint16 + ) + err := hcsGetProcessProperties(process.handle, &propertiesp, &resultp) + err = processHcsResult(err, resultp) + if err != nil { + return nil, err + } + + if propertiesp == nil { + return nil, ErrUnexpectedValue + } + propertiesRaw := convertAndFreeCoTaskMemBytes(propertiesp) + + properties := &processStatus{} + if err := json.Unmarshal(propertiesRaw, properties); err != nil { + return nil, err + } + + logrus.Debugf(title+" succeeded processid=%d, properties=%s", process.processID, propertiesRaw) + return properties, nil +} + +// Stdio returns the stdin, stdout, and stderr pipes, respectively. Closing +// these pipes does not close the underlying pipes; it should be possible to +// call this multiple times to get multiple interfaces. +func (process *process) Stdio() (io.WriteCloser, io.ReadCloser, io.ReadCloser, error) { + process.handleLock.RLock() + defer process.handleLock.RUnlock() + operation := "Stdio" + title := "HCSShim::Process::" + operation + logrus.Debugf(title+" processid=%d", process.processID) + + if process.handle == 0 { + return nil, nil, nil, makeProcessError(process, operation, "", ErrAlreadyClosed) + } + + var stdIn, stdOut, stdErr syscall.Handle + + if process.cachedPipes == nil { + var ( + processInfo hcsProcessInformation + resultp *uint16 + ) + err := hcsGetProcessInfo(process.handle, &processInfo, &resultp) + err = processHcsResult(err, resultp) + if err != nil { + return nil, nil, nil, makeProcessError(process, operation, "", err) + } + + stdIn, stdOut, stdErr = processInfo.StdInput, processInfo.StdOutput, processInfo.StdError + } else { + // Use cached pipes + stdIn, stdOut, stdErr = process.cachedPipes.stdIn, process.cachedPipes.stdOut, process.cachedPipes.stdErr + + // Invalidate the cache + process.cachedPipes = nil + } + + pipes, err := makeOpenFiles([]syscall.Handle{stdIn, stdOut, stdErr}) + if err != nil { + return nil, nil, nil, makeProcessError(process, operation, "", err) + } + + logrus.Debugf(title+" succeeded processid=%d", process.processID) + return pipes[0], pipes[1], pipes[2], nil +} + +// CloseStdin closes the write side of the stdin pipe so that the process is +// notified on the read side that there is no more data in stdin. +func (process *process) CloseStdin() error { + process.handleLock.RLock() + defer process.handleLock.RUnlock() + operation := "CloseStdin" + title := "HCSShim::Process::" + operation + logrus.Debugf(title+" processid=%d", process.processID) + + if process.handle == 0 { + return makeProcessError(process, operation, "", ErrAlreadyClosed) + } + + modifyRequest := processModifyRequest{ + Operation: modifyCloseHandle, + CloseHandle: &closeHandle{ + Handle: stdIn, + }, + } + + modifyRequestb, err := json.Marshal(modifyRequest) + if err != nil { + return err + } + + modifyRequestStr := string(modifyRequestb) + + var resultp *uint16 + err = hcsModifyProcess(process.handle, modifyRequestStr, &resultp) + err = processHcsResult(err, resultp) + if err != nil { + return makeProcessError(process, operation, "", err) + } + + logrus.Debugf(title+" succeeded processid=%d", process.processID) + return nil +} + +// Close cleans up any state associated with the process but does not kill +// or wait on it. +func (process *process) Close() error { + process.handleLock.Lock() + defer process.handleLock.Unlock() + operation := "Close" + title := "HCSShim::Process::" + operation + logrus.Debugf(title+" processid=%d", process.processID) + + // Don't double free this + if process.handle == 0 { + return nil + } + + if err := process.unregisterCallback(); err != nil { + return makeProcessError(process, operation, "", err) + } + + if err := hcsCloseProcess(process.handle); err != nil { + return makeProcessError(process, operation, "", err) + } + + process.handle = 0 + + logrus.Debugf(title+" succeeded processid=%d", process.processID) + return nil +} + +func (process *process) registerCallback() error { + context := ¬ifcationWatcherContext{ + channels: newChannels(), + } + + callbackMapLock.Lock() + callbackNumber := nextCallback + nextCallback++ + callbackMap[callbackNumber] = context + callbackMapLock.Unlock() + + var callbackHandle hcsCallback + err := hcsRegisterProcessCallback(process.handle, notificationWatcherCallback, callbackNumber, &callbackHandle) + if err != nil { + return err + } + context.handle = callbackHandle + process.callbackNumber = callbackNumber + + return nil +} + +func (process *process) unregisterCallback() error { + callbackNumber := process.callbackNumber + + callbackMapLock.RLock() + context := callbackMap[callbackNumber] + callbackMapLock.RUnlock() + + if context == nil { + return nil + } + + handle := context.handle + + if handle == 0 { + return nil + } + + // hcsUnregisterProcessCallback has its own syncronization + // to wait for all callbacks to complete. We must NOT hold the callbackMapLock. + err := hcsUnregisterProcessCallback(handle) + if err != nil { + return err + } + + closeChannels(context.channels) + + callbackMapLock.Lock() + callbackMap[callbackNumber] = nil + callbackMapLock.Unlock() + + handle = 0 + + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/processimage.go b/vendor/github.com/Microsoft/hcsshim/processimage.go new file mode 100644 index 00000000000..fadb1b92c5c --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/processimage.go @@ -0,0 +1,23 @@ +package hcsshim + +import "os" + +// ProcessBaseLayer post-processes a base layer that has had its files extracted. +// The files should have been extracted to \Files. +func ProcessBaseLayer(path string) error { + err := processBaseImage(path) + if err != nil { + return &os.PathError{Op: "ProcessBaseLayer", Path: path, Err: err} + } + return nil +} + +// ProcessUtilityVMImage post-processes a utility VM image that has had its files extracted. +// The files should have been extracted to \Files. +func ProcessUtilityVMImage(path string) error { + err := processUtilityImage(path) + if err != nil { + return &os.PathError{Op: "ProcessUtilityVMImage", Path: path, Err: err} + } + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/unpreparelayer.go b/vendor/github.com/Microsoft/hcsshim/unpreparelayer.go new file mode 100644 index 00000000000..e8a3b507bf5 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/unpreparelayer.go @@ -0,0 +1,27 @@ +package hcsshim + +import "github.com/sirupsen/logrus" + +// UnprepareLayer disables the filesystem filter for the read-write layer with +// the given id. +func UnprepareLayer(info DriverInfo, layerId string) error { + title := "hcsshim::UnprepareLayer " + logrus.Debugf(title+"flavour %d layerId %s", info.Flavour, layerId) + + // Convert info to API calling convention + infop, err := convertDriverInfo(info) + if err != nil { + logrus.Error(err) + return err + } + + err = unprepareLayer(&infop, layerId) + if err != nil { + err = makeErrorf(err, title, "layerId=%s flavour=%d", layerId, info.Flavour) + logrus.Error(err) + return err + } + + logrus.Debugf(title+"succeeded flavour %d layerId=%s", info.Flavour, layerId) + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/utils.go b/vendor/github.com/Microsoft/hcsshim/utils.go new file mode 100644 index 00000000000..bd6e2d94abb --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/utils.go @@ -0,0 +1,33 @@ +package hcsshim + +import ( + "io" + "syscall" + + "github.com/Microsoft/go-winio" +) + +// makeOpenFiles calls winio.MakeOpenFile for each handle in a slice but closes all the handles +// if there is an error. +func makeOpenFiles(hs []syscall.Handle) (_ []io.ReadWriteCloser, err error) { + fs := make([]io.ReadWriteCloser, len(hs)) + for i, h := range hs { + if h != syscall.Handle(0) { + if err == nil { + fs[i], err = winio.MakeOpenFile(h) + } + if err != nil { + syscall.Close(h) + } + } + } + if err != nil { + for _, f := range fs { + if f != nil { + f.Close() + } + } + return nil, err + } + return fs, nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/version.go b/vendor/github.com/Microsoft/hcsshim/version.go new file mode 100644 index 00000000000..ae10c23d42b --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/version.go @@ -0,0 +1,7 @@ +package hcsshim + +// IsTP4 returns whether the currently running Windows build is at least TP4. +func IsTP4() bool { + // HNSCall was not present in TP4 + return procHNSCall.Find() != nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/waithelper.go b/vendor/github.com/Microsoft/hcsshim/waithelper.go new file mode 100644 index 00000000000..b7be20ea0cd --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/waithelper.go @@ -0,0 +1,63 @@ +package hcsshim + +import ( + "time" + + "github.com/sirupsen/logrus" +) + +func processAsyncHcsResult(err error, resultp *uint16, callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) error { + err = processHcsResult(err, resultp) + if IsPending(err) { + return waitForNotification(callbackNumber, expectedNotification, timeout) + } + + return err +} + +func waitForNotification(callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) error { + callbackMapLock.RLock() + channels := callbackMap[callbackNumber].channels + callbackMapLock.RUnlock() + + expectedChannel := channels[expectedNotification] + if expectedChannel == nil { + logrus.Errorf("unknown notification type in waitForNotification %x", expectedNotification) + return ErrInvalidNotificationType + } + + var c <-chan time.Time + if timeout != nil { + timer := time.NewTimer(*timeout) + c = timer.C + defer timer.Stop() + } + + select { + case err, ok := <-expectedChannel: + if !ok { + return ErrHandleClose + } + return err + case err, ok := <-channels[hcsNotificationSystemExited]: + if !ok { + return ErrHandleClose + } + // If the expected notification is hcsNotificationSystemExited which of the two selects + // chosen is random. Return the raw error if hcsNotificationSystemExited is expected + if channels[hcsNotificationSystemExited] == expectedChannel { + return err + } + return ErrUnexpectedContainerExit + case _, ok := <-channels[hcsNotificationServiceDisconnect]: + if !ok { + return ErrHandleClose + } + // hcsNotificationServiceDisconnect should never be an expected notification + // it does not need the same handling as hcsNotificationSystemExited + return ErrUnexpectedProcessAbort + case <-c: + return ErrTimeout + } + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/zhcsshim.go b/vendor/github.com/Microsoft/hcsshim/zhcsshim.go new file mode 100644 index 00000000000..5d1a851ae87 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/zhcsshim.go @@ -0,0 +1,1042 @@ +// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT + +package hcsshim + +import ( + "syscall" + "unsafe" + + "github.com/Microsoft/go-winio" + "golang.org/x/sys/windows" +) + +var _ unsafe.Pointer + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return nil + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + +var ( + modole32 = windows.NewLazySystemDLL("ole32.dll") + modiphlpapi = windows.NewLazySystemDLL("iphlpapi.dll") + modvmcompute = windows.NewLazySystemDLL("vmcompute.dll") + + procCoTaskMemFree = modole32.NewProc("CoTaskMemFree") + procSetCurrentThreadCompartmentId = modiphlpapi.NewProc("SetCurrentThreadCompartmentId") + procActivateLayer = modvmcompute.NewProc("ActivateLayer") + procCopyLayer = modvmcompute.NewProc("CopyLayer") + procCreateLayer = modvmcompute.NewProc("CreateLayer") + procCreateSandboxLayer = modvmcompute.NewProc("CreateSandboxLayer") + procExpandSandboxSize = modvmcompute.NewProc("ExpandSandboxSize") + procDeactivateLayer = modvmcompute.NewProc("DeactivateLayer") + procDestroyLayer = modvmcompute.NewProc("DestroyLayer") + procExportLayer = modvmcompute.NewProc("ExportLayer") + procGetLayerMountPath = modvmcompute.NewProc("GetLayerMountPath") + procGetBaseImages = modvmcompute.NewProc("GetBaseImages") + procImportLayer = modvmcompute.NewProc("ImportLayer") + procLayerExists = modvmcompute.NewProc("LayerExists") + procNameToGuid = modvmcompute.NewProc("NameToGuid") + procPrepareLayer = modvmcompute.NewProc("PrepareLayer") + procUnprepareLayer = modvmcompute.NewProc("UnprepareLayer") + procProcessBaseImage = modvmcompute.NewProc("ProcessBaseImage") + procProcessUtilityImage = modvmcompute.NewProc("ProcessUtilityImage") + procImportLayerBegin = modvmcompute.NewProc("ImportLayerBegin") + procImportLayerNext = modvmcompute.NewProc("ImportLayerNext") + procImportLayerWrite = modvmcompute.NewProc("ImportLayerWrite") + procImportLayerEnd = modvmcompute.NewProc("ImportLayerEnd") + procExportLayerBegin = modvmcompute.NewProc("ExportLayerBegin") + procExportLayerNext = modvmcompute.NewProc("ExportLayerNext") + procExportLayerRead = modvmcompute.NewProc("ExportLayerRead") + procExportLayerEnd = modvmcompute.NewProc("ExportLayerEnd") + procHcsEnumerateComputeSystems = modvmcompute.NewProc("HcsEnumerateComputeSystems") + procHcsCreateComputeSystem = modvmcompute.NewProc("HcsCreateComputeSystem") + procHcsOpenComputeSystem = modvmcompute.NewProc("HcsOpenComputeSystem") + procHcsCloseComputeSystem = modvmcompute.NewProc("HcsCloseComputeSystem") + procHcsStartComputeSystem = modvmcompute.NewProc("HcsStartComputeSystem") + procHcsShutdownComputeSystem = modvmcompute.NewProc("HcsShutdownComputeSystem") + procHcsTerminateComputeSystem = modvmcompute.NewProc("HcsTerminateComputeSystem") + procHcsPauseComputeSystem = modvmcompute.NewProc("HcsPauseComputeSystem") + procHcsResumeComputeSystem = modvmcompute.NewProc("HcsResumeComputeSystem") + procHcsGetComputeSystemProperties = modvmcompute.NewProc("HcsGetComputeSystemProperties") + procHcsModifyComputeSystem = modvmcompute.NewProc("HcsModifyComputeSystem") + procHcsRegisterComputeSystemCallback = modvmcompute.NewProc("HcsRegisterComputeSystemCallback") + procHcsUnregisterComputeSystemCallback = modvmcompute.NewProc("HcsUnregisterComputeSystemCallback") + procHcsCreateProcess = modvmcompute.NewProc("HcsCreateProcess") + procHcsOpenProcess = modvmcompute.NewProc("HcsOpenProcess") + procHcsCloseProcess = modvmcompute.NewProc("HcsCloseProcess") + procHcsTerminateProcess = modvmcompute.NewProc("HcsTerminateProcess") + procHcsGetProcessInfo = modvmcompute.NewProc("HcsGetProcessInfo") + procHcsGetProcessProperties = modvmcompute.NewProc("HcsGetProcessProperties") + procHcsModifyProcess = modvmcompute.NewProc("HcsModifyProcess") + procHcsGetServiceProperties = modvmcompute.NewProc("HcsGetServiceProperties") + procHcsRegisterProcessCallback = modvmcompute.NewProc("HcsRegisterProcessCallback") + procHcsUnregisterProcessCallback = modvmcompute.NewProc("HcsUnregisterProcessCallback") + procHcsModifyServiceSettings = modvmcompute.NewProc("HcsModifyServiceSettings") + procHNSCall = modvmcompute.NewProc("HNSCall") +) + +func coTaskMemFree(buffer unsafe.Pointer) { + syscall.Syscall(procCoTaskMemFree.Addr(), 1, uintptr(buffer), 0, 0) + return +} + +func SetCurrentThreadCompartmentId(compartmentId uint32) (hr error) { + r0, _, _ := syscall.Syscall(procSetCurrentThreadCompartmentId.Addr(), 1, uintptr(compartmentId), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func activateLayer(info *driverInfo, id string) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + return _activateLayer(info, _p0) +} + +func _activateLayer(info *driverInfo, id *uint16) (hr error) { + if hr = procActivateLayer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procActivateLayer.Addr(), 2, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func copyLayer(info *driverInfo, srcId string, dstId string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(srcId) + if hr != nil { + return + } + var _p1 *uint16 + _p1, hr = syscall.UTF16PtrFromString(dstId) + if hr != nil { + return + } + return _copyLayer(info, _p0, _p1, descriptors) +} + +func _copyLayer(info *driverInfo, srcId *uint16, dstId *uint16, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { + var _p2 *WC_LAYER_DESCRIPTOR + if len(descriptors) > 0 { + _p2 = &descriptors[0] + } + if hr = procCopyLayer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procCopyLayer.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(srcId)), uintptr(unsafe.Pointer(dstId)), uintptr(unsafe.Pointer(_p2)), uintptr(len(descriptors)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func createLayer(info *driverInfo, id string, parent string) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + var _p1 *uint16 + _p1, hr = syscall.UTF16PtrFromString(parent) + if hr != nil { + return + } + return _createLayer(info, _p0, _p1) +} + +func _createLayer(info *driverInfo, id *uint16, parent *uint16) (hr error) { + if hr = procCreateLayer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procCreateLayer.Addr(), 3, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(parent))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func createSandboxLayer(info *driverInfo, id string, parent string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + var _p1 *uint16 + _p1, hr = syscall.UTF16PtrFromString(parent) + if hr != nil { + return + } + return _createSandboxLayer(info, _p0, _p1, descriptors) +} + +func _createSandboxLayer(info *driverInfo, id *uint16, parent *uint16, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { + var _p2 *WC_LAYER_DESCRIPTOR + if len(descriptors) > 0 { + _p2 = &descriptors[0] + } + if hr = procCreateSandboxLayer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procCreateSandboxLayer.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(parent)), uintptr(unsafe.Pointer(_p2)), uintptr(len(descriptors)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func expandSandboxSize(info *driverInfo, id string, size uint64) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + return _expandSandboxSize(info, _p0, size) +} + +func _expandSandboxSize(info *driverInfo, id *uint16, size uint64) (hr error) { + if hr = procExpandSandboxSize.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procExpandSandboxSize.Addr(), 3, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(size)) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func deactivateLayer(info *driverInfo, id string) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + return _deactivateLayer(info, _p0) +} + +func _deactivateLayer(info *driverInfo, id *uint16) (hr error) { + if hr = procDeactivateLayer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procDeactivateLayer.Addr(), 2, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func destroyLayer(info *driverInfo, id string) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + return _destroyLayer(info, _p0) +} + +func _destroyLayer(info *driverInfo, id *uint16) (hr error) { + if hr = procDestroyLayer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procDestroyLayer.Addr(), 2, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func exportLayer(info *driverInfo, id string, path string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + var _p1 *uint16 + _p1, hr = syscall.UTF16PtrFromString(path) + if hr != nil { + return + } + return _exportLayer(info, _p0, _p1, descriptors) +} + +func _exportLayer(info *driverInfo, id *uint16, path *uint16, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { + var _p2 *WC_LAYER_DESCRIPTOR + if len(descriptors) > 0 { + _p2 = &descriptors[0] + } + if hr = procExportLayer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procExportLayer.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(_p2)), uintptr(len(descriptors)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func getLayerMountPath(info *driverInfo, id string, length *uintptr, buffer *uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + return _getLayerMountPath(info, _p0, length, buffer) +} + +func _getLayerMountPath(info *driverInfo, id *uint16, length *uintptr, buffer *uint16) (hr error) { + if hr = procGetLayerMountPath.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procGetLayerMountPath.Addr(), 4, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(length)), uintptr(unsafe.Pointer(buffer)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func getBaseImages(buffer **uint16) (hr error) { + if hr = procGetBaseImages.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procGetBaseImages.Addr(), 1, uintptr(unsafe.Pointer(buffer)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func importLayer(info *driverInfo, id string, path string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + var _p1 *uint16 + _p1, hr = syscall.UTF16PtrFromString(path) + if hr != nil { + return + } + return _importLayer(info, _p0, _p1, descriptors) +} + +func _importLayer(info *driverInfo, id *uint16, path *uint16, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { + var _p2 *WC_LAYER_DESCRIPTOR + if len(descriptors) > 0 { + _p2 = &descriptors[0] + } + if hr = procImportLayer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procImportLayer.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(_p2)), uintptr(len(descriptors)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func layerExists(info *driverInfo, id string, exists *uint32) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + return _layerExists(info, _p0, exists) +} + +func _layerExists(info *driverInfo, id *uint16, exists *uint32) (hr error) { + if hr = procLayerExists.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procLayerExists.Addr(), 3, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(exists))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func nameToGuid(name string, guid *GUID) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(name) + if hr != nil { + return + } + return _nameToGuid(_p0, guid) +} + +func _nameToGuid(name *uint16, guid *GUID) (hr error) { + if hr = procNameToGuid.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procNameToGuid.Addr(), 2, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(guid)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func prepareLayer(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + return _prepareLayer(info, _p0, descriptors) +} + +func _prepareLayer(info *driverInfo, id *uint16, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { + var _p1 *WC_LAYER_DESCRIPTOR + if len(descriptors) > 0 { + _p1 = &descriptors[0] + } + if hr = procPrepareLayer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procPrepareLayer.Addr(), 4, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(_p1)), uintptr(len(descriptors)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func unprepareLayer(info *driverInfo, id string) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + return _unprepareLayer(info, _p0) +} + +func _unprepareLayer(info *driverInfo, id *uint16) (hr error) { + if hr = procUnprepareLayer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procUnprepareLayer.Addr(), 2, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func processBaseImage(path string) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(path) + if hr != nil { + return + } + return _processBaseImage(_p0) +} + +func _processBaseImage(path *uint16) (hr error) { + if hr = procProcessBaseImage.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procProcessBaseImage.Addr(), 1, uintptr(unsafe.Pointer(path)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func processUtilityImage(path string) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(path) + if hr != nil { + return + } + return _processUtilityImage(_p0) +} + +func _processUtilityImage(path *uint16) (hr error) { + if hr = procProcessUtilityImage.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procProcessUtilityImage.Addr(), 1, uintptr(unsafe.Pointer(path)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func importLayerBegin(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + return _importLayerBegin(info, _p0, descriptors, context) +} + +func _importLayerBegin(info *driverInfo, id *uint16, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) { + var _p1 *WC_LAYER_DESCRIPTOR + if len(descriptors) > 0 { + _p1 = &descriptors[0] + } + if hr = procImportLayerBegin.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procImportLayerBegin.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(_p1)), uintptr(len(descriptors)), uintptr(unsafe.Pointer(context)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func importLayerNext(context uintptr, fileName string, fileInfo *winio.FileBasicInfo) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(fileName) + if hr != nil { + return + } + return _importLayerNext(context, _p0, fileInfo) +} + +func _importLayerNext(context uintptr, fileName *uint16, fileInfo *winio.FileBasicInfo) (hr error) { + if hr = procImportLayerNext.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procImportLayerNext.Addr(), 3, uintptr(context), uintptr(unsafe.Pointer(fileName)), uintptr(unsafe.Pointer(fileInfo))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func importLayerWrite(context uintptr, buffer []byte) (hr error) { + var _p0 *byte + if len(buffer) > 0 { + _p0 = &buffer[0] + } + if hr = procImportLayerWrite.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procImportLayerWrite.Addr(), 3, uintptr(context), uintptr(unsafe.Pointer(_p0)), uintptr(len(buffer))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func importLayerEnd(context uintptr) (hr error) { + if hr = procImportLayerEnd.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procImportLayerEnd.Addr(), 1, uintptr(context), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func exportLayerBegin(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + return _exportLayerBegin(info, _p0, descriptors, context) +} + +func _exportLayerBegin(info *driverInfo, id *uint16, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) { + var _p1 *WC_LAYER_DESCRIPTOR + if len(descriptors) > 0 { + _p1 = &descriptors[0] + } + if hr = procExportLayerBegin.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procExportLayerBegin.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(_p1)), uintptr(len(descriptors)), uintptr(unsafe.Pointer(context)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func exportLayerNext(context uintptr, fileName **uint16, fileInfo *winio.FileBasicInfo, fileSize *int64, deleted *uint32) (hr error) { + if hr = procExportLayerNext.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procExportLayerNext.Addr(), 5, uintptr(context), uintptr(unsafe.Pointer(fileName)), uintptr(unsafe.Pointer(fileInfo)), uintptr(unsafe.Pointer(fileSize)), uintptr(unsafe.Pointer(deleted)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func exportLayerRead(context uintptr, buffer []byte, bytesRead *uint32) (hr error) { + var _p0 *byte + if len(buffer) > 0 { + _p0 = &buffer[0] + } + if hr = procExportLayerRead.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procExportLayerRead.Addr(), 4, uintptr(context), uintptr(unsafe.Pointer(_p0)), uintptr(len(buffer)), uintptr(unsafe.Pointer(bytesRead)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func exportLayerEnd(context uintptr) (hr error) { + if hr = procExportLayerEnd.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procExportLayerEnd.Addr(), 1, uintptr(context), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsEnumerateComputeSystems(query string, computeSystems **uint16, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(query) + if hr != nil { + return + } + return _hcsEnumerateComputeSystems(_p0, computeSystems, result) +} + +func _hcsEnumerateComputeSystems(query *uint16, computeSystems **uint16, result **uint16) (hr error) { + if hr = procHcsEnumerateComputeSystems.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsEnumerateComputeSystems.Addr(), 3, uintptr(unsafe.Pointer(query)), uintptr(unsafe.Pointer(computeSystems)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsCreateComputeSystem(id string, configuration string, identity syscall.Handle, computeSystem *hcsSystem, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + var _p1 *uint16 + _p1, hr = syscall.UTF16PtrFromString(configuration) + if hr != nil { + return + } + return _hcsCreateComputeSystem(_p0, _p1, identity, computeSystem, result) +} + +func _hcsCreateComputeSystem(id *uint16, configuration *uint16, identity syscall.Handle, computeSystem *hcsSystem, result **uint16) (hr error) { + if hr = procHcsCreateComputeSystem.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcsCreateComputeSystem.Addr(), 5, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(configuration)), uintptr(identity), uintptr(unsafe.Pointer(computeSystem)), uintptr(unsafe.Pointer(result)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsOpenComputeSystem(id string, computeSystem *hcsSystem, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(id) + if hr != nil { + return + } + return _hcsOpenComputeSystem(_p0, computeSystem, result) +} + +func _hcsOpenComputeSystem(id *uint16, computeSystem *hcsSystem, result **uint16) (hr error) { + if hr = procHcsOpenComputeSystem.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsOpenComputeSystem.Addr(), 3, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(computeSystem)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsCloseComputeSystem(computeSystem hcsSystem) (hr error) { + if hr = procHcsCloseComputeSystem.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsCloseComputeSystem.Addr(), 1, uintptr(computeSystem), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsStartComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(options) + if hr != nil { + return + } + return _hcsStartComputeSystem(computeSystem, _p0, result) +} + +func _hcsStartComputeSystem(computeSystem hcsSystem, options *uint16, result **uint16) (hr error) { + if hr = procHcsStartComputeSystem.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsStartComputeSystem.Addr(), 3, uintptr(computeSystem), uintptr(unsafe.Pointer(options)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsShutdownComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(options) + if hr != nil { + return + } + return _hcsShutdownComputeSystem(computeSystem, _p0, result) +} + +func _hcsShutdownComputeSystem(computeSystem hcsSystem, options *uint16, result **uint16) (hr error) { + if hr = procHcsShutdownComputeSystem.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsShutdownComputeSystem.Addr(), 3, uintptr(computeSystem), uintptr(unsafe.Pointer(options)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsTerminateComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(options) + if hr != nil { + return + } + return _hcsTerminateComputeSystem(computeSystem, _p0, result) +} + +func _hcsTerminateComputeSystem(computeSystem hcsSystem, options *uint16, result **uint16) (hr error) { + if hr = procHcsTerminateComputeSystem.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsTerminateComputeSystem.Addr(), 3, uintptr(computeSystem), uintptr(unsafe.Pointer(options)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsPauseComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(options) + if hr != nil { + return + } + return _hcsPauseComputeSystem(computeSystem, _p0, result) +} + +func _hcsPauseComputeSystem(computeSystem hcsSystem, options *uint16, result **uint16) (hr error) { + if hr = procHcsPauseComputeSystem.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsPauseComputeSystem.Addr(), 3, uintptr(computeSystem), uintptr(unsafe.Pointer(options)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsResumeComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(options) + if hr != nil { + return + } + return _hcsResumeComputeSystem(computeSystem, _p0, result) +} + +func _hcsResumeComputeSystem(computeSystem hcsSystem, options *uint16, result **uint16) (hr error) { + if hr = procHcsResumeComputeSystem.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsResumeComputeSystem.Addr(), 3, uintptr(computeSystem), uintptr(unsafe.Pointer(options)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsGetComputeSystemProperties(computeSystem hcsSystem, propertyQuery string, properties **uint16, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(propertyQuery) + if hr != nil { + return + } + return _hcsGetComputeSystemProperties(computeSystem, _p0, properties, result) +} + +func _hcsGetComputeSystemProperties(computeSystem hcsSystem, propertyQuery *uint16, properties **uint16, result **uint16) (hr error) { + if hr = procHcsGetComputeSystemProperties.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcsGetComputeSystemProperties.Addr(), 4, uintptr(computeSystem), uintptr(unsafe.Pointer(propertyQuery)), uintptr(unsafe.Pointer(properties)), uintptr(unsafe.Pointer(result)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsModifyComputeSystem(computeSystem hcsSystem, configuration string, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(configuration) + if hr != nil { + return + } + return _hcsModifyComputeSystem(computeSystem, _p0, result) +} + +func _hcsModifyComputeSystem(computeSystem hcsSystem, configuration *uint16, result **uint16) (hr error) { + if hr = procHcsModifyComputeSystem.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsModifyComputeSystem.Addr(), 3, uintptr(computeSystem), uintptr(unsafe.Pointer(configuration)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsRegisterComputeSystemCallback(computeSystem hcsSystem, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) { + if hr = procHcsRegisterComputeSystemCallback.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcsRegisterComputeSystemCallback.Addr(), 4, uintptr(computeSystem), uintptr(callback), uintptr(context), uintptr(unsafe.Pointer(callbackHandle)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsUnregisterComputeSystemCallback(callbackHandle hcsCallback) (hr error) { + if hr = procHcsUnregisterComputeSystemCallback.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsUnregisterComputeSystemCallback.Addr(), 1, uintptr(callbackHandle), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsCreateProcess(computeSystem hcsSystem, processParameters string, processInformation *hcsProcessInformation, process *hcsProcess, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(processParameters) + if hr != nil { + return + } + return _hcsCreateProcess(computeSystem, _p0, processInformation, process, result) +} + +func _hcsCreateProcess(computeSystem hcsSystem, processParameters *uint16, processInformation *hcsProcessInformation, process *hcsProcess, result **uint16) (hr error) { + if hr = procHcsCreateProcess.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcsCreateProcess.Addr(), 5, uintptr(computeSystem), uintptr(unsafe.Pointer(processParameters)), uintptr(unsafe.Pointer(processInformation)), uintptr(unsafe.Pointer(process)), uintptr(unsafe.Pointer(result)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsOpenProcess(computeSystem hcsSystem, pid uint32, process *hcsProcess, result **uint16) (hr error) { + if hr = procHcsOpenProcess.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcsOpenProcess.Addr(), 4, uintptr(computeSystem), uintptr(pid), uintptr(unsafe.Pointer(process)), uintptr(unsafe.Pointer(result)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsCloseProcess(process hcsProcess) (hr error) { + if hr = procHcsCloseProcess.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsCloseProcess.Addr(), 1, uintptr(process), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsTerminateProcess(process hcsProcess, result **uint16) (hr error) { + if hr = procHcsTerminateProcess.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsTerminateProcess.Addr(), 2, uintptr(process), uintptr(unsafe.Pointer(result)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsGetProcessInfo(process hcsProcess, processInformation *hcsProcessInformation, result **uint16) (hr error) { + if hr = procHcsGetProcessInfo.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsGetProcessInfo.Addr(), 3, uintptr(process), uintptr(unsafe.Pointer(processInformation)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsGetProcessProperties(process hcsProcess, processProperties **uint16, result **uint16) (hr error) { + if hr = procHcsGetProcessProperties.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsGetProcessProperties.Addr(), 3, uintptr(process), uintptr(unsafe.Pointer(processProperties)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsModifyProcess(process hcsProcess, settings string, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(settings) + if hr != nil { + return + } + return _hcsModifyProcess(process, _p0, result) +} + +func _hcsModifyProcess(process hcsProcess, settings *uint16, result **uint16) (hr error) { + if hr = procHcsModifyProcess.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsModifyProcess.Addr(), 3, uintptr(process), uintptr(unsafe.Pointer(settings)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsGetServiceProperties(propertyQuery string, properties **uint16, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(propertyQuery) + if hr != nil { + return + } + return _hcsGetServiceProperties(_p0, properties, result) +} + +func _hcsGetServiceProperties(propertyQuery *uint16, properties **uint16, result **uint16) (hr error) { + if hr = procHcsGetServiceProperties.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsGetServiceProperties.Addr(), 3, uintptr(unsafe.Pointer(propertyQuery)), uintptr(unsafe.Pointer(properties)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsRegisterProcessCallback(process hcsProcess, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) { + if hr = procHcsRegisterProcessCallback.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcsRegisterProcessCallback.Addr(), 4, uintptr(process), uintptr(callback), uintptr(context), uintptr(unsafe.Pointer(callbackHandle)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsUnregisterProcessCallback(callbackHandle hcsCallback) (hr error) { + if hr = procHcsUnregisterProcessCallback.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsUnregisterProcessCallback.Addr(), 1, uintptr(callbackHandle), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func hcsModifyServiceSettings(settings string, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(settings) + if hr != nil { + return + } + return _hcsModifyServiceSettings(_p0, result) +} + +func _hcsModifyServiceSettings(settings *uint16, result **uint16) (hr error) { + if hr = procHcsModifyServiceSettings.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcsModifyServiceSettings.Addr(), 2, uintptr(unsafe.Pointer(settings)), uintptr(unsafe.Pointer(result)), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} + +func _hnsCall(method string, path string, object string, response **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(method) + if hr != nil { + return + } + var _p1 *uint16 + _p1, hr = syscall.UTF16PtrFromString(path) + if hr != nil { + return + } + var _p2 *uint16 + _p2, hr = syscall.UTF16PtrFromString(object) + if hr != nil { + return + } + return __hnsCall(_p0, _p1, _p2, response) +} + +func __hnsCall(method *uint16, path *uint16, object *uint16, response **uint16) (hr error) { + if hr = procHNSCall.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHNSCall.Addr(), 4, uintptr(unsafe.Pointer(method)), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(object)), uintptr(unsafe.Pointer(response)), 0, 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + return +} From a8d797afdd933547275b00a69842f357c374e7de Mon Sep 17 00:00:00 2001 From: Madhan Raj Mookkandy Date: Mon, 28 Aug 2017 14:22:47 -0700 Subject: [PATCH 4/4] Add exception to golint check (*) Fix cleanup of NodePort resources. (*) Fix the logic to select existing policies Fix review comment Fix Bazel Update GoDep License Fix NodePort forwarding to target port Fix Darwin Build break. +1 Implement IsCompatible to validate kernel support for kernel mode --- Godeps/LICENSES | 31 +------------------ cmd/kube-proxy/app/BUILD | 17 +++------- .../app/{server_linux.go => server_others.go} | 1 + hack/.golint_failures | 1 + pkg/proxy/winkernel/proxier.go | 16 +++++++--- 5 files changed, 20 insertions(+), 46 deletions(-) rename cmd/kube-proxy/app/{server_linux.go => server_others.go} (99%) diff --git a/Godeps/LICENSES b/Godeps/LICENSES index 9696e91425e..fb5a03e98a4 100644 --- a/Godeps/LICENSES +++ b/Godeps/LICENSES @@ -68231,7 +68231,7 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -= vendor/github.com/Microsoft/hcsshim/LICENSE d4c2cbbea5ee1e7c86dff68a7073718e - += vendor/github.com/Microsoft/hcsshim/LICENSE d4c2cbbea5ee1e7c86dff68a7073718e ================================================================================ @@ -81007,35 +81007,6 @@ THE SOFTWARE. ================================================================================ -================================================================================ -= vendor/github.com/sirupsen/logrus licensed under: = - -The MIT License (MIT) - -Copyright (c) 2014 Simon Eskildsen - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -= vendor/github.com/sirupsen/logrus/LICENSE 8dadfef729c08ec4e631c4f6fc5d43a0 - -================================================================================ - - ================================================================================ = vendor/github.com/spf13/afero licensed under: = diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index 06540166fc9..33a20675600 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -11,10 +11,8 @@ go_library( srcs = [ "conntrack.go", "server.go", + "server_others.go", ] + select({ - "@io_bazel_rules_go//go/platform:linux_amd64": [ - "server_linux.go", - ], "@io_bazel_rules_go//go/platform:windows_amd64": [ "server_windows.go", ], @@ -37,6 +35,7 @@ go_library( "//pkg/proxy/ipvs:go_default_library", "//pkg/proxy/userspace:go_default_library", "//pkg/util/configz:go_default_library", + "//pkg/util/dbus:go_default_library", "//pkg/util/iptables:go_default_library", "//pkg/util/ipvs:go_default_library", "//pkg/util/mount:go_default_library", @@ -55,6 +54,8 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library", @@ -64,20 +65,12 @@ go_library( "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library", + "//vendor/k8s.io/utils/exec:go_default_library", ] + select({ - "@io_bazel_rules_go//go/platform:linux_amd64": [ - "//pkg/util/dbus:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - ], "@io_bazel_rules_go//go/platform:windows_amd64": [ "//pkg/proxy/winkernel:go_default_library", "//pkg/proxy/winuserspace:go_default_library", "//pkg/util/netsh:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], "//conditions:default": [], }), diff --git a/cmd/kube-proxy/app/server_linux.go b/cmd/kube-proxy/app/server_others.go similarity index 99% rename from cmd/kube-proxy/app/server_linux.go rename to cmd/kube-proxy/app/server_others.go index c92e2f1feeb..b0f0eace571 100644 --- a/cmd/kube-proxy/app/server_linux.go +++ b/cmd/kube-proxy/app/server_others.go @@ -51,6 +51,7 @@ import ( "github.com/golang/glog" ) +// NewProxyServer returns a new ProxyServer. func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndExit bool, scheme *runtime.Scheme, master string) (*ProxyServer, error) { if config == nil { return nil, errors.New("config is required") diff --git a/hack/.golint_failures b/hack/.golint_failures index a803ec0fb5d..b6283032826 100644 --- a/hack/.golint_failures +++ b/hack/.golint_failures @@ -301,6 +301,7 @@ pkg/proxy pkg/proxy/iptables pkg/proxy/userspace pkg/proxy/util +pkg/proxy/winkernel pkg/proxy/winuserspace pkg/quota/evaluator/core pkg/quota/generic diff --git a/pkg/proxy/winkernel/proxier.go b/pkg/proxy/winkernel/proxier.go index dc36003ce29..d1f3b55182e 100644 --- a/pkg/proxy/winkernel/proxier.go +++ b/pkg/proxy/winkernel/proxier.go @@ -65,8 +65,12 @@ func CanUseWinKernelProxier(kcompat KernelCompatTester) (bool, error) { type WindowsKernelCompatTester struct{} -// Todo : Fix the below API to query the OS version +// IsCompatible returns true if winkernel can support this mode of proxy func (lkct WindowsKernelCompatTester) IsCompatible() error { + _, err := hcsshim.HNSListPolicyListRequest() + if err != nil { + return fmt.Errorf("Windows kernel is not compatible for Kernel mode") + } return nil } @@ -531,6 +535,10 @@ func (svcInfo *serviceInfo) deleteAllHnsLoadBalancerPolicy() { // Remove the Hns Policy corresponding to this service deleteHnsLoadBalancerPolicy(svcInfo.hnsID) svcInfo.hnsID = "" + + deleteHnsLoadBalancerPolicy(svcInfo.nodePorthnsID) + svcInfo.nodePorthnsID = "" + for _, externalIp := range svcInfo.externalIPs { deleteHnsLoadBalancerPolicy(externalIp.hnsID) externalIp.hnsID = "" @@ -576,7 +584,7 @@ func getHnsLoadBalancer(endpoints []hcsshim.HNSEndpoint, isILB bool, vip string, } if elbPolicy.Protocol == protocol && elbPolicy.InternalPort == internalPort && elbPolicy.ExternalPort == externalPort && elbPolicy.ILB == isILB { if len(vip) > 0 { - if len(elbPolicy.VIPs) > 0 && elbPolicy.VIPs[0] != vip { + if len(elbPolicy.VIPs) == 0 || elbPolicy.VIPs[0] != vip { continue } } @@ -1049,7 +1057,7 @@ func (proxier *Proxier) syncProxyRules() { false, "", // VIP has to be empty to automatically select the nodeIP Enum(svcInfo.protocol), - uint16(svcInfo.port), + uint16(svcInfo.targetPort), uint16(svcInfo.nodePort), ) if err != nil { @@ -1058,7 +1066,7 @@ func (proxier *Proxier) syncProxyRules() { } svcInfo.nodePorthnsID = hnsLoadBalancer.ID - glog.V(3).Infof("Hns LoadBalancer resource created for cluster ip resources %v, Id [%s]", svcInfo.clusterIP, hnsLoadBalancer.ID) + glog.V(3).Infof("Hns LoadBalancer resource created for nodePort resources %v, Id [%s]", svcInfo.clusterIP, hnsLoadBalancer.ID) } // Create a Load Balancer Policy for each external IP