diff --git a/cmd/kubelet/app/options/container_runtime.go b/cmd/kubelet/app/options/container_runtime.go index 4225e8dd507..63508da3c15 100644 --- a/cmd/kubelet/app/options/container_runtime.go +++ b/cmd/kubelet/app/options/container_runtime.go @@ -29,8 +29,6 @@ const ( // When these values are updated, also update test/e2e/framework/util.go defaultPodSandboxImageName = "k8s.gcr.io/pause" defaultPodSandboxImageVersion = "3.1" - // From pkg/kubelet/rkt/rkt.go to avoid circular import - defaultRktAPIServiceEndpoint = "localhost:15441" ) var ( @@ -54,7 +52,6 @@ func NewContainerRuntimeOptions() *config.ContainerRuntimeOptions { DockerDisableSharedPID: true, PodSandboxImage: defaultPodSandboxImage, ImagePullProgressDeadline: metav1.Duration{Duration: 1 * time.Minute}, - RktAPIEndpoint: defaultRktAPIServiceEndpoint, ExperimentalDockershim: false, } } diff --git a/hack/.golint_failures b/hack/.golint_failures index 2f2baf2eb08..70ddf57a792 100644 --- a/hack/.golint_failures +++ b/hack/.golint_failures @@ -196,7 +196,6 @@ pkg/kubelet/prober/results pkg/kubelet/prober/testing pkg/kubelet/qos pkg/kubelet/remote -pkg/kubelet/rkt pkg/kubelet/secret pkg/kubelet/server pkg/kubelet/server/portforward diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index 6a8b3028001..6662eafe72e 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -74,7 +74,6 @@ go_library( "//pkg/kubelet/prober:go_default_library", "//pkg/kubelet/prober/results:go_default_library", "//pkg/kubelet/remote:go_default_library", - "//pkg/kubelet/rkt:go_default_library", "//pkg/kubelet/secret:go_default_library", "//pkg/kubelet/server:go_default_library", "//pkg/kubelet/server/portforward:go_default_library", @@ -281,7 +280,6 @@ filegroup( "//pkg/kubelet/prober:all-srcs", "//pkg/kubelet/qos:all-srcs", "//pkg/kubelet/remote:all-srcs", - "//pkg/kubelet/rkt:all-srcs", "//pkg/kubelet/secret:all-srcs", "//pkg/kubelet/server:all-srcs", "//pkg/kubelet/stats:all-srcs", diff --git a/pkg/kubelet/cadvisor/helpers_linux.go b/pkg/kubelet/cadvisor/helpers_linux.go index bff89894c5e..dba38022d25 100644 --- a/pkg/kubelet/cadvisor/helpers_linux.go +++ b/pkg/kubelet/cadvisor/helpers_linux.go @@ -38,8 +38,6 @@ func (i *imageFsInfoProvider) ImageFsInfoLabel() (string, error) { switch i.runtime { case types.DockerContainerRuntime: return cadvisorfs.LabelDockerImages, nil - case types.RktContainerRuntime: - return cadvisorfs.LabelRktImages, nil case types.RemoteContainerRuntime: // This is a temporary workaround to get stats for cri-o from cadvisor // and should be removed. diff --git a/pkg/kubelet/cadvisor/util.go b/pkg/kubelet/cadvisor/util.go index e4107d5b4a7..16596daa9d9 100644 --- a/pkg/kubelet/cadvisor/util.go +++ b/pkg/kubelet/cadvisor/util.go @@ -75,7 +75,6 @@ func EphemeralStorageCapacityFromFsInfo(info cadvisorapi2.FsInfo) v1.ResourceLis // https://github.com/kubernetes/kubernetes/issues/51798 // UsingLegacyCadvisorStats returns true if container stats are provided by cadvisor instead of through the CRI func UsingLegacyCadvisorStats(runtime, runtimeEndpoint string) bool { - return runtime == kubetypes.RktContainerRuntime || - (runtime == kubetypes.DockerContainerRuntime && goruntime.GOOS == "linux") || + return (runtime == kubetypes.DockerContainerRuntime && goruntime.GOOS == "linux") || runtimeEndpoint == CrioSocket } diff --git a/pkg/kubelet/config/flags.go b/pkg/kubelet/config/flags.go index fc8d3a85ba8..b3fdf60c995 100644 --- a/pkg/kubelet/config/flags.go +++ b/pkg/kubelet/config/flags.go @@ -67,16 +67,6 @@ type ContainerRuntimeOptions struct { // CNIBinDir is the full path of the directory in which to search for // CNI plugin binaries CNIBinDir string - - // rkt-specific options. - - // rktPath is the path of rkt binary. Leave empty to use the first rkt in $PATH. - RktPath string - // rktApiEndpoint is the endpoint of the rkt API service to communicate with. - RktAPIEndpoint string - // rktStage1Image is the image to use as stage1. Local paths and - // http/https URLs are supported. - RktStage1Image string } func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) { @@ -95,17 +85,9 @@ func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.DockerEndpoint, "docker-endpoint", s.DockerEndpoint, "Use this for the docker endpoint to communicate with") fs.DurationVar(&s.ImagePullProgressDeadline.Duration, "image-pull-progress-deadline", s.ImagePullProgressDeadline.Duration, "If no pulling progress is made before this deadline, the image pulling will be cancelled.") - // Network plugin settings. Shared by both docker and rkt. + // Network plugin settings for Docker. fs.StringVar(&s.NetworkPluginName, "network-plugin", s.NetworkPluginName, " The name of the network plugin to be invoked for various events in kubelet/pod lifecycle") fs.StringVar(&s.CNIConfDir, "cni-conf-dir", s.CNIConfDir, " The full path of the directory in which to search for CNI config files. Default: /etc/cni/net.d") fs.StringVar(&s.CNIBinDir, "cni-bin-dir", s.CNIBinDir, " A comma-separated list of full paths of directories in which to search for CNI plugin binaries. Default: /opt/cni/bin") fs.Int32Var(&s.NetworkPluginMTU, "network-plugin-mtu", s.NetworkPluginMTU, " The MTU to be passed to the network plugin, to override the default. Set to 0 to use the default 1460 MTU.") - - // Rkt-specific settings. - fs.StringVar(&s.RktPath, "rkt-path", s.RktPath, "Path of rkt binary. Leave empty to use the first rkt in $PATH. Only used if --container-runtime='rkt'.") - fs.MarkDeprecated("rkt-path", "will be removed in a future version. Rktnetes has been deprecated in favor of rktlet (https://github.com/kubernetes-incubator/rktlet).") - fs.StringVar(&s.RktAPIEndpoint, "rkt-api-endpoint", s.RktAPIEndpoint, "The endpoint of the rkt API service to communicate with. Only used if --container-runtime='rkt'.") - fs.MarkDeprecated("rkt-api-endpoint", "will be removed in a future version. Rktnetes has been deprecated in favor of rktlet (https://github.com/kubernetes-incubator/rktlet).") - fs.StringVar(&s.RktStage1Image, "rkt-stage1-image", s.RktStage1Image, "image to use as stage1. Local paths and http/https URLs are supported. If empty, the 'stage1.aci' in the same directory as '--rkt-path' will be used.") - fs.MarkDeprecated("rkt-stage1-image", "will be removed in a future version. Rktnetes has been deprecated in favor of rktlet (https://github.com/kubernetes-incubator/rktlet).") } diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 7f7d81165b5..57a1d273a3f 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -87,7 +87,6 @@ import ( "k8s.io/kubernetes/pkg/kubelet/prober" proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results" "k8s.io/kubernetes/pkg/kubelet/remote" - "k8s.io/kubernetes/pkg/kubelet/rkt" "k8s.io/kubernetes/pkg/kubelet/secret" "k8s.io/kubernetes/pkg/kubelet/server" serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats" @@ -600,145 +599,104 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, var nl *NoOpLegacyHost pluginSettings.LegacyRuntimeHost = nl - if containerRuntime == kubetypes.RktContainerRuntime { - glog.Warningln("rktnetes has been deprecated in favor of rktlet. Please see https://github.com/kubernetes-incubator/rktlet for more information.") + if containerRuntime == "rkt" { + glog.Fatalln("rktnetes has been deprecated in favor of rktlet. Please see https://github.com/kubernetes-incubator/rktlet for more information.") } - // rktnetes cannot be run with CRI. - if containerRuntime != kubetypes.RktContainerRuntime { - // kubelet defers to the runtime shim to setup networking. Setting - // this to nil will prevent it from trying to invoke the plugin. - // It's easier to always probe and initialize plugins till cri - // becomes the default. - klet.networkPlugin = nil - // if left at nil, that means it is unneeded - var legacyLogProvider kuberuntime.LegacyLogProvider + // kubelet defers to the runtime shim to setup networking. Setting + // this to nil will prevent it from trying to invoke the plugin. + // It's easier to always probe and initialize plugins till cri + // becomes the default. + klet.networkPlugin = nil + // if left at nil, that means it is unneeded + var legacyLogProvider kuberuntime.LegacyLogProvider - switch containerRuntime { - case kubetypes.DockerContainerRuntime: - // Create and start the CRI shim running as a grpc server. - streamingConfig := getStreamingConfig(kubeCfg, kubeDeps) - ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig, - &pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, - crOptions.DockerDisableSharedPID) - if err != nil { - return nil, err - } - // For now, the CRI shim redirects the streaming requests to the - // kubelet, which handles the requests using DockerService.. - klet.criHandler = ds - - // The unix socket for kubelet <-> dockershim communication. - glog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q", - remoteRuntimeEndpoint, - remoteImageEndpoint) - glog.V(2).Infof("Starting the GRPC server for the docker CRI shim.") - server := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds) - if err := server.Start(); err != nil { - return nil, err - } - - // Create dockerLegacyService when the logging driver is not supported. - supported, err := ds.IsCRISupportedLogDriver() - if err != nil { - return nil, err - } - if !supported { - klet.dockerLegacyService = ds - legacyLogProvider = ds - } - case kubetypes.RemoteContainerRuntime: - // No-op. - break - default: - return nil, fmt.Errorf("unsupported CRI runtime: %q", containerRuntime) - } - runtimeService, imageService, err := getRuntimeAndImageServices(remoteRuntimeEndpoint, remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout) + switch containerRuntime { + case kubetypes.DockerContainerRuntime: + // Create and start the CRI shim running as a grpc server. + streamingConfig := getStreamingConfig(kubeCfg, kubeDeps) + ds, err := dockershim.NewDockerService(kubeDeps.DockerClientConfig, crOptions.PodSandboxImage, streamingConfig, + &pluginSettings, runtimeCgroups, kubeCfg.CgroupDriver, crOptions.DockershimRootDirectory, + crOptions.DockerDisableSharedPID) if err != nil { return nil, err } - klet.runtimeService = runtimeService - runtime, err := kuberuntime.NewKubeGenericRuntimeManager( - kubecontainer.FilterEventRecorder(kubeDeps.Recorder), - klet.livenessManager, - seccompProfileRoot, - containerRefManager, - machineInfo, - klet, - kubeDeps.OSInterface, - klet, - httpClient, - imageBackOff, - kubeCfg.SerializeImagePulls, - float32(kubeCfg.RegistryPullQPS), - int(kubeCfg.RegistryBurst), - kubeCfg.CPUCFSQuota, - runtimeService, - imageService, - kubeDeps.ContainerManager.InternalContainerLifecycle(), - legacyLogProvider, - ) - if err != nil { - return nil, err - } - klet.containerRuntime = runtime - klet.runner = runtime + // For now, the CRI shim redirects the streaming requests to the + // kubelet, which handles the requests using DockerService.. + klet.criHandler = ds - if cadvisor.UsingLegacyCadvisorStats(containerRuntime, remoteRuntimeEndpoint) { - klet.StatsProvider = stats.NewCadvisorStatsProvider( - klet.cadvisor, - klet.resourceAnalyzer, - klet.podManager, - klet.runtimeCache, - klet.containerRuntime) - } else { - klet.StatsProvider = stats.NewCRIStatsProvider( - klet.cadvisor, - klet.resourceAnalyzer, - klet.podManager, - klet.runtimeCache, - runtimeService, - imageService, - stats.NewLogMetricsService()) + // The unix socket for kubelet <-> dockershim communication. + glog.V(5).Infof("RemoteRuntimeEndpoint: %q, RemoteImageEndpoint: %q", + remoteRuntimeEndpoint, + remoteImageEndpoint) + glog.V(2).Infof("Starting the GRPC server for the docker CRI shim.") + server := dockerremote.NewDockerServer(remoteRuntimeEndpoint, ds) + if err := server.Start(); err != nil { + return nil, err } - } else { - // rkt uses the legacy, non-CRI, integration. Configure it the old way. - // TODO: Include hairpin mode settings in rkt? - conf := &rkt.Config{ - Path: crOptions.RktPath, - Stage1Image: crOptions.RktStage1Image, - InsecureOptions: "image,ondisk", - } - runtime, err := rkt.New( - crOptions.RktAPIEndpoint, - conf, - klet, - kubeDeps.Recorder, - containerRefManager, - klet, - klet.livenessManager, - httpClient, - klet.networkPlugin, - hairpinMode == kubeletconfiginternal.HairpinVeth, - utilexec.New(), - kubecontainer.RealOS{}, - imageBackOff, - kubeCfg.SerializeImagePulls, - float32(kubeCfg.RegistryPullQPS), - int(kubeCfg.RegistryBurst), - kubeCfg.RuntimeRequestTimeout.Duration, - ) + + // Create dockerLegacyService when the logging driver is not supported. + supported, err := ds.IsCRISupportedLogDriver() if err != nil { return nil, err } - klet.containerRuntime = runtime - klet.runner = kubecontainer.DirectStreamingRunner(runtime) + if !supported { + klet.dockerLegacyService = ds + legacyLogProvider = ds + } + case kubetypes.RemoteContainerRuntime: + // No-op. + break + default: + return nil, fmt.Errorf("unsupported CRI runtime: %q", containerRuntime) + } + runtimeService, imageService, err := getRuntimeAndImageServices(remoteRuntimeEndpoint, remoteImageEndpoint, kubeCfg.RuntimeRequestTimeout) + if err != nil { + return nil, err + } + klet.runtimeService = runtimeService + runtime, err := kuberuntime.NewKubeGenericRuntimeManager( + kubecontainer.FilterEventRecorder(kubeDeps.Recorder), + klet.livenessManager, + seccompProfileRoot, + containerRefManager, + machineInfo, + klet, + kubeDeps.OSInterface, + klet, + httpClient, + imageBackOff, + kubeCfg.SerializeImagePulls, + float32(kubeCfg.RegistryPullQPS), + int(kubeCfg.RegistryBurst), + kubeCfg.CPUCFSQuota, + runtimeService, + imageService, + kubeDeps.ContainerManager.InternalContainerLifecycle(), + legacyLogProvider, + ) + if err != nil { + return nil, err + } + klet.containerRuntime = runtime + klet.runner = runtime + + if cadvisor.UsingLegacyCadvisorStats(containerRuntime, remoteRuntimeEndpoint) { klet.StatsProvider = stats.NewCadvisorStatsProvider( klet.cadvisor, klet.resourceAnalyzer, klet.podManager, klet.runtimeCache, klet.containerRuntime) + } else { + klet.StatsProvider = stats.NewCRIStatsProvider( + klet.cadvisor, + klet.resourceAnalyzer, + klet.podManager, + klet.runtimeCache, + runtimeService, + imageService, + stats.NewLogMetricsService()) } klet.pleg = pleg.NewGenericPLEG(klet.containerRuntime, plegChannelCapacity, plegRelistPeriod, klet.podCache, clock.RealClock{}) @@ -2108,34 +2066,30 @@ func (kl *Kubelet) updateRuntimeUp() { glog.Errorf("Container runtime sanity check failed: %v", err) return } - // rkt uses the legacy, non-CRI integration. Don't check the runtime - // conditions for it. - if kl.containerRuntimeName != kubetypes.RktContainerRuntime { - if s == nil { - glog.Errorf("Container runtime status is nil") - return - } - // Periodically log the whole runtime status for debugging. - // TODO(random-liu): Consider to send node event when optional - // condition is unmet. - glog.V(4).Infof("Container runtime status: %v", s) - networkReady := s.GetRuntimeCondition(kubecontainer.NetworkReady) - if networkReady == nil || !networkReady.Status { - glog.Errorf("Container runtime network not ready: %v", networkReady) - kl.runtimeState.setNetworkState(fmt.Errorf("runtime network not ready: %v", networkReady)) - } else { - // Set nil if the container runtime network is ready. - kl.runtimeState.setNetworkState(nil) - } - // TODO(random-liu): Add runtime error in runtimeState, and update it - // when runtime is not ready, so that the information in RuntimeReady - // condition will be propagated to NodeReady condition. - runtimeReady := s.GetRuntimeCondition(kubecontainer.RuntimeReady) - // If RuntimeReady is not set or is false, report an error. - if runtimeReady == nil || !runtimeReady.Status { - glog.Errorf("Container runtime not ready: %v", runtimeReady) - return - } + if s == nil { + glog.Errorf("Container runtime status is nil") + return + } + // Periodically log the whole runtime status for debugging. + // TODO(random-liu): Consider to send node event when optional + // condition is unmet. + glog.V(4).Infof("Container runtime status: %v", s) + networkReady := s.GetRuntimeCondition(kubecontainer.NetworkReady) + if networkReady == nil || !networkReady.Status { + glog.Errorf("Container runtime network not ready: %v", networkReady) + kl.runtimeState.setNetworkState(fmt.Errorf("runtime network not ready: %v", networkReady)) + } else { + // Set nil if the container runtime network is ready. + kl.runtimeState.setNetworkState(nil) + } + // TODO(random-liu): Add runtime error in runtimeState, and update it + // when runtime is not ready, so that the information in RuntimeReady + // condition will be propagated to NodeReady condition. + runtimeReady := s.GetRuntimeCondition(kubecontainer.RuntimeReady) + // If RuntimeReady is not set or is false, report an error. + if runtimeReady == nil || !runtimeReady.Status { + glog.Errorf("Container runtime not ready: %v", runtimeReady) + return } kl.oneTimeInitializer.Do(kl.initializeRuntimeDependentModules) kl.runtimeState.setRuntimeSync(kl.clock.Now()) diff --git a/pkg/kubelet/rkt/BUILD b/pkg/kubelet/rkt/BUILD deleted file mode 100644 index efd1a3de510..00000000000 --- a/pkg/kubelet/rkt/BUILD +++ /dev/null @@ -1,108 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) - -go_library( - name = "go_default_library", - srcs = [ - "cap.go", - "config.go", - "container_id.go", - "doc.go", - "image.go", - "log.go", - "rkt.go", - "systemd.go", - "version.go", - ], - importpath = "k8s.io/kubernetes/pkg/kubelet/rkt", - deps = [ - "//pkg/credentialprovider:go_default_library", - "//pkg/credentialprovider/secrets:go_default_library", - "//pkg/kubelet/container:go_default_library", - "//pkg/kubelet/events:go_default_library", - "//pkg/kubelet/images:go_default_library", - "//pkg/kubelet/leaky:go_default_library", - "//pkg/kubelet/lifecycle:go_default_library", - "//pkg/kubelet/network:go_default_library", - "//pkg/kubelet/network/hairpin:go_default_library", - "//pkg/kubelet/prober/results:go_default_library", - "//pkg/kubelet/types:go_default_library", - "//pkg/kubelet/util/format:go_default_library", - "//pkg/securitycontext:go_default_library", - "//pkg/util/parsers:go_default_library", - "//pkg/util/selinux:go_default_library", - "//pkg/util/strings:go_default_library", - "//pkg/util/term:go_default_library", - "//pkg/util/version:go_default_library", - "//vendor/github.com/appc/spec/schema:go_default_library", - "//vendor/github.com/appc/spec/schema/types:go_default_library", - "//vendor/github.com/coreos/go-systemd/dbus:go_default_library", - "//vendor/github.com/coreos/go-systemd/unit:go_default_library", - "//vendor/github.com/coreos/rkt/api/v1alpha:go_default_library", - "//vendor/github.com/docker/docker/api/types:go_default_library", - "//vendor/github.com/golang/glog:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", - "//vendor/k8s.io/api/core/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", - "//vendor/k8s.io/client-go/tools/record:go_default_library", - "//vendor/k8s.io/client-go/tools/remotecommand:go_default_library", - "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - ], -) - -go_test( - name = "go_default_test", - srcs = [ - "fake_rkt_interface_test.go", - "rkt_test.go", - ], - embed = [":go_default_library"], - deps = [ - "//pkg/kubelet/container:go_default_library", - "//pkg/kubelet/container/testing:go_default_library", - "//pkg/kubelet/lifecycle:go_default_library", - "//pkg/kubelet/network:go_default_library", - "//pkg/kubelet/network/kubenet:go_default_library", - "//pkg/kubelet/network/testing:go_default_library", - "//pkg/kubelet/types:go_default_library", - "//vendor/github.com/appc/spec/schema:go_default_library", - "//vendor/github.com/appc/spec/schema/types:go_default_library", - "//vendor/github.com/coreos/go-systemd/dbus:go_default_library", - "//vendor/github.com/coreos/go-systemd/unit:go_default_library", - "//vendor/github.com/coreos/rkt/api/v1alpha:go_default_library", - "//vendor/github.com/golang/mock/gomock:go_default_library", - "//vendor/github.com/stretchr/testify/assert:go_default_library", - "//vendor/google.golang.org/grpc:go_default_library", - "//vendor/k8s.io/api/core/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library", - "//vendor/k8s.io/client-go/util/testing:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", - "//vendor/k8s.io/utils/exec/testing:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/pkg/kubelet/rkt/OWNERS b/pkg/kubelet/rkt/OWNERS deleted file mode 100644 index 9b15beaf072..00000000000 --- a/pkg/kubelet/rkt/OWNERS +++ /dev/null @@ -1,3 +0,0 @@ -approvers: -- euank -- yifan-gu diff --git a/pkg/kubelet/rkt/cap.go b/pkg/kubelet/rkt/cap.go deleted file mode 100644 index bbf7cdb552f..00000000000 --- a/pkg/kubelet/rkt/cap.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rkt - -// TODO(yifan): Export this to higher level package. -const ( - CAP_CHOWN = iota - CAP_DAC_OVERRIDE - CAP_DAC_READ_SEARCH - CAP_FOWNER - CAP_FSETID - CAP_KILL - CAP_SETGID - CAP_SETUID - CAP_SETPCAP - CAP_LINUX_IMMUTABLE - CAP_NET_BIND_SERVICE - CAP_NET_BROADCAST - CAP_NET_ADMIN - CAP_NET_RAW - CAP_IPC_LOCK - CAP_IPC_OWNER - CAP_SYS_MODULE - CAP_SYS_RAWIO - CAP_SYS_CHROOT - CAP_SYS_PTRACE - CAP_SYS_PACCT - CAP_SYS_ADMIN - CAP_SYS_BOOT - CAP_SYS_NICE - CAP_SYS_RESOURCE - CAP_SYS_TIME - CAP_SYS_TTY_CONFIG - CAP_MKNOD - CAP_LEASE - CAP_AUDIT_WRITE - CAP_AUDIT_CONTROL - CAP_SETFCAP - CAP_MAC_OVERRIDE - CAP_MAC_ADMIN - CAP_SYSLOG - CAP_WAKE_ALARM - CAP_BLOCK_SUSPEND - CAP_AUDIT_READ -) - -// TODO(yifan): Export this to higher level package. -var capabilityList = map[int]string{ - CAP_CHOWN: "CAP_CHOWN", - CAP_DAC_OVERRIDE: "CAP_DAC_OVERRIDE", - CAP_DAC_READ_SEARCH: "CAP_DAC_READ_SEARCH", - CAP_FOWNER: "CAP_FOWNER", - CAP_FSETID: "CAP_FSETID", - CAP_KILL: "CAP_KILL", - CAP_SETGID: "CAP_SETGID", - CAP_SETUID: "CAP_SETUID", - CAP_SETPCAP: "CAP_SETPCAP", - CAP_LINUX_IMMUTABLE: "CAP_LINUX_IMMUTABLE", - CAP_NET_BIND_SERVICE: "CAP_NET_BIND_SERVICE", - CAP_NET_BROADCAST: "CAP_NET_BROADCAST", - CAP_NET_ADMIN: "CAP_NET_ADMIN", - CAP_NET_RAW: "CAP_NET_RAW", - CAP_IPC_LOCK: "CAP_IPC_LOCK", - CAP_IPC_OWNER: "CAP_IPC_OWNER", - CAP_SYS_MODULE: "CAP_SYS_MODULE", - CAP_SYS_RAWIO: "CAP_SYS_RAWIO", - CAP_SYS_CHROOT: "CAP_SYS_CHROOT", - CAP_SYS_PTRACE: "CAP_SYS_PTRACE", - CAP_SYS_PACCT: "CAP_SYS_PACCT", - CAP_SYS_ADMIN: "CAP_SYS_ADMIN", - CAP_SYS_BOOT: "CAP_SYS_BOOT", - CAP_SYS_NICE: "CAP_SYS_NICE", - CAP_SYS_RESOURCE: "CAP_SYS_RESOURCE", - CAP_SYS_TIME: "CAP_SYS_TIME", - CAP_SYS_TTY_CONFIG: "CAP_SYS_TTY_CONFIG", - CAP_MKNOD: "CAP_MKNOD", - CAP_LEASE: "CAP_LEASE", - CAP_AUDIT_WRITE: "CAP_AUDIT_WRITE", - CAP_AUDIT_CONTROL: "CAP_AUDIT_CONTROL", - CAP_SETFCAP: "CAP_SETFCAP", - CAP_MAC_OVERRIDE: "CAP_MAC_OVERRIDE", - CAP_MAC_ADMIN: "CAP_MAC_ADMIN", - CAP_SYSLOG: "CAP_SYSLOG", - CAP_WAKE_ALARM: "CAP_WAKE_ALARM", - CAP_BLOCK_SUSPEND: "CAP_BLOCK_SUSPEND", - CAP_AUDIT_READ: "CAP_AUDIT_READ", -} - -// allCapabilities returns the capability list with all capabilities. -func allCapabilities() []string { - var capabilities []string - for _, cap := range capabilityList { - capabilities = append(capabilities, cap) - } - return capabilities -} diff --git a/pkg/kubelet/rkt/config.go b/pkg/kubelet/rkt/config.go deleted file mode 100644 index 993b79b4fc4..00000000000 --- a/pkg/kubelet/rkt/config.go +++ /dev/null @@ -1,108 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rkt - -import ( - "context" - "fmt" - - rktapi "github.com/coreos/rkt/api/v1alpha" -) - -// Config stores the global configuration for the rkt runtime. -// Detailed documents can be found at: -// https://github.com/coreos/rkt/blob/master/Documentation/commands.md#global-options -type Config struct { - // The absolute path to the binary, or leave empty to find it in $PATH. - Path string - // The rkt data directory. - Dir string - // The image to use as stage1. - Stage1Image string - // The debug flag for rkt. - Debug bool - // Comma-separated list of security features to disable. - // Allowed values: "none", "image", "tls", "ondisk", "http", "all". - InsecureOptions string - // The local config directory. - LocalConfigDir string - // The user config directory. - UserConfigDir string - // The system config directory. - SystemConfigDir string -} - -// buildGlobalOptions returns an array of global command line options. -func (c *Config) buildGlobalOptions() []string { - var result []string - if c == nil { - return result - } - - if c.Debug { - result = append(result, "--debug=true") - } - if c.InsecureOptions != "" { - result = append(result, fmt.Sprintf("--insecure-options=%s", c.InsecureOptions)) - } - if c.LocalConfigDir != "" { - result = append(result, fmt.Sprintf("--local-config=%s", c.LocalConfigDir)) - } - if c.UserConfigDir != "" { - result = append(result, fmt.Sprintf("--user-config=%s", c.UserConfigDir)) - } - if c.SystemConfigDir != "" { - result = append(result, fmt.Sprintf("--system-config=%s", c.SystemConfigDir)) - } - if c.Dir != "" { - result = append(result, fmt.Sprintf("--dir=%s", c.Dir)) - } - return result -} - -// getConfig gets configurations from the rkt API service -// and merge it with the existing config. The merge rule is -// that the fields in the provided config will override the -// result that get from the rkt api service. -func (r *Runtime) getConfig(cfg *Config) (*Config, error) { - ctx, cancel := context.WithTimeout(context.Background(), r.requestTimeout) - defer cancel() - resp, err := r.apisvc.GetInfo(ctx, &rktapi.GetInfoRequest{}) - if err != nil { - return nil, err - } - - flags := resp.Info.GlobalFlags - - if flags.Dir != "" { - cfg.Dir = flags.Dir - } - if flags.LocalConfigDir != "" { - cfg.LocalConfigDir = flags.LocalConfigDir - } - if flags.UserConfigDir != "" { - cfg.UserConfigDir = flags.UserConfigDir - } - if flags.SystemConfigDir != "" { - cfg.SystemConfigDir = flags.SystemConfigDir - } - if flags.InsecureFlags != "" { - cfg.InsecureOptions = fmt.Sprintf("%s,%s", cfg.InsecureOptions, flags.InsecureFlags) - } - - return cfg, nil -} diff --git a/pkg/kubelet/rkt/container_id.go b/pkg/kubelet/rkt/container_id.go deleted file mode 100644 index ac37c1815c1..00000000000 --- a/pkg/kubelet/rkt/container_id.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rkt - -import ( - "fmt" - "strings" - - kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" -) - -// containerID defines the ID of rkt containers, it will -// be returned to kubelet, and kubelet will use this for -// container level operations. -type containerID struct { - uuid string // rkt uuid of the pod. - appName string // Name of the app in that pod. -} - -// buildContainerID constructs the containers's ID using containerID, -// which consists of the pod uuid and the container name. -// The result can be used to uniquely identify a container. -func buildContainerID(c *containerID) kubecontainer.ContainerID { - return kubecontainer.ContainerID{ - Type: RktType, - ID: fmt.Sprintf("%s:%s", c.uuid, c.appName), - } -} - -// parseContainerID parses the containerID into pod uuid and the container name. The -// results can be used to get more information of the container. -func parseContainerID(id kubecontainer.ContainerID) (*containerID, error) { - tuples := strings.Split(id.ID, ":") - if len(tuples) != 2 { - return nil, fmt.Errorf("rkt: cannot parse container ID for: %q, required format is [UUID:APPNAME]", id) - } - return &containerID{ - uuid: tuples[0], - appName: tuples[1], - }, nil -} diff --git a/pkg/kubelet/rkt/doc.go b/pkg/kubelet/rkt/doc.go deleted file mode 100644 index ac3ca36369d..00000000000 --- a/pkg/kubelet/rkt/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package rkt contains the Containerruntime interface implementation for rkt. -package rkt // import "k8s.io/kubernetes/pkg/kubelet/rkt" diff --git a/pkg/kubelet/rkt/fake_rkt_interface_test.go b/pkg/kubelet/rkt/fake_rkt_interface_test.go deleted file mode 100644 index 077e4ebd8b8..00000000000 --- a/pkg/kubelet/rkt/fake_rkt_interface_test.go +++ /dev/null @@ -1,218 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rkt - -import ( - "context" - "fmt" - "strconv" - "strings" - "sync" - - "github.com/coreos/go-systemd/dbus" - rktapi "github.com/coreos/rkt/api/v1alpha" - "google.golang.org/grpc" - "k8s.io/apimachinery/pkg/types" - kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" -) - -// fakeRktInterface mocks the rktapi.PublicAPIClient interface for testing purpose. -type fakeRktInterface struct { - sync.Mutex - info rktapi.Info - images []*rktapi.Image - podFilters []*rktapi.PodFilter - pods []*rktapi.Pod - called []string - err error -} - -func newFakeRktInterface() *fakeRktInterface { - return &fakeRktInterface{} -} - -func (f *fakeRktInterface) CleanCalls() { - f.Lock() - defer f.Unlock() - f.called = nil -} - -func (f *fakeRktInterface) GetInfo(ctx context.Context, in *rktapi.GetInfoRequest, opts ...grpc.CallOption) (*rktapi.GetInfoResponse, error) { - f.Lock() - defer f.Unlock() - - f.called = append(f.called, "GetInfo") - return &rktapi.GetInfoResponse{Info: &f.info}, f.err -} - -func (f *fakeRktInterface) ListPods(ctx context.Context, in *rktapi.ListPodsRequest, opts ...grpc.CallOption) (*rktapi.ListPodsResponse, error) { - f.Lock() - defer f.Unlock() - - f.called = append(f.called, "ListPods") - f.podFilters = in.Filters - return &rktapi.ListPodsResponse{Pods: f.pods}, f.err -} - -func (f *fakeRktInterface) InspectPod(ctx context.Context, in *rktapi.InspectPodRequest, opts ...grpc.CallOption) (*rktapi.InspectPodResponse, error) { - f.Lock() - defer f.Unlock() - - f.called = append(f.called, "InspectPod") - for _, pod := range f.pods { - if pod.Id == in.Id { - return &rktapi.InspectPodResponse{Pod: pod}, f.err - } - } - return &rktapi.InspectPodResponse{}, fmt.Errorf("pod %q not found", in.Id) -} - -func (f *fakeRktInterface) ListImages(ctx context.Context, in *rktapi.ListImagesRequest, opts ...grpc.CallOption) (*rktapi.ListImagesResponse, error) { - f.Lock() - defer f.Unlock() - - f.called = append(f.called, "ListImages") - return &rktapi.ListImagesResponse{Images: f.images}, f.err -} - -func (f *fakeRktInterface) InspectImage(ctx context.Context, in *rktapi.InspectImageRequest, opts ...grpc.CallOption) (*rktapi.InspectImageResponse, error) { - return nil, fmt.Errorf("Not implemented") -} - -func (f *fakeRktInterface) ListenEvents(ctx context.Context, in *rktapi.ListenEventsRequest, opts ...grpc.CallOption) (rktapi.PublicAPI_ListenEventsClient, error) { - return nil, fmt.Errorf("Not implemented") -} - -func (f *fakeRktInterface) GetLogs(ctx context.Context, in *rktapi.GetLogsRequest, opts ...grpc.CallOption) (rktapi.PublicAPI_GetLogsClient, error) { - return nil, fmt.Errorf("Not implemented") -} - -// fakeSystemd mocks the systemdInterface for testing purpose. -// TODO(yifan): Remove this once we have a package for launching rkt pods. -// See https://github.com/coreos/rkt/issues/1769. -type fakeSystemd struct { - sync.Mutex - called []string - resetFailedUnits []string - version string - err error -} - -func newFakeSystemd() *fakeSystemd { - return &fakeSystemd{} -} - -func (f *fakeSystemd) CleanCalls() { - f.Lock() - defer f.Unlock() - f.called = nil -} - -func (f *fakeSystemd) Version() (systemdVersion, error) { - f.Lock() - defer f.Unlock() - - f.called = append(f.called, "Version") - v, _ := strconv.Atoi(f.version) - return systemdVersion(v), f.err -} - -func (f *fakeSystemd) ListUnits() ([]dbus.UnitStatus, error) { - return nil, fmt.Errorf("Not implemented") -} - -func (f *fakeSystemd) StopUnit(name string, mode string, ch chan<- string) (int, error) { - return 0, fmt.Errorf("Not implemented") -} - -func (f *fakeSystemd) RestartUnit(name string, mode string, ch chan<- string) (int, error) { - return 0, fmt.Errorf("Not implemented") -} - -func (f *fakeSystemd) ResetFailedUnit(name string) error { - f.called = append(f.called, "ResetFailedUnit") - f.resetFailedUnits = append(f.resetFailedUnits, name) - return f.err -} - -type fakeRktCli struct { - sync.Mutex - cmds []string - result []string - err error -} - -func newFakeRktCli() *fakeRktCli { - return &fakeRktCli{ - cmds: []string{}, - result: []string{}, - } -} - -func (f *fakeRktCli) RunCommand(config *Config, args ...string) (result []string, err error) { - f.Lock() - defer f.Unlock() - cmd := append([]string{"rkt"}, args...) - f.cmds = append(f.cmds, strings.Join(cmd, " ")) - return f.result, f.err -} - -func (f *fakeRktCli) Reset() { - f.cmds = []string{} - f.result = []string{} - f.err = nil -} - -type fakePodDeletionProvider struct { - pods map[types.UID]struct{} -} - -func newFakePodDeletionProvider() *fakePodDeletionProvider { - return &fakePodDeletionProvider{ - pods: make(map[types.UID]struct{}), - } -} - -func (f *fakePodDeletionProvider) IsPodDeleted(uid types.UID) bool { - _, found := f.pods[uid] - return !found -} - -type fakeUnitGetter struct { - networkNamespace kubecontainer.ContainerID -} - -func newfakeUnitGetter() *fakeUnitGetter { - return &fakeUnitGetter{ - networkNamespace: kubecontainer.ContainerID{}, - } -} - -func (f *fakeUnitGetter) getNetworkNamespace(uid types.UID, latestPod *rktapi.Pod) (kubecontainer.ContainerID, error) { - return kubecontainer.ContainerID{ID: "42"}, nil -} - -func (f *fakeUnitGetter) getKubernetesDirective(serviceFilePath string) (podServiceDirective, error) { - podService := podServiceDirective{ - id: "fake", - name: "fake", - namespace: "fake", - hostNetwork: true, - networkNamespace: kubecontainer.ContainerID{ID: "42"}, - } - return podService, nil -} diff --git a/pkg/kubelet/rkt/image.go b/pkg/kubelet/rkt/image.go deleted file mode 100644 index 25d17ca8e0b..00000000000 --- a/pkg/kubelet/rkt/image.go +++ /dev/null @@ -1,294 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This file contains all image related functions for rkt runtime. -package rkt - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path" - "path/filepath" - "sort" - "strings" - - appcschema "github.com/appc/spec/schema" - appctypes "github.com/appc/spec/schema/types" - rktapi "github.com/coreos/rkt/api/v1alpha" - dockertypes "github.com/docker/docker/api/types" - "github.com/golang/glog" - - "k8s.io/api/core/v1" - "k8s.io/kubernetes/pkg/credentialprovider" - credentialprovidersecrets "k8s.io/kubernetes/pkg/credentialprovider/secrets" - kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" - "k8s.io/kubernetes/pkg/util/parsers" -) - -// PullImage invokes 'rkt fetch' to download an aci. -// TODO(yifan): Now we only support docker images, this should be changed -// once the format of image is landed, see: -// -// http://issue.k8s.io/7203 -// -func (r *Runtime) PullImage(image kubecontainer.ImageSpec, pullSecrets []v1.Secret) (string, error) { - img := image.Image - // TODO(yifan): The credential operation is a copy from dockertools package, - // Need to resolve the code duplication. - repoToPull, _, _, err := parsers.ParseImageName(img) - if err != nil { - return "", err - } - - keyring, err := credentialprovidersecrets.MakeDockerKeyring(pullSecrets, r.dockerKeyring) - if err != nil { - return "", err - } - - creds, ok := keyring.Lookup(repoToPull) - if !ok { - glog.V(1).Infof("Pulling image %s without credentials", img) - } - - userConfigDir, err := ioutil.TempDir("", "rktnetes-user-config-dir-") - if err != nil { - return "", fmt.Errorf("rkt: Cannot create a temporary user-config directory: %v", err) - } - defer os.RemoveAll(userConfigDir) - - config := *r.config - config.UserConfigDir = userConfigDir - - if err := r.writeDockerAuthConfig(img, creds, userConfigDir); err != nil { - return "", err - } - - // Today, `--no-store` will fetch the remote image regardless of whether the content of the image - // has changed or not. This causes performance downgrades when the image tag is ':latest' and - // the image pull policy is 'always'. The issue is tracked in https://github.com/coreos/rkt/issues/2937. - if _, err := r.cli.RunCommand(&config, "fetch", "--no-store", dockerPrefix+img); err != nil { - glog.Errorf("Failed to fetch: %v", err) - return "", err - } - return r.getImageID(img) -} - -func (r *Runtime) GetImageRef(image kubecontainer.ImageSpec) (string, error) { - images, err := r.listImages(image.Image, false) - if err != nil { - return "", err - } - if len(images) == 0 { - return "", nil - } - return images[0].Id, nil -} - -// ListImages lists all the available appc images on the machine by invoking 'rkt image list'. -func (r *Runtime) ListImages() ([]kubecontainer.Image, error) { - ctx, cancel := context.WithTimeout(context.Background(), r.requestTimeout) - defer cancel() - listResp, err := r.apisvc.ListImages(ctx, &rktapi.ListImagesRequest{}) - if err != nil { - return nil, fmt.Errorf("couldn't list images: %v", err) - } - - images := make([]kubecontainer.Image, len(listResp.Images)) - for i, image := range listResp.Images { - images[i] = kubecontainer.Image{ - ID: image.Id, - RepoTags: []string{buildImageName(image)}, - Size: image.Size, - } - } - return images, nil -} - -// RemoveImage removes an on-disk image using 'rkt image rm'. -func (r *Runtime) RemoveImage(image kubecontainer.ImageSpec) error { - imageID, err := r.getImageID(image.Image) - if err != nil { - return err - } - if _, err := r.cli.RunCommand(nil, "image", "rm", imageID); err != nil { - return err - } - return nil -} - -// buildImageName constructs the image name for kubecontainer.Image. -// If the annotations contain the docker2aci metadata for this image, those are -// used instead as they may be more accurate in some cases, namely if a -// non-appc valid character is present -func buildImageName(img *rktapi.Image) string { - registry := "" - repository := "" - for _, anno := range img.Annotations { - if anno.Key == appcDockerRegistryURL { - registry = anno.Value - } - if anno.Key == appcDockerRepository { - repository = anno.Value - } - } - if registry != "" && repository != "" { - // TODO(euank): This could do the special casing for dockerhub and library images - return fmt.Sprintf("%s/%s:%s", registry, repository, img.Version) - } - - return fmt.Sprintf("%s:%s", img.Name, img.Version) -} - -// getImageID tries to find the image ID for the given image name. -// imageName should be in the form of 'name[:version]', e.g., 'example.com/app:latest'. -// The name should matches the result of 'rkt image list'. If the version is empty, -// then 'latest' is assumed. -func (r *Runtime) getImageID(imageName string) (string, error) { - images, err := r.listImages(imageName, false) - if err != nil { - return "", err - } - if len(images) == 0 { - return "", fmt.Errorf("cannot find the image %q", imageName) - } - return images[0].Id, nil -} - -type sortByImportTime []*rktapi.Image - -func (s sortByImportTime) Len() int { return len(s) } -func (s sortByImportTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s sortByImportTime) Less(i, j int) bool { return s[i].ImportTimestamp < s[j].ImportTimestamp } - -// listImages lists the images that have the given name. If detail is true, -// then image manifest is also included in the result. -// Note that there could be more than one images that have the given name, we -// will return the result reversely sorted by the import time, so that the latest -// image comes first. -func (r *Runtime) listImages(image string, detail bool) ([]*rktapi.Image, error) { - repoToPull, tag, _, err := parsers.ParseImageName(image) - if err != nil { - return nil, err - } - - imageFilters := []*rktapi.ImageFilter{ - { - // TODO(yifan): Add a field in the ImageFilter to match the whole name, - // not just keywords. - // https://github.com/coreos/rkt/issues/1872#issuecomment-166456938 - Keywords: []string{repoToPull}, - Labels: []*rktapi.KeyValue{{Key: "version", Value: tag}}, - }, - } - - // If the repo name is not a valid ACIdentifier (namely if it has a port), - // then it will have a different name in the store. Search for both the - // original name and this modified name in case we choose to also change the - // api-service to do this un-conversion on its end. - if appcRepoToPull, err := appctypes.SanitizeACIdentifier(repoToPull); err != nil { - glog.Warningf("could not convert %v to an aci identifier: %v", err) - } else { - imageFilters = append(imageFilters, &rktapi.ImageFilter{ - Keywords: []string{appcRepoToPull}, - Labels: []*rktapi.KeyValue{{Key: "version", Value: tag}}, - }) - } - - ctx, cancel := context.WithTimeout(context.Background(), r.requestTimeout) - defer cancel() - listResp, err := r.apisvc.ListImages(ctx, &rktapi.ListImagesRequest{ - Detail: detail, - Filters: imageFilters, - }) - if err != nil { - return nil, fmt.Errorf("couldn't list images: %v", err) - } - - // TODO(yifan): Let the API service to sort the result: - // See https://github.com/coreos/rkt/issues/1911. - sort.Sort(sort.Reverse(sortByImportTime(listResp.Images))) - return listResp.Images, nil -} - -// getImageManifest retrieves the image manifest for the given image. -func (r *Runtime) getImageManifest(image string) (*appcschema.ImageManifest, error) { - var manifest appcschema.ImageManifest - - images, err := r.listImages(image, true) - if err != nil { - return nil, err - } - if len(images) == 0 { - return nil, fmt.Errorf("cannot find the image %q", image) - } - - return &manifest, json.Unmarshal(images[0].Manifest, &manifest) -} - -// TODO(yifan): This is very racy, inefficient, and unsafe, we need to provide -// different namespaces. See: https://github.com/coreos/rkt/issues/836. -func (r *Runtime) writeDockerAuthConfig(image string, credsSlice []credentialprovider.LazyAuthConfiguration, userConfigDir string) error { - if len(credsSlice) == 0 { - return nil - } - - creds := dockertypes.AuthConfig{} - // TODO handle multiple creds - if len(credsSlice) >= 1 { - creds = credentialprovider.LazyProvide(credsSlice[0]) - } - - registry := "index.docker.io" - // Image spec: [/]/[: 0 { - set, err := appctypes.NewLinuxCapabilitiesRetainSet(addCaps...) - if err != nil { - return err - } - isolator, err := set.AsIsolator() - if err != nil { - return err - } - isolators = append(isolators, *isolator) - } - if len(dropCaps) > 0 { - set, err := appctypes.NewLinuxCapabilitiesRevokeSet(dropCaps...) - if err != nil { - return err - } - isolator, err := set.AsIsolator() - if err != nil { - return err - } - isolators = append(isolators, *isolator) - } - } - - // Resources isolators. - type resource struct { - limit string - request string - } - - // If limit is empty, populate it with request and vice versa. - resources := make(map[v1.ResourceName]*resource) - for name, quantity := range c.Resources.Limits { - resources[name] = &resource{limit: quantity.String(), request: quantity.String()} - } - for name, quantity := range c.Resources.Requests { - r, ok := resources[name] - if ok { - r.request = quantity.String() - continue - } - resources[name] = &resource{limit: quantity.String(), request: quantity.String()} - } - - for name, res := range resources { - switch name { - case v1.ResourceCPU: - cpu, err := appctypes.NewResourceCPUIsolator(res.request, res.limit) - if err != nil { - return err - } - isolators = append(isolators, cpu.AsIsolator()) - case v1.ResourceMemory: - memory, err := appctypes.NewResourceMemoryIsolator(res.request, res.limit) - if err != nil { - return err - } - isolators = append(isolators, memory.AsIsolator()) - default: - return fmt.Errorf("resource type not supported: %v", name) - } - } - - if ok := securitycontext.AddNoNewPrivileges(ctx); ok { - isolator, err := newNoNewPrivilegesIsolator(true) - if err != nil { - return err - } - isolators = append(isolators, *isolator) - } - - mergeIsolators(app, isolators) - return nil -} - -// mergeIsolators replaces the app.Isolators with isolators. -func mergeIsolators(app *appctypes.App, isolators []appctypes.Isolator) { - for _, is := range isolators { - found := false - for j, js := range app.Isolators { - if is.Name.Equals(js.Name) { - switch is.Name { - case appctypes.LinuxCapabilitiesRetainSetName: - // TODO(yifan): More fine grain merge for capability set instead of override. - fallthrough - case appctypes.LinuxCapabilitiesRevokeSetName: - fallthrough - case appctypes.ResourceCPUName: - fallthrough - case appctypes.ResourceMemoryName: - app.Isolators[j] = is - default: - panic(fmt.Sprintf("unexpected isolator name: %v", is.Name)) - } - found = true - break - } - } - if !found { - app.Isolators = append(app.Isolators, is) - } - } -} - -// mergeEnv merges the optEnv with the image's environments. -// The environments defined in the image will be overridden by -// the ones with the same name in optEnv. -func mergeEnv(app *appctypes.App, optEnv []kubecontainer.EnvVar) { - envMap := make(map[string]string) - for _, e := range app.Environment { - envMap[e.Name] = e.Value - } - for _, e := range optEnv { - envMap[e.Name] = e.Value - } - app.Environment = nil - for name, value := range envMap { - app.Environment = append(app.Environment, appctypes.EnvironmentVariable{ - Name: name, - Value: value, - }) - } -} - -// mergeMounts merges the mountPoints with the image's mount points. -// The mount points defined in the image will be overridden by the ones -// with the same container path. -func mergeMounts(app *appctypes.App, mountPoints []appctypes.MountPoint) { - mountMap := make(map[string]appctypes.MountPoint) - for _, m := range app.MountPoints { - mountMap[m.Path] = m - } - for _, m := range mountPoints { - mountMap[m.Path] = m - } - app.MountPoints = nil - for _, mount := range mountMap { - app.MountPoints = append(app.MountPoints, mount) - } -} - -// mergePortMappings merges the containerPorts with the image's container ports. -// The port mappings defined in the image will be overridden by the ones -// with the same name in optPortMappings. -func mergePortMappings(app *appctypes.App, containerPorts []appctypes.Port) { - portMap := make(map[appctypes.ACName]appctypes.Port) - for _, p := range app.Ports { - portMap[p.Name] = p - } - for _, p := range containerPorts { - portMap[p.Name] = p - } - app.Ports = nil - for _, port := range portMap { - app.Ports = append(app.Ports, port) - } -} - -func verifyNonRoot(app *appctypes.App, ctx *v1.SecurityContext) error { - if ctx != nil && ctx.RunAsNonRoot != nil && *ctx.RunAsNonRoot { - if ctx.RunAsUser != nil && *ctx.RunAsUser == 0 { - return fmt.Errorf("container's runAsUser breaks non-root policy") - } - if ctx.RunAsUser == nil && app.User == "0" { - return fmt.Errorf("container has no runAsUser and image will run as root") - } - } - return nil -} - -func setSupplementalGIDs(app *appctypes.App, podCtx *v1.PodSecurityContext, supplementalGids []int64) { - if podCtx != nil || len(supplementalGids) != 0 { - app.SupplementaryGIDs = app.SupplementaryGIDs[:0] - } - if podCtx != nil { - for _, v := range podCtx.SupplementalGroups { - app.SupplementaryGIDs = append(app.SupplementaryGIDs, int(v)) - } - if podCtx.FSGroup != nil { - app.SupplementaryGIDs = append(app.SupplementaryGIDs, int(*podCtx.FSGroup)) - } - } - for _, v := range supplementalGids { - app.SupplementaryGIDs = append(app.SupplementaryGIDs, int(v)) - } -} - -// setApp merges the container spec with the image's manifest. -func setApp(imgManifest *appcschema.ImageManifest, c *v1.Container, - mountPoints []appctypes.MountPoint, containerPorts []appctypes.Port, envs []kubecontainer.EnvVar, - ctx *v1.SecurityContext, podCtx *v1.PodSecurityContext, supplementalGids []int64) error { - - app := imgManifest.App - - // Set up Exec. - var command, args []string - cmd, ok := imgManifest.Annotations.Get(appcDockerEntrypoint) - if ok { - err := json.Unmarshal([]byte(cmd), &command) - if err != nil { - return fmt.Errorf("cannot unmarshal ENTRYPOINT %q: %v", cmd, err) - } - } - ag, ok := imgManifest.Annotations.Get(appcDockerCmd) - if ok { - err := json.Unmarshal([]byte(ag), &args) - if err != nil { - return fmt.Errorf("cannot unmarshal CMD %q: %v", ag, err) - } - } - userCommand, userArgs := kubecontainer.ExpandContainerCommandAndArgs(c, envs) - - if len(userCommand) > 0 { - command = userCommand - args = nil // If 'command' is specified, then drop the default args. - } - if len(userArgs) > 0 { - args = userArgs - } - - exec := append(command, args...) - if len(exec) > 0 { - app.Exec = exec - } - - // Set UID and GIDs. - if err := verifyNonRoot(app, ctx); err != nil { - return err - } - if ctx != nil && ctx.RunAsUser != nil { - app.User = strconv.Itoa(int(*ctx.RunAsUser)) - } - setSupplementalGIDs(app, podCtx, supplementalGids) - - // If 'User' or 'Group' are still empty at this point, - // then apply the root UID and GID. - // TODO(yifan): If only the GID is empty, rkt should be able to determine the GID - // using the /etc/passwd file in the image. - // See https://github.com/appc/docker2aci/issues/175. - // Maybe we can remove this check in the future. - if app.User == "" { - app.User = "0" - app.Group = "0" - } - if app.Group == "" { - return fmt.Errorf("cannot determine the GID of the app %q", imgManifest.Name) - } - - // Set working directory. - if len(c.WorkingDir) > 0 { - app.WorkingDirectory = c.WorkingDir - } - - // Notes that we don't create Mounts section in the pod manifest here, - // as Mounts will be automatically generated by rkt. - mergeMounts(app, mountPoints) - mergeEnv(app, envs) - mergePortMappings(app, containerPorts) - - return setIsolators(app, c, ctx) -} - -// makePodManifest transforms a kubelet pod spec to the rkt pod manifest. -func (r *Runtime) makePodManifest(pod *v1.Pod, podIP string, pullSecrets []v1.Secret) (*appcschema.PodManifest, error) { - manifest := appcschema.BlankPodManifest() - - ctx, cancel := context.WithTimeout(context.Background(), r.requestTimeout) - defer cancel() - listResp, err := r.apisvc.ListPods(ctx, &rktapi.ListPodsRequest{ - Detail: true, - Filters: kubernetesPodFilters(pod.UID), - }) - if err != nil { - return nil, fmt.Errorf("couldn't list pods: %v", err) - } - - restartCount := 0 - for _, pod := range listResp.Pods { - manifest := &appcschema.PodManifest{} - err = json.Unmarshal(pod.Manifest, manifest) - if err != nil { - glog.Warningf("rkt: error unmatshaling pod manifest: %v", err) - continue - } - - if countString, ok := manifest.Annotations.Get(k8sRktRestartCountAnno); ok { - num, err := strconv.Atoi(countString) - if err != nil { - glog.Warningf("rkt: error reading restart count on pod: %v", err) - continue - } - if num+1 > restartCount { - restartCount = num + 1 - } - } - } - - requiresPrivileged := false - manifest.Annotations.Set(*appctypes.MustACIdentifier(k8sRktKubeletAnno), k8sRktKubeletAnnoValue) - manifest.Annotations.Set(*appctypes.MustACIdentifier(types.KubernetesPodUIDLabel), string(pod.UID)) - manifest.Annotations.Set(*appctypes.MustACIdentifier(types.KubernetesPodNameLabel), pod.Name) - manifest.Annotations.Set(*appctypes.MustACIdentifier(types.KubernetesPodNamespaceLabel), pod.Namespace) - manifest.Annotations.Set(*appctypes.MustACIdentifier(types.KubernetesContainerNameLabel), leaky.PodInfraContainerName) - manifest.Annotations.Set(*appctypes.MustACIdentifier(k8sRktRestartCountAnno), strconv.Itoa(restartCount)) - if stage1Name, ok := pod.Annotations[k8sRktStage1NameAnno]; ok { - requiresPrivileged = true - manifest.Annotations.Set(*appctypes.MustACIdentifier(k8sRktStage1NameAnno), stage1Name) - } - - for _, c := range pod.Spec.Containers { - err := r.newAppcRuntimeApp(pod, podIP, c, requiresPrivileged, pullSecrets, manifest) - if err != nil { - return nil, err - } - } - - // TODO(yifan): Set pod-level isolators once it's supported in kubernetes. - return manifest, nil -} - -func copyfile(src, dst string) error { - data, err := ioutil.ReadFile(src) - if err != nil { - return err - } - return ioutil.WriteFile(dst, data, 0644) -} - -// TODO(yifan): Can make rkt handle this when '--net=host'. See https://github.com/coreos/rkt/issues/2430. -func makeHostNetworkMount(opts *kubecontainer.RunContainerOptions) (*kubecontainer.Mount, *kubecontainer.Mount, error) { - mountHosts, mountResolvConf := true, true - for _, mnt := range opts.Mounts { - switch mnt.ContainerPath { - case etcHostsPath: - mountHosts = false - case etcResolvConfPath: - mountResolvConf = false - } - } - - var hostsMount, resolvMount kubecontainer.Mount - if mountHosts { - hostsPath := filepath.Join(opts.PodContainerDir, "etc-hosts") - if err := copyfile(etcHostsPath, hostsPath); err != nil { - return nil, nil, err - } - hostsMount = kubecontainer.Mount{ - Name: "kubernetes-hostnetwork-hosts-conf", - ContainerPath: etcHostsPath, - HostPath: hostsPath, - } - opts.Mounts = append(opts.Mounts, hostsMount) - } - - if mountResolvConf { - resolvPath := filepath.Join(opts.PodContainerDir, "etc-resolv-conf") - if err := copyfile(etcResolvConfPath, resolvPath); err != nil { - return nil, nil, err - } - resolvMount = kubecontainer.Mount{ - Name: "kubernetes-hostnetwork-resolv-conf", - ContainerPath: etcResolvConfPath, - HostPath: resolvPath, - } - opts.Mounts = append(opts.Mounts, resolvMount) - } - return &hostsMount, &resolvMount, nil -} - -// podFinishedMarkerPath returns the path to a file which should be used to -// indicate the pod exiting, and the time thereof. -// If the file at the path does not exist, the pod should not be exited. If it -// does exist, then the ctime of the file should indicate the time the pod -// exited. -func podFinishedMarkerPath(podDir string, rktUID string) string { - return filepath.Join(podDir, "finished-"+rktUID) -} - -func podFinishedMarkCommand(touchPath, podDir, rktUID string) string { - // TODO, if the path has a `'` character in it, this breaks. - return touchPath + " " + podFinishedMarkerPath(podDir, rktUID) -} - -// podFinishedAt returns the time that a pod exited, or a zero time if it has -// not. -func (r *Runtime) podFinishedAt(podUID kubetypes.UID, rktUID string) time.Time { - markerFile := podFinishedMarkerPath(r.runtimeHelper.GetPodDir(podUID), rktUID) - stat, err := r.os.Stat(markerFile) - if err != nil { - if !os.IsNotExist(err) { - glog.Warningf("rkt: unexpected fs error checking pod finished marker: %v", err) - } - return time.Time{} - } - return stat.ModTime() -} - -func (r *Runtime) makeContainerLogMount(opts *kubecontainer.RunContainerOptions, container *v1.Container) (*kubecontainer.Mount, error) { - if opts.PodContainerDir == "" || container.TerminationMessagePath == "" { - return nil, nil - } - - // In docker runtime, the container log path contains the container ID. - // However, for rkt runtime, we cannot get the container ID before the - // the container is launched, so here we generate a random uuid to enable - // us to map a container's termination message path to a unique log file - // on the disk. - randomUID := uuid.NewUUID() - containerLogPath := path.Join(opts.PodContainerDir, string(randomUID)) - fs, err := r.os.Create(containerLogPath) - if err != nil { - return nil, err - } - - if err := fs.Close(); err != nil { - return nil, err - } - - mnt := kubecontainer.Mount{ - // Use a random name for the termination message mount, so that - // when a container restarts, it will not overwrite the old termination - // message. - Name: fmt.Sprintf("termination-message-%s", randomUID), - ContainerPath: container.TerminationMessagePath, - HostPath: containerLogPath, - ReadOnly: false, - } - opts.Mounts = append(opts.Mounts, mnt) - - return &mnt, nil -} - -func (r *Runtime) newAppcRuntimeApp(pod *v1.Pod, podIP string, c v1.Container, requiresPrivileged bool, pullSecrets []v1.Secret, manifest *appcschema.PodManifest) error { - var annotations appctypes.Annotations = []appctypes.Annotation{ - { - Name: *appctypes.MustACIdentifier(k8sRktContainerHashAnno), - Value: strconv.FormatUint(kubecontainer.HashContainerLegacy(&c), 10), - }, - { - Name: *appctypes.MustACIdentifier(types.KubernetesContainerNameLabel), - Value: c.Name, - }, - } - - if requiresPrivileged && !securitycontext.HasPrivilegedRequest(&c) { - return fmt.Errorf("cannot make %q: running a custom stage1 requires a privileged security context", format.Pod(pod)) - } - imageRef, _, err := r.imagePuller.EnsureImageExists(pod, &c, pullSecrets) - if err != nil { - return err - } - imgManifest, err := r.getImageManifest(c.Image) - if err != nil { - return err - } - - if imgManifest.App == nil { - imgManifest.App = new(appctypes.App) - } - - hash, err := appctypes.NewHash(imageRef) - if err != nil { - return err - } - - // TODO: determine how this should be handled for rkt - opts, _, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &c, podIP) - if err != nil { - return err - } - - // Create additional mount for termination message path. - mount, err := r.makeContainerLogMount(opts, &c) - if err != nil { - return err - } - mounts := append(opts.Mounts, *mount) - annotations = append(annotations, appctypes.Annotation{ - Name: *appctypes.MustACIdentifier(k8sRktTerminationMessagePathAnno), - Value: mount.HostPath, - }) - - // If run in 'hostnetwork' mode, then copy the host's /etc/resolv.conf and /etc/hosts, - // and add mounts. - if kubecontainer.IsHostNetworkPod(pod) { - hostsMount, resolvMount, err := makeHostNetworkMount(opts) - if err != nil { - return err - } - mounts = append(mounts, *hostsMount, *resolvMount) - } - - supplementalGids := r.runtimeHelper.GetExtraSupplementalGroupsForPod(pod) - ctx := securitycontext.DetermineEffectiveSecurityContext(pod, &c) - - volumes, mountPoints := convertKubeMounts(mounts) - containerPorts, hostPorts := convertKubePortMappings(opts.PortMappings) - - if err := setApp(imgManifest, &c, mountPoints, containerPorts, opts.Envs, ctx, pod.Spec.SecurityContext, supplementalGids); err != nil { - return err - } - - ra := appcschema.RuntimeApp{ - Name: convertToACName(c.Name), - Image: appcschema.RuntimeImage{ID: *hash}, - App: imgManifest.App, - Annotations: annotations, - } - - if c.SecurityContext != nil && c.SecurityContext.ReadOnlyRootFilesystem != nil { - ra.ReadOnlyRootFS = *c.SecurityContext.ReadOnlyRootFilesystem - } - - manifest.Apps = append(manifest.Apps, ra) - manifest.Volumes = append(manifest.Volumes, volumes...) - manifest.Ports = append(manifest.Ports, hostPorts...) - - return nil -} - -func runningKubernetesPodFilters(uid kubetypes.UID) []*rktapi.PodFilter { - return []*rktapi.PodFilter{ - { - States: []rktapi.PodState{ - rktapi.PodState_POD_STATE_RUNNING, - }, - Annotations: []*rktapi.KeyValue{ - { - Key: k8sRktKubeletAnno, - Value: k8sRktKubeletAnnoValue, - }, - { - Key: types.KubernetesPodUIDLabel, - Value: string(uid), - }, - }, - }, - } -} - -func kubernetesPodFilters(uid kubetypes.UID) []*rktapi.PodFilter { - return []*rktapi.PodFilter{ - { - Annotations: []*rktapi.KeyValue{ - { - Key: k8sRktKubeletAnno, - Value: k8sRktKubeletAnnoValue, - }, - { - Key: types.KubernetesPodUIDLabel, - Value: string(uid), - }, - }, - }, - } -} - -func kubernetesPodsFilters() []*rktapi.PodFilter { - return []*rktapi.PodFilter{ - { - Annotations: []*rktapi.KeyValue{ - { - Key: k8sRktKubeletAnno, - Value: k8sRktKubeletAnnoValue, - }, - }, - }, - } -} - -func newUnitOption(section, name, value string) *unit.UnitOption { - return &unit.UnitOption{Section: section, Name: name, Value: value} -} - -// apiPodToruntimePod converts an v1.Pod to kubelet/container.Pod. -func apiPodToruntimePod(uuid string, pod *v1.Pod) *kubecontainer.Pod { - p := &kubecontainer.Pod{ - ID: pod.UID, - Name: pod.Name, - Namespace: pod.Namespace, - } - for i := range pod.Spec.Containers { - c := &pod.Spec.Containers[i] - p.Containers = append(p.Containers, &kubecontainer.Container{ - ID: buildContainerID(&containerID{uuid, c.Name}), - Name: c.Name, - Image: c.Image, - Hash: kubecontainer.HashContainerLegacy(c), - }) - } - return p -} - -// serviceFilePath returns the absolute path of the service file. -func serviceFilePath(serviceName string) string { - return path.Join(systemdServiceDir, serviceName) -} - -// shouldCreateNetns returns true if: -// The pod does not run in host network. And -// The pod runs inside a netns created outside of rkt. -func (r *Runtime) shouldCreateNetns(pod *v1.Pod) bool { - return !kubecontainer.IsHostNetworkPod(pod) && r.network.PluginName() != network.DefaultPluginName -} - -// usesRktHostNetwork returns true if: -// The pod runs in the host network. Or -// The pod runs inside a netns created outside of rkt. -func (r *Runtime) usesRktHostNetwork(pod *v1.Pod) bool { - return kubecontainer.IsHostNetworkPod(pod) || r.shouldCreateNetns(pod) -} - -// generateRunCommand crafts a 'rkt run-prepared' command with necessary parameters. -func (r *Runtime) generateRunCommand(pod *v1.Pod, uuid, networkNamespaceID string) (string, error) { - config := *r.config - privileged := true - - for _, c := range pod.Spec.Containers { - ctx := securitycontext.DetermineEffectiveSecurityContext(pod, &c) - if ctx == nil || ctx.Privileged == nil || *ctx.Privileged == false { - privileged = false - break - } - } - - // Use "all-run" insecure option (https://github.com/coreos/rkt/pull/2983) to take care - // of privileged pod. - // TODO(yifan): Have more granular app-level control of the insecure options. - // See: https://github.com/coreos/rkt/issues/2996. - if privileged { - config.InsecureOptions = fmt.Sprintf("%s,%s", config.InsecureOptions, "all-run") - } - - runPrepared := buildCommand(&config, "run-prepared").Args - - var hostname string - var err error - - osInfos, err := getOSReleaseInfo() - if err != nil { - glog.Warningf("rkt: Failed to read the os release info: %v", err) - } else { - // Overlay fs is not supported for SELinux yet on many distros. - // See https://github.com/coreos/rkt/issues/1727#issuecomment-173203129. - // For now, coreos carries a patch to support it: https://github.com/coreos/coreos-overlay/pull/1703 - if osInfos["ID"] != "coreos" && pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.SELinuxOptions != nil { - runPrepared = append(runPrepared, "--no-overlay=true") - } - } - - // Apply '--net=host' to pod that is running on host network or inside a network namespace. - if r.usesRktHostNetwork(pod) { - runPrepared = append(runPrepared, "--net=host") - } else { - runPrepared = append(runPrepared, fmt.Sprintf("--net=%s", defaultNetworkName)) - } - - if kubecontainer.IsHostNetworkPod(pod) { - // TODO(yifan): Let runtimeHelper.GeneratePodHostNameAndDomain() to handle this. - hostname, err = r.os.Hostname() - if err != nil { - return "", err - } - } else { - // Setup DNS. - dnsConfig, err := r.runtimeHelper.GetPodDNS(pod) - if err != nil { - return "", err - } - for _, server := range dnsConfig.Servers { - runPrepared = append(runPrepared, fmt.Sprintf("--dns=%s", server)) - } - for _, search := range dnsConfig.Searches { - runPrepared = append(runPrepared, fmt.Sprintf("--dns-search=%s", search)) - } - if len(dnsConfig.Servers) > 0 || len(dnsConfig.Searches) > 0 { - runPrepared = append(runPrepared, fmt.Sprintf("--dns-opt=%s", defaultDNSOption)) - } - - // TODO(yifan): host domain is not being used. - hostname, _, err = r.runtimeHelper.GeneratePodHostNameAndDomain(pod) - if err != nil { - return "", err - } - } - - runPrepared = append(runPrepared, fmt.Sprintf("--hostname=%s", hostname)) - runPrepared = append(runPrepared, uuid) - - if r.shouldCreateNetns(pod) { - // Drop the `rkt run-prepared` into the network namespace we - // created. - // TODO: switch to 'ip netns exec' once we can depend on a new - // enough version that doesn't have bugs like - // https://bugzilla.redhat.com/show_bug.cgi?id=882047 - nsenterExec := []string{r.nsenterPath, "--net=" + netnsPathFromName(networkNamespaceID), "--"} - runPrepared = append(nsenterExec, runPrepared...) - } - - return strings.Join(runPrepared, " "), nil -} - -func (r *Runtime) cleanupPodNetwork(pod *v1.Pod, networkNamespace kubecontainer.ContainerID) error { - // No-op if the pod is not running in a created netns. - if !r.shouldCreateNetns(pod) { - return nil - } - - glog.V(3).Infof("Calling network plugin %s to tear down pod for %s", r.network.PluginName(), format.Pod(pod)) - teardownErr := r.network.TearDownPod(pod.Namespace, pod.Name, networkNamespace) - if teardownErr != nil { - glog.Error(teardownErr) - } - - if _, err := r.execer.Command("ip", "netns", "del", networkNamespace.ID).Output(); err != nil { - return fmt.Errorf("rkt: Failed to remove network namespace for pod %s: %v", format.Pod(pod), err) - } - - return teardownErr -} - -func (r *Runtime) preparePodArgs(manifest *appcschema.PodManifest, manifestFileName string) []string { - // Order of precedence for the stage1: - // 1) pod annotation (stage1 name) - // 2) kubelet configured stage1 (stage1 path) - // 3) empty; whatever rkt's compiled to default to - stage1ImageCmd := "" - if r.config.Stage1Image != "" { - stage1ImageCmd = "--stage1-name=" + r.config.Stage1Image - } - if stage1Name, ok := manifest.Annotations.Get(k8sRktStage1NameAnno); ok { - stage1ImageCmd = "--stage1-name=" + stage1Name - } - - // Run 'rkt prepare' to get the rkt UUID. - cmds := []string{"prepare", "--quiet", "--pod-manifest", manifestFileName} - if stage1ImageCmd != "" { - cmds = append(cmds, stage1ImageCmd) - } - return cmds -} - -func (r *Runtime) getSelinuxContext(opt *v1.SELinuxOptions) (string, error) { - selinuxRunner := selinux.NewSELinuxRunner() - str, err := selinuxRunner.Getfilecon(r.config.Dir) - if err != nil { - return "", err - } - - ctx := strings.SplitN(str, ":", 4) - if len(ctx) != 4 { - return "", fmt.Errorf("malformated selinux context") - } - - if opt.User != "" { - ctx[0] = opt.User - } - if opt.Role != "" { - ctx[1] = opt.Role - } - if opt.Type != "" { - ctx[2] = opt.Type - } - if opt.Level != "" { - ctx[3] = opt.Level - } - - return strings.Join(ctx, ":"), nil -} - -// From the generateName or the podName return a basename for improving the logging with the Journal -// journalctl -t podBaseName -func constructSyslogIdentifier(generateName string, podName string) string { - if len(generateName) > 1 && generateName[len(generateName)-1] == '-' { - return generateName[0 : len(generateName)-1] - } - if len(generateName) > 0 { - return generateName - } - return podName -} - -// Setup additional systemd field specified in the Pod Annotation -func setupSystemdCustomFields(annotations map[string]string, unitOptionArray []*unit.UnitOption) ([]*unit.UnitOption, error) { - // LimitNOFILE - if strSize := annotations[k8sRktLimitNoFileAnno]; strSize != "" { - size, err := strconv.Atoi(strSize) - if err != nil { - return unitOptionArray, err - } - if size < 1 { - return unitOptionArray, fmt.Errorf("invalid value for %s: %s", k8sRktLimitNoFileAnno, strSize) - } - unitOptionArray = append(unitOptionArray, newUnitOption("Service", "LimitNOFILE", strSize)) - } - - return unitOptionArray, nil -} - -// preparePod will: -// -// 1. Invoke 'rkt prepare' to prepare the pod, and get the rkt pod uuid. -// 2. Create the unit file and save it under systemdUnitDir. -// -// On success, it will return a string that represents name of the unit file -// and the runtime pod. -func (r *Runtime) preparePod(pod *v1.Pod, podIP string, pullSecrets []v1.Secret, networkNamespaceID string) (string, *kubecontainer.Pod, error) { - // Generate the appc pod manifest from the k8s pod spec. - manifest, err := r.makePodManifest(pod, podIP, pullSecrets) - if err != nil { - return "", nil, err - } - manifestFile, err := ioutil.TempFile("", fmt.Sprintf("manifest-%s-", pod.Name)) - if err != nil { - return "", nil, err - } - defer func() { - manifestFile.Close() - if err := r.os.Remove(manifestFile.Name()); err != nil { - glog.Warningf("rkt: Cannot remove temp manifest file %q: %v", manifestFile.Name(), err) - } - }() - - data, err := json.Marshal(manifest) - if err != nil { - return "", nil, err - } - - glog.V(4).Infof("Generating pod manifest for pod %q: %v", format.Pod(pod), string(data)) - // Since File.Write returns error if the written length is less than len(data), - // so check error is enough for us. - if _, err := manifestFile.Write(data); err != nil { - return "", nil, err - } - - prepareCmd := r.preparePodArgs(manifest, manifestFile.Name()) - output, err := r.cli.RunCommand(nil, prepareCmd...) - if err != nil { - return "", nil, err - } - if len(output) != 1 { - return "", nil, fmt.Errorf("invalid output from 'rkt prepare': %v", output) - } - uuid := output[0] - glog.V(4).Infof("'rkt prepare' returns %q", uuid) - - // Create systemd service file for the rkt pod. - runPrepared, err := r.generateRunCommand(pod, uuid, networkNamespaceID) - if err != nil { - return "", nil, fmt.Errorf("failed to generate 'rkt run-prepared' command: %v", err) - } - - // TODO handle pod.Spec.HostPID - // TODO handle pod.Spec.HostIPC - - // TODO per container finishedAt, not just per pod - markPodFinished := podFinishedMarkCommand(r.touchPath, r.runtimeHelper.GetPodDir(pod.UID), uuid) - - hostNetwork := kubecontainer.IsHostNetworkPod(pod) - units := []*unit.UnitOption{ - newUnitOption("Service", "ExecStart", runPrepared), - newUnitOption("Service", "ExecStopPost", markPodFinished), - // This enables graceful stop. - newUnitOption("Service", "KillMode", "mixed"), - newUnitOption("Service", "TimeoutStopSec", fmt.Sprintf("%ds", getPodTerminationGracePeriodInSecond(pod))), - // Ops helpers - newUnitOption("Unit", "Description", pod.Name), - newUnitOption("Service", "SyslogIdentifier", constructSyslogIdentifier(pod.GenerateName, pod.Name)), - // Track pod info for garbage collection - newUnitOption(unitKubernetesSection, unitPodUID, string(pod.UID)), - newUnitOption(unitKubernetesSection, unitPodName, pod.Name), - newUnitOption(unitKubernetesSection, unitPodNamespace, pod.Namespace), - newUnitOption(unitKubernetesSection, unitPodHostNetwork, fmt.Sprintf("%v", hostNetwork)), - newUnitOption(unitKubernetesSection, unitPodNetworkNamespace, networkNamespaceID), - } - - if pod.Spec.SecurityContext != nil && pod.Spec.SecurityContext.SELinuxOptions != nil { - opt := pod.Spec.SecurityContext.SELinuxOptions - selinuxContext, err := r.getSelinuxContext(opt) - if err != nil { - glog.Errorf("rkt: Failed to construct selinux context with selinux option %q: %v", opt, err) - return "", nil, err - } - units = append(units, newUnitOption("Service", "SELinuxContext", selinuxContext)) - } - - units, err = setupSystemdCustomFields(pod.Annotations, units) - if err != nil { - glog.Warningf("fail to add custom systemd fields provided by pod Annotations: %q", err) - } - - serviceName := makePodServiceFileName(uuid) - glog.V(4).Infof("rkt: Creating service file %q for pod %q", serviceName, format.Pod(pod)) - serviceFile, err := r.os.Create(serviceFilePath(serviceName)) - if err != nil { - return "", nil, err - } - if _, err := io.Copy(serviceFile, unit.Serialize(units)); err != nil { - return "", nil, err - } - serviceFile.Close() - - return serviceName, apiPodToruntimePod(uuid, pod), nil -} - -// generateEvents is a helper function that generates some container -// life cycle events for containers in a pod. -func (r *Runtime) generateEvents(runtimePod *kubecontainer.Pod, reason string, failure error) { - // Set up container references. - for _, c := range runtimePod.Containers { - containerID := c.ID - id, err := parseContainerID(containerID) - if err != nil { - glog.Warningf("Invalid container ID %q", containerID) - continue - } - - ref, ok := r.containerRefManager.GetRef(containerID) - if !ok { - glog.Warningf("No ref for container %q", containerID) - continue - } - - // Note that 'rkt id' is the pod id. - uuid := utilstrings.ShortenString(id.uuid, 8) - switch reason { - case "Created": - r.recorder.Eventf(ref, v1.EventTypeNormal, events.CreatedContainer, "Created with rkt id %v", uuid) - case "Started": - r.recorder.Eventf(ref, v1.EventTypeNormal, events.StartedContainer, "Started with rkt id %v", uuid) - case "Failed": - r.recorder.Eventf(ref, v1.EventTypeWarning, events.FailedToStartContainer, "Failed to start with rkt id %v with error %v", uuid, failure) - case "Killing": - r.recorder.Eventf(ref, v1.EventTypeNormal, events.KillingContainer, "Killing with rkt id %v", uuid) - default: - glog.Errorf("rkt: Unexpected event %q", reason) - } - } - return -} - -// Generate a Network Namespace based on a New UUID -// to run the Pod and all of its containers inside a dedicated unique namespace -func generateNetworkNamespaceUUID() kubecontainer.ContainerID { - return kubecontainer.ContainerID{ID: fmt.Sprintf("%s%s", kubernetesUnitPrefix, uuid.NewUUID())} -} - -func netnsPathFromName(netnsName string) string { - return fmt.Sprintf("/var/run/netns/%s", netnsName) -} - -// setupPodNetwork creates a network namespace for the given pod and calls -// configured NetworkPlugin's setup function on it. -// It returns the namespace name, configured IP (if available), and an error if -// one occurred. -// -// If the pod is running in host network or is running using the no-op plugin, then nothing will be done. -func (r *Runtime) setupPodNetwork(pod *v1.Pod) (kubecontainer.ContainerID, string, error) { - glog.V(3).Infof("Calling network plugin %s to set up pod for %s", r.network.PluginName(), format.Pod(pod)) - - var networkNamespace kubecontainer.ContainerID - - // No-op if the pod is not running in a created netns. - if !r.shouldCreateNetns(pod) { - return networkNamespace, "", nil - } - - networkNamespace = generateNetworkNamespaceUUID() - glog.V(5).Infof("New network namespace %q generated for pod %s", networkNamespace.ID, format.Pod(pod)) - - // Create the network namespace for the pod - _, err := r.execer.Command("ip", "netns", "add", networkNamespace.ID).Output() - if err != nil { - return networkNamespace, "", fmt.Errorf("failed to create pod network namespace: %v", err) - } - - // Set up networking with the network plugin - err = r.network.SetUpPod(pod.Namespace, pod.Name, networkNamespace, pod.Annotations) - if err != nil { - return networkNamespace, "", err - } - status, err := r.network.GetPodNetworkStatus(pod.Namespace, pod.Name, networkNamespace) - if err != nil { - return networkNamespace, "", err - } - - if r.configureHairpinMode { - if err = hairpin.SetUpContainerPath(netnsPathFromName(networkNamespace.ID), network.DefaultInterfaceName); err != nil { - glog.Warningf("Hairpin setup failed for pod %q: %v", format.Pod(pod), err) - } - } - - return networkNamespace, status.IP.String(), nil -} - -// For a hostPath volume: rkt doesn't create any missing volume on the node/host so we need to create it -func createHostPathVolumes(pod *v1.Pod) (err error) { - for _, v := range pod.Spec.Volumes { - if v.VolumeSource.HostPath != nil { - _, err = os.Stat(v.HostPath.Path) - if os.IsNotExist(err) { - if err = os.MkdirAll(v.HostPath.Path, os.ModePerm); err != nil { - glog.Errorf("Create volume HostPath %q for Pod %q failed: %q", v.HostPath.Path, format.Pod(pod), err.Error()) - return err - } - glog.V(4).Infof("Created volume HostPath %q for Pod %q", v.HostPath.Path, format.Pod(pod)) - } - } - } - return nil -} - -// RunPod first creates the unit file for a pod, and then -// starts the unit over d-bus. -func (r *Runtime) RunPod(pod *v1.Pod, pullSecrets []v1.Secret) error { - glog.V(4).Infof("Rkt starts to run pod: name %q.", format.Pod(pod)) - - var err error - var networkNamespace kubecontainer.ContainerID - var podIP string - - err = createHostPathVolumes(pod) - if err != nil { - return err - } - - networkNamespace, podIP, err = r.setupPodNetwork(pod) - if err != nil { - r.cleanupPodNetwork(pod, networkNamespace) - return err - } - - name, runtimePod, prepareErr := r.preparePod(pod, podIP, pullSecrets, networkNamespace.ID) - - // Set container references and generate events. - // If preparedPod fails, then send out 'failed' events for each container. - // Otherwise, store the container references so we can use them later to send events. - for i, c := range pod.Spec.Containers { - ref, err := kubecontainer.GenerateContainerRef(pod, &c) - if err != nil { - glog.Errorf("Couldn't make a ref to pod %q, container %v: '%v'", format.Pod(pod), c.Name, err) - continue - } - if prepareErr != nil { - r.recorder.Eventf(ref, v1.EventTypeWarning, events.FailedToCreateContainer, "Failed to create rkt container with error: %v", prepareErr) - continue - } - containerID := runtimePod.Containers[i].ID - r.containerRefManager.SetRef(containerID, ref) - } - - if prepareErr != nil { - r.cleanupPodNetwork(pod, networkNamespace) - return prepareErr - } - - r.generateEvents(runtimePod, "Created", nil) - - // RestartUnit has the same effect as StartUnit if the unit is not running, besides it can restart - // a unit if the unit file is changed and reloaded. - reschan := make(chan string) - _, err = r.systemd.RestartUnit(name, "replace", reschan) - if err != nil { - r.generateEvents(runtimePod, "Failed", err) - r.cleanupPodNetwork(pod, networkNamespace) - return err - } - - res := <-reschan - if res != "done" { - err := fmt.Errorf("Failed to restart unit %q: %s", name, res) - r.generateEvents(runtimePod, "Failed", err) - r.cleanupPodNetwork(pod, networkNamespace) - return err - } - - r.generateEvents(runtimePod, "Started", nil) - - // This is a temporary solution until we have a clean design on how - // kubelet handles events. See https://github.com/kubernetes/kubernetes/issues/23084. - if err := r.runLifecycleHooks(pod, runtimePod, lifecyclePostStartHook); err != nil { - if errKill := r.KillPod(pod, *runtimePod, nil); errKill != nil { - return errors.NewAggregate([]error{err, errKill}) - } - r.cleanupPodNetwork(pod, networkNamespace) - return err - } - - return nil -} - -func (r *Runtime) runPreStopHook(containerID kubecontainer.ContainerID, pod *v1.Pod, container *v1.Container) error { - glog.V(4).Infof("rkt: Running pre-stop hook for container %q of pod %q", container.Name, format.Pod(pod)) - msg, err := r.runner.Run(containerID, pod, container, container.Lifecycle.PreStop) - if err != nil { - ref, ok := r.containerRefManager.GetRef(containerID) - if !ok { - glog.Warningf("No ref for container %q", containerID) - } else { - r.recorder.Eventf(ref, v1.EventTypeWarning, events.FailedPreStopHook, msg) - } - } - return err -} - -func (r *Runtime) runPostStartHook(containerID kubecontainer.ContainerID, pod *v1.Pod, container *v1.Container) error { - glog.V(4).Infof("rkt: Running post-start hook for container %q of pod %q", container.Name, format.Pod(pod)) - cid, err := parseContainerID(containerID) - if err != nil { - return fmt.Errorf("cannot parse container ID %v", containerID) - } - - isContainerRunning := func() (done bool, err error) { - ctx, cancel := context.WithTimeout(context.Background(), r.requestTimeout) - defer cancel() - resp, err := r.apisvc.InspectPod(ctx, &rktapi.InspectPodRequest{Id: cid.uuid}) - if err != nil { - return false, fmt.Errorf("failed to inspect rkt pod %q for pod %q", cid.uuid, format.Pod(pod)) - } - - for _, app := range resp.Pod.Apps { - if app.Name == cid.appName { - return app.State == rktapi.AppState_APP_STATE_RUNNING, nil - } - } - return false, fmt.Errorf("failed to find container %q in rkt pod %q", cid.appName, cid.uuid) - } - - // TODO(yifan): Polling the pod's state for now. - timeout := time.Second * 5 - pollInterval := time.Millisecond * 500 - if err := utilwait.Poll(pollInterval, timeout, isContainerRunning); err != nil { - return fmt.Errorf("rkt: Pod %q doesn't become running in %v: %v", format.Pod(pod), timeout, err) - } - - msg, err := r.runner.Run(containerID, pod, container, container.Lifecycle.PostStart) - if err != nil { - ref, ok := r.containerRefManager.GetRef(containerID) - if !ok { - glog.Warningf("No ref for container %q", containerID) - } else { - r.recorder.Eventf(ref, v1.EventTypeWarning, events.FailedPostStartHook, msg) - } - } - return err -} - -type lifecycleHookType string - -const ( - lifecyclePostStartHook lifecycleHookType = "post-start" - lifecyclePreStopHook lifecycleHookType = "pre-stop" -) - -func (r *Runtime) runLifecycleHooks(pod *v1.Pod, runtimePod *kubecontainer.Pod, typ lifecycleHookType) error { - var wg sync.WaitGroup - var errlist []error - errCh := make(chan error, len(pod.Spec.Containers)) - - wg.Add(len(pod.Spec.Containers)) - - for i, c := range pod.Spec.Containers { - var hookFunc func(kubecontainer.ContainerID, *v1.Pod, *v1.Container) error - - switch typ { - case lifecyclePostStartHook: - if c.Lifecycle != nil && c.Lifecycle.PostStart != nil { - hookFunc = r.runPostStartHook - } - case lifecyclePreStopHook: - if c.Lifecycle != nil && c.Lifecycle.PreStop != nil { - hookFunc = r.runPreStopHook - } - default: - errCh <- fmt.Errorf("Unrecognized lifecycle hook type %q for container %q in pod %q", typ, c.Name, format.Pod(pod)) - } - - if hookFunc == nil { - wg.Done() - continue - } - - container := &pod.Spec.Containers[i] - runtimeContainer := runtimePod.FindContainerByName(container.Name) - if runtimeContainer == nil { - // Container already gone. - wg.Done() - continue - } - containerID := runtimeContainer.ID - - go func() { - defer wg.Done() - if err := hookFunc(containerID, pod, container); err != nil { - glog.Errorf("rkt: Failed to run %s hook for container %q of pod %q: %v", typ, container.Name, format.Pod(pod), err) - errCh <- err - } else { - glog.V(4).Infof("rkt: %s hook completed successfully for container %q of pod %q", typ, container.Name, format.Pod(pod)) - } - }() - } - - wg.Wait() - close(errCh) - - for err := range errCh { - errlist = append(errlist, err) - } - return errors.NewAggregate(errlist) -} - -// convertRktPod will convert a rktapi.Pod to a kubecontainer.Pod -func (r *Runtime) convertRktPod(rktpod *rktapi.Pod) (*kubecontainer.Pod, error) { - manifest := &appcschema.PodManifest{} - err := json.Unmarshal(rktpod.Manifest, manifest) - if err != nil { - return nil, err - } - - podUID, ok := manifest.Annotations.Get(types.KubernetesPodUIDLabel) - if !ok { - return nil, fmt.Errorf("pod is missing annotation %s", types.KubernetesPodUIDLabel) - } - podName, ok := manifest.Annotations.Get(types.KubernetesPodNameLabel) - if !ok { - return nil, fmt.Errorf("pod is missing annotation %s", types.KubernetesPodNameLabel) - } - podNamespace, ok := manifest.Annotations.Get(types.KubernetesPodNamespaceLabel) - if !ok { - return nil, fmt.Errorf("pod is missing annotation %s", types.KubernetesPodNamespaceLabel) - } - - kubepod := &kubecontainer.Pod{ - ID: kubetypes.UID(podUID), - Name: podName, - Namespace: podNamespace, - } - - for i, app := range rktpod.Apps { - // The order of the apps is determined by the rkt pod manifest. - // TODO(yifan): Let the server to unmarshal the annotations? https://github.com/coreos/rkt/issues/1872 - hashStr, ok := manifest.Apps[i].Annotations.Get(k8sRktContainerHashAnno) - if !ok { - return nil, fmt.Errorf("app %q is missing annotation %s", app.Name, k8sRktContainerHashAnno) - } - containerHash, err := strconv.ParseUint(hashStr, 10, 64) - if err != nil { - return nil, fmt.Errorf("couldn't parse container's hash %q: %v", hashStr, err) - } - - kubepod.Containers = append(kubepod.Containers, &kubecontainer.Container{ - ID: buildContainerID(&containerID{rktpod.Id, app.Name}), - Name: app.Name, - // By default, the version returned by rkt API service will be "latest" if not specified. - Image: fmt.Sprintf("%s:%s", app.Image.Name, app.Image.Version), - ImageID: app.Image.Id, - Hash: containerHash, - State: appStateToContainerState(app.State), - }) - } - - return kubepod, nil -} - -// GetPods runs 'rkt list' to get the list of rkt pods. -// Then it will use the result to construct a list of container runtime pods. -// If all is false, then only running pods will be returned, otherwise all pods will be -// returned. -func (r *Runtime) GetPods(all bool) ([]*kubecontainer.Pod, error) { - glog.V(4).Infof("Rkt getting pods") - - listReq := &rktapi.ListPodsRequest{ - Detail: true, - Filters: []*rktapi.PodFilter{ - { - Annotations: []*rktapi.KeyValue{ - { - Key: k8sRktKubeletAnno, - Value: k8sRktKubeletAnnoValue, - }, - }, - }, - }, - } - if !all { - listReq.Filters[0].States = []rktapi.PodState{rktapi.PodState_POD_STATE_RUNNING} - } - ctx, cancel := context.WithTimeout(context.Background(), r.requestTimeout) - defer cancel() - listResp, err := r.apisvc.ListPods(ctx, listReq) - if err != nil { - return nil, fmt.Errorf("couldn't list pods: %v", err) - } - - pods := make(map[kubetypes.UID]*kubecontainer.Pod) - var podIDs []kubetypes.UID - for _, pod := range listResp.Pods { - pod, err := r.convertRktPod(pod) - if err != nil { - glog.Warningf("rkt: Cannot construct pod from unit file: %v.", err) - continue - } - - // Group pods together. - oldPod, found := pods[pod.ID] - if !found { - pods[pod.ID] = pod - podIDs = append(podIDs, pod.ID) - continue - } - - oldPod.Containers = append(oldPod.Containers, pod.Containers...) - } - - // Convert map to list, using the consistent order from the podIDs array. - var result []*kubecontainer.Pod - for _, id := range podIDs { - result = append(result, pods[id]) - } - - return result, nil -} - -func getPodTerminationGracePeriodInSecond(pod *v1.Pod) int64 { - var gracePeriod int64 - switch { - case pod.DeletionGracePeriodSeconds != nil: - gracePeriod = *pod.DeletionGracePeriodSeconds - case pod.Spec.TerminationGracePeriodSeconds != nil: - gracePeriod = *pod.Spec.TerminationGracePeriodSeconds - } - if gracePeriod < minimumGracePeriodInSeconds { - gracePeriod = minimumGracePeriodInSeconds - } - return gracePeriod -} - -func (r *Runtime) waitPreStopHooks(pod *v1.Pod, runningPod *kubecontainer.Pod) { - gracePeriod := getPodTerminationGracePeriodInSecond(pod) - - done := make(chan struct{}) - go func() { - if err := r.runLifecycleHooks(pod, runningPod, lifecyclePreStopHook); err != nil { - glog.Errorf("rkt: Some pre-stop hooks failed for pod %q: %v", format.Pod(pod), err) - } - close(done) - }() - - select { - case <-time.After(time.Duration(gracePeriod) * time.Second): - glog.V(2).Infof("rkt: Some pre-stop hooks did not complete in %d seconds for pod %q", gracePeriod, format.Pod(pod)) - case <-done: - } -} - -// KillPod invokes 'systemctl kill' to kill the unit that runs the pod. -// TODO: add support for gracePeriodOverride which is used in eviction scenarios -func (r *Runtime) KillPod(pod *v1.Pod, runningPod kubecontainer.Pod, gracePeriodOverride *int64) error { - glog.V(4).Infof("Rkt is killing pod: name %q.", runningPod.Name) - - if len(runningPod.Containers) == 0 { - glog.V(4).Infof("rkt: Pod %q is already being killed, no action will be taken", runningPod.Name) - return nil - } - - if pod != nil { - r.waitPreStopHooks(pod, &runningPod) - } - - containerID, err := parseContainerID(runningPod.Containers[0].ID) - if err != nil { - glog.Errorf("rkt: Failed to get rkt uuid of the pod %q: %v", runningPod.Name, err) - return err - } - serviceName := makePodServiceFileName(containerID.uuid) - serviceFile := serviceFilePath(serviceName) - - r.generateEvents(&runningPod, "Killing", nil) - for _, c := range runningPod.Containers { - r.containerRefManager.ClearRef(c.ID) - } - - // Since all service file have 'KillMode=mixed', the processes in - // the unit's cgroup will receive a SIGKILL if the normal stop timeouts. - reschan := make(chan string) - if _, err = r.systemd.StopUnit(serviceName, "replace", reschan); err != nil { - glog.Errorf("rkt: Failed to stop unit %q: %v", serviceName, err) - return err - } - - res := <-reschan - if res != "done" { - err := fmt.Errorf("invalid result: %s", res) - glog.Errorf("rkt: Failed to stop unit %q: %v", serviceName, err) - return err - } - - // Clean up networking. Use the service file to get pod details since 'pod' can be nil. - if err := r.cleanupPodNetworkFromServiceFile(serviceFile); err != nil { - glog.Errorf("rkt: failed to tear down network for unit %q: %v", serviceName, err) - return err - } - - return nil -} - -func (r *Runtime) Type() string { - return RktType -} - -func (r *Runtime) Version() (kubecontainer.Version, error) { - r.versions.RLock() - defer r.versions.RUnlock() - return r.versions.binVersion, nil -} - -func (r *Runtime) APIVersion() (kubecontainer.Version, error) { - r.versions.RLock() - defer r.versions.RUnlock() - return r.versions.apiVersion, nil -} - -// Status returns error if rkt is unhealthy, nil otherwise. -func (r *Runtime) Status() (*kubecontainer.RuntimeStatus, error) { - return nil, r.checkVersion(minimumRktBinVersion, minimumRktApiVersion, minimumSystemdVersion) -} - -// SyncPod syncs the running pod to match the specified desired pod. -func (r *Runtime) SyncPod(pod *v1.Pod, _ v1.PodStatus, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) { - var err error - defer func() { - if err != nil { - result.Fail(err) - } - }() - // TODO: (random-liu) Stop using running pod in SyncPod() - runningPod := kubecontainer.ConvertPodStatusToRunningPod(r.Type(), podStatus) - // Add references to all containers. - unidentifiedContainers := make(map[kubecontainer.ContainerID]*kubecontainer.Container) - for _, c := range runningPod.Containers { - unidentifiedContainers[c.ID] = c - } - - restartPod := false - for _, container := range pod.Spec.Containers { - expectedHash := kubecontainer.HashContainerLegacy(&container) - - c := runningPod.FindContainerByName(container.Name) - if c == nil { - if kubecontainer.ShouldContainerBeRestarted(&container, pod, podStatus) { - glog.V(3).Infof("Container %+v is dead, but RestartPolicy says that we should restart it.", container) - // TODO(yifan): Containers in one pod are fate-sharing at this moment, see: - // https://github.com/appc/spec/issues/276. - restartPod = true - break - } - continue - } - - // TODO: check for non-root image directives. See ../docker/manager.go#SyncPod - - // TODO(yifan): Take care of host network change. - containerChanged := c.Hash != 0 && c.Hash != expectedHash - if containerChanged { - glog.Infof("Pod %q container %q hash changed (%d vs %d), it will be killed and re-created.", format.Pod(pod), container.Name, c.Hash, expectedHash) - restartPod = true - break - } - - liveness, found := r.livenessManager.Get(c.ID) - if found && liveness != proberesults.Success && pod.Spec.RestartPolicy != v1.RestartPolicyNever { - glog.Infof("Pod %q container %q is unhealthy, it will be killed and re-created.", format.Pod(pod), container.Name) - restartPod = true - break - } - - delete(unidentifiedContainers, c.ID) - } - - // If there is any unidentified containers, restart the pod. - if len(unidentifiedContainers) > 0 { - restartPod = true - } - - if restartPod { - // Kill the pod only if the pod is actually running. - if len(runningPod.Containers) > 0 { - if err = r.KillPod(pod, runningPod, nil); err != nil { - return - } - } - if err = r.RunPod(pod, pullSecrets); err != nil { - return - } - } - return -} - -// Sort rkt pods by creation time. -type podsByCreatedAt []*rktapi.Pod - -func (s podsByCreatedAt) Len() int { return len(s) } -func (s podsByCreatedAt) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s podsByCreatedAt) Less(i, j int) bool { return s[i].CreatedAt < s[j].CreatedAt } - -// getPodUID returns the pod's API UID, it returns -// empty UID if the UID cannot be determined. -func getPodUID(pod *rktapi.Pod) kubetypes.UID { - for _, anno := range pod.Annotations { - if anno.Key == types.KubernetesPodUIDLabel { - return kubetypes.UID(anno.Value) - } - } - return kubetypes.UID("") -} - -// podIsActive returns true if the pod is embryo, preparing or running. -// If a pod is prepared, it is not guaranteed to be active (e.g. the systemd -// service might fail). -func podIsActive(pod *rktapi.Pod) bool { - return pod.State == rktapi.PodState_POD_STATE_EMBRYO || - pod.State == rktapi.PodState_POD_STATE_PREPARING || - pod.State == rktapi.PodState_POD_STATE_RUNNING -} - -// GetNetNS returns the network namespace path for the given container -func (r *Runtime) GetNetNS(containerID kubecontainer.ContainerID) (string, error) { - // Currently the containerID is a UUID for a network namespace - // This hack is a way to create an unique network namespace for each new starting/restarting Pod - // We can do this because we played the same trick in - // `networkPlugin.SetUpPod` and `networkPlugin.TearDownPod`. - // See https://github.com/kubernetes/kubernetes/issues/45149 - return netnsPathFromName(containerID.ID), nil -} - -func (r *Runtime) GetPodContainerID(pod *kubecontainer.Pod) (kubecontainer.ContainerID, error) { - return kubecontainer.ContainerID{ID: string(pod.ID)}, nil -} - -func (r *Runtime) getKubernetesDirective(serviceFilePath string) (podService podServiceDirective, err error) { - f, err := os.Open(serviceFilePath) - if err != nil { - return podService, err - } - defer f.Close() - - opts, err := unit.Deserialize(f) - if err != nil { - return podService, err - } - - var hostnetwork, networkNamespace string - for _, o := range opts { - if o.Section != unitKubernetesSection { - continue - } - switch o.Name { - case unitPodUID: - podService.id = o.Value - case unitPodName: - podService.name = o.Value - case unitPodNamespace: - podService.namespace = o.Value - case unitPodHostNetwork: - hostnetwork = o.Value - case unitPodNetworkNamespace: - networkNamespace = o.Value - } - - if podService.id != "" && podService.name != "" && podService.namespace != "" && hostnetwork != "" && networkNamespace != "" { - podService.hostNetwork, err = strconv.ParseBool(hostnetwork) - podService.networkNamespace = kubecontainer.ContainerID{ID: networkNamespace} - if err != nil { - return podService, err - } - return podService, nil - } - } - - return podService, fmt.Errorf("failed to parse pod from file %s", serviceFilePath) -} - -func (r *Runtime) DeleteContainer(containerID kubecontainer.ContainerID) error { - return fmt.Errorf("unimplemented") -} - -// Collects all the systemd units for k8s Pods -func (r *Runtime) getPodSystemdServiceFiles() ([]os.FileInfo, error) { - // Get all the current units - files, err := r.os.ReadDir(systemdServiceDir) - if err != nil { - glog.Errorf("rkt: Failed to read the systemd service directory: %v", err) - return files, err - } - - // Keep only k8s unit files - k8sSystemdServiceFiles := files[:0] - for _, f := range files { - if strings.HasPrefix(f.Name(), kubernetesUnitPrefix) { - k8sSystemdServiceFiles = append(k8sSystemdServiceFiles, f) - } - } - return k8sSystemdServiceFiles, err -} - -// GarbageCollect collects the pods/containers. -// After one GC iteration: -// - The deleted pods will be removed. -// - If the number of containers exceeds gcPolicy.MaxContainers, -// then containers whose ages are older than gcPolicy.minAge will -// be removed. -func (r *Runtime) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool, _ bool) error { - var errlist []error - var totalInactiveContainers int - var inactivePods []*rktapi.Pod - var removeCandidates []*rktapi.Pod - var allPods = map[string]*rktapi.Pod{} - - glog.V(4).Infof("rkt: Garbage collecting triggered with policy %v", gcPolicy) - - // GC all inactive systemd service files and pods. - files, err := r.getPodSystemdServiceFiles() - if err != nil { - return err - } - - ctx, cancel := context.WithTimeout(context.Background(), r.requestTimeout) - defer cancel() - resp, err := r.apisvc.ListPods(ctx, &rktapi.ListPodsRequest{Filters: kubernetesPodsFilters()}) - if err != nil { - glog.Errorf("rkt: Failed to list pods: %v", err) - return err - } - - // Mark inactive pods. - for _, pod := range resp.Pods { - allPods[pod.Id] = pod - if !podIsActive(pod) { - uid := getPodUID(pod) - if uid == kubetypes.UID("") { - glog.Errorf("rkt: Cannot get the UID of pod %q, pod is broken, will remove it", pod.Id) - removeCandidates = append(removeCandidates, pod) - continue - } - if r.podDeletionProvider.IsPodDeleted(uid) && allSourcesReady { - removeCandidates = append(removeCandidates, pod) - continue - } - - inactivePods = append(inactivePods, pod) - totalInactiveContainers = totalInactiveContainers + len(pod.Apps) - } - } - - // Remove any orphan service files. - for _, f := range files { - serviceName := f.Name() - rktUUID := getRktUUIDFromServiceFileName(serviceName) - if _, ok := allPods[rktUUID]; !ok { - glog.V(4).Infof("rkt: No rkt pod found for service file %q, will remove it", serviceName) - - if err := r.cleanupByPodId(rktUUID); err != nil { - errlist = append(errlist, fmt.Errorf("rkt: Failed to clean up rkt pod %q: %v", rktUUID, err)) - } - } - } - - sort.Sort(podsByCreatedAt(inactivePods)) - - // Enforce GCPolicy.MaxContainers. - for _, pod := range inactivePods { - if totalInactiveContainers <= gcPolicy.MaxContainers { - break - } - creationTime := time.Unix(0, pod.CreatedAt) - if creationTime.Add(gcPolicy.MinAge).Before(time.Now()) { - // The pod is old and we are exceeding the MaxContainers limit. - // Delete the pod. - removeCandidates = append(removeCandidates, pod) - totalInactiveContainers = totalInactiveContainers - len(pod.Apps) - } - } - - // Remove pods and their service files. - for _, pod := range removeCandidates { - if err := r.removePod(pod); err != nil { - errlist = append(errlist, fmt.Errorf("rkt: Failed to clean up rkt pod %q: %v", pod.Id, err)) - } - } - - return errors.NewAggregate(errlist) -} - -// Read kubernetes pod UUID, namespace, netns and name from systemd service file and -// use that to clean up any pod network that may still exist. -func (r *Runtime) cleanupPodNetworkFromServiceFile(serviceFilePath string) error { - podService, err := r.unitGetter.getKubernetesDirective(serviceFilePath) - if err != nil { - return err - } - return r.cleanupPodNetwork(&v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - UID: kubetypes.UID(podService.id), - Name: podService.name, - Namespace: podService.namespace, - }, - Spec: v1.PodSpec{ - HostNetwork: podService.hostNetwork, - }, - }, podService.networkNamespace) -} - -// Remove the touched file created by ExecStartPost in the systemd service file -func (r *Runtime) removeFinishedMarkerFile(serviceName string) error { - serviceFile := serviceFilePath(serviceName) - podDetail, err := r.unitGetter.getKubernetesDirective(serviceFile) - if err != nil { - return err - } - podDir := r.runtimeHelper.GetPodDir(kubetypes.UID(podDetail.id)) - finishedFile := podFinishedMarkerPath(podDir, getRktUUIDFromServiceFileName(serviceName)) - return r.os.Remove(finishedFile) -} - -// Iter over each container in the pod to delete its termination log file -func (r *Runtime) removeTerminationFiles(pod *rktapi.Pod) (errlist []error) { - // container == app - for _, app := range pod.Apps { - for _, annotation := range app.Annotations { - if annotation.GetKey() == k8sRktTerminationMessagePathAnno { - if err := r.os.Remove(annotation.GetValue()); err != nil { - errlist = append(errlist, fmt.Errorf("rkt: Failed to remove for pod %q container file %v", pod.Id, err)) - } - } - } - } - return errlist -} - -func (r *Runtime) cleanupByPodId(podID string) (errlist []error) { - serviceName := makePodServiceFileName(podID) - serviceFile := serviceFilePath(serviceName) - - if err := r.cleanupPodNetworkFromServiceFile(serviceFile); err != nil { - errlist = append(errlist, fmt.Errorf("rkt: Failed to clean up pod network from service %q: %v, the network may not be around already", serviceName, err)) - } - - // GC finished marker, termination-log file, systemd service files as well. - if err := r.systemd.ResetFailedUnit(serviceName); err != nil { - errlist = append(errlist, fmt.Errorf("rkt: Failed to reset the failed systemd service %q: %v", serviceName, err)) - } - if err := r.removeFinishedMarkerFile(serviceName); err != nil { - errlist = append(errlist, fmt.Errorf("rkt: Failed to remove finished file %q for unit %q: %v", serviceName, podID, err)) - } - if err := r.os.Remove(serviceFile); err != nil { - errlist = append(errlist, fmt.Errorf("rkt: Failed to remove service file %q for pod %q: %v", serviceFile, podID, err)) - } - return errlist -} - -// removePod calls 'rkt rm $UUID' to delete a rkt pod, -// it also remove the systemd service file, -// the finished-* marker and the termination-log files -// related to the pod. -func (r *Runtime) removePod(pod *rktapi.Pod) error { - var errlist []error - glog.V(4).Infof("rkt: GC is removing pod %q", pod) - - if err := r.cleanupByPodId(pod.Id); err != nil { - errlist = append(errlist, fmt.Errorf("rkt: Failed to remove pod %q: %v", pod.Id, err)) - } - if err := r.removeTerminationFiles(pod); err != nil { - errlist = append(errlist, fmt.Errorf("rkt: Failed to clean up pod TerminationMessageFile %q: %v", pod.Id, err)) - } - - if _, err := r.cli.RunCommand(nil, "rm", pod.Id); err != nil { - errlist = append(errlist, fmt.Errorf("rkt: Failed to remove pod %q: %v", pod.Id, err)) - } - - return errors.NewAggregate(errlist) -} - -// rktExitError implements /pkg/util/exec.ExitError interface. -type rktExitError struct{ *exec.ExitError } - -var _ utilexec.ExitError = &rktExitError{} - -func (r *rktExitError) ExitStatus() int { - if status, ok := r.Sys().(syscall.WaitStatus); ok { - return status.ExitStatus() - } - return 0 -} - -func newRktExitError(e error) error { - if exitErr, ok := e.(*exec.ExitError); ok { - return &rktExitError{exitErr} - } - return e -} - -func (r *Runtime) AttachContainer(containerID kubecontainer.ContainerID, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error { - return fmt.Errorf("unimplemented") -} - -// Note: In rkt, the container ID is in the form of "UUID:appName", where UUID is -// the rkt UUID, and appName is the container name. -// TODO(yifan): If the rkt is using lkvm as the stage1 image, then this function will fail. -func (r *Runtime) ExecInContainer(containerID kubecontainer.ContainerID, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error { - glog.V(4).Infof("Rkt execing in container.") - - id, err := parseContainerID(containerID) - if err != nil { - return err - } - args := []string{"enter", fmt.Sprintf("--app=%s", id.appName), id.uuid} - args = append(args, cmd...) - command := buildCommand(r.config, args...) - - if tty { - p, err := kubecontainer.StartPty(command) - if err != nil { - return err - } - defer p.Close() - - // make sure to close the stdout stream - defer stdout.Close() - - kubecontainer.HandleResizing(resize, func(size remotecommand.TerminalSize) { - term.SetSize(p.Fd(), size) - }) - - if stdin != nil { - go io.Copy(p, stdin) - } - if stdout != nil { - go io.Copy(stdout, p) - } - return newRktExitError(command.Wait()) - } - if stdin != nil { - // Use an os.Pipe here as it returns true *os.File objects. - // This way, if you run 'kubectl exec -i bash' (no tty) and type 'exit', - // the call below to command.Run() can unblock because its Stdin is the read half - // of the pipe. - r, w, err := r.os.Pipe() - if err != nil { - return newRktExitError(err) - } - go io.Copy(w, stdin) - - command.Stdin = r - } - if stdout != nil { - command.Stdout = stdout - } - if stderr != nil { - command.Stderr = stderr - } - return newRktExitError(command.Run()) -} - -// PortForward executes socat in the pod's network namespace and copies -// data between stream (representing the user's local connection on their -// computer) and the specified port in the container. -// -// TODO: -// - match cgroups of container -// - should we support nsenter + socat on the host? (current impl) -// - should we support nsenter + socat in a container, running with elevated privs and --pid=host? -// -// TODO(yifan): Merge with the same function in dockertools. -func (r *Runtime) PortForward(pod *kubecontainer.Pod, port int32, stream io.ReadWriteCloser) error { - glog.V(4).Infof("Rkt port forwarding in container.") - - ctx, cancel := context.WithTimeout(context.Background(), r.requestTimeout) - defer cancel() - listResp, err := r.apisvc.ListPods(ctx, &rktapi.ListPodsRequest{ - Detail: true, - Filters: runningKubernetesPodFilters(pod.ID), - }) - if err != nil { - return fmt.Errorf("couldn't list pods: %v", err) - } - - if len(listResp.Pods) != 1 { - var podlist []string - for _, p := range listResp.Pods { - podlist = append(podlist, p.Id) - } - return fmt.Errorf("more than one running rkt pod for the kubernetes pod [%s]", strings.Join(podlist, ", ")) - } - listPod := listResp.Pods[0] - - socatPath, lookupErr := exec.LookPath("socat") - if lookupErr != nil { - return fmt.Errorf("unable to do port forwarding: socat not found.") - } - - // Check in config and in annotations if we're running kvm flavor - isKvm := strings.Contains(r.config.Stage1Image, "kvm") - for _, anno := range listPod.Annotations { - if anno.Key == k8sRktStage1NameAnno { - isKvm = strings.Contains(anno.Value, "kvm") - break - } - } - - var args []string - var fwCaller string - if isKvm { - podNetworks := listPod.GetNetworks() - if podNetworks == nil { - return fmt.Errorf("unable to get networks") - } - args = []string{"-", fmt.Sprintf("TCP4:%s:%d", podNetworks[0].Ipv4, port)} - fwCaller = socatPath - } else { - args = []string{"-t", fmt.Sprintf("%d", listPod.Pid), "-n", socatPath, "-", fmt.Sprintf("TCP4:localhost:%d", port)} - nsenterPath, lookupErr := exec.LookPath("nsenter") - if lookupErr != nil { - return fmt.Errorf("unable to do port forwarding: nsenter not found") - } - fwCaller = nsenterPath - } - - command := exec.Command(fwCaller, args...) - command.Stdout = stream - - // If we use Stdin, command.Run() won't return until the goroutine that's copying - // from stream finishes. Unfortunately, if you have a client like telnet connected - // via port forwarding, as long as the user's telnet client is connected to the user's - // local listener that port forwarding sets up, the telnet session never exits. This - // means that even if socat has finished running, command.Run() won't ever return - // (because the client still has the connection and stream open). - // - // The work around is to use StdinPipe(), as Wait() (called by Run()) closes the pipe - // when the command (socat) exits. - inPipe, err := command.StdinPipe() - if err != nil { - return fmt.Errorf("unable to do port forwarding: error creating stdin pipe: %v", err) - } - go func() { - io.Copy(inPipe, stream) - inPipe.Close() - }() - - return command.Run() -} - -// UpdatePodCIDR updates the runtimeconfig with the podCIDR. -// Currently no-ops, just implemented to satisfy the cri. -func (r *Runtime) UpdatePodCIDR(podCIDR string) error { - return nil -} - -// appStateToContainerState converts rktapi.AppState to kubecontainer.ContainerState. -func appStateToContainerState(state rktapi.AppState) kubecontainer.ContainerState { - switch state { - case rktapi.AppState_APP_STATE_RUNNING: - return kubecontainer.ContainerStateRunning - case rktapi.AppState_APP_STATE_EXITED: - return kubecontainer.ContainerStateExited - } - return kubecontainer.ContainerStateUnknown -} - -// getPodInfo returns the pod manifest, creation time and restart count of the pod. -func getPodInfo(pod *rktapi.Pod) (podManifest *appcschema.PodManifest, restartCount int, err error) { - // TODO(yifan): The manifest is only used for getting the annotations. - // Consider to let the server to unmarshal the annotations. - var manifest appcschema.PodManifest - if err = json.Unmarshal(pod.Manifest, &manifest); err != nil { - return - } - - if countString, ok := manifest.Annotations.Get(k8sRktRestartCountAnno); ok { - restartCount, err = strconv.Atoi(countString) - if err != nil { - return - } - } - - return &manifest, restartCount, nil -} - -// populateContainerStatus fills the container status according to the app's information. -func populateContainerStatus(pod rktapi.Pod, app rktapi.App, runtimeApp appcschema.RuntimeApp, restartCount int, finishedTime time.Time) (*kubecontainer.ContainerStatus, error) { - hashStr, ok := runtimeApp.Annotations.Get(k8sRktContainerHashAnno) - if !ok { - return nil, fmt.Errorf("No container hash in pod manifest") - } - - hashNum, err := strconv.ParseUint(hashStr, 10, 64) - if err != nil { - return nil, err - } - - var reason, message string - if app.State == rktapi.AppState_APP_STATE_EXITED { - if app.ExitCode == 0 { - reason = "Completed" - } else { - reason = "Error" - } - } - - terminationMessagePath, ok := runtimeApp.Annotations.Get(k8sRktTerminationMessagePathAnno) - if ok { - if data, err := ioutil.ReadFile(terminationMessagePath); err != nil { - message = fmt.Sprintf("Error on reading termination-log %s: %v", terminationMessagePath, err) - } else { - message = string(data) - } - } - - createdTime := time.Unix(0, pod.CreatedAt) - startedTime := time.Unix(0, pod.StartedAt) - - return &kubecontainer.ContainerStatus{ - ID: buildContainerID(&containerID{uuid: pod.Id, appName: app.Name}), - Name: app.Name, - State: appStateToContainerState(app.State), - CreatedAt: createdTime, - StartedAt: startedTime, - FinishedAt: finishedTime, - ExitCode: int(app.ExitCode), - // By default, the version returned by rkt API service will be "latest" if not specified. - Image: fmt.Sprintf("%s:%s", app.Image.Name, app.Image.Version), - ImageID: "rkt://" + app.Image.Id, // TODO(yifan): Add the prefix only in v1.PodStatus. - Hash: hashNum, - // TODO(yifan): Note that now all apps share the same restart count, this might - // change once apps don't share the same lifecycle. - // See https://github.com/appc/spec/pull/547. - RestartCount: restartCount, - Reason: reason, - Message: message, - }, nil -} - -// from a running systemd unit, return the network namespace of a Pod -// this field is inside the X-Kubernetes directive -func (r *Runtime) getNetworkNamespace(uid kubetypes.UID, latestPod *rktapi.Pod) (networkNamespace kubecontainer.ContainerID, err error) { - serviceFiles, err := r.getPodSystemdServiceFiles() - if err != nil { - return networkNamespace, err - } - - for _, f := range serviceFiles { - fileName := f.Name() - if latestPod.Id == getRktUUIDFromServiceFileName(fileName) { - podService, err := r.unitGetter.getKubernetesDirective(serviceFilePath(fileName)) - if err != nil { - return networkNamespace, err - } - return podService.networkNamespace, nil - } - } - - return networkNamespace, fmt.Errorf("Pod %q containing rktPod %q haven't find a corresponding NetworkNamespace in %d systemd units", uid, latestPod.Id, len(serviceFiles)) -} - -// GetPodStatus returns the status for a pod specified by a given UID, name, -// and namespace. It will attempt to find pod's information via a request to -// the rkt api server. -// An error will be returned if the api server returns an error. If the api -// server doesn't error, but doesn't provide meaningful information about the -// pod, a status with no information (other than the passed in arguments) is -// returned anyways. -func (r *Runtime) GetPodStatus(uid kubetypes.UID, name, namespace string) (*kubecontainer.PodStatus, error) { - podStatus := &kubecontainer.PodStatus{ - ID: uid, - Name: name, - Namespace: namespace, - } - - ctx, cancel := context.WithTimeout(context.Background(), r.requestTimeout) - defer cancel() - listResp, err := r.apisvc.ListPods(ctx, &rktapi.ListPodsRequest{ - Detail: true, - Filters: kubernetesPodFilters(uid), - }) - if err != nil { - return nil, fmt.Errorf("couldn't list pods: %v", err) - } - - var latestPod *rktapi.Pod - var latestRestartCount int = -1 - - // In this loop, we group all containers from all pods together, - // also we try to find the latest pod, so we can fill other info of the pod below. - for _, pod := range listResp.Pods { - manifest, restartCount, err := getPodInfo(pod) - if err != nil { - glog.Warningf("rkt: Couldn't get necessary info from the rkt pod, (uuid %q): %v", pod.Id, err) - continue - } - - if restartCount > latestRestartCount { - latestPod = pod - latestRestartCount = restartCount - } - - finishedTime := r.podFinishedAt(uid, pod.Id) - for i, app := range pod.Apps { - // The order of the apps is determined by the rkt pod manifest. - cs, err := populateContainerStatus(*pod, *app, manifest.Apps[i], restartCount, finishedTime) - if err != nil { - glog.Warningf("rkt: Failed to populate container status(uuid %q, app %q): %v", pod.Id, app.Name, err) - continue - } - podStatus.ContainerStatuses = append(podStatus.ContainerStatuses, cs) - } - } - - if latestPod == nil { - glog.Warningf("No latestPod: rkt api-svc returns [%d]rktPods, cannot fill podStatus.IP", len(listResp.Pods)) - return podStatus, nil - } - - // If we are running no-op network plugin, then get the pod IP from the rkt pod status. - if r.network.PluginName() == network.DefaultPluginName { - for _, n := range latestPod.Networks { - if n.Name == defaultNetworkName { - podStatus.IP = n.Ipv4 - break - } - } - return podStatus, nil - } - - networkNamespace, err := r.unitGetter.getNetworkNamespace(uid, latestPod) - if err != nil { - glog.Warningf("networkNamespace: %v", err) - } - status, err := r.network.GetPodNetworkStatus(namespace, name, networkNamespace) - if err != nil { - glog.Warningf("rkt: %v", err) - } else if status != nil { - // status can be nil when the pod is running on the host network, - // in which case the pod IP will be populated by the upper layer. - podStatus.IP = status.IP.String() - } - - return podStatus, nil -} - -// getOSReleaseInfo reads /etc/os-release and returns a map -// that contains the key value pairs in that file. -func getOSReleaseInfo() (map[string]string, error) { - result := make(map[string]string) - - path := "/etc/os-release" - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - if len(strings.TrimSpace(line)) == 0 { - // Skips empty lines - continue - } - - info := strings.SplitN(line, "=", 2) - if len(info) != 2 { - glog.Warningf("Unexpected entry in os-release %q", line) - continue - } - result[info[0]] = info[1] - } - if err := scanner.Err(); err != nil { - return nil, err - } - return result, nil -} - -// convertKubeMounts creates appc volumes and mount points according to the given mounts. -// Only one volume will be created for every unique host path. -// Only one mount point will be created for every unique container path. -func convertKubeMounts(mounts []kubecontainer.Mount) ([]appctypes.Volume, []appctypes.MountPoint) { - volumeMap := make(map[string]*appctypes.Volume) - mountPointMap := make(map[string]*appctypes.MountPoint) - - for _, mnt := range mounts { - readOnly := mnt.ReadOnly - - if _, existed := volumeMap[mnt.HostPath]; !existed { - volumeMap[mnt.HostPath] = &appctypes.Volume{ - Name: *appctypes.MustACName(string(uuid.NewUUID())), - Kind: "host", - Source: mnt.HostPath, - ReadOnly: &readOnly, - } - } - - if _, existed := mountPointMap[mnt.ContainerPath]; existed { - glog.Warningf("Multiple mount points with the same container path %v, ignore it", mnt) - continue - } - - mountPointMap[mnt.ContainerPath] = &appctypes.MountPoint{ - Name: volumeMap[mnt.HostPath].Name, - Path: mnt.ContainerPath, - ReadOnly: readOnly, - } - } - - volumes := make([]appctypes.Volume, 0, len(volumeMap)) - mountPoints := make([]appctypes.MountPoint, 0, len(mountPointMap)) - - for _, vol := range volumeMap { - volumes = append(volumes, *vol) - } - for _, mnt := range mountPointMap { - mountPoints = append(mountPoints, *mnt) - } - - return volumes, mountPoints -} - -// convertKubePortMappings creates appc container ports and host ports according to the given port mappings. -// The container ports and host ports are mapped by PortMapping.Name. -func convertKubePortMappings(portMappings []kubecontainer.PortMapping) ([]appctypes.Port, []appctypes.ExposedPort) { - containerPorts := make([]appctypes.Port, 0, len(portMappings)) - hostPorts := make([]appctypes.ExposedPort, 0, len(portMappings)) - - for _, p := range portMappings { - // This matches the docker code's behaviour. - if p.HostPort == 0 { - continue - } - - portName := convertToACName(p.Name) - containerPorts = append(containerPorts, appctypes.Port{ - Name: portName, - Protocol: string(p.Protocol), - Port: uint(p.ContainerPort), - }) - - hostPorts = append(hostPorts, appctypes.ExposedPort{ - Name: portName, - HostPort: uint(p.HostPort), - }) - } - - return containerPorts, hostPorts -} - -func newNoNewPrivilegesIsolator(v bool) (*appctypes.Isolator, error) { - b := fmt.Sprintf(`{"name": "%s", "value": %t}`, appctypes.LinuxNoNewPrivilegesName, v) - - i := &appctypes.Isolator{ - Name: appctypes.LinuxNoNewPrivilegesName, - } - if err := i.UnmarshalJSON([]byte(b)); err != nil { - return nil, err - } - - return i, nil -} diff --git a/pkg/kubelet/rkt/rkt_test.go b/pkg/kubelet/rkt/rkt_test.go deleted file mode 100644 index 8e5f6cdd812..00000000000 --- a/pkg/kubelet/rkt/rkt_test.go +++ /dev/null @@ -1,2130 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rkt - -import ( - "encoding/json" - "fmt" - "net" - "os" - "path/filepath" - "sort" - "strings" - "testing" - "time" - - appcschema "github.com/appc/spec/schema" - appctypes "github.com/appc/spec/schema/types" - "github.com/coreos/go-systemd/unit" - rktapi "github.com/coreos/rkt/api/v1alpha" - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kubetypes "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/errors" - utiltesting "k8s.io/client-go/util/testing" - kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" - containertesting "k8s.io/kubernetes/pkg/kubelet/container/testing" - "k8s.io/kubernetes/pkg/kubelet/lifecycle" - "k8s.io/kubernetes/pkg/kubelet/network" - "k8s.io/kubernetes/pkg/kubelet/network/kubenet" - nettest "k8s.io/kubernetes/pkg/kubelet/network/testing" - "k8s.io/kubernetes/pkg/kubelet/types" - "k8s.io/utils/exec" - fakeexec "k8s.io/utils/exec/testing" -) - -func mustMarshalPodManifest(man *appcschema.PodManifest) []byte { - manblob, err := json.Marshal(man) - if err != nil { - panic(err) - } - return manblob -} - -func mustMarshalImageManifest(man *appcschema.ImageManifest) []byte { - manblob, err := json.Marshal(man) - if err != nil { - panic(err) - } - return manblob -} - -func mustRktHash(hash string) *appctypes.Hash { - h, err := appctypes.NewHash(hash) - if err != nil { - panic(err) - } - return h -} - -func makeRktPod(rktPodState rktapi.PodState, - rktPodID, podUID, podName, podNamespace string, podCreatedAt, podStartedAt int64, - podRestartCount string, appNames, imgIDs, imgNames, - containerHashes []string, appStates []rktapi.AppState, - exitcodes []int32, ips map[string]string) *rktapi.Pod { - - podManifest := &appcschema.PodManifest{ - ACKind: appcschema.PodManifestKind, - ACVersion: appcschema.AppContainerVersion, - Annotations: appctypes.Annotations{ - appctypes.Annotation{ - Name: *appctypes.MustACIdentifier(k8sRktKubeletAnno), - Value: k8sRktKubeletAnnoValue, - }, - appctypes.Annotation{ - Name: *appctypes.MustACIdentifier(types.KubernetesPodUIDLabel), - Value: podUID, - }, - appctypes.Annotation{ - Name: *appctypes.MustACIdentifier(types.KubernetesPodNameLabel), - Value: podName, - }, - appctypes.Annotation{ - Name: *appctypes.MustACIdentifier(types.KubernetesPodNamespaceLabel), - Value: podNamespace, - }, - appctypes.Annotation{ - Name: *appctypes.MustACIdentifier(k8sRktRestartCountAnno), - Value: podRestartCount, - }, - }, - } - - appNum := len(appNames) - if appNum != len(imgNames) || - appNum != len(imgIDs) || - appNum != len(containerHashes) || - appNum != len(appStates) { - panic("inconsistent app number") - } - - apps := make([]*rktapi.App, appNum) - for i := range appNames { - apps[i] = &rktapi.App{ - Name: appNames[i], - State: appStates[i], - Image: &rktapi.Image{ - Id: imgIDs[i], - Name: imgNames[i], - Version: "latest", - Manifest: mustMarshalImageManifest( - &appcschema.ImageManifest{ - ACKind: appcschema.ImageManifestKind, - ACVersion: appcschema.AppContainerVersion, - Name: *appctypes.MustACIdentifier(imgNames[i]), - Annotations: appctypes.Annotations{ - appctypes.Annotation{ - Name: *appctypes.MustACIdentifier(k8sRktContainerHashAnno), - Value: containerHashes[i], - }, - }, - }, - ), - }, - ExitCode: exitcodes[i], - } - podManifest.Apps = append(podManifest.Apps, appcschema.RuntimeApp{ - Name: *appctypes.MustACName(appNames[i]), - Image: appcschema.RuntimeImage{ID: *mustRktHash("sha512-foo")}, - Annotations: appctypes.Annotations{ - appctypes.Annotation{ - Name: *appctypes.MustACIdentifier(k8sRktContainerHashAnno), - Value: containerHashes[i], - }, - }, - }) - } - - var networks []*rktapi.Network - for name, ip := range ips { - networks = append(networks, &rktapi.Network{Name: name, Ipv4: ip}) - } - - return &rktapi.Pod{ - Id: rktPodID, - State: rktPodState, - Apps: apps, - Manifest: mustMarshalPodManifest(podManifest), - StartedAt: podStartedAt, - CreatedAt: podCreatedAt, - Networks: networks, - } -} - -func TestCheckVersion(t *testing.T) { - fr := newFakeRktInterface() - fs := newFakeSystemd() - r := &Runtime{apisvc: fr, systemd: fs} - - fr.info = rktapi.Info{ - RktVersion: "1.2.3+git", - AppcVersion: "1.2.4+git", - ApiVersion: "1.2.6-alpha", - } - fs.version = "100" - tests := []struct { - minimumRktBinVersion string - minimumRktApiVersion string - minimumSystemdVersion string - err error - calledGetInfo bool - calledSystemVersion bool - }{ - // Good versions. - { - "1.2.3", - "1.2.5", - "99", - nil, - true, - true, - }, - // Good versions. - { - "1.2.3+git", - "1.2.6-alpha", - "100", - nil, - true, - true, - }, - // Requires greater binary version. - { - "1.2.4", - "1.2.6-alpha", - "100", - fmt.Errorf("rkt: binary version is too old(%v), requires at least %v", fr.info.RktVersion, "1.2.4"), - true, - true, - }, - // Requires greater API version. - { - "1.2.3", - "1.2.6", - "100", - fmt.Errorf("rkt: API version is too old(%v), requires at least %v", fr.info.ApiVersion, "1.2.6"), - true, - true, - }, - // Requires greater API version. - { - "1.2.3", - "1.2.7", - "100", - fmt.Errorf("rkt: API version is too old(%v), requires at least %v", fr.info.ApiVersion, "1.2.7"), - true, - true, - }, - // Requires greater systemd version. - { - "1.2.3", - "1.2.7", - "101", - fmt.Errorf("rkt: systemd version(%v) is too old, requires at least %v", fs.version, "101"), - false, - true, - }, - } - - for i, tt := range tests { - testCaseHint := fmt.Sprintf("test case #%d", i) - err := r.checkVersion(tt.minimumRktBinVersion, tt.minimumRktApiVersion, tt.minimumSystemdVersion) - assert.Equal(t, tt.err, err, testCaseHint) - - if tt.calledGetInfo { - assert.Equal(t, fr.called, []string{"GetInfo"}, testCaseHint) - } - if tt.calledSystemVersion { - assert.Equal(t, fs.called, []string{"Version"}, testCaseHint) - } - if err == nil { - assert.Equal(t, fr.info.RktVersion, r.versions.binVersion.String(), testCaseHint) - assert.Equal(t, fr.info.ApiVersion, r.versions.apiVersion.String(), testCaseHint) - } - fr.CleanCalls() - fs.CleanCalls() - } -} - -func TestListImages(t *testing.T) { - fr := newFakeRktInterface() - fs := newFakeSystemd() - r := &Runtime{apisvc: fr, systemd: fs} - - tests := []struct { - images []*rktapi.Image - expected []kubecontainer.Image - }{ - {nil, []kubecontainer.Image{}}, - { - []*rktapi.Image{ - { - Id: "sha512-a2fb8f390702", - Name: "quay.io/coreos/alpine-sh", - Version: "latest", - }, - }, - []kubecontainer.Image{ - { - ID: "sha512-a2fb8f390702", - RepoTags: []string{"quay.io/coreos/alpine-sh:latest"}, - }, - }, - }, - { - []*rktapi.Image{ - { - Id: "sha512-a2fb8f390702", - Name: "quay.io/coreos/alpine-sh", - Version: "latest", - Size: 400, - }, - { - Id: "sha512-c6b597f42816", - Name: "coreos.com/rkt/stage1-coreos", - Version: "0.10.0", - Size: 400, - }, - }, - []kubecontainer.Image{ - { - ID: "sha512-a2fb8f390702", - RepoTags: []string{"quay.io/coreos/alpine-sh:latest"}, - Size: 400, - }, - { - ID: "sha512-c6b597f42816", - RepoTags: []string{"coreos.com/rkt/stage1-coreos:0.10.0"}, - Size: 400, - }, - }, - }, - { - []*rktapi.Image{ - { - Id: "sha512-a2fb8f390702", - Name: "quay.io_443/coreos/alpine-sh", - Version: "latest", - Annotations: []*rktapi.KeyValue{ - { - Key: appcDockerRegistryURL, - Value: "quay.io:443", - }, - { - Key: appcDockerRepository, - Value: "coreos/alpine-sh", - }, - }, - Size: 400, - }, - }, - []kubecontainer.Image{ - { - ID: "sha512-a2fb8f390702", - RepoTags: []string{"quay.io:443/coreos/alpine-sh:latest"}, - Size: 400, - }, - }, - }, - } - - for i, tt := range tests { - fr.images = tt.images - - images, err := r.ListImages() - if err != nil { - t.Errorf("%v", err) - } - assert.Equal(t, tt.expected, images) - assert.Equal(t, fr.called, []string{"ListImages"}, fmt.Sprintf("test case %d: unexpected called list", i)) - - fr.CleanCalls() - } -} - -func TestGetPods(t *testing.T) { - fr := newFakeRktInterface() - fs := newFakeSystemd() - r := &Runtime{apisvc: fr, systemd: fs} - - ns := func(seconds int64) int64 { - return seconds * 1e9 - } - - tests := []struct { - pods []*rktapi.Pod - result []*kubecontainer.Pod - }{ - // No pods. - {}, - // One pod. - { - []*rktapi.Pod{ - makeRktPod(rktapi.PodState_POD_STATE_RUNNING, - "uuid-4002", "42", "guestbook", "default", - ns(10), ns(10), "7", - []string{"app-1", "app-2"}, - []string{"img-id-1", "img-id-2"}, - []string{"img-name-1", "img-name-2"}, - []string{"1001", "1002"}, - []rktapi.AppState{rktapi.AppState_APP_STATE_RUNNING, rktapi.AppState_APP_STATE_EXITED}, - []int32{0, 0}, - nil, - ), - }, - []*kubecontainer.Pod{ - { - ID: "42", - Name: "guestbook", - Namespace: "default", - Containers: []*kubecontainer.Container{ - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-1"), - Name: "app-1", - Image: "img-name-1:latest", - ImageID: "img-id-1", - Hash: 1001, - State: "running", - }, - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-2"), - Name: "app-2", - Image: "img-name-2:latest", - ImageID: "img-id-2", - Hash: 1002, - State: "exited", - }, - }, - }, - }, - }, - // Multiple pods. - { - []*rktapi.Pod{ - makeRktPod(rktapi.PodState_POD_STATE_RUNNING, - "uuid-4002", "42", "guestbook", "default", - ns(10), ns(20), "7", - []string{"app-1", "app-2"}, - []string{"img-id-1", "img-id-2"}, - []string{"img-name-1", "img-name-2"}, - []string{"1001", "1002"}, - []rktapi.AppState{rktapi.AppState_APP_STATE_RUNNING, rktapi.AppState_APP_STATE_EXITED}, - []int32{0, 0}, - nil, - ), - makeRktPod(rktapi.PodState_POD_STATE_EXITED, - "uuid-4003", "43", "guestbook", "default", - ns(30), ns(40), "7", - []string{"app-11", "app-22"}, - []string{"img-id-11", "img-id-22"}, - []string{"img-name-11", "img-name-22"}, - []string{"10011", "10022"}, - []rktapi.AppState{rktapi.AppState_APP_STATE_EXITED, rktapi.AppState_APP_STATE_EXITED}, - []int32{0, 0}, - nil, - ), - makeRktPod(rktapi.PodState_POD_STATE_EXITED, - "uuid-4004", "43", "guestbook", "default", - ns(50), ns(60), "8", - []string{"app-11", "app-22"}, - []string{"img-id-11", "img-id-22"}, - []string{"img-name-11", "img-name-22"}, - []string{"10011", "10022"}, - []rktapi.AppState{rktapi.AppState_APP_STATE_RUNNING, rktapi.AppState_APP_STATE_RUNNING}, - []int32{0, 0}, - nil, - ), - }, - []*kubecontainer.Pod{ - { - ID: "42", - Name: "guestbook", - Namespace: "default", - Containers: []*kubecontainer.Container{ - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-1"), - Name: "app-1", - Image: "img-name-1:latest", - ImageID: "img-id-1", - Hash: 1001, - State: "running", - }, - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-2"), - Name: "app-2", - Image: "img-name-2:latest", - ImageID: "img-id-2", - Hash: 1002, - State: "exited", - }, - }, - }, - { - ID: "43", - Name: "guestbook", - Namespace: "default", - Containers: []*kubecontainer.Container{ - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4003:app-11"), - Name: "app-11", - Image: "img-name-11:latest", - ImageID: "img-id-11", - Hash: 10011, - State: "exited", - }, - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4003:app-22"), - Name: "app-22", - Image: "img-name-22:latest", - ImageID: "img-id-22", - Hash: 10022, - State: "exited", - }, - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4004:app-11"), - Name: "app-11", - Image: "img-name-11:latest", - ImageID: "img-id-11", - Hash: 10011, - State: "running", - }, - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4004:app-22"), - Name: "app-22", - Image: "img-name-22:latest", - ImageID: "img-id-22", - Hash: 10022, - State: "running", - }, - }, - }, - }, - }, - } - - for i, tt := range tests { - testCaseHint := fmt.Sprintf("test case #%d", i) - fr.pods = tt.pods - - pods, err := r.GetPods(true) - if err != nil { - t.Errorf("test case #%d: unexpected error: %v", i, err) - } - - assert.Equal(t, tt.result, pods, testCaseHint) - assert.Equal(t, []string{"ListPods"}, fr.called, fmt.Sprintf("test case %d: unexpected called list", i)) - - fr.CleanCalls() - } -} - -func TestGetPodsFilters(t *testing.T) { - fr := newFakeRktInterface() - fs := newFakeSystemd() - r := &Runtime{apisvc: fr, systemd: fs} - - for _, test := range []struct { - All bool - ExpectedFilters []*rktapi.PodFilter - }{ - { - true, - []*rktapi.PodFilter{ - { - Annotations: []*rktapi.KeyValue{ - { - Key: k8sRktKubeletAnno, - Value: k8sRktKubeletAnnoValue, - }, - }, - }, - }, - }, - { - false, - []*rktapi.PodFilter{ - { - States: []rktapi.PodState{rktapi.PodState_POD_STATE_RUNNING}, - Annotations: []*rktapi.KeyValue{ - { - Key: k8sRktKubeletAnno, - Value: k8sRktKubeletAnnoValue, - }, - }, - }, - }, - }, - } { - _, err := r.GetPods(test.All) - if err != nil { - t.Errorf("%v", err) - } - assert.Equal(t, test.ExpectedFilters, fr.podFilters, "filters didn't match when all=%b", test.All) - } -} - -func TestGetPodStatus(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - fr := newFakeRktInterface() - fs := newFakeSystemd() - fug := newfakeUnitGetter() - fnp := nettest.NewMockNetworkPlugin(ctrl) - fos := &containertesting.FakeOS{} - frh := &containertesting.FakeRuntimeHelper{} - r := &Runtime{ - apisvc: fr, - systemd: fs, - runtimeHelper: frh, - os: fos, - network: network.NewPluginManager(fnp), - unitGetter: fug, - } - - ns := func(seconds int64) int64 { - return seconds * 1e9 - } - - tests := []struct { - networkPluginName string - pods []*rktapi.Pod - result *kubecontainer.PodStatus - }{ - // # case 0, No pods. - { - kubenet.KubenetPluginName, - nil, - &kubecontainer.PodStatus{ID: "42", Name: "guestbook", Namespace: "default"}, - }, - // # case 1, One pod. - { - kubenet.KubenetPluginName, - []*rktapi.Pod{ - makeRktPod(rktapi.PodState_POD_STATE_RUNNING, - "uuid-4002", "42", "guestbook", "default", - ns(10), ns(20), "7", - []string{"app-1", "app-2"}, - []string{"img-id-1", "img-id-2"}, - []string{"img-name-1", "img-name-2"}, - []string{"1001", "1002"}, - []rktapi.AppState{rktapi.AppState_APP_STATE_RUNNING, rktapi.AppState_APP_STATE_EXITED}, - []int32{0, 0}, - nil, - ), - }, - &kubecontainer.PodStatus{ - ID: "42", - Name: "guestbook", - Namespace: "default", - IP: "10.10.10.42", - ContainerStatuses: []*kubecontainer.ContainerStatus{ - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-1"), - Name: "app-1", - State: kubecontainer.ContainerStateRunning, - CreatedAt: time.Unix(10, 0), - StartedAt: time.Unix(20, 0), - FinishedAt: time.Unix(0, 30), - Image: "img-name-1:latest", - ImageID: "rkt://img-id-1", - Hash: 1001, - RestartCount: 7, - }, - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-2"), - Name: "app-2", - State: kubecontainer.ContainerStateExited, - CreatedAt: time.Unix(10, 0), - StartedAt: time.Unix(20, 0), - FinishedAt: time.Unix(0, 30), - Image: "img-name-2:latest", - ImageID: "rkt://img-id-2", - Hash: 1002, - RestartCount: 7, - Reason: "Completed", - }, - }, - }, - }, - // # case 2, One pod with no-op network plugin name. - { - network.DefaultPluginName, - []*rktapi.Pod{ - makeRktPod(rktapi.PodState_POD_STATE_RUNNING, - "uuid-4002", "42", "guestbook", "default", - ns(10), ns(20), "7", - []string{"app-1", "app-2"}, - []string{"img-id-1", "img-id-2"}, - []string{"img-name-1", "img-name-2"}, - []string{"1001", "1002"}, - []rktapi.AppState{rktapi.AppState_APP_STATE_RUNNING, rktapi.AppState_APP_STATE_EXITED}, - []int32{0, 0}, - map[string]string{defaultNetworkName: "10.10.10.22"}, - ), - }, - &kubecontainer.PodStatus{ - ID: "42", - Name: "guestbook", - Namespace: "default", - IP: "10.10.10.22", - ContainerStatuses: []*kubecontainer.ContainerStatus{ - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-1"), - Name: "app-1", - State: kubecontainer.ContainerStateRunning, - CreatedAt: time.Unix(10, 0), - StartedAt: time.Unix(20, 0), - FinishedAt: time.Unix(0, 30), - Image: "img-name-1:latest", - ImageID: "rkt://img-id-1", - Hash: 1001, - RestartCount: 7, - }, - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-2"), - Name: "app-2", - State: kubecontainer.ContainerStateExited, - CreatedAt: time.Unix(10, 0), - StartedAt: time.Unix(20, 0), - FinishedAt: time.Unix(0, 30), - Image: "img-name-2:latest", - ImageID: "rkt://img-id-2", - Hash: 1002, - RestartCount: 7, - Reason: "Completed", - }, - }, - }, - }, - // # case 3, Multiple pods. - { - kubenet.KubenetPluginName, - []*rktapi.Pod{ - makeRktPod(rktapi.PodState_POD_STATE_EXITED, - "uuid-4002", "42", "guestbook", "default", - ns(10), ns(20), "7", - []string{"app-1", "app-2"}, - []string{"img-id-1", "img-id-2"}, - []string{"img-name-1", "img-name-2"}, - []string{"1001", "1002"}, - []rktapi.AppState{rktapi.AppState_APP_STATE_RUNNING, rktapi.AppState_APP_STATE_EXITED}, - []int32{0, 0}, - nil, - ), - makeRktPod(rktapi.PodState_POD_STATE_RUNNING, // The latest pod is running. - "uuid-4003", "42", "guestbook", "default", - ns(10), ns(20), "10", - []string{"app-1", "app-2"}, - []string{"img-id-1", "img-id-2"}, - []string{"img-name-1", "img-name-2"}, - []string{"1001", "1002"}, - []rktapi.AppState{rktapi.AppState_APP_STATE_RUNNING, rktapi.AppState_APP_STATE_EXITED}, - []int32{0, 1}, - nil, - ), - }, - &kubecontainer.PodStatus{ - ID: "42", - Name: "guestbook", - Namespace: "default", - IP: "10.10.10.42", - // Result should contain all containers. - ContainerStatuses: []*kubecontainer.ContainerStatus{ - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-1"), - Name: "app-1", - State: kubecontainer.ContainerStateRunning, - CreatedAt: time.Unix(10, 0), - StartedAt: time.Unix(20, 0), - FinishedAt: time.Unix(0, 30), - Image: "img-name-1:latest", - ImageID: "rkt://img-id-1", - Hash: 1001, - RestartCount: 7, - }, - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4002:app-2"), - Name: "app-2", - State: kubecontainer.ContainerStateExited, - CreatedAt: time.Unix(10, 0), - StartedAt: time.Unix(20, 0), - FinishedAt: time.Unix(0, 30), - Image: "img-name-2:latest", - ImageID: "rkt://img-id-2", - Hash: 1002, - RestartCount: 7, - Reason: "Completed", - }, - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4003:app-1"), - Name: "app-1", - State: kubecontainer.ContainerStateRunning, - CreatedAt: time.Unix(10, 0), - StartedAt: time.Unix(20, 0), - FinishedAt: time.Unix(0, 30), - Image: "img-name-1:latest", - ImageID: "rkt://img-id-1", - Hash: 1001, - RestartCount: 10, - }, - { - ID: kubecontainer.BuildContainerID("rkt", "uuid-4003:app-2"), - Name: "app-2", - State: kubecontainer.ContainerStateExited, - CreatedAt: time.Unix(10, 0), - StartedAt: time.Unix(20, 0), - FinishedAt: time.Unix(0, 30), - Image: "img-name-2:latest", - ImageID: "rkt://img-id-2", - Hash: 1002, - RestartCount: 10, - ExitCode: 1, - Reason: "Error", - }, - }, - }, - }, - } - - for i, tt := range tests { - testCaseHint := fmt.Sprintf("test case #%d", i) - fr.pods = tt.pods - - podTimes := map[string]time.Time{} - for _, pod := range tt.pods { - podTimes[podFinishedMarkerPath(r.runtimeHelper.GetPodDir(tt.result.ID), pod.Id)] = tt.result.ContainerStatuses[0].FinishedAt - } - - ctrl := gomock.NewController(t) - - r.os.(*containertesting.FakeOS).StatFn = func(name string) (os.FileInfo, error) { - podTime, ok := podTimes[name] - if !ok { - t.Errorf("osStat called with %v, but only knew about %#v", name, podTimes) - } - mockFI := containertesting.NewMockFileInfo(ctrl) - mockFI.EXPECT().ModTime().Return(podTime) - return mockFI, nil - } - - if tt.networkPluginName == network.DefaultPluginName { - fnp.EXPECT().Name().Return(tt.networkPluginName) - } - - if tt.pods != nil && tt.networkPluginName == kubenet.KubenetPluginName { - fnp.EXPECT().Name().Return(tt.networkPluginName) - if tt.result.IP != "" { - fnp.EXPECT().GetPodNetworkStatus("default", "guestbook", kubecontainer.ContainerID{ID: "42"}). - Return(&network.PodNetworkStatus{IP: net.ParseIP(tt.result.IP)}, nil) - } else { - fnp.EXPECT().GetPodNetworkStatus("default", "guestbook", kubecontainer.ContainerID{ID: "42"}). - Return(nil, fmt.Errorf("no such network")) - // Plugin name only requested again in error case - fnp.EXPECT().Name().Return(tt.networkPluginName) - } - } - - status, err := r.GetPodStatus("42", "guestbook", "default") - if err != nil { - t.Errorf("test case #%d: unexpected error: %v", i, err) - } - - assert.Equal(t, tt.result, status, testCaseHint) - assert.Equal(t, []string{"ListPods"}, fr.called, testCaseHint) - fug.networkNamespace = kubecontainer.ContainerID{} - fr.CleanCalls() - ctrl.Finish() - } -} - -func generateCapRetainIsolator(t *testing.T, caps ...string) appctypes.Isolator { - retain, err := appctypes.NewLinuxCapabilitiesRetainSet(caps...) - if err != nil { - t.Fatalf("Error generating cap retain isolator: %v", err) - } - isolator, err := retain.AsIsolator() - if err != nil { - t.Fatalf("Error generating cap retain isolator: %v", err) - } - return *isolator -} - -func generateCapRevokeIsolator(t *testing.T, caps ...string) appctypes.Isolator { - revoke, err := appctypes.NewLinuxCapabilitiesRevokeSet(caps...) - if err != nil { - t.Fatalf("Error generating cap revoke isolator: %v", err) - } - isolator, err := revoke.AsIsolator() - if err != nil { - t.Fatalf("Error generating cap revoke isolator: %v", err) - } - return *isolator -} - -func generateCPUIsolator(t *testing.T, request, limit string) appctypes.Isolator { - cpu, err := appctypes.NewResourceCPUIsolator(request, limit) - if err != nil { - t.Fatalf("Error generating cpu resource isolator: %v", err) - } - return cpu.AsIsolator() -} - -func generateMemoryIsolator(t *testing.T, request, limit string) appctypes.Isolator { - memory, err := appctypes.NewResourceMemoryIsolator(request, limit) - if err != nil { - t.Fatalf("Error generating memory resource isolator: %v", err) - } - return memory.AsIsolator() -} - -func baseApp(t *testing.T) *appctypes.App { - return &appctypes.App{ - User: "0", - Group: "0", - Exec: appctypes.Exec{"/bin/foo", "bar"}, - SupplementaryGIDs: []int{4, 5, 6}, - WorkingDirectory: "/foo", - Environment: []appctypes.EnvironmentVariable{ - {Name: "env-foo", Value: "bar"}, - }, - MountPoints: []appctypes.MountPoint{ - {Name: *appctypes.MustACName("mnt-foo"), Path: "/mnt-foo", ReadOnly: false}, - }, - Ports: []appctypes.Port{ - {Name: *appctypes.MustACName("port-foo"), Protocol: "TCP", Port: 4242}, - }, - Isolators: []appctypes.Isolator{ - generateCapRetainIsolator(t, "CAP_SYS_ADMIN"), - generateCapRevokeIsolator(t, "CAP_NET_ADMIN"), - generateCPUIsolator(t, "100m", "200m"), - generateMemoryIsolator(t, "10M", "20M"), - }, - } -} - -func baseImageManifest(t *testing.T) *appcschema.ImageManifest { - img := &appcschema.ImageManifest{App: baseApp(t)} - entrypoint, err := json.Marshal([]string{"/bin/foo"}) - if err != nil { - t.Fatal(err) - } - cmd, err := json.Marshal([]string{"bar"}) - if err != nil { - t.Fatal(err) - } - img.Annotations.Set(*appctypes.MustACIdentifier(appcDockerEntrypoint), string(entrypoint)) - img.Annotations.Set(*appctypes.MustACIdentifier(appcDockerCmd), string(cmd)) - return img -} - -func baseAppWithRootUserGroup(t *testing.T) *appctypes.App { - app := baseApp(t) - app.User, app.Group = "0", "0" - app.Isolators = append(app.Isolators) - return app -} - -type envByName []appctypes.EnvironmentVariable - -func (s envByName) Len() int { return len(s) } -func (s envByName) Less(i, j int) bool { return s[i].Name < s[j].Name } -func (s envByName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -type mountsByName []appctypes.MountPoint - -func (s mountsByName) Len() int { return len(s) } -func (s mountsByName) Less(i, j int) bool { return s[i].Name < s[j].Name } -func (s mountsByName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -type portsByName []appctypes.Port - -func (s portsByName) Len() int { return len(s) } -func (s portsByName) Less(i, j int) bool { return s[i].Name < s[j].Name } -func (s portsByName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -type isolatorsByName []appctypes.Isolator - -func (s isolatorsByName) Len() int { return len(s) } -func (s isolatorsByName) Less(i, j int) bool { return s[i].Name < s[j].Name } -func (s isolatorsByName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func sortAppFields(app *appctypes.App) { - sort.Sort(envByName(app.Environment)) - sort.Sort(mountsByName(app.MountPoints)) - sort.Sort(portsByName(app.Ports)) - sort.Sort(isolatorsByName(app.Isolators)) -} - -type sortedStringList []string - -func (s sortedStringList) Len() int { return len(s) } -func (s sortedStringList) Less(i, j int) bool { return s[i] < s[j] } -func (s sortedStringList) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -func TestSetApp(t *testing.T) { - tmpDir, err := utiltesting.MkTmpdir("rkt_test") - if err != nil { - t.Fatalf("error creating temp dir: %v", err) - } - defer os.RemoveAll(tmpDir) - - rootUser := int64(0) - nonRootUser := int64(42) - runAsNonRootTrue := true - fsgid := int64(3) - - tests := []struct { - container *v1.Container - mountPoints []appctypes.MountPoint - containerPorts []appctypes.Port - envs []kubecontainer.EnvVar - ctx *v1.SecurityContext - podCtx *v1.PodSecurityContext - supplementalGids []int64 - expect *appctypes.App - err error - }{ - // Nothing should change, but the "User" and "Group" should be filled. - { - container: &v1.Container{}, - mountPoints: []appctypes.MountPoint{}, - containerPorts: []appctypes.Port{}, - envs: []kubecontainer.EnvVar{}, - ctx: nil, - podCtx: nil, - supplementalGids: nil, - expect: baseAppWithRootUserGroup(t), - err: nil, - }, - - // error verifying non-root. - { - container: &v1.Container{}, - mountPoints: []appctypes.MountPoint{}, - containerPorts: []appctypes.Port{}, - envs: []kubecontainer.EnvVar{}, - ctx: &v1.SecurityContext{ - RunAsNonRoot: &runAsNonRootTrue, - RunAsUser: &rootUser, - }, - podCtx: nil, - supplementalGids: nil, - expect: nil, - err: fmt.Errorf("container has no runAsUser and image will run as root"), - }, - - // app's args should be changed. - { - container: &v1.Container{ - Args: []string{"foo"}, - }, - mountPoints: []appctypes.MountPoint{}, - containerPorts: []appctypes.Port{}, - envs: []kubecontainer.EnvVar{}, - ctx: nil, - podCtx: nil, - supplementalGids: nil, - expect: &appctypes.App{ - Exec: appctypes.Exec{"/bin/foo", "foo"}, - User: "0", - Group: "0", - SupplementaryGIDs: []int{4, 5, 6}, - WorkingDirectory: "/foo", - Environment: []appctypes.EnvironmentVariable{ - {Name: "env-foo", Value: "bar"}, - }, - MountPoints: []appctypes.MountPoint{ - {Name: *appctypes.MustACName("mnt-foo"), Path: "/mnt-foo", ReadOnly: false}, - }, - Ports: []appctypes.Port{ - {Name: *appctypes.MustACName("port-foo"), Protocol: "TCP", Port: 4242}, - }, - Isolators: []appctypes.Isolator{ - generateCapRetainIsolator(t, "CAP_SYS_ADMIN"), - generateCapRevokeIsolator(t, "CAP_NET_ADMIN"), - generateCPUIsolator(t, "100m", "200m"), - generateMemoryIsolator(t, "10M", "20M"), - }, - }, - err: nil, - }, - - // app should be changed. - { - container: &v1.Container{ - Command: []string{"/bin/bar", "$(env-bar)"}, - WorkingDir: tmpDir, - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("50m"), v1.ResourceMemory: resource.MustParse("50M")}, - Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("5m"), v1.ResourceMemory: resource.MustParse("5M")}, - }, - }, - mountPoints: []appctypes.MountPoint{ - {Name: *appctypes.MustACName("mnt-bar"), Path: "/mnt-bar", ReadOnly: true}, - }, - containerPorts: []appctypes.Port{ - {Name: *appctypes.MustACName("port-bar"), Protocol: "TCP", Port: 1234}, - }, - envs: []kubecontainer.EnvVar{ - {Name: "env-bar", Value: "foo"}, - }, - ctx: &v1.SecurityContext{ - Capabilities: &v1.Capabilities{ - Add: []v1.Capability{"CAP_SYS_CHROOT", "CAP_SYS_BOOT"}, - Drop: []v1.Capability{"CAP_SETUID", "CAP_SETGID"}, - }, - RunAsUser: &nonRootUser, - RunAsNonRoot: &runAsNonRootTrue, - }, - podCtx: &v1.PodSecurityContext{ - SupplementalGroups: []int64{ - int64(1), - int64(2), - }, - FSGroup: &fsgid, - }, - supplementalGids: []int64{4}, - expect: &appctypes.App{ - Exec: appctypes.Exec{"/bin/bar", "foo"}, - User: "42", - Group: "0", - SupplementaryGIDs: []int{1, 2, 3, 4}, - WorkingDirectory: tmpDir, - Environment: []appctypes.EnvironmentVariable{ - {Name: "env-foo", Value: "bar"}, - {Name: "env-bar", Value: "foo"}, - }, - MountPoints: []appctypes.MountPoint{ - {Name: *appctypes.MustACName("mnt-foo"), Path: "/mnt-foo", ReadOnly: false}, - {Name: *appctypes.MustACName("mnt-bar"), Path: "/mnt-bar", ReadOnly: true}, - }, - Ports: []appctypes.Port{ - {Name: *appctypes.MustACName("port-foo"), Protocol: "TCP", Port: 4242}, - {Name: *appctypes.MustACName("port-bar"), Protocol: "TCP", Port: 1234}, - }, - Isolators: []appctypes.Isolator{ - generateCapRetainIsolator(t, "CAP_SYS_CHROOT", "CAP_SYS_BOOT"), - generateCapRevokeIsolator(t, "CAP_SETUID", "CAP_SETGID"), - generateCPUIsolator(t, "5m", "50m"), - generateMemoryIsolator(t, "5M", "50M"), - }, - }, - }, - - // app should be changed. (env, mounts, ports, are overrided). - { - container: &v1.Container{ - Name: "hello-world", - Command: []string{"/bin/hello", "$(env-foo)"}, - Args: []string{"hello", "world", "$(env-bar)"}, - WorkingDir: tmpDir, - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("50m")}, - Requests: v1.ResourceList{v1.ResourceMemory: resource.MustParse("5M")}, - }, - }, - mountPoints: []appctypes.MountPoint{ - {Name: *appctypes.MustACName("mnt-foo"), Path: "/mnt-foo", ReadOnly: true}, - }, - containerPorts: []appctypes.Port{ - {Name: *appctypes.MustACName("port-foo"), Protocol: "TCP", Port: 1234}, - }, - envs: []kubecontainer.EnvVar{ - {Name: "env-foo", Value: "foo"}, - {Name: "env-bar", Value: "bar"}, - }, - ctx: &v1.SecurityContext{ - Capabilities: &v1.Capabilities{ - Add: []v1.Capability{"CAP_SYS_CHROOT", "CAP_SYS_BOOT"}, - Drop: []v1.Capability{"CAP_SETUID", "CAP_SETGID"}, - }, - RunAsUser: &nonRootUser, - RunAsNonRoot: &runAsNonRootTrue, - }, - podCtx: &v1.PodSecurityContext{ - SupplementalGroups: []int64{ - int64(1), - int64(2), - }, - FSGroup: &fsgid, - }, - supplementalGids: []int64{4}, - expect: &appctypes.App{ - Exec: appctypes.Exec{"/bin/hello", "foo", "hello", "world", "bar"}, - User: "42", - Group: "0", - SupplementaryGIDs: []int{1, 2, 3, 4}, - WorkingDirectory: tmpDir, - Environment: []appctypes.EnvironmentVariable{ - {Name: "env-foo", Value: "foo"}, - {Name: "env-bar", Value: "bar"}, - }, - MountPoints: []appctypes.MountPoint{ - {Name: *appctypes.MustACName("mnt-foo"), Path: "/mnt-foo", ReadOnly: true}, - }, - Ports: []appctypes.Port{ - {Name: *appctypes.MustACName("port-foo"), Protocol: "TCP", Port: 1234}, - }, - Isolators: []appctypes.Isolator{ - generateCapRetainIsolator(t, "CAP_SYS_CHROOT", "CAP_SYS_BOOT"), - generateCapRevokeIsolator(t, "CAP_SETUID", "CAP_SETGID"), - generateCPUIsolator(t, "50m", "50m"), - generateMemoryIsolator(t, "5M", "5M"), - }, - }, - }, - } - - for i, tt := range tests { - testCaseHint := fmt.Sprintf("test case #%d", i) - img := baseImageManifest(t) - - err := setApp(img, tt.container, - tt.mountPoints, tt.containerPorts, tt.envs, - tt.ctx, tt.podCtx, tt.supplementalGids) - - if err == nil && tt.err != nil || err != nil && tt.err == nil { - t.Errorf("%s: expect %v, saw %v", testCaseHint, tt.err, err) - } - if err == nil { - sortAppFields(tt.expect) - sortAppFields(img.App) - assert.Equal(t, tt.expect, img.App, testCaseHint) - } - } -} - -func TestGenerateRunCommand(t *testing.T) { - hostName := "test-hostname" - boolTrue := true - boolFalse := false - pluginDirs := []string{"/tmp"} - - tests := []struct { - networkPlugin network.NetworkPlugin - pod *v1.Pod - uuid string - netnsName string - - dnsServers []string - dnsSearches []string - hostName string - err error - - expect string - }{ - // Case #0, returns error. - { - kubenet.NewPlugin(pluginDirs), - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-name-foo", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{{Name: "container-foo"}}, - }, - }, - "rkt-uuid-foo", - "default", - []string{}, - []string{}, - "", - fmt.Errorf("failed to get cluster dns"), - "", - }, - // Case #1, returns no dns, with private-net. - { - kubenet.NewPlugin(pluginDirs), - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-name-foo", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{{Name: "container-foo"}}, - }, - }, - "rkt-uuid-foo", - "default", - []string{}, - []string{}, - "pod-hostname-foo", - nil, - "/usr/bin/nsenter --net=/var/run/netns/default -- /bin/rkt/rkt --insecure-options=image,ondisk --local-config=/var/rkt/local/data --dir=/var/data run-prepared --net=host --hostname=pod-hostname-foo rkt-uuid-foo", - }, - // Case #2, returns no dns, with host-net. - { - kubenet.NewPlugin(pluginDirs), - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-name-foo", - }, - Spec: v1.PodSpec{ - HostNetwork: true, - - Containers: []v1.Container{{Name: "container-foo"}}, - }, - }, - "rkt-uuid-foo", - "", - []string{}, - []string{}, - "", - nil, - fmt.Sprintf("/bin/rkt/rkt --insecure-options=image,ondisk --local-config=/var/rkt/local/data --dir=/var/data run-prepared --net=host --hostname=%s rkt-uuid-foo", hostName), - }, - // Case #3, returns dns, dns searches, with private-net. - { - kubenet.NewPlugin(pluginDirs), - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-name-foo", - }, - Spec: v1.PodSpec{ - HostNetwork: false, - - Containers: []v1.Container{{Name: "container-foo"}}, - }, - }, - "rkt-uuid-foo", - "default", - []string{"127.0.0.1"}, - []string{"."}, - "pod-hostname-foo", - nil, - "/usr/bin/nsenter --net=/var/run/netns/default -- /bin/rkt/rkt --insecure-options=image,ondisk --local-config=/var/rkt/local/data --dir=/var/data run-prepared --net=host --dns=127.0.0.1 --dns-search=. --dns-opt=ndots:5 --hostname=pod-hostname-foo rkt-uuid-foo", - }, - // Case #4, returns no dns, dns searches, with host-network. - { - kubenet.NewPlugin(pluginDirs), - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-name-foo", - }, - Spec: v1.PodSpec{ - HostNetwork: true, - - Containers: []v1.Container{{Name: "container-foo"}}, - }, - }, - "rkt-uuid-foo", - "", - []string{"127.0.0.1"}, - []string{"."}, - "pod-hostname-foo", - nil, - fmt.Sprintf("/bin/rkt/rkt --insecure-options=image,ondisk --local-config=/var/rkt/local/data --dir=/var/data run-prepared --net=host --hostname=%s rkt-uuid-foo", hostName), - }, - // Case #5, with no-op plugin, returns --net=rkt.kubernetes.io, with dns and dns search. - { - &network.NoopNetworkPlugin{}, - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-name-foo", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{{Name: "container-foo"}}, - }, - }, - "rkt-uuid-foo", - "default", - []string{"127.0.0.1"}, - []string{"."}, - "pod-hostname-foo", - nil, - "/bin/rkt/rkt --insecure-options=image,ondisk --local-config=/var/rkt/local/data --dir=/var/data run-prepared --net=rkt.kubernetes.io --dns=127.0.0.1 --dns-search=. --dns-opt=ndots:5 --hostname=pod-hostname-foo rkt-uuid-foo", - }, - // Case #6, if all containers are privileged, the result should have 'insecure-options=all-run' - { - kubenet.NewPlugin(pluginDirs), - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-name-foo", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - {Name: "container-foo", SecurityContext: &v1.SecurityContext{Privileged: &boolTrue}}, - {Name: "container-bar", SecurityContext: &v1.SecurityContext{Privileged: &boolTrue}}, - }, - }, - }, - "rkt-uuid-foo", - "default", - []string{}, - []string{}, - "pod-hostname-foo", - nil, - "/usr/bin/nsenter --net=/var/run/netns/default -- /bin/rkt/rkt --insecure-options=image,ondisk,all-run --local-config=/var/rkt/local/data --dir=/var/data run-prepared --net=host --hostname=pod-hostname-foo rkt-uuid-foo", - }, - // Case #7, if not all containers are privileged, the result should not have 'insecure-options=all-run' - { - kubenet.NewPlugin(pluginDirs), - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-name-foo", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - {Name: "container-foo", SecurityContext: &v1.SecurityContext{Privileged: &boolTrue}}, - {Name: "container-bar", SecurityContext: &v1.SecurityContext{Privileged: &boolFalse}}, - }, - }, - }, - "rkt-uuid-foo", - "default", - []string{}, - []string{}, - "pod-hostname-foo", - nil, - "/usr/bin/nsenter --net=/var/run/netns/default -- /bin/rkt/rkt --insecure-options=image,ondisk --local-config=/var/rkt/local/data --dir=/var/data run-prepared --net=host --hostname=pod-hostname-foo rkt-uuid-foo", - }, - } - - rkt := &Runtime{ - nsenterPath: "/usr/bin/nsenter", - os: &containertesting.FakeOS{HostName: hostName}, - config: &Config{ - Path: "/bin/rkt/rkt", - Stage1Image: "/bin/rkt/stage1-coreos.aci", - Dir: "/var/data", - InsecureOptions: "image,ondisk", - LocalConfigDir: "/var/rkt/local/data", - }, - } - - for i, tt := range tests { - testCaseHint := fmt.Sprintf("test case #%d", i) - rkt.network = network.NewPluginManager(tt.networkPlugin) - rkt.runtimeHelper = &containertesting.FakeRuntimeHelper{ - DNSServers: tt.dnsServers, - DNSSearches: tt.dnsSearches, - HostName: tt.hostName, - Err: tt.err, - } - rkt.execer = &fakeexec.FakeExec{CommandScript: []fakeexec.FakeCommandAction{func(cmd string, args ...string) exec.Cmd { - return fakeexec.InitFakeCmd(&fakeexec.FakeCmd{}, cmd, args...) - }}} - - // a command should be created of this form, but the returned command shouldn't be called (asserted by having no expectations on it) - - result, err := rkt.generateRunCommand(tt.pod, tt.uuid, tt.netnsName) - assert.Equal(t, tt.err, err, testCaseHint) - assert.Equal(t, tt.expect, result, testCaseHint) - } -} - -func TestLifeCycleHooks(t *testing.T) { - runner := lifecycle.NewFakeHandlerRunner() - fr := newFakeRktInterface() - fs := newFakeSystemd() - - rkt := &Runtime{ - runner: runner, - apisvc: fr, - systemd: fs, - containerRefManager: kubecontainer.NewRefManager(), - } - - tests := []struct { - pod *v1.Pod - runtimePod *kubecontainer.Pod - postStartRuns []string - preStopRuns []string - err error - }{ - { - // Case 0, container without any hooks. - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-1", - Namespace: "ns-1", - UID: "uid-1", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - {Name: "container-name-1"}, - }, - }, - }, - &kubecontainer.Pod{ - Containers: []*kubecontainer.Container{ - {ID: kubecontainer.BuildContainerID("rkt", "id-1")}, - }, - }, - []string{}, - []string{}, - nil, - }, - { - // Case 1, containers with post-start and pre-stop hooks. - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-1", - Namespace: "ns-1", - UID: "uid-1", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "container-name-1", - Lifecycle: &v1.Lifecycle{ - PostStart: &v1.Handler{ - Exec: &v1.ExecAction{}, - }, - }, - }, - { - Name: "container-name-2", - Lifecycle: &v1.Lifecycle{ - PostStart: &v1.Handler{ - HTTPGet: &v1.HTTPGetAction{}, - }, - }, - }, - { - Name: "container-name-3", - Lifecycle: &v1.Lifecycle{ - PreStop: &v1.Handler{ - Exec: &v1.ExecAction{}, - }, - }, - }, - { - Name: "container-name-4", - Lifecycle: &v1.Lifecycle{ - PreStop: &v1.Handler{ - HTTPGet: &v1.HTTPGetAction{}, - }, - }, - }, - }, - }, - }, - &kubecontainer.Pod{ - Containers: []*kubecontainer.Container{ - { - ID: kubecontainer.ParseContainerID("rkt://uuid:container-name-4"), - Name: "container-name-4", - }, - { - ID: kubecontainer.ParseContainerID("rkt://uuid:container-name-3"), - Name: "container-name-3", - }, - { - ID: kubecontainer.ParseContainerID("rkt://uuid:container-name-2"), - Name: "container-name-2", - }, - { - ID: kubecontainer.ParseContainerID("rkt://uuid:container-name-1"), - Name: "container-name-1", - }, - }, - }, - []string{ - "exec on pod: pod-1_ns-1(uid-1), container: container-name-1: rkt://uuid:container-name-1", - "http-get on pod: pod-1_ns-1(uid-1), container: container-name-2: rkt://uuid:container-name-2", - }, - []string{ - "exec on pod: pod-1_ns-1(uid-1), container: container-name-3: rkt://uuid:container-name-3", - "http-get on pod: pod-1_ns-1(uid-1), container: container-name-4: rkt://uuid:container-name-4", - }, - nil, - }, - { - // Case 2, one container with invalid hooks. - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod-1", - Namespace: "ns-1", - UID: "uid-1", - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "container-name-1", - Lifecycle: &v1.Lifecycle{ - PostStart: &v1.Handler{}, - PreStop: &v1.Handler{}, - }, - }, - }, - }, - }, - &kubecontainer.Pod{ - Containers: []*kubecontainer.Container{ - { - ID: kubecontainer.ParseContainerID("rkt://uuid:container-name-1"), - Name: "container-name-1", - }, - }, - }, - []string{}, - []string{}, - errors.NewAggregate([]error{fmt.Errorf("Invalid handler: %v", &v1.Handler{})}), - }, - } - - for i, tt := range tests { - testCaseHint := fmt.Sprintf("test case #%d", i) - - pod := &rktapi.Pod{Id: "uuid"} - for _, c := range tt.runtimePod.Containers { - pod.Apps = append(pod.Apps, &rktapi.App{ - Name: c.Name, - State: rktapi.AppState_APP_STATE_RUNNING, - }) - } - fr.pods = []*rktapi.Pod{pod} - - // Run post-start hooks - err := rkt.runLifecycleHooks(tt.pod, tt.runtimePod, lifecyclePostStartHook) - assert.Equal(t, tt.err, err, testCaseHint) - - sort.Sort(sortedStringList(tt.postStartRuns)) - sort.Sort(sortedStringList(runner.HandlerRuns)) - - assert.Equal(t, tt.postStartRuns, runner.HandlerRuns, testCaseHint) - - runner.Reset() - - // Run pre-stop hooks. - err = rkt.runLifecycleHooks(tt.pod, tt.runtimePod, lifecyclePreStopHook) - assert.Equal(t, tt.err, err, testCaseHint) - - sort.Sort(sortedStringList(tt.preStopRuns)) - sort.Sort(sortedStringList(runner.HandlerRuns)) - - assert.Equal(t, tt.preStopRuns, runner.HandlerRuns, testCaseHint) - - runner.Reset() - } -} - -func TestImageStats(t *testing.T) { - fr := newFakeRktInterface() - rkt := &Runtime{apisvc: fr} - - fr.images = []*rktapi.Image{ - {Size: 100}, - {Size: 200}, - {Size: 300}, - } - - result, err := rkt.ImageStats() - assert.NoError(t, err) - assert.Equal(t, result, &kubecontainer.ImageStats{TotalStorageBytes: 600}) -} - -func TestGarbageCollect(t *testing.T) { - fr := newFakeRktInterface() - fs := newFakeSystemd() - cli := newFakeRktCli() - fakeOS := containertesting.NewFakeOS() - deletionProvider := newFakePodDeletionProvider() - fug := newfakeUnitGetter() - frh := &containertesting.FakeRuntimeHelper{} - - rkt := &Runtime{ - os: fakeOS, - cli: cli, - apisvc: fr, - podDeletionProvider: deletionProvider, - systemd: fs, - containerRefManager: kubecontainer.NewRefManager(), - unitGetter: fug, - runtimeHelper: frh, - } - - fakeApp := &rktapi.App{Name: "app-foo"} - - tests := []struct { - gcPolicy kubecontainer.ContainerGCPolicy - apiPods []*v1.Pod - pods []*rktapi.Pod - serviceFilesOnDisk []string - expectedCommands []string - expectedDeletedFiles []string - }{ - // All running pods, should not be gc'd. - // Dead, new pods should not be gc'd. - // Dead, old pods should be gc'd. - // Deleted pods should be gc'd. - // Service files without corresponded pods should be removed. - { - kubecontainer.ContainerGCPolicy{ - MinAge: 0, - MaxContainers: 0, - }, - []*v1.Pod{ - {ObjectMeta: metav1.ObjectMeta{UID: "pod-uid-1"}}, - {ObjectMeta: metav1.ObjectMeta{UID: "pod-uid-2"}}, - {ObjectMeta: metav1.ObjectMeta{UID: "pod-uid-3"}}, - {ObjectMeta: metav1.ObjectMeta{UID: "pod-uid-4"}}, - }, - []*rktapi.Pod{ - { - Id: "deleted-foo", - State: rktapi.PodState_POD_STATE_EXITED, - CreatedAt: time.Now().Add(time.Hour).UnixNano(), - StartedAt: time.Now().Add(time.Hour).UnixNano(), - Apps: []*rktapi.App{fakeApp}, - Annotations: []*rktapi.KeyValue{ - { - Key: types.KubernetesPodUIDLabel, - Value: "pod-uid-0", - }, - }, - }, - { - Id: "running-foo", - State: rktapi.PodState_POD_STATE_RUNNING, - CreatedAt: 0, - StartedAt: 0, - Apps: []*rktapi.App{fakeApp}, - Annotations: []*rktapi.KeyValue{ - { - Key: types.KubernetesPodUIDLabel, - Value: "pod-uid-1", - }, - }, - }, - { - Id: "running-bar", - State: rktapi.PodState_POD_STATE_RUNNING, - CreatedAt: 0, - StartedAt: 0, - Apps: []*rktapi.App{fakeApp}, - Annotations: []*rktapi.KeyValue{ - { - Key: types.KubernetesPodUIDLabel, - Value: "pod-uid-2", - }, - }, - }, - { - Id: "dead-old", - State: rktapi.PodState_POD_STATE_EXITED, - CreatedAt: 0, - StartedAt: 0, - Apps: []*rktapi.App{fakeApp}, - Annotations: []*rktapi.KeyValue{ - { - Key: types.KubernetesPodUIDLabel, - Value: "pod-uid-3", - }, - }, - }, - { - Id: "dead-new", - State: rktapi.PodState_POD_STATE_EXITED, - CreatedAt: time.Now().Add(time.Hour).UnixNano(), - StartedAt: time.Now().Add(time.Hour).UnixNano(), - Apps: []*rktapi.App{fakeApp}, - Annotations: []*rktapi.KeyValue{ - { - Key: types.KubernetesPodUIDLabel, - Value: "pod-uid-4", - }, - }, - }, - }, - []string{"k8s_dead-old.service", "k8s_deleted-foo.service", "k8s_non-existing-bar.service"}, - []string{"rkt rm dead-old", "rkt rm deleted-foo"}, - []string{"/poddir/fake/finished-dead-old", "/poddir/fake/finished-deleted-foo", "/poddir/fake/finished-non-existing-bar", "/run/systemd/system/k8s_dead-old.service", "/run/systemd/system/k8s_deleted-foo.service", "/run/systemd/system/k8s_non-existing-bar.service"}, - }, - // gcPolicy.MaxContainers should be enforced. - // Oldest ones are removed first. - { - kubecontainer.ContainerGCPolicy{ - MinAge: 0, - MaxContainers: 1, - }, - []*v1.Pod{ - {ObjectMeta: metav1.ObjectMeta{UID: "pod-uid-0"}}, - {ObjectMeta: metav1.ObjectMeta{UID: "pod-uid-1"}}, - {ObjectMeta: metav1.ObjectMeta{UID: "pod-uid-2"}}, - }, - []*rktapi.Pod{ - { - Id: "dead-2", - State: rktapi.PodState_POD_STATE_EXITED, - CreatedAt: 2, - StartedAt: 2, - Apps: []*rktapi.App{fakeApp}, - Annotations: []*rktapi.KeyValue{ - { - Key: types.KubernetesPodUIDLabel, - Value: "pod-uid-2", - }, - }, - }, - { - Id: "dead-1", - State: rktapi.PodState_POD_STATE_EXITED, - CreatedAt: 1, - StartedAt: 1, - Apps: []*rktapi.App{fakeApp}, - Annotations: []*rktapi.KeyValue{ - { - Key: types.KubernetesPodUIDLabel, - Value: "pod-uid-1", - }, - }, - }, - { - Id: "dead-0", - State: rktapi.PodState_POD_STATE_EXITED, - CreatedAt: 0, - StartedAt: 0, - Apps: []*rktapi.App{fakeApp}, - Annotations: []*rktapi.KeyValue{ - { - Key: types.KubernetesPodUIDLabel, - Value: "pod-uid-0", - }, - }, - }, - }, - []string{"k8s_dead-0.service", "k8s_dead-1.service", "k8s_dead-2.service"}, - []string{"rkt rm dead-0", "rkt rm dead-1"}, - []string{"/poddir/fake/finished-dead-0", "/poddir/fake/finished-dead-1", "/run/systemd/system/k8s_dead-0.service", "/run/systemd/system/k8s_dead-1.service"}, - }, - } - - for i, tt := range tests { - testCaseHint := fmt.Sprintf("test case #%d", i) - - ctrl := gomock.NewController(t) - - fakeOS.ReadDirFn = func(dirname string) ([]os.FileInfo, error) { - serviceFileNames := tt.serviceFilesOnDisk - var fileInfos []os.FileInfo - - for _, name := range serviceFileNames { - mockFI := containertesting.NewMockFileInfo(ctrl) - // we need to specify two calls - // first: get all systemd units - // second: filter only the files with a k8s_ prefix - mockFI.EXPECT().Name().Return(name) - mockFI.EXPECT().Name().Return(name) - fileInfos = append(fileInfos, mockFI) - } - return fileInfos, nil - } - - fr.pods = tt.pods - for _, p := range tt.apiPods { - deletionProvider.pods[p.UID] = struct{}{} - } - - allSourcesReady := true - evictNonDeletedPods := false - err := rkt.GarbageCollect(tt.gcPolicy, allSourcesReady, evictNonDeletedPods) - assert.NoError(t, err, testCaseHint) - - sort.Sort(sortedStringList(tt.expectedCommands)) - sort.Sort(sortedStringList(cli.cmds)) - - assert.Equal(t, tt.expectedCommands, cli.cmds, testCaseHint) - - sort.Sort(sortedStringList(tt.expectedDeletedFiles)) - sort.Sort(sortedStringList(fakeOS.Removes)) - sort.Sort(sortedStringList(fs.resetFailedUnits)) - - assert.Equal(t, tt.expectedDeletedFiles, fakeOS.Removes, testCaseHint) - var expectedService []string - for _, f := range tt.expectedDeletedFiles { - unit := filepath.Base(f) - if strings.HasSuffix(unit, ".service") && strings.HasPrefix(unit, kubernetesUnitPrefix) { - expectedService = append(expectedService, unit) - } - } - assert.Equal(t, expectedService, fs.resetFailedUnits, testCaseHint) - - // Cleanup after each test. - cli.Reset() - ctrl.Finish() - fakeOS.Removes = []string{} - fs.resetFailedUnits = []string{} - deletionProvider.pods = make(map[kubetypes.UID]struct{}) - } -} - -type annotationsByName []appctypes.Annotation - -func (a annotationsByName) Len() int { return len(a) } -func (a annotationsByName) Less(x, y int) bool { return a[x].Name < a[y].Name } -func (a annotationsByName) Swap(x, y int) { a[x], a[y] = a[y], a[x] } - -func TestMakePodManifestAnnotations(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - fr := newFakeRktInterface() - fs := newFakeSystemd() - r := &Runtime{apisvc: fr, systemd: fs} - - testCases := []struct { - in *v1.Pod - out *appcschema.PodManifest - outerr error - }{ - { - in: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - UID: "uid-1", - Name: "name-1", - Namespace: "namespace-1", - Annotations: map[string]string{ - k8sRktStage1NameAnno: "stage1-override-img", - }, - }, - }, - out: &appcschema.PodManifest{ - Annotations: []appctypes.Annotation{ - { - Name: "io.kubernetes.container.name", - Value: "POD", - }, - { - Name: appctypes.ACIdentifier(k8sRktStage1NameAnno), - Value: "stage1-override-img", - }, - { - Name: appctypes.ACIdentifier(types.KubernetesPodUIDLabel), - Value: "uid-1", - }, - { - Name: appctypes.ACIdentifier(types.KubernetesPodNameLabel), - Value: "name-1", - }, - { - Name: appctypes.ACIdentifier(k8sRktKubeletAnno), - Value: "true", - }, - { - Name: appctypes.ACIdentifier(types.KubernetesPodNamespaceLabel), - Value: "namespace-1", - }, - { - Name: appctypes.ACIdentifier(k8sRktRestartCountAnno), - Value: "0", - }, - }, - }, - }, - } - - for i, testCase := range testCases { - hint := fmt.Sprintf("case #%d", i) - - result, err := r.makePodManifest(testCase.in, "", []v1.Secret{}) - assert.Equal(t, testCase.outerr, err, hint) - if err == nil { - sort.Sort(annotationsByName(result.Annotations)) - sort.Sort(annotationsByName(testCase.out.Annotations)) - assert.Equal(t, testCase.out.Annotations, result.Annotations, hint) - } - } -} - -func TestPreparePodArgs(t *testing.T) { - r := &Runtime{ - config: &Config{}, - } - - testCases := []struct { - manifest appcschema.PodManifest - stage1Config string - cmd []string - }{ - { - appcschema.PodManifest{ - Annotations: appctypes.Annotations{ - { - Name: k8sRktStage1NameAnno, - Value: "stage1-image", - }, - }, - }, - "", - []string{"prepare", "--quiet", "--pod-manifest", "file", "--stage1-name=stage1-image"}, - }, - { - appcschema.PodManifest{ - Annotations: appctypes.Annotations{ - { - Name: k8sRktStage1NameAnno, - Value: "stage1-image", - }, - }, - }, - "stage1-image0", - []string{"prepare", "--quiet", "--pod-manifest", "file", "--stage1-name=stage1-image"}, - }, - { - appcschema.PodManifest{ - Annotations: appctypes.Annotations{}, - }, - "stage1-image0", - []string{"prepare", "--quiet", "--pod-manifest", "file", "--stage1-name=stage1-image0"}, - }, - { - appcschema.PodManifest{ - Annotations: appctypes.Annotations{}, - }, - "", - []string{"prepare", "--quiet", "--pod-manifest", "file"}, - }, - } - - for i, testCase := range testCases { - r.config.Stage1Image = testCase.stage1Config - cmd := r.preparePodArgs(&testCase.manifest, "file") - assert.Equal(t, testCase.cmd, cmd, fmt.Sprintf("Test case #%d", i)) - } -} - -func TestConstructSyslogIdentifier(t *testing.T) { - testCases := []struct { - podName string - podGenerateName string - identifier string - }{ - { - "prometheus-node-exporter-rv90m", - "prometheus-node-exporter-", - "prometheus-node-exporter", - }, - { - "simplepod", - "", - "simplepod", - }, - { - "p", - "", - "p", - }, - } - for i, testCase := range testCases { - identifier := constructSyslogIdentifier(testCase.podGenerateName, testCase.podName) - assert.Equal(t, testCase.identifier, identifier, fmt.Sprintf("Test case #%d", i)) - } -} - -func TestGetPodSystemdServiceFiles(t *testing.T) { - fs := containertesting.NewFakeOS() - r := &Runtime{os: fs} - - testCases := []struct { - serviceFilesOnDisk []string - expected []string - }{ - { - []string{"one.service", "two.service", "k8s_513ce947-8f6e-4d27-8c03-99f97b78d680.service", "k8s_184482df-8630-4d41-b84f-302684871758.service", "k8s_f4a244d8-5ec2-4f59-b7dd-c9e130d6e7a3.service", "k8s_f5aad446-5598-488f-93a4-5a27e03e7fcb.service"}, - []string{"k8s_513ce947-8f6e-4d27-8c03-99f97b78d680.service", "k8s_184482df-8630-4d41-b84f-302684871758.service", "k8s_f4a244d8-5ec2-4f59-b7dd-c9e130d6e7a3.service", "k8s_f5aad446-5598-488f-93a4-5a27e03e7fcb.service"}, - }, - { - []string{"one.service", "two.service"}, - []string{}, - }, - { - []string{"one.service", "k8s_513ce947-8f6e-4d27-8c03-99f97b78d680.service"}, - []string{"k8s_513ce947-8f6e-4d27-8c03-99f97b78d680.service"}, - }, - } - for i, tt := range testCases { - ctrl := gomock.NewController(t) - - fs.ReadDirFn = func(dirname string) ([]os.FileInfo, error) { - serviceFileNames := tt.serviceFilesOnDisk - var fileInfos []os.FileInfo - - for _, name := range serviceFileNames { - mockFI := containertesting.NewMockFileInfo(ctrl) - // we need to specify two calls - // first: get all systemd units - // second: filter only the files with a k8s_ prefix - mockFI.EXPECT().Name().Return(name) - mockFI.EXPECT().Name().Return(name) - fileInfos = append(fileInfos, mockFI) - } - return fileInfos, nil - } - serviceFiles, err := r.getPodSystemdServiceFiles() - if err != nil { - t.Errorf("%v", err) - } - for _, f := range serviceFiles { - assert.Contains(t, tt.expected, f.Name(), fmt.Sprintf("Test case #%d", i)) - - } - } -} - -func TestSetupSystemdCustomFields(t *testing.T) { - testCases := []struct { - unitOpts []*unit.UnitOption - podAnnotations map[string]string - expectedValues []string - raiseErr bool - }{ - // without annotation - { - []*unit.UnitOption{ - {Section: "Service", Name: "ExecStart", Value: "/bin/true"}, - }, - map[string]string{}, - []string{"/bin/true"}, - false, - }, - // with valid annotation for LimitNOFile - { - []*unit.UnitOption{ - {Section: "Service", Name: "ExecStart", Value: "/bin/true"}, - }, - map[string]string{k8sRktLimitNoFileAnno: "1024"}, - []string{"/bin/true", "1024"}, - false, - }, - // with invalid annotation for LimitNOFile - { - []*unit.UnitOption{ - {Section: "Service", Name: "ExecStart", Value: "/bin/true"}, - }, - map[string]string{k8sRktLimitNoFileAnno: "-1"}, - []string{"/bin/true"}, - true, - }, - } - - for i, tt := range testCases { - raiseErr := false - newUnitsOpts, err := setupSystemdCustomFields(tt.podAnnotations, tt.unitOpts) - if err != nil { - raiseErr = true - } - assert.Equal(t, tt.raiseErr, raiseErr, fmt.Sprintf("Test case #%d", i)) - for _, opt := range newUnitsOpts { - assert.Equal(t, "Service", opt.Section, fmt.Sprintf("Test case #%d", i)) - assert.Contains(t, tt.expectedValues, opt.Value, fmt.Sprintf("Test case #%d", i)) - } - } -} diff --git a/pkg/kubelet/rkt/systemd.go b/pkg/kubelet/rkt/systemd.go deleted file mode 100644 index 73768e5ea8c..00000000000 --- a/pkg/kubelet/rkt/systemd.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rkt - -import ( - "fmt" - "strconv" - "strings" - - "github.com/coreos/go-systemd/dbus" -) - -// systemdVersion is a type wraps the int to implement kubecontainer.Version interface. -type systemdVersion int - -func (s systemdVersion) String() string { - return fmt.Sprintf("%d", s) -} - -func (s systemdVersion) Compare(other string) (int, error) { - v, err := strconv.Atoi(other) - if err != nil { - return -1, err - } - if int(s) < v { - return -1, nil - } else if int(s) > v { - return 1, nil - } - return 0, nil -} - -// systemdInterface is an abstraction of the go-systemd/dbus to make -// it mockable for testing. -// TODO(yifan): Eventually we should move these functionalities to: -// 1. a package for launching/stopping rkt pods. -// 2. rkt api-service interface for listing pods. -// See https://github.com/coreos/rkt/issues/1769. -type systemdInterface interface { - // Version returns the version of the systemd. - Version() (systemdVersion, error) - // ListUnits lists all the loaded units. - ListUnits() ([]dbus.UnitStatus, error) - // StopUnits stops the unit with the given name. - StopUnit(name string, mode string, ch chan<- string) (int, error) - // RestartUnit restarts the unit with the given name. - RestartUnit(name string, mode string, ch chan<- string) (int, error) - // ResetFailedUnit resets the "failed" state of a specific unit. - ResetFailedUnit(name string) error -} - -// systemd implements the systemdInterface using dbus and systemctl. -// All the functions other then Version() are already implemented by go-systemd/dbus. -type systemd struct { - *dbus.Conn -} - -// newSystemd creates a systemd object that implements systemdInterface. -func newSystemd() (*systemd, error) { - dbusConn, err := dbus.New() - if err != nil { - return nil, err - } - return &systemd{dbusConn}, nil -} - -// Version returns the version of the systemd. -func (s *systemd) Version() (systemdVersion, error) { - versionStr, err := s.Conn.GetManagerProperty("Version") - if err != nil { - return -1, err - } - result, err := strconv.Atoi(strings.Trim(versionStr, "\"")) - if err != nil { - return -1, err - } - return systemdVersion(result), nil -} diff --git a/pkg/kubelet/rkt/version.go b/pkg/kubelet/rkt/version.go deleted file mode 100644 index d94e7ea943c..00000000000 --- a/pkg/kubelet/rkt/version.go +++ /dev/null @@ -1,111 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rkt - -import ( - "context" - "fmt" - "sync" - - rktapi "github.com/coreos/rkt/api/v1alpha" - - utilversion "k8s.io/kubernetes/pkg/util/version" -) - -type versions struct { - sync.RWMutex - binVersion *utilversion.Version - apiVersion *utilversion.Version - systemdVersion systemdVersion -} - -func newRktVersion(version string) (*utilversion.Version, error) { - return utilversion.ParseSemantic(version) -} - -func (r *Runtime) getVersions() error { - r.versions.Lock() - defer r.versions.Unlock() - - // Get systemd version. - var err error - r.versions.systemdVersion, err = r.systemd.Version() - if err != nil { - return err - } - - ctx, cancel := context.WithTimeout(context.Background(), r.requestTimeout) - defer cancel() - // Example for the version strings returned by GetInfo(): - // RktVersion:"0.10.0+gitb7349b1" AppcVersion:"0.7.1" ApiVersion:"1.0.0-alpha" - resp, err := r.apisvc.GetInfo(ctx, &rktapi.GetInfoRequest{}) - if err != nil { - return err - } - - // Get rkt binary version. - r.versions.binVersion, err = newRktVersion(resp.Info.RktVersion) - if err != nil { - return err - } - - // Get rkt API version. - r.versions.apiVersion, err = newRktVersion(resp.Info.ApiVersion) - if err != nil { - return err - } - return nil -} - -// checkVersion tests whether the rkt/systemd/rkt-api-service that meet the version requirement. -// If all version requirements are met, it returns nil. -func (r *Runtime) checkVersion(minimumRktBinVersion, minimumRktApiVersion, minimumSystemdVersion string) error { - if err := r.getVersions(); err != nil { - return err - } - - r.versions.RLock() - defer r.versions.RUnlock() - - // Check systemd version. - result, err := r.versions.systemdVersion.Compare(minimumSystemdVersion) - if err != nil { - return err - } - if result < 0 { - return fmt.Errorf("rkt: systemd version(%v) is too old, requires at least %v", r.versions.systemdVersion, minimumSystemdVersion) - } - - // Check rkt binary version. - result, err = r.versions.binVersion.Compare(minimumRktBinVersion) - if err != nil { - return err - } - if result < 0 { - return fmt.Errorf("rkt: binary version is too old(%v), requires at least %v", r.versions.binVersion, minimumRktBinVersion) - } - - // Check rkt API version. - result, err = r.versions.apiVersion.Compare(minimumRktApiVersion) - if err != nil { - return err - } - if result < 0 { - return fmt.Errorf("rkt: API version is too old(%v), requires at least %v", r.versions.apiVersion, minimumRktApiVersion) - } - return nil -} diff --git a/pkg/kubelet/sysctl/runtime.go b/pkg/kubelet/sysctl/runtime.go index 7ccbbdf5328..a7aa49e9ba5 100644 --- a/pkg/kubelet/sysctl/runtime.go +++ b/pkg/kubelet/sysctl/runtime.go @@ -31,7 +31,6 @@ const ( dockerMinimumAPIVersion = "1.24.0" dockerTypeName = "docker" - rktTypeName = "rkt" ) // TODO: The admission logic in this file is runtime-dependent. It should be @@ -72,14 +71,6 @@ func NewRuntimeAdmitHandler(runtime container.Runtime) (*runtimeAdmitHandler, er Message: "Docker API version before 1.24 does not support sysctls", }, }, nil - case rktTypeName: - return &runtimeAdmitHandler{ - result: lifecycle.PodAdmitResult{ - Admit: false, - Reason: UnsupportedReason, - Message: "Rkt does not support sysctls", - }, - }, nil default: // Return admit for other runtimes. return &runtimeAdmitHandler{ diff --git a/pkg/kubelet/types/constants.go b/pkg/kubelet/types/constants.go index 2c83908a7c4..f2f703a8d2b 100644 --- a/pkg/kubelet/types/constants.go +++ b/pkg/kubelet/types/constants.go @@ -22,7 +22,6 @@ const ( // different container runtimes DockerContainerRuntime = "docker" - RktContainerRuntime = "rkt" RemoteContainerRuntime = "remote" // User visible keys for managing node allocatable enforcement on the node.