From 09285864db0d925b650488abb98cdea797eed09a Mon Sep 17 00:00:00 2001 From: Cesar Wong Date: Thu, 7 Apr 2016 13:43:52 -0400 Subject: [PATCH 1/7] Initial windows container runtime --- pkg/kubelet/cadvisor/cadvisor_stub.go | 74 +++ pkg/kubelet/cm/container_manager_stub.go | 4 +- .../cm/container_manager_unsupported.go | 64 --- pkg/kubelet/dockertools/docker_manager.go | 440 ++++++++++++++++-- pkg/volume/volume.go | 57 ++- 5 files changed, 541 insertions(+), 98 deletions(-) create mode 100644 pkg/kubelet/cadvisor/cadvisor_stub.go delete mode 100644 pkg/kubelet/cm/container_manager_unsupported.go diff --git a/pkg/kubelet/cadvisor/cadvisor_stub.go b/pkg/kubelet/cadvisor/cadvisor_stub.go new file mode 100644 index 00000000000..f2cb6e3e73f --- /dev/null +++ b/pkg/kubelet/cadvisor/cadvisor_stub.go @@ -0,0 +1,74 @@ +// +build darwin windows + +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cadvisor + +import ( + "github.com/google/cadvisor/events" + cadvisorapi "github.com/google/cadvisor/info/v1" + cadvisorapiv2 "github.com/google/cadvisor/info/v2" +) + +type cadvisorStub struct { +} + +var _ Interface = new(cadvisorStub) + +func New(port uint, runtime string) (Interface, error) { + return &cadvisorStub{}, nil +} + +func (cu *cadvisorStub) Start() error { + return nil +} + +func (cu *cadvisorStub) DockerContainer(name string, req *cadvisorapi.ContainerInfoRequest) (cadvisorapi.ContainerInfo, error) { + return cadvisorapi.ContainerInfo{}, nil +} + +func (cu *cadvisorStub) ContainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) { + return &cadvisorapi.ContainerInfo{}, nil +} + +func (cu *cadvisorStub) ContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) { + return make(map[string]cadvisorapiv2.ContainerInfo), nil +} + +func (cu *cadvisorStub) SubcontainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (map[string]*cadvisorapi.ContainerInfo, error) { + return nil, nil +} + +func (cu *cadvisorStub) MachineInfo() (*cadvisorapi.MachineInfo, error) { + return &cadvisorapi.MachineInfo{}, nil +} + +func (cu *cadvisorStub) VersionInfo() (*cadvisorapi.VersionInfo, error) { + return &cadvisorapi.VersionInfo{}, nil +} + +func (cu *cadvisorStub) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) { + return cadvisorapiv2.FsInfo{}, nil +} + +func (cu *cadvisorStub) RootFsInfo() (cadvisorapiv2.FsInfo, error) { + return cadvisorapiv2.FsInfo{}, nil +} + +func (cu *cadvisorStub) WatchEvents(request *events.Request) (*events.EventChannel, error) { + return &events.EventChannel{}, nil +} diff --git a/pkg/kubelet/cm/container_manager_stub.go b/pkg/kubelet/cm/container_manager_stub.go index 186d773dbe9..94c71729253 100644 --- a/pkg/kubelet/cm/container_manager_stub.go +++ b/pkg/kubelet/cm/container_manager_stub.go @@ -1,3 +1,5 @@ +// +build !linux + /* Copyright 2015 The Kubernetes Authors. @@ -23,8 +25,6 @@ import ( type containerManagerStub struct{} -var _ ContainerManager = &containerManagerStub{} - func (cm *containerManagerStub) Start(_ *api.Node) error { glog.V(2).Infof("Starting stub container manager") return nil diff --git a/pkg/kubelet/cm/container_manager_unsupported.go b/pkg/kubelet/cm/container_manager_unsupported.go deleted file mode 100644 index 4118a998f8c..00000000000 --- a/pkg/kubelet/cm/container_manager_unsupported.go +++ /dev/null @@ -1,64 +0,0 @@ -// +build !linux - -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cm - -import ( - "fmt" - - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/kubelet/cadvisor" - "k8s.io/kubernetes/pkg/util/mount" -) - -type unsupportedContainerManager struct { -} - -var _ ContainerManager = &unsupportedContainerManager{} - -func (unsupportedContainerManager) Start(_ *api.Node) error { - return fmt.Errorf("Container Manager is unsupported in this build") -} - -func (unsupportedContainerManager) SystemCgroupsLimit() api.ResourceList { - return api.ResourceList{} -} - -func (unsupportedContainerManager) GetNodeConfig() NodeConfig { - return NodeConfig{} -} - -func (unsupportedContainerManager) GetMountedSubsystems() *CgroupSubsystems { - return &CgroupSubsystems{} -} - -func (unsupportedContainerManager) GetQOSContainersInfo() QOSContainersInfo { - return QOSContainersInfo{} -} - -func (cm *unsupportedContainerManager) Status() Status { - return Status{} -} - -func (cm *unsupportedContainerManager) NewPodContainerManager() PodContainerManager { - return &unsupportedPodContainerManager{} -} - -func NewContainerManager(_ mount.Interface, _ cadvisor.Interface, _ NodeConfig) (ContainerManager, error) { - return &unsupportedContainerManager{}, nil -} diff --git a/pkg/kubelet/dockertools/docker_manager.go b/pkg/kubelet/dockertools/docker_manager.go index 90ece063cc6..b5083f50ce7 100644 --- a/pkg/kubelet/dockertools/docker_manager.go +++ b/pkg/kubelet/dockertools/docker_manager.go @@ -115,6 +115,10 @@ var ( defaultSeccompOpt = []dockerOpt{{"seccomp", "unconfined", ""}} ) +type WindowsDockerManager struct { + *DockerManager +} + type DockerManager struct { client DockerInterface recorder record.EventRecorder @@ -202,6 +206,71 @@ func PodInfraContainerEnv(env map[string]string) kubecontainer.Option { } } +type windowsRuntimeHelper struct { + kubecontainer.RuntimeHelper +} + +func (h *windowsRuntimeHelper) GenerateRunContainerOptions(pod *api.Pod, container *api.Container, podIP string) (*kubecontainer.RunContainerOptions, error) { + return &kubecontainer.RunContainerOptions{ + Hostname: "test-pod", + }, nil +} + +func NewWindowsDockerManager( + client DockerInterface, + recorder record.EventRecorder, + livenessManager proberesults.Manager, + containerRefManager *kubecontainer.RefManager, + podGetter podGetter, + machineInfo *cadvisorapi.MachineInfo, + podInfraContainerImage string, + qps float32, + burst int, + containerLogsDir string, + osInterface kubecontainer.OSInterface, + networkPlugin network.NetworkPlugin, + runtimeHelper kubecontainer.RuntimeHelper, + httpClient types.HttpGetter, + execHandler ExecHandler, + oomAdjuster *oom.OOMAdjuster, + procFs procfs.ProcFSInterface, + cpuCFSQuota bool, + imageBackOff *flowcontrol.Backoff, + serializeImagePulls bool, + enableCustomMetrics bool, + hairpinMode bool, + seccompProfileRoot string, + options ...kubecontainer.Option) *WindowsDockerManager { + dockerManager := NewDockerManager(client, + recorder, + livenessManager, + containerRefManager, + podGetter, + machineInfo, + podInfraContainerImage, + qps, + burst, + containerLogsDir, + osInterface, + networkPlugin, + runtimeHelper, + httpClient, + execHandler, + oomAdjuster, + procFs, + cpuCFSQuota, + imageBackOff, + serializeImagePulls, + enableCustomMetrics, + hairpinMode, + seccompProfileRoot, + options...) + + return &WindowsDockerManager{ + DockerManager: dockerManager, + } +} + func NewDockerManager( client DockerInterface, recorder record.EventRecorder, @@ -345,10 +414,13 @@ func (dm *DockerManager) determineContainerIP(podNamespace, podName string, cont if container.NetworkSettings != nil { result = container.NetworkSettings.IPAddress - - // Fall back to IPv6 address if no IPv4 address is present - if result == "" { - result = container.NetworkSettings.GlobalIPv6Address + if len(result) == 0 { + for _, network := range container.NetworkSettings.Networks { + if len(network.IPAddress) > 0 { + result = network.IPAddress + break + } + } } } @@ -431,13 +503,11 @@ func (dm *DockerManager) inspectContainer(id string, podName, podNamespace strin // Container that are running, restarting and paused status.State = kubecontainer.ContainerStateRunning status.StartedAt = startedAt - if containerName == PodInfraContainerName { - ip, err = dm.determineContainerIP(podNamespace, podName, iResult) - // Kubelet doesn't handle the network error scenario - if err != nil { - status.State = kubecontainer.ContainerStateUnknown - status.Message = fmt.Sprintf("Network error: %#v", err) - } + ip, err = dm.determineContainerIP(podNamespace, podName, iResult) + // Kubelet doesn't handle the network error scenario + if err != nil { + status.State = kubecontainer.ContainerStateUnknown + status.Message = fmt.Sprintf("Network error: %#v", err) } return &status, ip, nil } @@ -637,6 +707,7 @@ func (dm *DockerManager) runContainer( cpuShares = milliCPUToShares(cpuRequest.MilliValue()) } var devices []dockercontainer.DeviceMapping + _ = devices if nvidiaGPULimit.Value() != 0 { // Experimental. For now, we hardcode /dev/nvidia0 no matter what the user asks for // (we only support one device per node). @@ -670,20 +741,23 @@ func (dm *DockerManager) runContainer( } } + _ = cpuShares + _ = memoryLimit + _ = securityOpts hc := &dockercontainer.HostConfig{ - Binds: binds, - NetworkMode: dockercontainer.NetworkMode(netMode), - IpcMode: dockercontainer.IpcMode(ipcMode), - UTSMode: dockercontainer.UTSMode(utsMode), - PidMode: dockercontainer.PidMode(pidMode), - ReadonlyRootfs: readOnlyRootFilesystem(container), - Resources: dockercontainer.Resources{ - Memory: memoryLimit, - MemorySwap: -1, - CPUShares: cpuShares, - Devices: devices, - }, - SecurityOpt: fmtSecurityOpts, + // Binds: binds, + NetworkMode: dockercontainer.NetworkMode(netMode), + // IpcMode: dockercontainer.IpcMode(ipcMode), + // UTSMode: dockercontainer.UTSMode(utsMode), + // PidMode: dockercontainer.PidMode(pidMode), + // ReadonlyRootfs: readOnlyRootFilesystem(container), + // Resources: dockercontainer.Resources{ + // Memory: memoryLimit, + // MemorySwap: -1, + // CPUShares: cpuShares, + // Devices: devices, + // }, + // SecurityOpt: securityOpts, } // Set sysctls if requested @@ -746,10 +820,10 @@ func (dm *DockerManager) runContainer( glog.V(3).Infof("Container %v/%v/%v: setting entrypoint \"%v\" and command \"%v\"", pod.Namespace, pod.Name, container.Name, dockerOpts.Config.Entrypoint, dockerOpts.Config.Cmd) - supplementalGids := dm.runtimeHelper.GetExtraSupplementalGroupsForPod(pod) - securityContextProvider := securitycontext.NewSimpleSecurityContextProvider() - securityContextProvider.ModifyContainerConfig(pod, container, dockerOpts.Config) - securityContextProvider.ModifyHostConfig(pod, container, dockerOpts.HostConfig, supplementalGids) + // supplementalGids := dm.runtimeHelper.GetExtraSupplementalGroupsForPod(pod) + // securityContextProvider := securitycontext.NewSimpleSecurityContextProvider() + // securityContextProvider.ModifyContainerConfig(pod, container, dockerOpts.Config) + // securityContextProvider.ModifyHostConfig(pod, container, dockerOpts.HostConfig, supplementalGids) createResp, err := dm.client.CreateContainer(dockerOpts) if err != nil { dm.recorder.Eventf(ref, api.EventTypeWarning, events.FailedToCreateContainer, "Failed to create docker container %q of pod %q with error: %v", container.Name, format.Pod(pod), err) @@ -925,6 +999,10 @@ func (dm *DockerManager) PullImage(image kubecontainer.ImageSpec, secrets []api. return dm.dockerPuller.Pull(image.Image, secrets) } +func (dm *WindowsDockerManager) PullImage(image kubecontainer.ImageSpec, secrets []api.Secret) error { + return fmt.Errorf("Image pull not yet supported") +} + // IsImagePresent checks whether the container image is already in the local storage. func (dm *DockerManager) IsImagePresent(image kubecontainer.ImageSpec) (bool, error) { return dm.dockerPuller.IsImagePresent(image.Image) @@ -1743,9 +1821,9 @@ func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Containe // Check if current docker version is higher than 1.10. Otherwise, we have to apply OOMScoreAdj instead of using docker API. // TODO: Remove this logic after we stop supporting docker version < 1.10. - if err = dm.applyOOMScoreAdjIfNeeded(pod, container, containerInfo); err != nil { - return kubecontainer.ContainerID{}, err - } + // if err = dm.applyOOMScoreAdjIfNeeded(pod, container, containerInfo); err != nil { + // return kubecontainer.ContainerID{}, err + // } // The addNDotsOption call appends the ndots option to the resolv.conf file generated by docker. // This resolv.conf file is shared by all containers of the same pod, and needs to be modified only once per pod. @@ -2070,6 +2148,116 @@ func (dm *DockerManager) computePodContainerChanges(pod *api.Pod, podStatus *kub }, nil } +func (dm *WindowsDockerManager) computePodContainerChanges(pod *api.Pod, podStatus *kubecontainer.PodStatus) (podContainerChangesSpec, error) { + start := time.Now() + defer func() { + metrics.ContainerManagerLatency.WithLabelValues("computePodContainerChanges").Observe(metrics.SinceInMicroseconds(start)) + }() + glog.V(5).Infof("Syncing Pod %q: %+v", format.Pod(pod), pod) + + containersToStart := make(map[int]string) + containersToKeep := make(map[kubecontainer.DockerID]int) + + // var err error + + // Skip podInfraContainer checking and creation + // var podInfraContainerID kubecontainer.DockerID + // var changed bool + // podInfraContainerStatus := podStatus.FindContainerStatusByName(PodInfraContainerName) + // if podInfraContainerStatus != nil && podInfraContainerStatus.State == kubecontainer.ContainerStateRunning { + // glog.V(4).Infof("Found pod infra container for %q", format.Pod(pod)) + // changed, err = dm.podInfraContainerChanged(pod, podInfraContainerStatus) + // if err != nil { + // return podContainerChangesSpec{}, err + // } + // } + + // createPodInfraContainer := true + // if podInfraContainerStatus == nil || podInfraContainerStatus.State != kubecontainer.ContainerStateRunning { + // glog.V(2).Infof("Need to restart pod infra container for %q because it is not found", format.Pod(pod)) + // } else if changed { + // glog.V(2).Infof("Need to restart pod infra container for %q because it is changed", format.Pod(pod)) + // } else { + // glog.V(4).Infof("Pod infra container looks good, keep it %q", format.Pod(pod)) + // createPodInfraContainer = false + // podInfraContainerID = kubecontainer.DockerID(podInfraContainerStatus.ID.ID) + // containersToKeep[podInfraContainerID] = -1 + // } + + for index, container := range pod.Spec.Containers { + expectedHash := kubecontainer.HashContainer(&container) + + containerStatus := podStatus.FindContainerStatusByName(container.Name) + if containerStatus == nil || containerStatus.State != kubecontainer.ContainerStateRunning { + if kubecontainer.ShouldContainerBeRestarted(&container, pod, podStatus) { + // If we are here it means that the container is dead and should be restarted, or never existed and should + // be created. We may be inserting this ID again if the container has changed and it has + // RestartPolicy::Always, but it's not a big deal. + message := fmt.Sprintf("Container %+v is dead, but RestartPolicy says that we should restart it.", container) + glog.V(3).Info(message) + containersToStart[index] = message + } + continue + } + + containerID := kubecontainer.DockerID(containerStatus.ID.ID) + hash := containerStatus.Hash + glog.V(3).Infof("pod %q container %q exists as %v", format.Pod(pod), container.Name, containerID) + + // if createPodInfraContainer { + // // createPodInfraContainer == true and Container exists + // // If we're creating infra container everything will be killed anyway + // // If RestartPolicy is Always or OnFailure we restart containers that were running before we + // // killed them when restarting Infra Container. + // if pod.Spec.RestartPolicy != api.RestartPolicyNever { + // message := fmt.Sprintf("Infra Container is being recreated. %q will be restarted.", container.Name) + // glog.V(1).Info(message) + // containersToStart[index] = message + // } + // continue + // } + + // At this point, the container is running and pod infra container is good. + // We will look for changes and check healthiness for the container. + containerChanged := hash != 0 && hash != expectedHash + if containerChanged { + message := fmt.Sprintf("pod %q container %q hash changed (%d vs %d), it will be killed and re-created.", format.Pod(pod), container.Name, hash, expectedHash) + glog.Info(message) + containersToStart[index] = message + continue + } + + liveness, found := dm.livenessManager.Get(containerStatus.ID) + if !found || liveness == proberesults.Success { + containersToKeep[containerID] = index + continue + } + if pod.Spec.RestartPolicy != api.RestartPolicyNever { + message := fmt.Sprintf("pod %q container %q is unhealthy, it will be killed and re-created.", format.Pod(pod), container.Name) + glog.Info(message) + containersToStart[index] = message + } + } + + // After the loop one of the following should be true: + // - createPodInfraContainer is true and containersToKeep is empty. + // (In fact, when createPodInfraContainer is false, containersToKeep will not be touched). + // - createPodInfraContainer is false and containersToKeep contains at least ID of Infra Container + + // If Infra container is the last running one, we don't want to keep it. + // if !createPodInfraContainer && len(containersToStart) == 0 && len(containersToKeep) == 1 { + // containersToKeep = make(map[kubecontainer.DockerID]int) + // } + + return podContainerChangesSpec{ + StartInfraContainer: false, + InfraChanged: false, + InfraContainerId: "", + ContainersToStart: containersToStart, + ContainersToKeep: containersToKeep, + }, nil +} + // Sync the running pod to match the specified desired pod. func (dm *DockerManager) SyncPod(pod *api.Pod, _ api.PodStatus, podStatus *kubecontainer.PodStatus, pullSecrets []api.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) { start := time.Now() @@ -2412,6 +2600,196 @@ func findActiveInitContainer(pod *api.Pod, podStatus *kubecontainer.PodStatus) ( return &pod.Spec.InitContainers[0], nil, false } +// Sync the running pod to match the specified desired pod. +func (dm *WindowsDockerManager) SyncPod(pod *api.Pod, _ api.PodStatus, podStatus *kubecontainer.PodStatus, pullSecrets []api.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) { + start := time.Now() + defer func() { + metrics.ContainerManagerLatency.WithLabelValues("SyncPod").Observe(metrics.SinceInMicroseconds(start)) + }() + + containerChanges, err := dm.computePodContainerChanges(pod, podStatus) + if err != nil { + result.Fail(err) + return + } + glog.V(3).Infof("Got container changes for pod %q: %+v", format.Pod(pod), containerChanges) + + // if containerChanges.InfraChanged { + // ref, err := api.GetReference(pod) + // if err != nil { + // glog.Errorf("Couldn't make a ref to pod %q: '%v'", format.Pod(pod), err) + // } + // dm.recorder.Eventf(ref, api.EventTypeNormal, "InfraChanged", "Pod infrastructure changed, it will be killed and re-created.") + // } + // if containerChanges.StartInfraContainer || (len(containerChanges.ContainersToKeep) == 0 && len(containerChanges.ContainersToStart) == 0) { + // if len(containerChanges.ContainersToKeep) == 0 && len(containerChanges.ContainersToStart) == 0 { + // glog.V(4).Infof("Killing Infra Container for %q because all other containers are dead.", format.Pod(pod)) + // } else { + // glog.V(4).Infof("Killing Infra Container for %q, will start new one", format.Pod(pod)) + // } + + // // Killing phase: if we want to start new infra container, or nothing is running kill everything (including infra container) + // // TODO(random-liu): We'll use pod status directly in the future + // killResult := dm.killPodWithSyncResult(pod, kubecontainer.ConvertPodStatusToRunningPod(podStatus)) + // result.AddPodSyncResult(killResult) + // if killResult.Error() != nil { + // return + // } + // } else { + // Otherwise kill any running containers in this pod which are not specified as ones to keep. + runningContainerStatues := podStatus.GetRunningContainerStatuses() + for _, containerStatus := range runningContainerStatues { + _, keep := containerChanges.ContainersToKeep[kubecontainer.DockerID(containerStatus.ID.ID)] + if !keep { + glog.V(3).Infof("Killing unwanted container %q(id=%q) for pod %q", containerStatus.Name, containerStatus.ID, format.Pod(pod)) + // attempt to find the appropriate container policy + var podContainer *api.Container + var killMessage string + for i, c := range pod.Spec.Containers { + if c.Name == containerStatus.Name { + podContainer = &pod.Spec.Containers[i] + killMessage = containerChanges.ContainersToStart[i] + break + } + } + killContainerResult := kubecontainer.NewSyncResult(kubecontainer.KillContainer, containerStatus.Name) + result.AddSyncResult(killContainerResult) + if err := dm.KillContainerInPod(containerStatus.ID, podContainer, pod, killMessage, nil); err != nil { + killContainerResult.Fail(kubecontainer.ErrKillContainer, err.Error()) + glog.Errorf("Error killing container %q(id=%q) for pod %q: %v", containerStatus.Name, containerStatus.ID, format.Pod(pod), err) + return + } + } + } + // } + + // We pass the value of the podIP down to runContainerInPod, which in turn + // passes it to various other functions, in order to facilitate + // functionality that requires this value (hosts file and downward API) + // and avoid races determining the pod IP in cases where a container + // requires restart but the podIP isn't in the status manager yet. + // + // We default to the IP in the passed-in pod status, and overwrite it if the + // infra container needs to be (re)started. + podIP := "" + if podStatus != nil { + podIP = podStatus.IP + } + + // If we should create infra container then we do it first. + // podInfraContainerID := containerChanges.InfraContainerId + // if containerChanges.StartInfraContainer && (len(containerChanges.ContainersToStart) > 0) { + // glog.V(4).Infof("Creating pod infra container for %q", format.Pod(pod)) + // startContainerResult := kubecontainer.NewSyncResult(kubecontainer.StartContainer, PodInfraContainerName) + // result.AddSyncResult(startContainerResult) + // var msg string + // podInfraContainerID, err, msg = dm.createPodInfraContainer(pod) + // if err != nil { + // startContainerResult.Fail(err, msg) + // glog.Errorf("Failed to create pod infra container: %v; Skipping pod %q", err, format.Pod(pod)) + // return + // } + + // setupNetworkResult := kubecontainer.NewSyncResult(kubecontainer.SetupNetwork, kubecontainer.GetPodFullName(pod)) + // result.AddSyncResult(setupNetworkResult) + // if !usesHostNetwork(pod) { + // // Call the networking plugin + // err = dm.networkPlugin.SetUpPod(pod.Namespace, pod.Name, podInfraContainerID) + // if err != nil { + // // TODO: (random-liu) There shouldn't be "Skipping pod" in sync result message + // message := fmt.Sprintf("Failed to setup network for pod %q using network plugins %q: %v; Skipping pod", format.Pod(pod), dm.networkPlugin.Name(), err) + // setupNetworkResult.Fail(kubecontainer.ErrSetupNetwork, message) + // glog.Error(message) + + // // Delete infra container + // killContainerResult := kubecontainer.NewSyncResult(kubecontainer.KillContainer, PodInfraContainerName) + // result.AddSyncResult(killContainerResult) + // if delErr := dm.KillContainerInPod(kubecontainer.ContainerID{ + // ID: string(podInfraContainerID), + // Type: "docker"}, nil, pod, message); delErr != nil { + // killContainerResult.Fail(kubecontainer.ErrKillContainer, delErr.Error()) + // glog.Warningf("Clear infra container failed for pod %q: %v", format.Pod(pod), delErr) + // } + // return + // } + + // // Setup the host interface unless the pod is on the host's network (FIXME: move to networkPlugin when ready) + // var podInfraContainer *docker.Container + // podInfraContainer, err = dm.client.InspectContainer(string(podInfraContainerID)) + // if err != nil { + // glog.Errorf("Failed to inspect pod infra container: %v; Skipping pod %q", err, format.Pod(pod)) + // result.Fail(err) + // return + // } + + // if dm.configureHairpinMode { + // if err = hairpin.SetUpContainer(podInfraContainer.State.Pid, network.DefaultInterfaceName); err != nil { + // glog.Warningf("Hairpin setup failed for pod %q: %v", format.Pod(pod), err) + // } + // } + + // // Overwrite the podIP passed in the pod status, since we just started the infra container. + // podIP = dm.determineContainerIP(pod.Name, pod.Namespace, podInfraContainer) + // } + // } + + // Start everything + for idx := range containerChanges.ContainersToStart { + container := &pod.Spec.Containers[idx] + startContainerResult := kubecontainer.NewSyncResult(kubecontainer.StartContainer, container.Name) + result.AddSyncResult(startContainerResult) + + // containerChanges.StartInfraContainer causes the containers to be restarted for config reasons + // ignore backoff + if !containerChanges.StartInfraContainer { + isInBackOff, err, msg := dm.doBackOff(pod, container, podStatus, backOff) + if isInBackOff { + startContainerResult.Fail(err, msg) + glog.V(4).Infof("Backing Off restarting container %+v in pod %v", container, format.Pod(pod)) + continue + } + } + glog.V(4).Infof("Creating container %+v in pod %v", container, format.Pod(pod)) + // err, msg := dm.imagePuller.PullImage(pod, container, pullSecrets) + // if err != nil { + // startContainerResult.Fail(err, msg) + // continue + // } + + // if container.SecurityContext != nil && container.SecurityContext.RunAsNonRoot != nil && *container.SecurityContext.RunAsNonRoot { + // err := dm.verifyNonRoot(container) + // if err != nil { + // startContainerResult.Fail(kubecontainer.ErrVerifyNonRoot, err.Error()) + // glog.Errorf("Error running pod %q container %q: %v", format.Pod(pod), container.Name, err) + // continue + // } + // } + // For a new container, the RestartCount should be 0 + restartCount := 0 + containerStatus := podStatus.FindContainerStatusByName(container.Name) + if containerStatus != nil { + restartCount = containerStatus.RestartCount + 1 + } + + // TODO(dawnchen): Check RestartPolicy.DelaySeconds before restart a container + // Note: when configuring the pod's containers anything that can be configured by pointing + // to the namespace of the infra container should use namespaceMode. This includes things like the net namespace + // and IPC namespace. PID mode cannot point to another container right now. + // See createPodInfraContainer for infra container setup. + // namespaceMode := fmt.Sprintf("container:%v", podInfraContainerID) + namespaceMode := "" + _, err = dm.runContainerInPod(pod, container, namespaceMode, namespaceMode, getPidMode(pod), podIP, restartCount) + if err != nil { + startContainerResult.Fail(kubecontainer.ErrRunContainer, err.Error()) + // TODO(bburns) : Perhaps blacklist a container after N failures? + glog.Errorf("Error running pod %q container %q: %v", format.Pod(pod), container.Name, err) + continue + } + // Successfully started the container; clear the entry in the failure + } + return +} + // verifyNonRoot returns an error if the container or image will run as the root user. func (dm *DockerManager) verifyNonRoot(container *api.Container) error { if securitycontext.HasRunAsUser(container) { diff --git a/pkg/volume/volume.go b/pkg/volume/volume.go index 54081c54534..93db6169cb9 100644 --- a/pkg/volume/volume.go +++ b/pkg/volume/volume.go @@ -17,6 +17,7 @@ limitations under the License. package volume import ( + "io" "io/ioutil" "os" "path" @@ -213,9 +214,63 @@ func RenameDirectory(oldPath, newName string) (string, error) { if err != nil { return "", err } - err = os.Rename(oldPath, newPath) + // os.Rename call fails on windows (https://github.com/golang/go/issues/14527) + // Replacing with copyFolder to the newPath and deleting the oldPath directory + // err = os.Rename(oldPath, newPath) + err = copyFolder(oldPath, newPath) + if err != nil { return "", err } + os.RemoveAll(oldPath) return newPath, nil } + +func copyFolder(source string, dest string) (err error) { + directory, _ := os.Open(source) + objects, err := directory.Readdir(-1) + + for _, obj := range objects { + sourcefilepointer := source + "/" + obj.Name() + destinationfilepointer := dest + "/" + obj.Name() + if obj.IsDir() { + err = copyFolder(sourcefilepointer, destinationfilepointer) + if err != nil { + return err + } + } else { + err = copyFile(sourcefilepointer, destinationfilepointer) + if err != nil { + return err + } + } + + } + return +} + +func copyFile(source string, dest string) (err error) { + sourcefile, err := os.Open(source) + if err != nil { + return err + } + + defer sourcefile.Close() + + destfile, err := os.Create(dest) + if err != nil { + return err + } + + defer destfile.Close() + + _, err = io.Copy(destfile, sourcefile) + if err == nil { + sourceinfo, err := os.Stat(source) + if err != nil { + err = os.Chmod(dest, sourceinfo.Mode()) + } + + } + return +} From 244152544ce4fa161f9e42efb9c6549dc4d9453f Mon Sep 17 00:00:00 2001 From: Alexander Brand Date: Wed, 24 Aug 2016 10:49:06 -0400 Subject: [PATCH 2/7] Changes to kubelet to support win containers --- pkg/apis/componentconfig/v1alpha1/defaults.go | 2 +- pkg/kubelet/cm/container_manager_stub.go | 4 +- pkg/kubelet/cm/container_manager_windows.go | 26 ++ pkg/kubelet/dockertools/docker_manager.go | 442 ++---------------- .../dockertools/docker_manager_linux.go | 24 + .../dockertools/docker_manager_windows.go | 33 ++ pkg/kubelet/kubelet_pods.go | 11 +- pkg/volume/volume.go | 14 +- 8 files changed, 134 insertions(+), 422 deletions(-) create mode 100644 pkg/kubelet/cm/container_manager_windows.go create mode 100644 pkg/kubelet/dockertools/docker_manager_linux.go create mode 100644 pkg/kubelet/dockertools/docker_manager_windows.go diff --git a/pkg/apis/componentconfig/v1alpha1/defaults.go b/pkg/apis/componentconfig/v1alpha1/defaults.go index a2d193eee2a..2044e89f72f 100644 --- a/pkg/apis/componentconfig/v1alpha1/defaults.go +++ b/pkg/apis/componentconfig/v1alpha1/defaults.go @@ -196,7 +196,7 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) { if obj.DockerExecHandlerName == "" { obj.DockerExecHandlerName = "native" } - if obj.DockerEndpoint == "" { + if obj.DockerEndpoint == "" && runtime.GOOS != "windows" { obj.DockerEndpoint = "unix:///var/run/docker.sock" } if obj.EventBurst == 0 { diff --git a/pkg/kubelet/cm/container_manager_stub.go b/pkg/kubelet/cm/container_manager_stub.go index 94c71729253..186d773dbe9 100644 --- a/pkg/kubelet/cm/container_manager_stub.go +++ b/pkg/kubelet/cm/container_manager_stub.go @@ -1,5 +1,3 @@ -// +build !linux - /* Copyright 2015 The Kubernetes Authors. @@ -25,6 +23,8 @@ import ( type containerManagerStub struct{} +var _ ContainerManager = &containerManagerStub{} + func (cm *containerManagerStub) Start(_ *api.Node) error { glog.V(2).Infof("Starting stub container manager") return nil diff --git a/pkg/kubelet/cm/container_manager_windows.go b/pkg/kubelet/cm/container_manager_windows.go new file mode 100644 index 00000000000..8b72179aa76 --- /dev/null +++ b/pkg/kubelet/cm/container_manager_windows.go @@ -0,0 +1,26 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cm + +import ( + "k8s.io/kubernetes/pkg/kubelet/cadvisor" + "k8s.io/kubernetes/pkg/util/mount" +) + +func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.Interface, nodeConfig NodeConfig) (ContainerManager, error) { + return NewStubContainerManager(), nil +} diff --git a/pkg/kubelet/dockertools/docker_manager.go b/pkg/kubelet/dockertools/docker_manager.go index b5083f50ce7..c91b9c09b58 100644 --- a/pkg/kubelet/dockertools/docker_manager.go +++ b/pkg/kubelet/dockertools/docker_manager.go @@ -115,10 +115,6 @@ var ( defaultSeccompOpt = []dockerOpt{{"seccomp", "unconfined", ""}} ) -type WindowsDockerManager struct { - *DockerManager -} - type DockerManager struct { client DockerInterface recorder record.EventRecorder @@ -206,71 +202,6 @@ func PodInfraContainerEnv(env map[string]string) kubecontainer.Option { } } -type windowsRuntimeHelper struct { - kubecontainer.RuntimeHelper -} - -func (h *windowsRuntimeHelper) GenerateRunContainerOptions(pod *api.Pod, container *api.Container, podIP string) (*kubecontainer.RunContainerOptions, error) { - return &kubecontainer.RunContainerOptions{ - Hostname: "test-pod", - }, nil -} - -func NewWindowsDockerManager( - client DockerInterface, - recorder record.EventRecorder, - livenessManager proberesults.Manager, - containerRefManager *kubecontainer.RefManager, - podGetter podGetter, - machineInfo *cadvisorapi.MachineInfo, - podInfraContainerImage string, - qps float32, - burst int, - containerLogsDir string, - osInterface kubecontainer.OSInterface, - networkPlugin network.NetworkPlugin, - runtimeHelper kubecontainer.RuntimeHelper, - httpClient types.HttpGetter, - execHandler ExecHandler, - oomAdjuster *oom.OOMAdjuster, - procFs procfs.ProcFSInterface, - cpuCFSQuota bool, - imageBackOff *flowcontrol.Backoff, - serializeImagePulls bool, - enableCustomMetrics bool, - hairpinMode bool, - seccompProfileRoot string, - options ...kubecontainer.Option) *WindowsDockerManager { - dockerManager := NewDockerManager(client, - recorder, - livenessManager, - containerRefManager, - podGetter, - machineInfo, - podInfraContainerImage, - qps, - burst, - containerLogsDir, - osInterface, - networkPlugin, - runtimeHelper, - httpClient, - execHandler, - oomAdjuster, - procFs, - cpuCFSQuota, - imageBackOff, - serializeImagePulls, - enableCustomMetrics, - hairpinMode, - seccompProfileRoot, - options...) - - return &WindowsDockerManager{ - DockerManager: dockerManager, - } -} - func NewDockerManager( client DockerInterface, recorder record.EventRecorder, @@ -410,19 +341,7 @@ var ( // that the container passed is the infrastructure container of a pod and the responsibility // of the caller to ensure that the correct container is passed. func (dm *DockerManager) determineContainerIP(podNamespace, podName string, container *dockertypes.ContainerJSON) (string, error) { - result := "" - - if container.NetworkSettings != nil { - result = container.NetworkSettings.IPAddress - if len(result) == 0 { - for _, network := range container.NetworkSettings.Networks { - if len(network.IPAddress) > 0 { - result = network.IPAddress - break - } - } - } - } + result := getContainerIP(container) networkMode := getDockerNetworkMode(container) isHostNetwork := networkMode == namespaceModeHost @@ -707,7 +626,6 @@ func (dm *DockerManager) runContainer( cpuShares = milliCPUToShares(cpuRequest.MilliValue()) } var devices []dockercontainer.DeviceMapping - _ = devices if nvidiaGPULimit.Value() != 0 { // Experimental. For now, we hardcode /dev/nvidia0 no matter what the user asks for // (we only support one device per node). @@ -741,23 +659,20 @@ func (dm *DockerManager) runContainer( } } - _ = cpuShares - _ = memoryLimit - _ = securityOpts hc := &dockercontainer.HostConfig{ - // Binds: binds, - NetworkMode: dockercontainer.NetworkMode(netMode), - // IpcMode: dockercontainer.IpcMode(ipcMode), - // UTSMode: dockercontainer.UTSMode(utsMode), - // PidMode: dockercontainer.PidMode(pidMode), - // ReadonlyRootfs: readOnlyRootFilesystem(container), - // Resources: dockercontainer.Resources{ - // Memory: memoryLimit, - // MemorySwap: -1, - // CPUShares: cpuShares, - // Devices: devices, - // }, - // SecurityOpt: securityOpts, + Binds: binds, + NetworkMode: dockercontainer.NetworkMode(netMode), + IpcMode: dockercontainer.IpcMode(ipcMode), + UTSMode: dockercontainer.UTSMode(utsMode), + PidMode: dockercontainer.PidMode(pidMode), + ReadonlyRootfs: readOnlyRootFilesystem(container), + Resources: dockercontainer.Resources{ + Memory: memoryLimit, + MemorySwap: -1, + CPUShares: cpuShares, + Devices: devices, + }, + SecurityOpt: fmtSecurityOpts, } // Set sysctls if requested @@ -820,10 +735,10 @@ func (dm *DockerManager) runContainer( glog.V(3).Infof("Container %v/%v/%v: setting entrypoint \"%v\" and command \"%v\"", pod.Namespace, pod.Name, container.Name, dockerOpts.Config.Entrypoint, dockerOpts.Config.Cmd) - // supplementalGids := dm.runtimeHelper.GetExtraSupplementalGroupsForPod(pod) - // securityContextProvider := securitycontext.NewSimpleSecurityContextProvider() - // securityContextProvider.ModifyContainerConfig(pod, container, dockerOpts.Config) - // securityContextProvider.ModifyHostConfig(pod, container, dockerOpts.HostConfig, supplementalGids) + supplementalGids := dm.runtimeHelper.GetExtraSupplementalGroupsForPod(pod) + securityContextProvider := securitycontext.NewSimpleSecurityContextProvider() + securityContextProvider.ModifyContainerConfig(pod, container, dockerOpts.Config) + securityContextProvider.ModifyHostConfig(pod, container, dockerOpts.HostConfig, supplementalGids) createResp, err := dm.client.CreateContainer(dockerOpts) if err != nil { dm.recorder.Eventf(ref, api.EventTypeWarning, events.FailedToCreateContainer, "Failed to create docker container %q of pod %q with error: %v", container.Name, format.Pod(pod), err) @@ -999,10 +914,6 @@ func (dm *DockerManager) PullImage(image kubecontainer.ImageSpec, secrets []api. return dm.dockerPuller.Pull(image.Image, secrets) } -func (dm *WindowsDockerManager) PullImage(image kubecontainer.ImageSpec, secrets []api.Secret) error { - return fmt.Errorf("Image pull not yet supported") -} - // IsImagePresent checks whether the container image is already in the local storage. func (dm *DockerManager) IsImagePresent(image kubecontainer.ImageSpec) (bool, error) { return dm.dockerPuller.IsImagePresent(image.Image) @@ -1821,9 +1732,9 @@ func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Containe // Check if current docker version is higher than 1.10. Otherwise, we have to apply OOMScoreAdj instead of using docker API. // TODO: Remove this logic after we stop supporting docker version < 1.10. - // if err = dm.applyOOMScoreAdjIfNeeded(pod, container, containerInfo); err != nil { - // return kubecontainer.ContainerID{}, err - // } + if err = dm.applyOOMScoreAdjIfNeeded(pod, container, containerInfo); err != nil { + return kubecontainer.ContainerID{}, err + } // The addNDotsOption call appends the ndots option to the resolv.conf file generated by docker. // This resolv.conf file is shared by all containers of the same pod, and needs to be modified only once per pod. @@ -2148,116 +2059,6 @@ func (dm *DockerManager) computePodContainerChanges(pod *api.Pod, podStatus *kub }, nil } -func (dm *WindowsDockerManager) computePodContainerChanges(pod *api.Pod, podStatus *kubecontainer.PodStatus) (podContainerChangesSpec, error) { - start := time.Now() - defer func() { - metrics.ContainerManagerLatency.WithLabelValues("computePodContainerChanges").Observe(metrics.SinceInMicroseconds(start)) - }() - glog.V(5).Infof("Syncing Pod %q: %+v", format.Pod(pod), pod) - - containersToStart := make(map[int]string) - containersToKeep := make(map[kubecontainer.DockerID]int) - - // var err error - - // Skip podInfraContainer checking and creation - // var podInfraContainerID kubecontainer.DockerID - // var changed bool - // podInfraContainerStatus := podStatus.FindContainerStatusByName(PodInfraContainerName) - // if podInfraContainerStatus != nil && podInfraContainerStatus.State == kubecontainer.ContainerStateRunning { - // glog.V(4).Infof("Found pod infra container for %q", format.Pod(pod)) - // changed, err = dm.podInfraContainerChanged(pod, podInfraContainerStatus) - // if err != nil { - // return podContainerChangesSpec{}, err - // } - // } - - // createPodInfraContainer := true - // if podInfraContainerStatus == nil || podInfraContainerStatus.State != kubecontainer.ContainerStateRunning { - // glog.V(2).Infof("Need to restart pod infra container for %q because it is not found", format.Pod(pod)) - // } else if changed { - // glog.V(2).Infof("Need to restart pod infra container for %q because it is changed", format.Pod(pod)) - // } else { - // glog.V(4).Infof("Pod infra container looks good, keep it %q", format.Pod(pod)) - // createPodInfraContainer = false - // podInfraContainerID = kubecontainer.DockerID(podInfraContainerStatus.ID.ID) - // containersToKeep[podInfraContainerID] = -1 - // } - - for index, container := range pod.Spec.Containers { - expectedHash := kubecontainer.HashContainer(&container) - - containerStatus := podStatus.FindContainerStatusByName(container.Name) - if containerStatus == nil || containerStatus.State != kubecontainer.ContainerStateRunning { - if kubecontainer.ShouldContainerBeRestarted(&container, pod, podStatus) { - // If we are here it means that the container is dead and should be restarted, or never existed and should - // be created. We may be inserting this ID again if the container has changed and it has - // RestartPolicy::Always, but it's not a big deal. - message := fmt.Sprintf("Container %+v is dead, but RestartPolicy says that we should restart it.", container) - glog.V(3).Info(message) - containersToStart[index] = message - } - continue - } - - containerID := kubecontainer.DockerID(containerStatus.ID.ID) - hash := containerStatus.Hash - glog.V(3).Infof("pod %q container %q exists as %v", format.Pod(pod), container.Name, containerID) - - // if createPodInfraContainer { - // // createPodInfraContainer == true and Container exists - // // If we're creating infra container everything will be killed anyway - // // If RestartPolicy is Always or OnFailure we restart containers that were running before we - // // killed them when restarting Infra Container. - // if pod.Spec.RestartPolicy != api.RestartPolicyNever { - // message := fmt.Sprintf("Infra Container is being recreated. %q will be restarted.", container.Name) - // glog.V(1).Info(message) - // containersToStart[index] = message - // } - // continue - // } - - // At this point, the container is running and pod infra container is good. - // We will look for changes and check healthiness for the container. - containerChanged := hash != 0 && hash != expectedHash - if containerChanged { - message := fmt.Sprintf("pod %q container %q hash changed (%d vs %d), it will be killed and re-created.", format.Pod(pod), container.Name, hash, expectedHash) - glog.Info(message) - containersToStart[index] = message - continue - } - - liveness, found := dm.livenessManager.Get(containerStatus.ID) - if !found || liveness == proberesults.Success { - containersToKeep[containerID] = index - continue - } - if pod.Spec.RestartPolicy != api.RestartPolicyNever { - message := fmt.Sprintf("pod %q container %q is unhealthy, it will be killed and re-created.", format.Pod(pod), container.Name) - glog.Info(message) - containersToStart[index] = message - } - } - - // After the loop one of the following should be true: - // - createPodInfraContainer is true and containersToKeep is empty. - // (In fact, when createPodInfraContainer is false, containersToKeep will not be touched). - // - createPodInfraContainer is false and containersToKeep contains at least ID of Infra Container - - // If Infra container is the last running one, we don't want to keep it. - // if !createPodInfraContainer && len(containersToStart) == 0 && len(containersToKeep) == 1 { - // containersToKeep = make(map[kubecontainer.DockerID]int) - // } - - return podContainerChangesSpec{ - StartInfraContainer: false, - InfraChanged: false, - InfraContainerId: "", - ContainersToStart: containersToStart, - ContainersToKeep: containersToKeep, - }, nil -} - // Sync the running pod to match the specified desired pod. func (dm *DockerManager) SyncPod(pod *api.Pod, _ api.PodStatus, podStatus *kubecontainer.PodStatus, pullSecrets []api.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) { start := time.Now() @@ -2509,7 +2310,14 @@ func (dm *DockerManager) tryContainerStart(container *api.Container, pod *api.Po restartCount = containerStatus.RestartCount + 1 } - _, err = dm.runContainerInPod(pod, container, namespaceMode, namespaceMode, pidMode, podIP, restartCount) + // Allow override of networking mode for specific platforms (e.g. Windows) + netMode := getNetworkingMode() + if netMode == "" { + // If not overriden, use the namespace mode + netMode = namespaceMode + } + + _, err = dm.runContainerInPod(pod, container, netMode, namespaceMode, pidMode, podIP, restartCount) if err != nil { // TODO(bburns) : Perhaps blacklist a container after N failures? return kubecontainer.ErrRunContainer, err.Error() @@ -2600,196 +2408,6 @@ func findActiveInitContainer(pod *api.Pod, podStatus *kubecontainer.PodStatus) ( return &pod.Spec.InitContainers[0], nil, false } -// Sync the running pod to match the specified desired pod. -func (dm *WindowsDockerManager) SyncPod(pod *api.Pod, _ api.PodStatus, podStatus *kubecontainer.PodStatus, pullSecrets []api.Secret, backOff *flowcontrol.Backoff) (result kubecontainer.PodSyncResult) { - start := time.Now() - defer func() { - metrics.ContainerManagerLatency.WithLabelValues("SyncPod").Observe(metrics.SinceInMicroseconds(start)) - }() - - containerChanges, err := dm.computePodContainerChanges(pod, podStatus) - if err != nil { - result.Fail(err) - return - } - glog.V(3).Infof("Got container changes for pod %q: %+v", format.Pod(pod), containerChanges) - - // if containerChanges.InfraChanged { - // ref, err := api.GetReference(pod) - // if err != nil { - // glog.Errorf("Couldn't make a ref to pod %q: '%v'", format.Pod(pod), err) - // } - // dm.recorder.Eventf(ref, api.EventTypeNormal, "InfraChanged", "Pod infrastructure changed, it will be killed and re-created.") - // } - // if containerChanges.StartInfraContainer || (len(containerChanges.ContainersToKeep) == 0 && len(containerChanges.ContainersToStart) == 0) { - // if len(containerChanges.ContainersToKeep) == 0 && len(containerChanges.ContainersToStart) == 0 { - // glog.V(4).Infof("Killing Infra Container for %q because all other containers are dead.", format.Pod(pod)) - // } else { - // glog.V(4).Infof("Killing Infra Container for %q, will start new one", format.Pod(pod)) - // } - - // // Killing phase: if we want to start new infra container, or nothing is running kill everything (including infra container) - // // TODO(random-liu): We'll use pod status directly in the future - // killResult := dm.killPodWithSyncResult(pod, kubecontainer.ConvertPodStatusToRunningPod(podStatus)) - // result.AddPodSyncResult(killResult) - // if killResult.Error() != nil { - // return - // } - // } else { - // Otherwise kill any running containers in this pod which are not specified as ones to keep. - runningContainerStatues := podStatus.GetRunningContainerStatuses() - for _, containerStatus := range runningContainerStatues { - _, keep := containerChanges.ContainersToKeep[kubecontainer.DockerID(containerStatus.ID.ID)] - if !keep { - glog.V(3).Infof("Killing unwanted container %q(id=%q) for pod %q", containerStatus.Name, containerStatus.ID, format.Pod(pod)) - // attempt to find the appropriate container policy - var podContainer *api.Container - var killMessage string - for i, c := range pod.Spec.Containers { - if c.Name == containerStatus.Name { - podContainer = &pod.Spec.Containers[i] - killMessage = containerChanges.ContainersToStart[i] - break - } - } - killContainerResult := kubecontainer.NewSyncResult(kubecontainer.KillContainer, containerStatus.Name) - result.AddSyncResult(killContainerResult) - if err := dm.KillContainerInPod(containerStatus.ID, podContainer, pod, killMessage, nil); err != nil { - killContainerResult.Fail(kubecontainer.ErrKillContainer, err.Error()) - glog.Errorf("Error killing container %q(id=%q) for pod %q: %v", containerStatus.Name, containerStatus.ID, format.Pod(pod), err) - return - } - } - } - // } - - // We pass the value of the podIP down to runContainerInPod, which in turn - // passes it to various other functions, in order to facilitate - // functionality that requires this value (hosts file and downward API) - // and avoid races determining the pod IP in cases where a container - // requires restart but the podIP isn't in the status manager yet. - // - // We default to the IP in the passed-in pod status, and overwrite it if the - // infra container needs to be (re)started. - podIP := "" - if podStatus != nil { - podIP = podStatus.IP - } - - // If we should create infra container then we do it first. - // podInfraContainerID := containerChanges.InfraContainerId - // if containerChanges.StartInfraContainer && (len(containerChanges.ContainersToStart) > 0) { - // glog.V(4).Infof("Creating pod infra container for %q", format.Pod(pod)) - // startContainerResult := kubecontainer.NewSyncResult(kubecontainer.StartContainer, PodInfraContainerName) - // result.AddSyncResult(startContainerResult) - // var msg string - // podInfraContainerID, err, msg = dm.createPodInfraContainer(pod) - // if err != nil { - // startContainerResult.Fail(err, msg) - // glog.Errorf("Failed to create pod infra container: %v; Skipping pod %q", err, format.Pod(pod)) - // return - // } - - // setupNetworkResult := kubecontainer.NewSyncResult(kubecontainer.SetupNetwork, kubecontainer.GetPodFullName(pod)) - // result.AddSyncResult(setupNetworkResult) - // if !usesHostNetwork(pod) { - // // Call the networking plugin - // err = dm.networkPlugin.SetUpPod(pod.Namespace, pod.Name, podInfraContainerID) - // if err != nil { - // // TODO: (random-liu) There shouldn't be "Skipping pod" in sync result message - // message := fmt.Sprintf("Failed to setup network for pod %q using network plugins %q: %v; Skipping pod", format.Pod(pod), dm.networkPlugin.Name(), err) - // setupNetworkResult.Fail(kubecontainer.ErrSetupNetwork, message) - // glog.Error(message) - - // // Delete infra container - // killContainerResult := kubecontainer.NewSyncResult(kubecontainer.KillContainer, PodInfraContainerName) - // result.AddSyncResult(killContainerResult) - // if delErr := dm.KillContainerInPod(kubecontainer.ContainerID{ - // ID: string(podInfraContainerID), - // Type: "docker"}, nil, pod, message); delErr != nil { - // killContainerResult.Fail(kubecontainer.ErrKillContainer, delErr.Error()) - // glog.Warningf("Clear infra container failed for pod %q: %v", format.Pod(pod), delErr) - // } - // return - // } - - // // Setup the host interface unless the pod is on the host's network (FIXME: move to networkPlugin when ready) - // var podInfraContainer *docker.Container - // podInfraContainer, err = dm.client.InspectContainer(string(podInfraContainerID)) - // if err != nil { - // glog.Errorf("Failed to inspect pod infra container: %v; Skipping pod %q", err, format.Pod(pod)) - // result.Fail(err) - // return - // } - - // if dm.configureHairpinMode { - // if err = hairpin.SetUpContainer(podInfraContainer.State.Pid, network.DefaultInterfaceName); err != nil { - // glog.Warningf("Hairpin setup failed for pod %q: %v", format.Pod(pod), err) - // } - // } - - // // Overwrite the podIP passed in the pod status, since we just started the infra container. - // podIP = dm.determineContainerIP(pod.Name, pod.Namespace, podInfraContainer) - // } - // } - - // Start everything - for idx := range containerChanges.ContainersToStart { - container := &pod.Spec.Containers[idx] - startContainerResult := kubecontainer.NewSyncResult(kubecontainer.StartContainer, container.Name) - result.AddSyncResult(startContainerResult) - - // containerChanges.StartInfraContainer causes the containers to be restarted for config reasons - // ignore backoff - if !containerChanges.StartInfraContainer { - isInBackOff, err, msg := dm.doBackOff(pod, container, podStatus, backOff) - if isInBackOff { - startContainerResult.Fail(err, msg) - glog.V(4).Infof("Backing Off restarting container %+v in pod %v", container, format.Pod(pod)) - continue - } - } - glog.V(4).Infof("Creating container %+v in pod %v", container, format.Pod(pod)) - // err, msg := dm.imagePuller.PullImage(pod, container, pullSecrets) - // if err != nil { - // startContainerResult.Fail(err, msg) - // continue - // } - - // if container.SecurityContext != nil && container.SecurityContext.RunAsNonRoot != nil && *container.SecurityContext.RunAsNonRoot { - // err := dm.verifyNonRoot(container) - // if err != nil { - // startContainerResult.Fail(kubecontainer.ErrVerifyNonRoot, err.Error()) - // glog.Errorf("Error running pod %q container %q: %v", format.Pod(pod), container.Name, err) - // continue - // } - // } - // For a new container, the RestartCount should be 0 - restartCount := 0 - containerStatus := podStatus.FindContainerStatusByName(container.Name) - if containerStatus != nil { - restartCount = containerStatus.RestartCount + 1 - } - - // TODO(dawnchen): Check RestartPolicy.DelaySeconds before restart a container - // Note: when configuring the pod's containers anything that can be configured by pointing - // to the namespace of the infra container should use namespaceMode. This includes things like the net namespace - // and IPC namespace. PID mode cannot point to another container right now. - // See createPodInfraContainer for infra container setup. - // namespaceMode := fmt.Sprintf("container:%v", podInfraContainerID) - namespaceMode := "" - _, err = dm.runContainerInPod(pod, container, namespaceMode, namespaceMode, getPidMode(pod), podIP, restartCount) - if err != nil { - startContainerResult.Fail(kubecontainer.ErrRunContainer, err.Error()) - // TODO(bburns) : Perhaps blacklist a container after N failures? - glog.Errorf("Error running pod %q container %q: %v", format.Pod(pod), container.Name, err) - continue - } - // Successfully started the container; clear the entry in the failure - } - return -} - // verifyNonRoot returns an error if the container or image will run as the root user. func (dm *DockerManager) verifyNonRoot(container *api.Container) error { if securitycontext.HasRunAsUser(container) { @@ -2996,7 +2614,7 @@ func (dm *DockerManager) GetPodStatus(uid kubetypes.UID, name, namespace string) } } containerStatuses = append(containerStatuses, result) - if ip != "" { + if containerProvidesPodIP(dockerName) && ip != "" { podStatus.IP = ip } } diff --git a/pkg/kubelet/dockertools/docker_manager_linux.go b/pkg/kubelet/dockertools/docker_manager_linux.go new file mode 100644 index 00000000000..2404cc7bbcd --- /dev/null +++ b/pkg/kubelet/dockertools/docker_manager_linux.go @@ -0,0 +1,24 @@ +package dockertools + +import dockertypes "github.com/docker/engine-api/types" + +func getContainerIP(container *dockertypes.ContainerJSON) string { + result := "" + if container.NetworkSettings != nil { + result = container.NetworkSettings.IPAddress + + // Fall back to IPv6 address if no IPv4 address is present + if result == "" { + result = container.NetworkSettings.GlobalIPv6Address + } + } + return result +} + +// We don't want to override the networking mode on Linux. +func getNetworkingMode() string { return "" } + +// Returns true if the container name matches the infrastructure's container name +func containerProvidesPodIP(name *KubeletContainerName) bool { + return name.ContainerName == PodInfraContainerName +} diff --git a/pkg/kubelet/dockertools/docker_manager_windows.go b/pkg/kubelet/dockertools/docker_manager_windows.go new file mode 100644 index 00000000000..89d2d460cf3 --- /dev/null +++ b/pkg/kubelet/dockertools/docker_manager_windows.go @@ -0,0 +1,33 @@ +package dockertools + +import ( + "os" + + dockertypes "github.com/docker/engine-api/types" +) + +func getContainerIP(container *dockertypes.ContainerJSON) string { + if container.NetworkSettings != nil { + for _, network := range container.NetworkSettings.Networks { + if network.IPAddress != "" { + return network.IPAddress + } + } + } + return "" +} + +func getNetworkingMode() string { + // Allow override via env variable. Otherwise, use a default "kubenet" network + netMode := os.Getenv("CONTAINER_NETWORK") + if netMode == "" { + netMode = "kubenet" + } + return netMode +} + +// Infrastructure containers are not supported on Windows. For this reason, we +// make sure to not grab the infra container's IP for the pod. +func containerProvidesPodIP(name *KubeletContainerName) bool { + return name.ContainerName != PodInfraContainerName +} diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index d0c7d6068fa..8574a0ee5c9 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -25,6 +25,7 @@ import ( "os" "path" "path/filepath" + "runtime" "sort" "strings" @@ -261,16 +262,18 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *api.Pod, container *api.Cont } } - opts.Mounts, err = makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes) - if err != nil { - return nil, err + if runtime.GOOS != "windows" { + opts.Mounts, err = makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes) + if err != nil { + return nil, err + } } opts.Envs, err = kl.makeEnvironmentVariables(pod, container, podIP) if err != nil { return nil, err } - if len(container.TerminationMessagePath) != 0 { + if len(container.TerminationMessagePath) != 0 && runtime.GOOS != "windows" { p := kl.getPodContainerDir(pod.UID, container.Name) if err := os.MkdirAll(p, 0750); err != nil { glog.Errorf("Error on creating %q: %v", p, err) diff --git a/pkg/volume/volume.go b/pkg/volume/volume.go index 93db6169cb9..09f4186931c 100644 --- a/pkg/volume/volume.go +++ b/pkg/volume/volume.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os" "path" + "runtime" "time" "k8s.io/kubernetes/pkg/api" @@ -214,15 +215,22 @@ func RenameDirectory(oldPath, newName string) (string, error) { if err != nil { return "", err } + // os.Rename call fails on windows (https://github.com/golang/go/issues/14527) // Replacing with copyFolder to the newPath and deleting the oldPath directory - // err = os.Rename(oldPath, newPath) - err = copyFolder(oldPath, newPath) + if runtime.GOOS == "windows" { + err = copyFolder(oldPath, newPath) + if err != nil { + return "", err + } + os.RemoveAll(oldPath) + return newPath, nil + } + err = os.Rename(oldPath, newPath) if err != nil { return "", err } - os.RemoveAll(oldPath) return newPath, nil } From 9ef35289e4dbb355dae36f3c67f86e388703abac Mon Sep 17 00:00:00 2001 From: Alexander Brand Date: Tue, 25 Oct 2016 11:49:05 -0400 Subject: [PATCH 3/7] Restore check when getting container IP --- pkg/kubelet/dockertools/docker_manager.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/kubelet/dockertools/docker_manager.go b/pkg/kubelet/dockertools/docker_manager.go index c91b9c09b58..a3e52bff3aa 100644 --- a/pkg/kubelet/dockertools/docker_manager.go +++ b/pkg/kubelet/dockertools/docker_manager.go @@ -422,11 +422,13 @@ func (dm *DockerManager) inspectContainer(id string, podName, podNamespace strin // Container that are running, restarting and paused status.State = kubecontainer.ContainerStateRunning status.StartedAt = startedAt - ip, err = dm.determineContainerIP(podNamespace, podName, iResult) - // Kubelet doesn't handle the network error scenario - if err != nil { - status.State = kubecontainer.ContainerStateUnknown - status.Message = fmt.Sprintf("Network error: %#v", err) + if containerProvidesPodIP(dockerName) { + ip, err = dm.determineContainerIP(podNamespace, podName, iResult) + // Kubelet doesn't handle the network error scenario + if err != nil { + status.State = kubecontainer.ContainerStateUnknown + status.Message = fmt.Sprintf("Network error: %#v", err) + } } return &status, ip, nil } From a659ac99b652b41ab37db65613c8c58364222c2e Mon Sep 17 00:00:00 2001 From: Jitendra Bhurat Date: Thu, 13 Oct 2016 12:01:15 -0400 Subject: [PATCH 4/7] Moved getSecurityOpts() function to OS specific manager variant --- pkg/kubelet/dockertools/docker_manager.go | 17 -------------- .../dockertools/docker_manager_linux.go | 23 ++++++++++++++++++- .../dockertools/docker_manager_windows.go | 7 ++++++ 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/pkg/kubelet/dockertools/docker_manager.go b/pkg/kubelet/dockertools/docker_manager.go index a3e52bff3aa..b7210b939b2 100644 --- a/pkg/kubelet/dockertools/docker_manager.go +++ b/pkg/kubelet/dockertools/docker_manager.go @@ -1113,23 +1113,6 @@ func (dm *DockerManager) fmtDockerOpts(opts []dockerOpt) ([]string, error) { return fmtOpts, nil } -func (dm *DockerManager) getSecurityOpts(pod *api.Pod, ctrName string) ([]dockerOpt, error) { - var securityOpts []dockerOpt - if seccompOpts, err := dm.getSeccompOpts(pod, ctrName); err != nil { - return nil, err - } else { - securityOpts = append(securityOpts, seccompOpts...) - } - - if appArmorOpts, err := dm.getAppArmorOpts(pod, ctrName); err != nil { - return nil, err - } else { - securityOpts = append(securityOpts, appArmorOpts...) - } - - return securityOpts, nil -} - type dockerOpt struct { // The key-value pair passed to docker. key, value string diff --git a/pkg/kubelet/dockertools/docker_manager_linux.go b/pkg/kubelet/dockertools/docker_manager_linux.go index 2404cc7bbcd..3d19e94d35a 100644 --- a/pkg/kubelet/dockertools/docker_manager_linux.go +++ b/pkg/kubelet/dockertools/docker_manager_linux.go @@ -1,6 +1,9 @@ package dockertools -import dockertypes "github.com/docker/engine-api/types" +import ( + dockertypes "github.com/docker/engine-api/types" + "k8s.io/kubernetes/pkg/api" +) func getContainerIP(container *dockertypes.ContainerJSON) string { result := "" @@ -22,3 +25,21 @@ func getNetworkingMode() string { return "" } func containerProvidesPodIP(name *KubeletContainerName) bool { return name.ContainerName == PodInfraContainerName } + +// Returns Seccomp and AppArmor Security options +func (dm *DockerManager) getSecurityOpts(pod *api.Pod, ctrName string) ([]dockerOpt, error) { + var securityOpts []dockerOpt + if seccompOpts, err := dm.getSeccompOpts(pod, ctrName); err != nil { + return nil, err + } else { + securityOpts = append(securityOpts, seccompOpts...) + } + + if appArmorOpts, err := dm.getAppArmorOpts(pod, ctrName); err != nil { + return nil, err + } else { + securityOpts = append(securityOpts, appArmorOpts...) + } + + return securityOpts, nil +} diff --git a/pkg/kubelet/dockertools/docker_manager_windows.go b/pkg/kubelet/dockertools/docker_manager_windows.go index 89d2d460cf3..da8b8293816 100644 --- a/pkg/kubelet/dockertools/docker_manager_windows.go +++ b/pkg/kubelet/dockertools/docker_manager_windows.go @@ -3,6 +3,8 @@ package dockertools import ( "os" + "k8s.io/kubernetes/pkg/api" + dockertypes "github.com/docker/engine-api/types" ) @@ -31,3 +33,8 @@ func getNetworkingMode() string { func containerProvidesPodIP(name *KubeletContainerName) bool { return name.ContainerName != PodInfraContainerName } + +// Returns nil as both Seccomp and AppArmor security options are not valid on Windows +func (dm *DockerManager) getSecurityOpts(pod *api.Pod, ctrName string) ([]dockerOpt, error) { + return nil, nil +} From 66a1ef25e07650fc9358e639b0fcccf200418624 Mon Sep 17 00:00:00 2001 From: Jitendra Bhurat Date: Tue, 1 Nov 2016 10:10:59 -0400 Subject: [PATCH 5/7] Fixing Volumes on Windows --- pkg/api/validation/validation.go | 2 -- pkg/kubelet/dockertools/docker_manager.go | 13 +++++++--- pkg/kubelet/kubelet_pods.go | 27 +++++++++++++++------ pkg/volume/secret/secret.go | 16 +++++++++++-- pkg/volume/util/atomic_writer.go | 16 +++++++++++-- pkg/volume/volume.go | 29 +++++++++++++++++++---- 6 files changed, 83 insertions(+), 20 deletions(-) diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index 871ff7b22b7..412b5970607 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -1463,8 +1463,6 @@ func validateVolumeMounts(mounts []api.VolumeMount, volumes sets.String, fldPath } if len(mnt.MountPath) == 0 { allErrs = append(allErrs, field.Required(idxPath.Child("mountPath"), "")) - } else if strings.Contains(mnt.MountPath, ":") { - allErrs = append(allErrs, field.Invalid(idxPath.Child("mountPath"), mnt.MountPath, "must not contain ':'")) } if mountpoints.Has(mnt.MountPath) { allErrs = append(allErrs, field.Invalid(idxPath.Child("mountPath"), mnt.MountPath, "must be unique")) diff --git a/pkg/kubelet/dockertools/docker_manager.go b/pkg/kubelet/dockertools/docker_manager.go index b7210b939b2..31540f8ef8b 100644 --- a/pkg/kubelet/dockertools/docker_manager.go +++ b/pkg/kubelet/dockertools/docker_manager.go @@ -28,6 +28,7 @@ import ( "os/exec" "path" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -57,7 +58,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/kubelet/util/cache" "k8s.io/kubernetes/pkg/kubelet/util/format" - "k8s.io/kubernetes/pkg/runtime" + kruntime "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/security/apparmor" "k8s.io/kubernetes/pkg/securitycontext" kubetypes "k8s.io/kubernetes/pkg/types" @@ -606,7 +607,7 @@ func (dm *DockerManager) runContainer( // TODO: This is kind of hacky, we should really just encode the bits we need. // TODO: This is hacky because the Kubelet should be parameterized to encode a specific version // and needs to be able to migrate this whenever we deprecate v1. Should be a member of DockerManager. - if data, err := runtime.Encode(api.Codecs.LegacyCodec(unversioned.GroupVersion{Group: api.GroupName, Version: "v1"}), pod); err == nil { + if data, err := kruntime.Encode(api.Codecs.LegacyCodec(unversioned.GroupVersion{Group: api.GroupName, Version: "v1"}), pod); err == nil { labels[kubernetesPodLabel] = string(data) } else { glog.Errorf("Failed to encode pod: %s for prestop hook", pod.Name) @@ -677,6 +678,12 @@ func (dm *DockerManager) runContainer( SecurityOpt: fmtSecurityOpts, } + // There is no /etc/resolv.conf in Windows, DNS and DNSSearch options would have to be passed to Docker runtime instead + if runtime.GOOS == "windows" { + hc.DNS = opts.DNS + hc.DNSSearch = opts.DNSSearch + } + // Set sysctls if requested sysctls, unsafeSysctls, err := api.SysctlsFromPodAnnotations(pod.Annotations) if err != nil { @@ -1586,7 +1593,7 @@ func containerAndPodFromLabels(inspect *dockertypes.ContainerJSON) (pod *api.Pod // the pod data may not be set if body, found := labels[kubernetesPodLabel]; found { pod = &api.Pod{} - if err = runtime.DecodeInto(api.Codecs.UniversalDecoder(), []byte(body), pod); err == nil { + if err = kruntime.DecodeInto(api.Codecs.UniversalDecoder(), []byte(body), pod); err == nil { name := labels[types.KubernetesContainerNameLabel] for ix := range pod.Spec.Containers { if pod.Spec.Containers[ix].Name == name { diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index 8574a0ee5c9..de9d2e2dfcb 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -82,7 +82,8 @@ func makeMounts(pod *api.Pod, podDir string, container *api.Container, hostName, // - container is not an infrastructure(pause) container // - container is not already mounting on /etc/hosts // When the pause container is being created, its IP is still unknown. Hence, PodIP will not have been set. - mountEtcHostsFile := (pod.Spec.SecurityContext == nil || !pod.Spec.SecurityContext.HostNetwork) && len(podIP) > 0 + // OS is not Windows + mountEtcHostsFile := (pod.Spec.SecurityContext == nil || !pod.Spec.SecurityContext.HostNetwork) && len(podIP) > 0 && runtime.GOOS != "windows" glog.V(3).Infof("container: %v/%v/%v podIP: %q creating hosts mount: %v", pod.Namespace, pod.Name, container.Name, podIP, mountEtcHostsFile) mounts := []kubecontainer.Mount{} for _, mount := range container.VolumeMounts { @@ -108,9 +109,21 @@ func makeMounts(pod *api.Pod, podDir string, container *api.Container, hostName, if mount.SubPath != "" { hostPath = filepath.Join(hostPath, mount.SubPath) } + + // Docker Volume Mounts fail on Windows if it is not of the form C:/ + containerPath := mount.MountPath + if runtime.GOOS == "windows" { + if strings.HasPrefix(hostPath, "/") && !strings.Contains(hostPath, ":") { + hostPath = "c:" + hostPath + } + if strings.HasPrefix(containerPath, "/") && !strings.Contains(containerPath, ":") { + containerPath = "c:" + containerPath + } + } + mounts = append(mounts, kubecontainer.Mount{ Name: mount.Name, - ContainerPath: mount.MountPath, + ContainerPath: containerPath, HostPath: hostPath, ReadOnly: mount.ReadOnly, SELinuxRelabel: relabelVolume, @@ -262,17 +275,17 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *api.Pod, container *api.Cont } } - if runtime.GOOS != "windows" { - opts.Mounts, err = makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes) - if err != nil { - return nil, err - } + opts.Mounts, err = makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes) + if err != nil { + return nil, err } opts.Envs, err = kl.makeEnvironmentVariables(pod, container, podIP) if err != nil { return nil, err } + // Disabling adding TerminationMessagePath on Windows as these files would be mounted as docker volume and + // Docker for Windows has a bug where only directories can be mounted if len(container.TerminationMessagePath) != 0 && runtime.GOOS != "windows" { p := kl.getPodContainerDir(pod.UID, container.Name) if err := os.MkdirAll(p, 0750); err != nil { diff --git a/pkg/volume/secret/secret.go b/pkg/volume/secret/secret.go index 116d519bce6..cbb7b39d756 100644 --- a/pkg/volume/secret/secret.go +++ b/pkg/volume/secret/secret.go @@ -18,6 +18,8 @@ package secret import ( "fmt" + "path/filepath" + "runtime" "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" @@ -157,7 +159,12 @@ func (sv *secretVolume) GetAttributes() volume.Attributes { } } func (b *secretVolumeMounter) SetUp(fsGroup *int64) error { - return b.SetUpAt(b.GetPath(), fsGroup) + // Update each Slash "/" character for Windows with seperator character + dir := b.GetPath() + if runtime.GOOS == "windows" { + dir = filepath.FromSlash(dir) + } + return b.SetUpAt(dir, fsGroup) } func (b *secretVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { @@ -269,7 +276,12 @@ type secretVolumeUnmounter struct { var _ volume.Unmounter = &secretVolumeUnmounter{} func (c *secretVolumeUnmounter) TearDown() error { - return c.TearDownAt(c.GetPath()) + // Update each Slash "/" character for Windows with seperator character + dir := c.GetPath() + if runtime.GOOS == "windows" { + dir = filepath.FromSlash(dir) + } + return c.TearDownAt(dir) } func (c *secretVolumeUnmounter) TearDownAt(dir string) error { diff --git a/pkg/volume/util/atomic_writer.go b/pkg/volume/util/atomic_writer.go index 8e2dad20eab..8b84710fc31 100644 --- a/pkg/volume/util/atomic_writer.go +++ b/pkg/volume/util/atomic_writer.go @@ -23,6 +23,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strings" "time" @@ -183,7 +184,14 @@ func (w *AtomicWriter) Write(payload map[string]FileProjection) error { } // (9) - if err = os.Rename(newDataDirPath, dataDirPath); err != nil { + if runtime.GOOS == "windows" { + os.Remove(dataDirPath) + err = os.Symlink(tsDirName, dataDirPath) + os.Remove(newDataDirPath) + } else { + err = os.Rename(newDataDirPath, dataDirPath) + } + if err != nil { os.Remove(newDataDirPath) os.RemoveAll(tsDir) glog.Errorf("%s: error renaming symbolic link for data directory %s: %v", w.logContext, newDataDirPath, err) @@ -303,7 +311,11 @@ func (w *AtomicWriter) pathsToRemove(payload map[string]FileProjection) (sets.St } relativePath := strings.TrimPrefix(path, w.targetDir) - relativePath = strings.TrimPrefix(relativePath, "/") + if runtime.GOOS == "windows" { + relativePath = strings.TrimPrefix(relativePath, "\\") + } else { + relativePath = strings.TrimPrefix(relativePath, "/") + } if strings.HasPrefix(relativePath, "..") { return nil } diff --git a/pkg/volume/volume.go b/pkg/volume/volume.go index 09f4186931c..4f4ccd62ff1 100644 --- a/pkg/volume/volume.go +++ b/pkg/volume/volume.go @@ -20,10 +20,11 @@ import ( "io" "io/ioutil" "os" - "path" + filepath "path/filepath" "runtime" "time" + "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/types" @@ -211,7 +212,7 @@ func (err deletedVolumeInUseError) Error() string { } func RenameDirectory(oldPath, newName string) (string, error) { - newPath, err := ioutil.TempDir(path.Dir(oldPath), newName) + newPath, err := ioutil.TempDir(filepath.Dir(oldPath), newName) if err != nil { return "", err } @@ -221,6 +222,7 @@ func RenameDirectory(oldPath, newName string) (string, error) { if runtime.GOOS == "windows" { err = copyFolder(oldPath, newPath) if err != nil { + glog.Errorf("Error copying folder from: %s to: %s with error: %v", oldPath, newPath, err) return "", err } os.RemoveAll(oldPath) @@ -235,12 +237,31 @@ func RenameDirectory(oldPath, newName string) (string, error) { } func copyFolder(source string, dest string) (err error) { + fi, err := os.Lstat(source) + if err != nil { + glog.Errorf("Error getting stats for %s. %v", source, err) + return err + } + + err = os.MkdirAll(dest, fi.Mode()) + if err != nil { + glog.Errorf("Unable to create %s directory %v", dest, err) + } + directory, _ := os.Open(source) + + defer directory.Close() + objects, err := directory.Readdir(-1) for _, obj := range objects { - sourcefilepointer := source + "/" + obj.Name() - destinationfilepointer := dest + "/" + obj.Name() + if obj.Mode()&os.ModeSymlink != 0 { + continue + } + + sourcefilepointer := source + "\\" + obj.Name() + destinationfilepointer := dest + "\\" + obj.Name() + if obj.IsDir() { err = copyFolder(sourcefilepointer, destinationfilepointer) if err != nil { From 9e6815e7c760209c816f02fcb5f816d9405d19b1 Mon Sep 17 00:00:00 2001 From: Paulo Pires Date: Tue, 1 Nov 2016 16:01:27 -0400 Subject: [PATCH 6/7] Fixed kubelet build. --- pkg/kubelet/cadvisor/cadvisor_stub.go | 74 ------------------ pkg/kubelet/cadvisor/cadvisor_unsupported.go | 2 +- pkg/kubelet/cadvisor/cadvisor_windows.go | 75 +++++++++++++++++++ .../cm/container_manager_unsupported.go | 64 ++++++++++++++++ .../cm/container_manager_unsupported_test.go | 2 +- pkg/kubelet/cm/container_manager_windows.go | 20 ++++- pkg/kubelet/dockertools/BUILD | 1 + .../dockertools/docker_manager_linux.go | 18 +++++ .../dockertools/docker_manager_unsupported.go | 42 +++++++++++ .../dockertools/docker_manager_windows.go | 18 +++++ 10 files changed, 238 insertions(+), 78 deletions(-) delete mode 100644 pkg/kubelet/cadvisor/cadvisor_stub.go create mode 100644 pkg/kubelet/cadvisor/cadvisor_windows.go create mode 100644 pkg/kubelet/cm/container_manager_unsupported.go create mode 100644 pkg/kubelet/dockertools/docker_manager_unsupported.go diff --git a/pkg/kubelet/cadvisor/cadvisor_stub.go b/pkg/kubelet/cadvisor/cadvisor_stub.go deleted file mode 100644 index f2cb6e3e73f..00000000000 --- a/pkg/kubelet/cadvisor/cadvisor_stub.go +++ /dev/null @@ -1,74 +0,0 @@ -// +build darwin windows - -/* -Copyright 2015 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cadvisor - -import ( - "github.com/google/cadvisor/events" - cadvisorapi "github.com/google/cadvisor/info/v1" - cadvisorapiv2 "github.com/google/cadvisor/info/v2" -) - -type cadvisorStub struct { -} - -var _ Interface = new(cadvisorStub) - -func New(port uint, runtime string) (Interface, error) { - return &cadvisorStub{}, nil -} - -func (cu *cadvisorStub) Start() error { - return nil -} - -func (cu *cadvisorStub) DockerContainer(name string, req *cadvisorapi.ContainerInfoRequest) (cadvisorapi.ContainerInfo, error) { - return cadvisorapi.ContainerInfo{}, nil -} - -func (cu *cadvisorStub) ContainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) { - return &cadvisorapi.ContainerInfo{}, nil -} - -func (cu *cadvisorStub) ContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) { - return make(map[string]cadvisorapiv2.ContainerInfo), nil -} - -func (cu *cadvisorStub) SubcontainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (map[string]*cadvisorapi.ContainerInfo, error) { - return nil, nil -} - -func (cu *cadvisorStub) MachineInfo() (*cadvisorapi.MachineInfo, error) { - return &cadvisorapi.MachineInfo{}, nil -} - -func (cu *cadvisorStub) VersionInfo() (*cadvisorapi.VersionInfo, error) { - return &cadvisorapi.VersionInfo{}, nil -} - -func (cu *cadvisorStub) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) { - return cadvisorapiv2.FsInfo{}, nil -} - -func (cu *cadvisorStub) RootFsInfo() (cadvisorapiv2.FsInfo, error) { - return cadvisorapiv2.FsInfo{}, nil -} - -func (cu *cadvisorStub) WatchEvents(request *events.Request) (*events.EventChannel, error) { - return &events.EventChannel{}, nil -} diff --git a/pkg/kubelet/cadvisor/cadvisor_unsupported.go b/pkg/kubelet/cadvisor/cadvisor_unsupported.go index fc3b3166870..dc7b9fbd4d6 100644 --- a/pkg/kubelet/cadvisor/cadvisor_unsupported.go +++ b/pkg/kubelet/cadvisor/cadvisor_unsupported.go @@ -1,4 +1,4 @@ -// +build !cgo !linux +// +build !linux,!windows /* Copyright 2015 The Kubernetes Authors. diff --git a/pkg/kubelet/cadvisor/cadvisor_windows.go b/pkg/kubelet/cadvisor/cadvisor_windows.go new file mode 100644 index 00000000000..a970f096d58 --- /dev/null +++ b/pkg/kubelet/cadvisor/cadvisor_windows.go @@ -0,0 +1,75 @@ +// +build windows + +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cadvisor + +import ( + "github.com/google/cadvisor/events" + cadvisorapi "github.com/google/cadvisor/info/v1" + cadvisorapiv2 "github.com/google/cadvisor/info/v2" +) + +type cadvisorClient struct { +} + +var _ Interface = new(cadvisorClient) + +// New creates a cAdvisor and exports its API on the specified port if port > 0. +func New(port uint, runtime string, rootPath string) (Interface, error) { + return &cadvisorClient{}, nil +} + +func (cu *cadvisorClient) Start() error { + return nil +} + +func (cu *cadvisorClient) DockerContainer(name string, req *cadvisorapi.ContainerInfoRequest) (cadvisorapi.ContainerInfo, error) { + return cadvisorapi.ContainerInfo{}, nil +} + +func (cu *cadvisorClient) ContainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (*cadvisorapi.ContainerInfo, error) { + return &cadvisorapi.ContainerInfo{}, nil +} + +func (cu *cadvisorClient) ContainerInfoV2(name string, options cadvisorapiv2.RequestOptions) (map[string]cadvisorapiv2.ContainerInfo, error) { + return make(map[string]cadvisorapiv2.ContainerInfo), nil +} + +func (cu *cadvisorClient) SubcontainerInfo(name string, req *cadvisorapi.ContainerInfoRequest) (map[string]*cadvisorapi.ContainerInfo, error) { + return nil, nil +} + +func (cu *cadvisorClient) MachineInfo() (*cadvisorapi.MachineInfo, error) { + return &cadvisorapi.MachineInfo{}, nil +} + +func (cu *cadvisorClient) VersionInfo() (*cadvisorapi.VersionInfo, error) { + return &cadvisorapi.VersionInfo{}, nil +} + +func (cu *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) { + return cadvisorapiv2.FsInfo{}, nil +} + +func (cu *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) { + return cadvisorapiv2.FsInfo{}, nil +} + +func (cu *cadvisorClient) WatchEvents(request *events.Request) (*events.EventChannel, error) { + return &events.EventChannel{}, nil +} diff --git a/pkg/kubelet/cm/container_manager_unsupported.go b/pkg/kubelet/cm/container_manager_unsupported.go new file mode 100644 index 00000000000..0db75bd2ada --- /dev/null +++ b/pkg/kubelet/cm/container_manager_unsupported.go @@ -0,0 +1,64 @@ +// +build !linux,!windows + +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cm + +import ( + "fmt" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/kubelet/cadvisor" + "k8s.io/kubernetes/pkg/util/mount" +) + +type unsupportedContainerManager struct { +} + +var _ ContainerManager = &unsupportedContainerManager{} + +func (unsupportedContainerManager) Start(_ *api.Node) error { + return fmt.Errorf("Container Manager is unsupported in this build") +} + +func (unsupportedContainerManager) SystemCgroupsLimit() api.ResourceList { + return api.ResourceList{} +} + +func (unsupportedContainerManager) GetNodeConfig() NodeConfig { + return NodeConfig{} +} + +func (unsupportedContainerManager) GetMountedSubsystems() *CgroupSubsystems { + return &CgroupSubsystems{} +} + +func (unsupportedContainerManager) GetQOSContainersInfo() QOSContainersInfo { + return QOSContainersInfo{} +} + +func (cm *unsupportedContainerManager) Status() Status { + return Status{} +} + +func (cm *unsupportedContainerManager) NewPodContainerManager() PodContainerManager { + return &unsupportedPodContainerManager{} +} + +func NewContainerManager(_ mount.Interface, _ cadvisor.Interface, _ NodeConfig) (ContainerManager, error) { + return &unsupportedContainerManager{}, nil +} diff --git a/pkg/kubelet/cm/container_manager_unsupported_test.go b/pkg/kubelet/cm/container_manager_unsupported_test.go index 9e09d32a895..4c4e5e25d8d 100644 --- a/pkg/kubelet/cm/container_manager_unsupported_test.go +++ b/pkg/kubelet/cm/container_manager_unsupported_test.go @@ -1,4 +1,4 @@ -// +build !linux +// +build !linux,!windows /* Copyright 2015 The Kubernetes Authors. diff --git a/pkg/kubelet/cm/container_manager_windows.go b/pkg/kubelet/cm/container_manager_windows.go index 8b72179aa76..be257e8a474 100644 --- a/pkg/kubelet/cm/container_manager_windows.go +++ b/pkg/kubelet/cm/container_manager_windows.go @@ -1,3 +1,5 @@ +// +build windows + /* Copyright 2015 The Kubernetes Authors. @@ -17,10 +19,24 @@ limitations under the License. package cm import ( + "github.com/golang/glog" + + "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/util/mount" ) -func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.Interface, nodeConfig NodeConfig) (ContainerManager, error) { - return NewStubContainerManager(), nil +type containerManagerImpl struct { + containerManagerStub +} + +var _ ContainerManager = &containerManagerImpl{} + +func (cm *containerManagerImpl) Start(_ *api.Node) error { + glog.V(2).Infof("Starting Windows stub container manager") + return nil +} + +func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.Interface, nodeConfig NodeConfig) (ContainerManager, error) { + return &containerManagerImpl{}, nil } diff --git a/pkg/kubelet/dockertools/BUILD b/pkg/kubelet/dockertools/BUILD index 690da58bb2b..7b919595f7b 100644 --- a/pkg/kubelet/dockertools/BUILD +++ b/pkg/kubelet/dockertools/BUILD @@ -17,6 +17,7 @@ go_library( "convert.go", "docker.go", "docker_manager.go", + "docker_manager_linux.go", "exec.go", "fake_docker_client.go", "fake_manager.go", diff --git a/pkg/kubelet/dockertools/docker_manager_linux.go b/pkg/kubelet/dockertools/docker_manager_linux.go index 3d19e94d35a..2a520468578 100644 --- a/pkg/kubelet/dockertools/docker_manager_linux.go +++ b/pkg/kubelet/dockertools/docker_manager_linux.go @@ -1,3 +1,21 @@ +// +build linux + +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package dockertools import ( diff --git a/pkg/kubelet/dockertools/docker_manager_unsupported.go b/pkg/kubelet/dockertools/docker_manager_unsupported.go new file mode 100644 index 00000000000..8182784a161 --- /dev/null +++ b/pkg/kubelet/dockertools/docker_manager_unsupported.go @@ -0,0 +1,42 @@ +// +build !linux,!windows + +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dockertools + +import ( + "k8s.io/kubernetes/pkg/api" + + dockertypes "github.com/docker/engine-api/types" +) + +func getContainerIP(container *dockertypes.ContainerJSON) string { + return "" +} + +func getNetworkingMode() string { + return "" +} + +func containerProvidesPodIP(name *KubeletContainerName) bool { + return false +} + +// Returns nil as both Seccomp and AppArmor security options are not valid on Windows +func (dm *DockerManager) getSecurityOpts(pod *api.Pod, ctrName string) ([]dockerOpt, error) { + return nil, nil +} diff --git a/pkg/kubelet/dockertools/docker_manager_windows.go b/pkg/kubelet/dockertools/docker_manager_windows.go index da8b8293816..cb32cc7aa72 100644 --- a/pkg/kubelet/dockertools/docker_manager_windows.go +++ b/pkg/kubelet/dockertools/docker_manager_windows.go @@ -1,3 +1,21 @@ +// +build windows + +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package dockertools import ( From 6daab2628000c65229f5617bfcb6f04feded5e03 Mon Sep 17 00:00:00 2001 From: Paulo Pires Date: Tue, 1 Nov 2016 17:45:08 -0400 Subject: [PATCH 7/7] Volume mount with colon in path is now allowed because it's how it works on Windows. --- pkg/api/validation/validation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/validation/validation_test.go b/pkg/api/validation/validation_test.go index 5630526b7be..1c270bd529b 100644 --- a/pkg/api/validation/validation_test.go +++ b/pkg/api/validation/validation_test.go @@ -2611,6 +2611,7 @@ func TestValidateVolumeMounts(t *testing.T) { {Name: "abc-123", MountPath: "/bab", SubPath: "baz"}, {Name: "abc-123", MountPath: "/bac", SubPath: ".baz"}, {Name: "abc-123", MountPath: "/bad", SubPath: "..baz"}, + {Name: "abc", MountPath: "c:/foo/bar"}, } if errs := validateVolumeMounts(successCase, volumes, field.NewPath("field")); len(errs) != 0 { t.Errorf("expected success: %v", errs) @@ -2620,7 +2621,6 @@ func TestValidateVolumeMounts(t *testing.T) { "empty name": {{Name: "", MountPath: "/foo"}}, "name not found": {{Name: "", MountPath: "/foo"}}, "empty mountpath": {{Name: "abc", MountPath: ""}}, - "colon mountpath": {{Name: "abc", MountPath: "foo:bar"}}, "mountpath collision": {{Name: "foo", MountPath: "/path/a"}, {Name: "bar", MountPath: "/path/a"}}, "absolute subpath": {{Name: "abc", MountPath: "/bar", SubPath: "/baz"}}, "subpath in ..": {{Name: "abc", MountPath: "/bar", SubPath: "../baz"}},