From 5e3d2b91385527f26edc44c98b3faaf38786949c Mon Sep 17 00:00:00 2001 From: CJ Cullen Date: Fri, 8 May 2015 11:47:33 -0700 Subject: [PATCH] Kubelet configure cbr0 instead of configure-vm.sh --- cluster/gce/configure-vm.sh | 10 +--- pkg/cloudprovider/gce/gce.go | 44 ++++------------- pkg/kubelet/container_bridge.go | 83 +++++++++++++++++++++++++++++++++ pkg/kubelet/kubelet.go | 21 +++++++++ 4 files changed, 114 insertions(+), 44 deletions(-) create mode 100644 pkg/kubelet/container_bridge.go diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 9fdb9c1e4d5..e54d6f23f76 100644 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -64,14 +64,6 @@ for k,v in yaml.load(sys.stdin).iteritems(): else KUBERNETES_MASTER="true" fi - - if [[ "${KUBERNETES_MASTER}" != "true" ]] && [[ -z "${MINION_IP_RANGE:-}" ]]; then - # This block of code should go away once the master can allocate CIDRs - until MINION_IP_RANGE=$(curl-metadata node-ip-range); do - echo 'Waiting for metadata MINION_IP_RANGE...' - sleep 3 - done - fi } function remove-docker-artifacts() { @@ -393,7 +385,7 @@ function salt-node-role() { grains: roles: - kubernetes-pool - cbr-cidr: '$(echo "$MINION_IP_RANGE" | sed -e "s/'/''/g")' + cbr-cidr: 10.123.45.0/30 cloud: gce EOF } diff --git a/pkg/cloudprovider/gce/gce.go b/pkg/cloudprovider/gce/gce.go index 14e60ca453c..15d3c81a789 100644 --- a/pkg/cloudprovider/gce/gce.go +++ b/pkg/cloudprovider/gce/gce.go @@ -17,7 +17,6 @@ limitations under the License. package gce_cloud import ( - "errors" "fmt" "io" "io/ioutil" @@ -43,10 +42,6 @@ import ( "google.golang.org/cloud/compute/metadata" ) -var ErrMetadataConflict = errors.New("Metadata already set at the same key") - -const podCIDRMetadataKey string = "node-ip-range" - // GCECloud is an implementation of Interface, TCPLoadBalancer and Instances for Google Compute Engine. type GCECloud struct { service *compute.Service @@ -562,44 +557,23 @@ func getMetadataValue(metadata *compute.Metadata, key string) (string, bool) { func (gce *GCECloud) Configure(name string, spec *api.NodeSpec) error { instanceName := canonicalizeInstanceName(name) - instance, err := gce.service.Instances.Get(gce.projectID, gce.zone, instanceName).Do() - if err != nil { - return err - } - if currentValue, ok := getMetadataValue(instance.Metadata, podCIDRMetadataKey); ok { - if currentValue == spec.PodCIDR { - // IP range already set to proper value. - return nil - } - return ErrMetadataConflict - } - // We are setting the metadata, so they can be picked-up by the configure-vm.sh script to start docker with the given CIDR for Pods. - instance.Metadata.Items = append(instance.Metadata.Items, - &compute.MetadataItems{ - Key: podCIDRMetadataKey, - Value: spec.PodCIDR, - }) - setMetadataCall := gce.service.Instances.SetMetadata(gce.projectID, gce.zone, instanceName, instance.Metadata) - setMetadataOp, err := setMetadataCall.Do() - if err != nil { - return err - } - err = gce.waitForZoneOp(setMetadataOp) - if err != nil { - return err - } - insertCall := gce.service.Routes.Insert(gce.projectID, &compute.Route{ + insertOp, err := gce.service.Routes.Insert(gce.projectID, &compute.Route{ Name: instanceName, DestRange: spec.PodCIDR, NextHopInstance: fmt.Sprintf("zones/%s/instances/%s", gce.zone, instanceName), Network: fmt.Sprintf("global/networks/%s", gce.networkName), Priority: 1000, - }) - insertOp, err := insertCall.Do() + }).Do() if err != nil { return err } - return gce.waitForGlobalOp(insertOp) + if err := gce.waitForGlobalOp(insertOp); err != nil { + if gapiErr, ok := err.(*googleapi.Error); ok && gapiErr.Code == http.StatusConflict { + // TODO (cjcullen): Make this actually check the route is correct. + return nil + } + } + return err } func (gce *GCECloud) Release(name string) error { diff --git a/pkg/kubelet/container_bridge.go b/pkg/kubelet/container_bridge.go new file mode 100644 index 00000000000..f1ea5cce908 --- /dev/null +++ b/pkg/kubelet/container_bridge.go @@ -0,0 +1,83 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "bytes" + "net" + "os/exec" + "regexp" + + "github.com/golang/glog" +) + +var cidrRegexp = regexp.MustCompile(`inet ([0-9a-fA-F.:]*/[0-9]*)`) + +func ensureCbr0(wantCIDR *net.IPNet) error { + if !cbr0CidrCorrect(wantCIDR) { + glog.V(5).Infof("Attempting to recreate cbr0 with address range: %s", wantCIDR) + + // delete cbr0 + if err := exec.Command("ip", "link", "set", "dev", "cbr0", "down").Run(); err != nil { + glog.Error(err) + return err + } + if err := exec.Command("brctl", "delbr", "cbr0").Run(); err != nil { + glog.Error(err) + return err + } + // recreate cbr0 with wantCIDR + if err := exec.Command("brctl", "addbr", "cbr0").Run(); err != nil { + glog.Error(err) + return err + } + if err := exec.Command("ip", "addr", "add", wantCIDR.String(), "dev", "cbr0").Run(); err != nil { + glog.Error(err) + return err + } + if err := exec.Command("ip", "link", "set", "dev", "cbr0", "up").Run(); err != nil { + glog.Error(err) + return err + } + // restart docker + if err := exec.Command("service", "docker", "restart").Run(); err != nil { + glog.Error(err) + return err + } + glog.V(5).Info("Recreated cbr0 and restarted docker") + } + return nil +} + +func cbr0CidrCorrect(wantCIDR *net.IPNet) bool { + output, err := exec.Command("ip", "addr", "show", "cbr0").Output() + if err != nil { + return false + } + match := cidrRegexp.FindSubmatch(output) + if len(match) < 2 { + return false + } + cbr0IP, cbr0CIDR, err := net.ParseCIDR(string(match[1])) + cbr0CIDR.IP = cbr0IP + if err != nil { + glog.Errorf("Couldn't parse CIDR: %q", match[1]) + return false + } + glog.V(5).Infof("Want cbr0 CIDR: %s, have cbr0 CIDR: %s", wantCIDR, cbr0CIDR) + return wantCIDR.IP.Equal(cbr0IP) && bytes.Equal(wantCIDR.Mask, cbr0CIDR.Mask) +} diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 011f33bac63..05923d65502 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -1555,6 +1555,23 @@ func (kl *Kubelet) updateRuntimeUp() { } } +func (kl *Kubelet) reconcileCbr0(podCIDR string) error { + if podCIDR == "" { + glog.V(5).Info("PodCIDR not set. Will not configure cbr0.") + return nil + } + _, cidr, err := net.ParseCIDR(podCIDR) + if err != nil { + return err + } + // Set cbr0 interface address to first address in IPNet + cidr.IP.To4()[3] += 1 + if err := ensureCbr0(cidr); err != nil { + return err + } + return nil +} + // updateNodeStatus updates node status to master with retries. func (kl *Kubelet) updateNodeStatus() error { for i := 0; i < nodeStatusUpdateRetry; i++ { @@ -1599,6 +1616,10 @@ func (kl *Kubelet) tryUpdateNodeStatus() error { return fmt.Errorf("no node instance returned for %q", kl.hostname) } + if err := kl.reconcileCbr0(node.Spec.PodCIDR); err != nil { + glog.Errorf("Error configuring cbr0: %v", err) + } + // TODO: Post NotReady if we cannot get MachineInfo from cAdvisor. This needs to start // cAdvisor locally, e.g. for test-cmd.sh, and in integration test. info, err := kl.GetCachedMachineInfo()