From e83fd7b8e7096d8a1a5974a12125c597aff9551a Mon Sep 17 00:00:00 2001 From: saadali Date: Mon, 19 Jan 2015 19:25:06 -0800 Subject: [PATCH] Make master service IP static (no longer randomly assigned) --- cmd/integration/integration.go | 2 +- cmd/kubernetes/kubernetes.go | 8 ++- pkg/master/master.go | 45 +++++++++++---- pkg/master/publish.go | 21 +++---- pkg/master/server/server.go | 20 +++---- pkg/registry/service/ip_allocator.go | 12 ++++ pkg/registry/service/ip_allocator_test.go | 69 +++++++++++++++++++++++ pkg/standalone/standalone.go | 2 +- test/integration/auth_test.go | 2 +- 9 files changed, 144 insertions(+), 37 deletions(-) diff --git a/cmd/integration/integration.go b/cmd/integration/integration.go index 72b10f039a8..8e4908b7e83 100644 --- a/cmd/integration/integration.go +++ b/cmd/integration/integration.go @@ -162,7 +162,7 @@ func startComponents(manifestURL string) (apiServerURL string) { AdmissionControl: admit.NewAlwaysAdmit(), ReadWritePort: portNumber, ReadOnlyPort: portNumber, - PublicAddress: host, + PublicAddress: net.ParseIP(host), }) handler.delegate = m.Handler diff --git a/cmd/kubernetes/kubernetes.go b/cmd/kubernetes/kubernetes.go index ad22881dcab..c1898f2059c 100644 --- a/cmd/kubernetes/kubernetes.go +++ b/cmd/kubernetes/kubernetes.go @@ -22,6 +22,7 @@ package main import ( "fmt" + "net" kubeletapp "github.com/GoogleCloudPlatform/kubernetes/cmd/kubelet/app" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" @@ -46,7 +47,7 @@ var ( masterServiceNamespace = flag.String("master_service_namespace", api.NamespaceDefault, "The namespace from which the kubernetes master services should be injected into pods") ) -func startComponents(etcdClient tools.EtcdClient, cl *client.Client, addr string, port int) { +func startComponents(etcdClient tools.EtcdClient, cl *client.Client, addr net.IP, port int) { machineList := []string{"localhost"} standalone.RunApiServer(cl, etcdClient, addr, port, *masterServiceNamespace) @@ -57,7 +58,7 @@ func startComponents(etcdClient tools.EtcdClient, cl *client.Client, addr string standalone.SimpleRunKubelet(cl, nil, dockerClient, machineList[0], "/tmp/kubernetes", "", "127.0.0.1", 10250, *masterServiceNamespace, kubeletapp.ProbeVolumePlugins()) } -func newApiClient(addr string, port int) *client.Client { +func newApiClient(addr net.IP, port int) *client.Client { apiServerURL := fmt.Sprintf("http://%s:%d", addr, port) cl := client.NewOrDie(&client.Config{Host: apiServerURL, Version: testapi.Version()}) return cl @@ -73,7 +74,8 @@ func main() { if err != nil { glog.Fatalf("Failed to connect to etcd: %v", err) } - startComponents(etcdClient, newApiClient(*addr, *port), *addr, *port) + address := net.ParseIP(*addr) + startComponents(etcdClient, newApiClient(address, *port), address, *port) glog.Infof("Kubernetes API Server is up and running on http://%s:%d", *addr, *port) select {} diff --git a/pkg/master/master.go b/pkg/master/master.go index cc81000fa52..a2ad26d8890 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -97,8 +97,8 @@ type Config struct { // Defaults to 443 if not set. ReadWritePort int - // If empty, the first result from net.InterfaceAddrs will be used. - PublicAddress string + // If nil, the first result from net.InterfaceAddrs will be used. + PublicAddress net.IP } // Master contains state for a Kubernetes cluster master/api server. @@ -133,9 +133,14 @@ type Master struct { v1beta3 bool nodeIPCache IPGetter - readOnlyServer string - readWriteServer string - masterServices *util.Runner + publicIP net.IP + publicReadOnlyPort int + publicReadWritePort int + serviceReadOnlyIP net.IP + serviceReadOnlyPort int + serviceReadWriteIP net.IP + serviceReadWritePort int + masterServices *util.Runner // "Outputs" Handler http.Handler @@ -176,7 +181,7 @@ func setDefaults(c *Config) { if c.ReadWritePort == 0 { c.ReadWritePort = 443 } - for c.PublicAddress == "" { + for c.PublicAddress == nil { // Find and use the first non-loopback address. // TODO: potentially it'd be useful to skip the docker interface if it // somehow is first in the list. @@ -196,7 +201,7 @@ func setDefaults(c *Config) { continue } found = true - c.PublicAddress = ip.String() + c.PublicAddress = ip glog.Infof("Will report %v as public IP address.", ip) break } @@ -244,6 +249,17 @@ func New(c *Config) *Master { glog.Fatalf("master.New() called with config.KubeletClient == nil") } + // Select the first two valid IPs from portalNet to use as the master service portalIPs + serviceReadOnlyIP, err := service.GetIndexedIP(c.PortalNet, 1) + if err != nil { + glog.Fatalf("Failed to generate service read-only IP for master service: %v", err) + } + serviceReadWriteIP, err := service.GetIndexedIP(c.PortalNet, 2) + if err != nil { + glog.Fatalf("Failed to generate service read-write IP for master service: %v", err) + } + glog.Infof("Setting master service IPs based on PortalNet subnet to %q (read-only) and %q (read-write).", serviceReadOnlyIP, serviceReadWriteIP) + m := &Master{ podRegistry: etcd.NewRegistry(c.EtcdHelper, boundPodFactory), controllerRegistry: etcd.NewRegistry(c.EtcdHelper, nil), @@ -268,9 +284,16 @@ func New(c *Config) *Master { v1beta3: c.EnableV1Beta3, nodeIPCache: NewIPCache(c.Cloud, util.RealClock{}, 30*time.Second), - masterCount: c.MasterCount, - readOnlyServer: net.JoinHostPort(c.PublicAddress, strconv.Itoa(int(c.ReadOnlyPort))), - readWriteServer: net.JoinHostPort(c.PublicAddress, strconv.Itoa(int(c.ReadWritePort))), + masterCount: c.MasterCount, + publicIP: c.PublicAddress, + publicReadOnlyPort: c.ReadOnlyPort, + publicReadWritePort: c.ReadWritePort, + serviceReadOnlyIP: serviceReadOnlyIP, + // TODO: serviceReadOnlyPort should be passed in as an argument, it may not always be 80 + serviceReadOnlyPort: 80, + serviceReadWriteIP: serviceReadWriteIP, + // TODO: serviceReadWritePort should be passed in as an argument, it may not always be 443 + serviceReadWritePort: 443, } if c.RestfulContainer != nil { @@ -443,7 +466,7 @@ func (m *Master) init(c *Config) { func (m *Master) InstallSwaggerAPI() { // Enable swagger UI and discovery API swaggerConfig := swagger.Config{ - WebServicesUrl: m.readWriteServer, + WebServicesUrl: net.JoinHostPort(m.publicIP.String(), strconv.Itoa(int(m.publicReadWritePort))), WebServices: m.handlerContainer.RegisteredWebServices(), // TODO: Parameterize the path? ApiPath: "/swaggerapi/", diff --git a/pkg/master/publish.go b/pkg/master/publish.go index 1f9477e0384..fecdbf3845d 100644 --- a/pkg/master/publish.go +++ b/pkg/master/publish.go @@ -18,6 +18,8 @@ package master import ( "fmt" + "net" + "strconv" "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" @@ -32,12 +34,11 @@ func (m *Master) serviceWriterLoop(stop chan struct{}) { // TODO: when it becomes possible to change this stuff, // stop polling and start watching. // TODO: add endpoints of all replicas, not just the elected master. - if m.readWriteServer != "" { - // TODO: the public port should be part of the argument here, port will not always be 443 - if err := m.createMasterServiceIfNeeded("kubernetes", 443); err != nil { + if m.serviceReadWriteIP != nil { + if err := m.createMasterServiceIfNeeded("kubernetes", m.serviceReadWriteIP, m.serviceReadWritePort); err != nil { glog.Errorf("Can't create rw service: %v", err) } - if err := m.ensureEndpointsContain("kubernetes", m.readWriteServer); err != nil { + if err := m.ensureEndpointsContain("kubernetes", net.JoinHostPort(m.publicIP.String(), strconv.Itoa(int(m.publicReadWritePort)))); err != nil { glog.Errorf("Can't create rw endpoints: %v", err) } } @@ -55,12 +56,11 @@ func (m *Master) roServiceWriterLoop(stop chan struct{}) { // Update service & endpoint records. // TODO: when it becomes possible to change this stuff, // stop polling and start watching. - if m.readOnlyServer != "" { - // TODO: the public port should be part of the argument here, port will not always be 80 - if err := m.createMasterServiceIfNeeded("kubernetes-ro", 80); err != nil { + if m.serviceReadOnlyIP != nil { + if err := m.createMasterServiceIfNeeded("kubernetes-ro", m.serviceReadOnlyIP, m.serviceReadOnlyPort); err != nil { glog.Errorf("Can't create ro service: %v", err) } - if err := m.ensureEndpointsContain("kubernetes-ro", m.readOnlyServer); err != nil { + if err := m.ensureEndpointsContain("kubernetes-ro", net.JoinHostPort(m.publicIP.String(), strconv.Itoa(int(m.publicReadOnlyPort)))); err != nil { glog.Errorf("Can't create ro endpoints: %v", err) } } @@ -75,7 +75,7 @@ func (m *Master) roServiceWriterLoop(stop chan struct{}) { // createMasterServiceIfNeeded will create the specified service if it // doesn't already exist. -func (m *Master) createMasterServiceIfNeeded(serviceName string, port int) error { +func (m *Master) createMasterServiceIfNeeded(serviceName string, serviceIP net.IP, servicePort int) error { ctx := api.NewDefaultContext() if _, err := m.serviceRegistry.GetService(ctx, serviceName); err == nil { // The service already exists. @@ -88,9 +88,10 @@ func (m *Master) createMasterServiceIfNeeded(serviceName string, port int) error Labels: map[string]string{"provider": "kubernetes", "component": "apiserver"}, }, Spec: api.ServiceSpec{ - Port: port, + Port: servicePort, // maintained by this code, not by the pod selector Selector: nil, + PortalIP: serviceIP.String(), }, } // Kids, don't do this at home: this is a hack. There's no good way to call the business diff --git a/pkg/master/server/server.go b/pkg/master/server/server.go index 9c95dfd4d53..34366097e42 100644 --- a/pkg/master/server/server.go +++ b/pkg/master/server/server.go @@ -47,7 +47,7 @@ import ( type APIServer struct { Port int Address util.IP - PublicAddressOverride string + PublicAddressOverride util.IP ReadOnlyPort int APIRate float32 APIBurst int @@ -80,6 +80,7 @@ func NewAPIServer() *APIServer { s := APIServer{ Port: 8080, Address: util.IP(net.ParseIP("127.0.0.1")), + PublicAddressOverride: util.IP(net.ParseIP("")), ReadOnlyPort: 7080, APIRate: 10.0, APIBurst: 200, @@ -127,11 +128,10 @@ func (s *APIServer) AddFlags(fs *pflag.FlagSet) { "further assumed that port 443 on the cluster's public address is proxied to this "+ "port. This is performed by nginx in the default setup.") fs.Var(&s.Address, "address", "The IP address on to serve on (set to 0.0.0.0 for all interfaces)") - fs.StringVar(&s.PublicAddressOverride, "public_address_override", s.PublicAddressOverride, ""+ - "Public serving address. Read only port will be opened on this address, "+ - "and it is assumed that port 443 at this address will be proxied/redirected "+ - "to '-address':'-port'. If blank, the address in the first listed interface "+ - "will be used.") + fs.Var(&s.PublicAddressOverride, "public_address_override", "Public serving address."+ + "Read only port will be opened on this address, and it is assumed that port "+ + "443 at this address will be proxied/redirected to '-address':'-port'. If "+ + "blank, the address in the first listed interface will be used.") fs.IntVar(&s.ReadOnlyPort, "read_only_port", s.ReadOnlyPort, ""+ "The port from which to serve read-only resources. If 0, don't serve on a "+ "read-only address. It is assumed that firewall rules are set up such that "+ @@ -251,7 +251,7 @@ func (s *APIServer) Run(_ []string) error { CorsAllowedOriginList: s.CorsAllowedOriginList, ReadOnlyPort: s.ReadOnlyPort, ReadWritePort: s.Port, - PublicAddress: s.PublicAddressOverride, + PublicAddress: net.IP(s.PublicAddressOverride), Authenticator: authenticator, Authorizer: authorizer, AdmissionControl: admissionController, @@ -263,11 +263,11 @@ func (s *APIServer) Run(_ []string) error { // We serve on 3 ports. See docs/reaching_the_api.md roLocation := "" if s.ReadOnlyPort != 0 { - roLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(config.ReadOnlyPort)) + roLocation = net.JoinHostPort(config.PublicAddress.String(), strconv.Itoa(config.ReadOnlyPort)) } secureLocation := "" if s.SecurePort != 0 { - secureLocation = net.JoinHostPort(config.PublicAddress, strconv.Itoa(s.SecurePort)) + secureLocation = net.JoinHostPort(config.PublicAddress.String(), strconv.Itoa(s.SecurePort)) } rwLocation := net.JoinHostPort(s.Address.String(), strconv.Itoa(int(s.Port))) @@ -317,7 +317,7 @@ func (s *APIServer) Run(_ []string) error { if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" { s.TLSCertFile = "/var/run/kubernetes/apiserver.crt" s.TLSPrivateKeyFile = "/var/run/kubernetes/apiserver.key" - if err := util.GenerateSelfSignedCert(config.PublicAddress, s.TLSCertFile, s.TLSPrivateKeyFile); err != nil { + if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile); err != nil { glog.Errorf("Unable to generate self signed cert: %v", err) } else { glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile) diff --git a/pkg/registry/service/ip_allocator.go b/pkg/registry/service/ip_allocator.go index 09746aa9bbe..c8fb4c59ac8 100644 --- a/pkg/registry/service/ip_allocator.go +++ b/pkg/registry/service/ip_allocator.go @@ -174,6 +174,18 @@ func (ipa *ipAllocator) AllocateNext() (net.IP, error) { return nil, fmt.Errorf("can't find a free IP in %s", ipa.subnet) } +// Returns the index-th IP from the specified subnet range. +// For example, subnet "10.0.0.0/24" with index "2" will return the IP "10.0.0.2". +// TODO(saad-ali): Move this (and any other functions that are independent of ipAllocator) to some +// place more generic. +func GetIndexedIP(subnet *net.IPNet, index int) (net.IP, error) { + ip := ipAdd(subnet.IP, index /* offset */) + if !subnet.Contains(ip) { + return nil, fmt.Errorf("can't generate IP with index %d from subnet. subnet too small. subnet: %q", index, subnet) + } + return ip, nil +} + func (ipa *ipAllocator) createRandomIp() net.IP { ip := ipa.subnet.IP mask := ipa.subnet.Mask diff --git a/pkg/registry/service/ip_allocator_test.go b/pkg/registry/service/ip_allocator_test.go index 8873e2c14a7..8cdfc4a3a80 100644 --- a/pkg/registry/service/ip_allocator_test.go +++ b/pkg/registry/service/ip_allocator_test.go @@ -214,6 +214,75 @@ func TestIPAdd(t *testing.T) { } } +func TestGenerateFirstTwoIPsFromSubnet(t *testing.T) { + // Arrange + testCases := []struct { + subnet string + expected1stIP string + expected2ndIP string + }{ + {"10.0.0.0/24", "10.0.0.1", "10.0.0.2"}, + {"10.10.10.10/24", "10.10.10.1", "10.10.10.2"}, + {"10.10.10.10/16", "10.10.0.1", "10.10.0.2"}, + {"10.10.10.10/8", "10.0.0.1", "10.0.0.2"}, + {"10.10.10.10/0", "0.0.0.1", "0.0.0.2"}, + {"192.168.100.1/16", "192.168.0.1", "192.168.0.2"}, + {"153.15.250.5/23", "153.15.250.1", "153.15.250.2"}, + {"2001:db8::/48", "2001:db8::1", "2001:db8::2"}, + {"2001:db8:123:255::/48", "2001:db8:123::1", "2001:db8:123::2"}, + {"12.12.0.0/30", "12.12.0.1", "12.12.0.2"}, + } + + // Act & Assert + for _, testCase := range testCases { + _, subnet, _ := net.ParseCIDR(testCase.subnet) + firstIP, err := GetIndexedIP(subnet, 1) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + secondIP, err := GetIndexedIP(subnet, 2) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if firstIP.String() != testCase.expected1stIP { + t.Errorf("Unexpected first IP: Expected <%q> Actual <%q>", testCase.expected1stIP, firstIP.String()) + } + if secondIP.String() != testCase.expected2ndIP { + t.Errorf("Unexpected second IP: Expected <%q> Actual <%q>", testCase.expected2ndIP, secondIP.String()) + } + } +} + +func TestGetIndexedIPSubnetTooSmall(t *testing.T) { + // Arrange + testCases := []struct { + subnet string + }{ + {"12.12.0.0/32"}, + {"12.12.0.0/31"}, + } + + // Act & Assert + for _, testCase := range testCases { + _, subnet, _ := net.ParseCIDR(testCase.subnet) + secondIP, err := GetIndexedIP(subnet, 2) + if err == nil { + t.Errorf("Expected error but no error occured for subnet: ", testCase.subnet) + } + thirdIP, err := GetIndexedIP(subnet, 3) + if err == nil { + t.Errorf("Expected error but no error occured for subnet: ", testCase.subnet) + } + if secondIP != nil { + t.Errorf("Unexpected second IP: Expected nil Actual <%q>", thirdIP.String()) + } + if thirdIP != nil { + t.Errorf("Unexpected third IP: Expected nil Actual <%q>", secondIP.String()) + } + + } +} + func TestCopyIP(t *testing.T) { ip1 := net.ParseIP("1.2.3.4") ip2 := copyIP(ip1) diff --git a/pkg/standalone/standalone.go b/pkg/standalone/standalone.go index 2696f86ea95..77a73f97841 100644 --- a/pkg/standalone/standalone.go +++ b/pkg/standalone/standalone.go @@ -85,7 +85,7 @@ func GetAPIServerClient(authPath string, apiServerList util.StringList) (*client } // RunApiServer starts an API server in a go routine. -func RunApiServer(cl *client.Client, etcdClient tools.EtcdClient, addr string, port int, masterServiceNamespace string) { +func RunApiServer(cl *client.Client, etcdClient tools.EtcdClient, addr net.IP, port int, masterServiceNamespace string) { handler := delegateHandler{} helper, err := master.NewEtcdHelper(etcdClient, "") diff --git a/test/integration/auth_test.go b/test/integration/auth_test.go index a2ca1471291..556f58e644d 100644 --- a/test/integration/auth_test.go +++ b/test/integration/auth_test.go @@ -106,7 +106,7 @@ var aService string = ` "apiVersion": "v1beta1", "id": "a", "port": 8000, - "portalIP": "10.0.0.1", + "portalIP": "10.0.0.100", "labels": { "name": "a" }, "selector": { "name": "a" } }