Merge pull request #8155 from lavalamp/no-ro

Remove ro service

cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest

@@ -106,9 +106,6 @@
       { "name": "https",
         "containerPort": {{secure_port}},
         "hostPort": {{secure_port}}},{
-       "name": "http",
-        "containerPort": 7080,
-        "hostPort": 7080},{
        "name": "local",
         "containerPort": 8080,
         "hostPort": 8080}

cmd/integration/integration.go

@@ -162,7 +162,6 @@ func startComponents(firstManifestURL, secondManifestURL, apiVersion string) (st
 		Authorizer:            apiserver.NewAlwaysAllowAuthorizer(),
 		AdmissionControl:      admit.NewAlwaysAdmit(),
 		ReadWritePort:         portNumber,
-		ReadOnlyPort:          portNumber,
 		PublicAddress:         publicAddress,
 		CacheTimeout:          2 * time.Second,
 		EnableV1:              true,
@@ -704,16 +703,13 @@ func runMasterServiceTest(client *client.Client) {
 	if err != nil {
 		glog.Fatalf("unexpected error listing services: %v", err)
 	}
-	var foundRW, foundRO bool
+	var foundRW bool
 	found := util.StringSet{}
 	for i := range svcList.Items {
 		found.Insert(svcList.Items[i].Name)
 		if svcList.Items[i].Name == "kubernetes" {
 			foundRW = true
 		}
-		if svcList.Items[i].Name == "kubernetes-ro" {
-			foundRO = true
-		}
 	}
 	if foundRW {
 		ep, err := client.Endpoints(api.NamespaceDefault).Get("kubernetes")
@@ -725,20 +721,7 @@ func runMasterServiceTest(client *client.Client) {
 		}
 	} else {
 		glog.Errorf("no RW service found: %v", found)
-	}
+		glog.Fatal("Kubernetes service test failed")
-	if foundRO {
-		ep, err := client.Endpoints(api.NamespaceDefault).Get("kubernetes-ro")
-		if err != nil {
-			glog.Fatalf("unexpected error listing endpoints for kubernetes service: %v", err)
-		}
-		if countEndpoints(ep) == 0 {
-			glog.Fatalf("no endpoints for kubernetes service: %v", ep)
-		}
-	} else {
-		glog.Errorf("no RO service found: %v", found)
-	}
-	if !foundRW || !foundRO {
-		glog.Fatalf("Kubernetes service test failed: %v", found)
 	}
 	glog.Infof("Master service test passed.")
 }
@@ -851,7 +834,7 @@ func runServiceTest(client *client.Client) {
 	for _, svc := range svcList.Items {
 		names.Insert(fmt.Sprintf("%s/%s", svc.Namespace, svc.Name))
 	}
-	if !names.HasAll("default/kubernetes", "default/kubernetes-ro", "default/service1", "default/service2", "other/service1") {
+	if !names.HasAll("default/kubernetes", "default/service1", "default/service2", "other/service1") {
 		glog.Fatalf("Unexpected service list: %#v", names)
 	}
 

cmd/kube-apiserver/app/server.go

@@ -59,7 +59,6 @@ type APIServer struct {
 	InsecurePort               int
 	BindAddress                util.IP
 	AdvertiseAddress           util.IP
-	ReadOnlyPort               int
 	SecurePort                 int
 	ExternalHost               string
 	APIRate                    float32
@@ -106,7 +105,6 @@ func NewAPIServer() *APIServer {
 		InsecurePort:           8080,
 		InsecureBindAddress:    util.IP(net.ParseIP("127.0.0.1")),
 		BindAddress:            util.IP(net.ParseIP("0.0.0.0")),
-		ReadOnlyPort:           7080,
 		SecurePort:             6443,
 		APIRate:                10.0,
 		APIBurst:               200,
@@ -156,10 +154,6 @@ func (s *APIServer) AddFlags(fs *pflag.FlagSet) {
 		"will be used. If --bind-address is unspecified, the host's default interface will "+
 		"be used.")
 	fs.Var(&s.BindAddress, "public-address-override", "DEPRECATED: see --bind-address instead")
-	fs.IntVar(&s.ReadOnlyPort, "read-only-port", s.ReadOnlyPort, ""+
-		"The port on which to serve read-only resources. If 0, don't serve read-only "+
-		"at all. It is assumed that firewall rules are set up such that this port is "+
-		"not reachable from outside of the cluster.")
 	fs.IntVar(&s.SecurePort, "secure-port", s.SecurePort, ""+
 		"The port on which to serve HTTPS with authentication and authorization. If 0, "+
 		"don't serve HTTPS at all.")
@@ -370,7 +364,6 @@ func (s *APIServer) Run(_ []string) error {
 		EnableIndex:            true,
 		APIPrefix:              s.APIPrefix,
 		CorsAllowedOriginList:  s.CorsAllowedOriginList,
-		ReadOnlyPort:           s.ReadOnlyPort,
 		ReadWritePort:          s.SecurePort,
 		PublicAddress:          net.IP(s.AdvertiseAddress),
 		Authenticator:          authenticator,
@@ -386,11 +379,7 @@ func (s *APIServer) Run(_ []string) error {
 	}
 	m := master.New(config)
 
-	// We serve on 3 ports.  See docs/accessing_the_api.md
+	// We serve on 2 ports.  See docs/accessing_the_api.md
-	roLocation := ""
-	if s.ReadOnlyPort != 0 {
-		roLocation = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.ReadOnlyPort))
-	}
 	secureLocation := ""
 	if s.SecurePort != 0 {
 		secureLocation = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.SecurePort))
@@ -406,28 +395,6 @@ func (s *APIServer) Run(_ []string) error {
 
 	longRunningRE := regexp.MustCompile(s.LongRunningRequestRE)
 
-	if roLocation != "" {
-		// Default settings allow 1 read-only request per second, allow up to 20 in a burst before enforcing.
-		rl := util.NewTokenBucketRateLimiter(s.APIRate, s.APIBurst)
-		readOnlyServer := &http.Server{
-			Addr:           roLocation,
-			Handler:        apiserver.MaxInFlightLimit(sem, longRunningRE, apiserver.RecoverPanics(apiserver.ReadOnly(apiserver.RateLimit(rl, m.InsecureHandler)))),
-			ReadTimeout:    ReadWriteTimeout,
-			WriteTimeout:   ReadWriteTimeout,
-			MaxHeaderBytes: 1 << 20,
-		}
-		glog.Infof("Serving read-only insecurely on %s", roLocation)
-		go func() {
-			defer util.HandleCrash()
-			for {
-				if err := readOnlyServer.ListenAndServe(); err != nil {
-					glog.Errorf("Unable to listen for read only traffic (%v); will try again.", err)
-				}
-				time.Sleep(15 * time.Second)
-			}
-		}()
-	}
-
 	if secureLocation != "" {
 		secureServer := &http.Server{
 			Addr:           secureLocation,

cmd/kubernetes/kubernetes.go

@@ -98,7 +98,6 @@ func runApiServer(etcdClient tools.EtcdClient, addr net.IP, port int, masterServ
 		Authorizer:            apiserver.NewAlwaysAllowAuthorizer(),
 
 		ReadWritePort:          port,
-		ReadOnlyPort:           port,
 		PublicAddress:          addr,
 		MasterServiceNamespace: masterServiceNamespace,
 	})

docs/accessing_the_api.md

@@ -4,27 +4,21 @@
 The Kubernetes API is served by the Kubernetes APIServer process.  Typically,
 there is one of these running on a single kubernetes-master node.
 
-By default the Kubernetes APIserver serves
+By default the Kubernetes APIserver serves HTTP on 2 ports:
-HTTP on 3 ports:
   1. Localhost Port
     - serves HTTP
     - default is port 8080, change with `-port` flag.
     - defaults IP is localhost, change with `-address` flag.
     - no authentication or authorization checks in HTTP
     - protected by need to have host access
-  2. ReadOnly Port
+  2. Secure Port
-    - default is port 7080, change with `-read_only_port`
-    - default IP is first non-localhost network interface, change with `-public_address_override`
-    - serves HTTP
-    - no authentication checks in HTTP
-    - only GET requests are allowed.
-    - requests are rate limited
-  3. Secure Port
     - default is port 443, change with `-secure_port`
     - default IP is first non-localhost network interface, change with `-public_address_override`
     - serves HTTPS.  Set cert with `-tls_cert_file` and key with `-tls_private_key_file`.
     - uses token-file or client-certificate based [authentication](./authentication.md).
     - uses policy-based [authorization](./authorization.md).
+  3. Removed: ReadOnly Port
+    - For security reasons, this had to be removed. Use the service account feature instead.
 
 ## Proxies and Firewall rules
 
@@ -46,11 +40,12 @@ variety of uses cases:
       on desktop machine.  Currently, accesses the Localhost Port via a proxy (nginx)
       running on the `kubernetes-master` machine.  Proxy uses bearer token authentication.
    2. Processes running in Containers on Kubernetes that need to do read from
-      the apiserver.  Currently, these can use Readonly Port.
+      the apiserver.  Currently, these can use a service account.
    3. Scheduler and Controller-manager processes, which need to do read-write
-      API operations.  Currently, these have to run on the
+      API operations.  Currently, these have to run on the operations on the
-      operations on the apiserver.  Currently, these have to run on the same
+      apiserver.  Currently, these have to run on the same host as the
-      host as the apiserver and use the Localhost Port.
+      apiserver and use the Localhost Port.  In the future, these will be
+      switched to using service accounts to avoid the need to be co-located.
    4. Kubelets, which need to do read-write API operations and are necessarily
       on different machines than the apiserver.  Kubelet uses the Secure Port
       to get their pods, to find the services that a pod can see, and to
@@ -59,16 +54,12 @@ variety of uses cases:
 
 ## Expected changes
    - Policy will limit the actions kubelets can do via the authed port.
-   - Kube-proxy currently uses the readonly port to read services and endpoints,
-     but will eventually use the auth port.
    - Kubelets will change from token-based authentication to cert-based-auth.
    - Scheduler and Controller-manager will use the Secure Port too.  They
      will then be able to run on different machines than the apiserver.
    - A general mechanism will be provided for [giving credentials to
      pods](
      https://github.com/GoogleCloudPlatform/kubernetes/issues/1907).
-   - The Readonly Port will no longer be needed and [will be removed](
-     https://github.com/GoogleCloudPlatform/kubernetes/issues/5921).
    - Clients, like kubectl, will all support token-based auth, and the
      Localhost will no longer be needed, and will not be the default.
      However, the localhost port may continue to be an option for

hack/test-cmd.sh

@@ -387,11 +387,11 @@ for version in "${kube_api_versions[@]}"; do
 
   ### Create redis-master service from JSON
   # Pre-condition: Only the default kubernetes services are running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:'
   # Command
   kubectl create -f examples/guestbook/redis-master-service.json "${kube_flags[@]}"
   # Post-condition: redis-master service is running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:redis-master:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:redis-master:'
   # Describe command should print detailed information
   kube::test::describe_object_assert services 'redis-master' "Name:" "Labels:" "Selector:" "IP:" "Port:" "Endpoints:" "Session Affinity:"
 
@@ -400,23 +400,23 @@ for version in "${kube_api_versions[@]}"; do
 
   ### Delete redis-master-service by id
   # Pre-condition: redis-master service is running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:redis-master:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:redis-master:'
   # Command
   kubectl delete service redis-master "${kube_flags[@]}"
   # Post-condition: Only the default kubernetes services are running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:'
 
   ### Create redis-master-service from dumped JSON
   # Pre-condition: Only the default kubernetes services are running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:'
   # Command
   echo "${output_service}" | kubectl create -f - "${kube_flags[@]}"
   # Post-condition: redis-master service is running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:redis-master:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:redis-master:'
 
   ### Create redis-master-${version}-test service
   # Pre-condition: redis-master-service service is running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:redis-master:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:redis-master:'
   # Command
   kubectl create -f - "${kube_flags[@]}" << __EOF__
 {
@@ -437,36 +437,36 @@ for version in "${kube_api_versions[@]}"; do
 }
 __EOF__
   # Post-condition:redis-master-service service is running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:redis-master:service-.*-test:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:redis-master:service-.*-test:'
 
   ### Identity
   kubectl get service "${kube_flags[@]}" service-${version}-test -o json | kubectl update "${kube_flags[@]}" -f -
 
   ### Delete services by id
   # Pre-condition: redis-master-service service is running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:redis-master:service-.*-test:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:redis-master:service-.*-test:'
   # Command
   kubectl delete service redis-master "${kube_flags[@]}"
   kubectl delete service "service-${version}-test" "${kube_flags[@]}"
   # Post-condition: Only the default kubernetes services are running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:'
 
   ### Create two services
   # Pre-condition: Only the default kubernetes services are running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:'
   # Command
   kubectl create -f examples/guestbook/redis-master-service.json "${kube_flags[@]}"
   kubectl create -f examples/guestbook/redis-slave-service.json "${kube_flags[@]}"
   # Post-condition: redis-master and redis-slave services are running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:redis-master:redis-slave:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:redis-master:redis-slave:'
 
   ### Delete multiple services at once
   # Pre-condition: redis-master and redis-slave services are running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:redis-master:redis-slave:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:redis-master:redis-slave:'
   # Command
   kubectl delete services redis-master redis-slave "${kube_flags[@]}" # delete multiple services at once
   # Post-condition: Only the default kubernetes services are running
-  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:kubernetes-ro:'
+  kube::test::get_object_assert services "{{range.items}}{{$id_field}}:{{end}}" 'kubernetes:'
 
 
   ###########################

pkg/master/controller.go

@@ -60,10 +60,6 @@ type Controller struct {
 	ServicePort       int
 	PublicServicePort int
 
-	ReadOnlyServiceIP         net.IP
-	ReadOnlyServicePort       int
-	PublicReadOnlyServicePort int
-
 	runner *util.Runner
 }
 
@@ -87,11 +83,8 @@ func (c *Controller) Start() {
 	if err := c.UpdateKubernetesService(); err != nil {
 		glog.Errorf("Unable to perform initial Kubernetes service initialization: %v", err)
 	}
-	if err := c.UpdateKubernetesROService(); err != nil {
-		glog.Errorf("Unable to perform initial Kubernetes RO service initialization: %v", err)
-	}
 
-	c.runner = util.NewRunner(c.RunKubernetesService, c.RunKubernetesROService, repairClusterIPs.RunUntil, repairNodePorts.RunUntil)
+	c.runner = util.NewRunner(c.RunKubernetesService, repairClusterIPs.RunUntil, repairNodePorts.RunUntil)
 	c.runner.Start()
 }
 
@@ -124,34 +117,6 @@ func (c *Controller) UpdateKubernetesService() error {
 	return nil
 }
 
-// RunKubernetesROService periodically updates the kubernetes RO service
-func (c *Controller) RunKubernetesROService(ch chan struct{}) {
-	util.Until(func() {
-		if err := c.UpdateKubernetesROService(); err != nil {
-			util.HandleError(fmt.Errorf("unable to sync kubernetes RO service: %v", err))
-		}
-	}, c.EndpointInterval, ch)
-}
-
-// UpdateKubernetesROService attempts to update the default Kube read-only service.
-func (c *Controller) UpdateKubernetesROService() error {
-	// Update service & endpoint records.
-	// TODO: when it becomes possible to change this stuff,
-	// stop polling and start watching.
-	if err := c.CreateNamespaceIfNeeded(api.NamespaceDefault); err != nil {
-		return err
-	}
-	if c.ReadOnlyServiceIP != nil {
-		if err := c.CreateMasterServiceIfNeeded("kubernetes-ro", c.ReadOnlyServiceIP, c.ReadOnlyServicePort); err != nil {
-			return err
-		}
-		if err := c.SetEndpoints("kubernetes-ro", c.PublicIP, c.PublicReadOnlyServicePort); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // CreateNamespaceIfNeeded will create the namespace that contains the master services if it doesn't already exist
 func (c *Controller) CreateNamespaceIfNeeded(ns string) error {
 	ctx := api.NewContext()

pkg/master/master.go

@@ -119,9 +119,6 @@ type Config struct {
 	// same value for this field. (Numbers > 1 currently untested.)
 	MasterCount int
 
-	// The port on PublicAddress where a read-only server will be installed.
-	// Defaults to 7080 if not set.
-	ReadOnlyPort int
 	// The port on PublicAddress where a read-write server will be installed.
 	// Defaults to 6443 if not set.
 	ReadWritePort int
@@ -178,10 +175,7 @@ type Master struct {
 	externalHost string
 	// clusterIP is the IP address of the master within the cluster.
 	clusterIP            net.IP
-	publicReadOnlyPort   int
 	publicReadWritePort  int
-	serviceReadOnlyIP    net.IP
-	serviceReadOnlyPort  int
 	serviceReadWriteIP   net.IP
 	serviceReadWritePort int
 	masterServices       *util.Runner
@@ -244,9 +238,6 @@ func setDefaults(c *Config) {
 		// Clearly, there will be at least one master.
 		c.MasterCount = 1
 	}
-	if c.ReadOnlyPort == 0 {
-		c.ReadOnlyPort = 7080
-	}
 	if c.ReadWritePort == 0 {
 		c.ReadWritePort = 6443
 	}
@@ -276,7 +267,6 @@ func setDefaults(c *Config) {
 //   ServiceClusterIPRange
 //   ServiceNodePortRange
 //   MasterCount
-//   ReadOnlyPort
 //   ReadWritePort
 //   PublicAddress
 // Certain config fields must be specified, including:
@@ -301,16 +291,12 @@ func New(c *Config) *Master {
 		glog.Fatalf("master.New() called with config.KubeletClient == nil")
 	}
 
-	// Select the first two valid IPs from serviceClusterIPRange to use as the master service IPs
+	// Select the first valid IP from serviceClusterIPRange to use as the master service IP.
-	serviceReadOnlyIP, err := ipallocator.GetIndexedIP(c.ServiceClusterIPRange, 1)
+	serviceReadWriteIP, err := ipallocator.GetIndexedIP(c.ServiceClusterIPRange, 1)
-	if err != nil {
-		glog.Fatalf("Failed to generate service read-only IP for master service: %v", err)
-	}
-	serviceReadWriteIP, err := ipallocator.GetIndexedIP(c.ServiceClusterIPRange, 2)
 	if err != nil {
 		glog.Fatalf("Failed to generate service read-write IP for master service: %v", err)
 	}
-	glog.V(4).Infof("Setting master service IPs based to %q (read-only) and %q (read-write).", serviceReadOnlyIP, serviceReadWriteIP)
+	glog.V(4).Infof("Setting master service IP to %q (read-write).", serviceReadWriteIP)
 
 	m := &Master{
 		serviceClusterIPRange: c.ServiceClusterIPRange,
@@ -335,11 +321,7 @@ func New(c *Config) *Master {
 		masterCount:         c.MasterCount,
 		externalHost:        c.ExternalHost,
 		clusterIP:           c.PublicAddress,
-		publicReadOnlyPort:  c.ReadOnlyPort,
 		publicReadWritePort: c.ReadWritePort,
-		serviceReadOnlyIP:   serviceReadOnlyIP,
-		// TODO: serviceReadOnlyPort should be passed in as an argument, it may not always be 80
-		serviceReadOnlyPort: 80,
 		serviceReadWriteIP:  serviceReadWriteIP,
 		// TODO: serviceReadWritePort should be passed in as an argument, it may not always be 443
 		serviceReadWritePort: 443,
@@ -360,6 +342,7 @@ func New(c *Config) *Master {
 	m.muxHelper = &apiserver.MuxHelper{m.mux, []string{}}
 
 	m.init(c)
+
 	return m
 }
 
@@ -618,10 +601,6 @@ func (m *Master) NewBootstrapController() *Controller {
 		ServiceIP:         m.serviceReadWriteIP,
 		ServicePort:       m.serviceReadWritePort,
 		PublicServicePort: m.publicReadWritePort,
-
-		ReadOnlyServiceIP:         m.serviceReadOnlyIP,
-		ReadOnlyServicePort:       m.serviceReadOnlyPort,
-		PublicReadOnlyServicePort: m.publicReadOnlyPort,
 	}
 }
 
@@ -639,10 +618,6 @@ func (m *Master) InstallSwaggerAPI() {
 		host := m.clusterIP.String()
 		if m.publicReadWritePort != 0 {
 			hostAndPort = net.JoinHostPort(host, strconv.Itoa(m.publicReadWritePort))
-		} else {
-			// Use the read only port.
-			hostAndPort = net.JoinHostPort(host, strconv.Itoa(m.publicReadOnlyPort))
-			protocol = "http://"
 		}
 	}
 	webServicesUrl := protocol + hostAndPort