Merge pull request #35797 from bacongobbler/registry-proxy

Automatic merge from submit-queue (batch tested with PRs 32663, 35797)

contribute deis/registry-proxy as a replacement for kube-registry-proxy

This PR is a proposal to replace the `kube-registry-proxy` addon code with [deis/registry-proxy](https://github.com/deis/registry-proxy). We have been running this component in production for several months ([since Workflow v2.3.0](15d4c1c298/workflow-v2.3.0/tpl/deis-registry-proxy-daemon.yaml)) without any issues.

There are several benefits that this proxy provides over the current implementation:
- it's the same code that is provided in [docker/distribution's contrib dir](https://github.com/docker/distribution/tree/master/contrib/compose) which I have personally used for both Docker v1 and v2 engine deployments without any issues
- the ability to [disable old Docker clients](https://github.com/deis/registry-proxy/blob/master/rootfs/etc/nginx/conf.d/default.conf.in#L19-L23) that are incompatible with the v2 registry
- better default connection timeouts, using best practices from the Docker community as a whole
- workarounds for bugs like https://github.com/docker/docker/issues/1486 (see https://github.com/deis/registry-proxy/blob/master/rootfs/etc/nginx/conf.d/default.conf.in#L15-L16)

Things that this PR differs from the current implementation:
- it's not HAProxy.

I'm not sure how the release process goes for this component, but I bumped the version to v0.4 and changed the maintainer to myself considering this is a massive overhaul. Please let me know if this is acceptable as a replacement or if we should perhaps consider this as an alternative implementation.

Happy Friday!

cluster/addons/registry/README.md

@@ -188,7 +188,7 @@ metadata:
 spec:
   containers:
   - name: kube-registry-proxy
-    image: gcr.io/google_containers/kube-registry-proxy:0.3
+    image: gcr.io/google_containers/kube-registry-proxy:0.4
     resources:
       limits:
         cpu: 100m

cluster/addons/registry/images/Dockerfile

@@ -12,15 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM haproxy:1.5
-MAINTAINER Muhammed Uluyol <uluyol@google.com>
+FROM nginx:1.11
+MAINTAINER Matthew Fisher <mfisher@deis.com>
 
-RUN apt-get update && apt-get install -y dnsutils
+RUN apt-get update \
+	&& apt-get install -y \
+		curl \
+		--no-install-recommends \
+	&& apt-get clean \
+	&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man /usr/share/doc
 
-ADD proxy.conf.insecure.in /proxy.conf.in
-ADD run_proxy.sh /usr/bin/run_proxy
+COPY rootfs /
 
-RUN chown root:users /usr/bin/run_proxy
-RUN chmod 755 /usr/bin/run_proxy
-
-CMD ["/usr/bin/run_proxy"]
+CMD ["/bin/boot"]

cluster/addons/registry/images/Makefile

@@ -14,7 +14,7 @@
 
 .PHONY: build push vet test clean
 
-TAG = 0.3
+TAG = 0.4
 REPO = gcr.io/google_containers/kube-registry-proxy
 
 build:

cluster/addons/registry/images/proxy.conf.in

@@ -1,17 +0,0 @@
-global
-  maxconn 1024
-
-defaults
-  mode http
-  retries 3
-  option redispatch
-  timeout client 1s
-  timeout server 5s
-  timeout connect 5s
-
-frontend forwarder
-  bind *:%FWDPORT%
-  default_backend registry
-
-backend registry
-  server kube-registry %HOST%:%PORT% ssl verify required ca-file %CA_FILE%

cluster/addons/registry/images/proxy.conf.insecure.in

@@ -1,17 +0,0 @@
-global
-  maxconn 1024
-
-defaults
-  mode http
-  retries 3
-  option redispatch
-  timeout client 1s
-  timeout server 5s
-  timeout connect 5s
-
-frontend forwarder
-  bind *:%FWDPORT%
-  default_backend registry
-
-backend registry
-  server kube-registry %HOST%:%PORT%

cluster/addons/registry/images/rootfs/bin/boot

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# fail if no hostname is provided
+REGISTRY_HOST=${REGISTRY_HOST:?no host}
+REGISTRY_PORT=${REGISTRY_PORT:-5000}
+
+# we are always listening on port 80
+# https://github.com/nginxinc/docker-nginx/blob/43c112100750cbd1e9f2160324c64988e7920ac9/stable/jessie/Dockerfile#L25
+PORT=80
+
+sed -e "s/%HOST%/$REGISTRY_HOST/g" \
+	-e "s/%PORT%/$REGISTRY_PORT/g" \
+	-e "s/%BIND_PORT%/$PORT/g" \
+	</etc/nginx/conf.d/default.conf.in >/etc/nginx/conf.d/default.conf
+
+# wait for registry to come online
+while ! curl -sS "$REGISTRY_HOST:$REGISTRY_PORT" &>/dev/null; do
+	printf "waiting for the registry (%s:%s) to come online...\n" "$REGISTRY_HOST" "$REGISTRY_PORT"
+	sleep 1
+done
+
+printf "starting proxy...\n"
+exec nginx -g "daemon off;" "$@"

cluster/addons/registry/images/rootfs/etc/nginx/conf.d/default.conf.in

@@ -0,0 +1,28 @@
+# Docker registry proxy for api version 2
+
+upstream docker-registry {
+    server %HOST%:%PORT%;
+}
+
+# No client auth or TLS
+# TODO(bacongobbler): experiment with authenticating the registry if it's using TLS
+server {
+    listen %BIND_PORT%;
+    server_name localhost;
+
+    # disable any limits to avoid HTTP 413 for large image uploads
+    client_max_body_size 0;
+
+    # required to avoid HTTP 411: see Issue #1486 (https://github.com/docker/docker/issues/1486)
+    chunked_transfer_encoding on;
+
+    location / {
+        # Do not allow connections from docker 1.5 and earlier
+        # docker pre-1.6.0 did not properly set the user agent on ping, catch "Go *" user agents
+        if ($http_user_agent ~ "^(docker\/1\.(3|4|5(?!\.[0-9]-dev))|Go ).*$" ) {
+            return 404;
+        }
+
+        include docker-registry.conf;
+    }
+}

cluster/addons/registry/images/rootfs/etc/nginx/docker-registry.conf

@@ -0,0 +1,6 @@
+proxy_pass                          http://docker-registry;
+proxy_set_header  Host              $http_host;   # required for docker client's sake
+proxy_set_header  X-Real-IP         $remote_addr; # pass on real client's IP
+proxy_set_header  X-Forwarded-For   $proxy_add_x_forwarded_for;
+proxy_set_header  X-Forwarded-Proto $scheme;
+proxy_read_timeout                  900;

cluster/addons/registry/images/rootfs/etc/nginx/nginx.conf

@@ -0,0 +1,26 @@
+user nginx;
+worker_processes auto;
+
+error_log   /var/log/nginx/error.log    warn;
+pid         /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include      /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log main;
+
+    sendfile on;
+
+    keepalive_timeout 65;
+
+    include /etc/nginx/conf.d/*.conf;
+}

cluster/addons/registry/images/run_proxy.sh

@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-REGISTRY_HOST=${REGISTRY_HOST:?no host}
-REGISTRY_PORT=${REGISTRY_PORT:-5000}
-REGISTRY_CA=${REGISTRY_CA:-/var/run/secrets/kubernetes.io/serviceaccount/ca.crt}
-FORWARD_PORT=${FORWARD_PORT:-5000}
-sed -e "s/%HOST%/$REGISTRY_HOST/g" \
-	-e "s/%PORT%/$REGISTRY_PORT/g" \
-	-e "s/%FWDPORT%/$FORWARD_PORT/g" \
-	-e "s|%CA_FILE%|$REGISTRY_CA|g" \
-	</proxy.conf.in >/proxy.conf
-
-# wait for registry to come online
-while ! host "$REGISTRY_HOST" &>/dev/null; do
-	printf "waiting for %s to come online\n" "$REGISTRY_HOST"
-	sleep 1
-done
-
-printf "starting proxy\n"
-exec haproxy -f /proxy.conf "$@"