From d8f839bb72d4a7fcbb40539a5f1c59e1b7009b6f Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Wed, 22 Apr 2015 15:11:23 +0100 Subject: [PATCH 1/9] coreos/azure: Simplify etcd, use Weave 0.10.0 --- .../kubernetes-cluster-etcd-node-template.yml | 10 +++++----- .../kubernetes-cluster-main-nodes-template.yml | 11 +++++------ .../coreos/azure/lib/deployment_logic/kubernetes.js | 13 ------------- 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-etcd-node-template.yml b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-etcd-node-template.yml index bf138e36794..72b52737f8b 100644 --- a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-etcd-node-template.yml +++ b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-etcd-node-template.yml @@ -32,14 +32,14 @@ coreos: Description=etcd 2 Documentation=https://github.com/coreos/etcd/ [Service] - Environment=ETCD_NAME=%host% + Environment=ETCD_NAME=%H Environment=ETCD_INITIAL_CLUSTER_TOKEN=etcd-cluster - Environment=ETCD_INITIAL_ADVERTISE_PEER_URLS=http://%host%:2380 - Environment=ETCD_LISTEN_PEER_URLS=http://%host%:2380 + Environment=ETCD_INITIAL_ADVERTISE_PEER_URLS=http://%H:2380 + Environment=ETCD_LISTEN_PEER_URLS=http://%H:2380 Environment=ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001 - Environment=ETCD_ADVERTISE_CLIENT_URLS=http://%host%:2379,http://%host%:4001 - Environment=ETCD_INITIAL_CLUSTER=%cluster% + Environment=ETCD_ADVERTISE_CLIENT_URLS=http://%H:2379,http://%H:4001 Environment=ETCD_INITIAL_CLUSTER_STATE=new + Environment=ETCD_INITIAL_CLUSTER=etcd-00=http://etcd-00:2380,etcd-01=http://etcd-01:2380,etcd-02=http://etcd-02:2380 ExecStart=/opt/bin/etcd2 Restart=always RestartSec=10 diff --git a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml index d5396783035..2b081ea20f2 100644 --- a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml +++ b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml @@ -120,7 +120,7 @@ coreos: Before=weave-helper.service Before=docker.service Description=Install Weave - Documentation=http://zettio.github.io/weave/ + Documentation=http://weaveworks.github.io/weave/ Requires=network-online.target [Service] Type=oneshot @@ -129,7 +129,7 @@ coreos: ExecStartPre=/usr/bin/curl \ --silent \ --location \ - https://github.com/weaveworks/weave/releases/download/v0.9.0/weave \ + https://github.com/weaveworks/weave/releases/download/latest_release/weave \ --output /opt/bin/weave ExecStartPre=/usr/bin/curl \ --silent \ @@ -150,7 +150,7 @@ coreos: After=install-weave.service After=docker.service Description=Weave Network Router - Documentation=http://zettio.github.io/weave/ + Documentation=http://weaveworks.github.io/weave/ Requires=docker.service Requires=install-weave.service [Service] @@ -165,13 +165,12 @@ coreos: After=install-weave.service After=docker.service Description=Weave Network Router - Documentation=http://zettio.github.io/weave/ + Documentation=http://weaveworks.github.io/weave/ Requires=docker.service Requires=install-weave.service [Service] EnvironmentFile=/etc/weave.%H.env - ExecStartPre=/usr/bin/docker pull zettio/weave:latest - ExecStartPre=/usr/bin/docker pull zettio/weavetools:latest + ExecStartPre=/opt/bin/weave setup ExecStartPre=/opt/bin/weave launch $WEAVE_PEERS ExecStart=/usr/bin/docker attach weave Restart=on-failure diff --git a/docs/getting-started-guides/coreos/azure/lib/deployment_logic/kubernetes.js b/docs/getting-started-guides/coreos/azure/lib/deployment_logic/kubernetes.js index 112d30570e9..d13f3701cd7 100644 --- a/docs/getting-started-guides/coreos/azure/lib/deployment_logic/kubernetes.js +++ b/docs/getting-started-guides/coreos/azure/lib/deployment_logic/kubernetes.js @@ -8,23 +8,10 @@ var cloud_config = require('../cloud_config.js'); exports.create_etcd_cloud_config = function (node_count, conf) { var input_file = './cloud_config_templates/kubernetes-cluster-etcd-node-template.yml'; - var peers = [ ]; - for (var i = 0; i < node_count; i++) { - peers.push(util.hostname(i, 'etcd') + '=http://' + util.hostname(i, 'etcd') + ':2380'); - } - var cluster = peers.join(','); - return _(node_count).times(function (n) { var output_file = util.join_output_file_path('kubernetes-cluster-etcd-node-' + n, 'generated.yml'); return cloud_config.process_template(input_file, output_file, function(data) { - for (var i = 0; i < data.coreos.units.length; i++) { - var unit = data.coreos.units[i]; - if (unit.name === 'etcd2.service') { - unit.content = _.replaceAll(_.replaceAll(unit.content, '%host%', util.hostname(n, 'etcd')), '%cluster%', cluster); - break; - } - } return data; }); }); From 55efa3f400410553e5c923d15684f4a86408cfe8 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Wed, 22 Apr 2015 15:37:13 +0100 Subject: [PATCH 2/9] coreos/azure: Pass `--local` to weave, as docker is not running yet --- .../kubernetes-cluster-main-nodes-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml index 2b081ea20f2..d880ffe888f 100644 --- a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml +++ b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml @@ -193,7 +193,7 @@ coreos: [Service] Type=oneshot EnvironmentFile=/etc/weave.%H.env - ExecStart=/opt/bin/weave create-bridge + ExecStart=/opt/bin/weave --local create-bridge ExecStart=/usr/bin/ip addr add dev weave $BRIDGE_ADDRESS_CIDR ExecStart=/usr/bin/ip route add $BREAKOUT_ROUTE dev weave scope link ExecStart=/usr/bin/ip route add 224.0.0.0/4 dev weave From 3b6b369b26de39244e59933a1076a7e709a07f48 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Wed, 22 Apr 2015 15:48:30 +0100 Subject: [PATCH 3/9] coreos/azure: Bump CLI tools version --- docs/getting-started-guides/coreos/azure/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started-guides/coreos/azure/package.json b/docs/getting-started-guides/coreos/azure/package.json index 1f28c503740..cf482b3ec07 100644 --- a/docs/getting-started-guides/coreos/azure/package.json +++ b/docs/getting-started-guides/coreos/azure/package.json @@ -9,7 +9,7 @@ "author": "Ilya Dmitrichenko ", "license": "Apache 2.0", "dependencies": { - "azure-cli": "^0.8.16", + "azure-cli": "^0.8.17", "colors": "^1.0.3", "js-yaml": "^3.2.5", "openssl-wrapper": "^0.2.1", From d882ad5411530f9a8de7e4015aa4ca28dfc7b4cc Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Wed, 22 Apr 2015 16:29:55 +0100 Subject: [PATCH 4/9] coreos/azure: Make examples work with Azure provider as-is --- .../kubernetes-cluster-main-nodes-template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml index d880ffe888f..9537978da28 100644 --- a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml +++ b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml @@ -221,6 +221,7 @@ coreos: ExecStartPost=/bin/ln -s /opt/kubernetes/server/bin/kubectl /opt/bin/ ExecStartPost=/bin/mv /tmp/kubernetes/examples/guestbook /home/core/guestbook-example ExecStartPost=/bin/rm -rf /tmp/kubernetes + ExecStartPost=/bin/sed 's/\("createExternalLoadBalancer":\) true/\1 false/' -i /home/core/guestbook-example/frontend-service.json RemainAfterExit=yes Type=oneshot [Install] From 7b8abf3cbb6de98da5052ca8dd8217fe6f34c84d Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Wed, 22 Apr 2015 17:11:26 +0100 Subject: [PATCH 5/9] coreos/azure: Make etcd cluster scalable --- .../kubernetes-cluster-etcd-node-template.yml | 1 - ...kubernetes-cluster-main-nodes-template.yml | 2 +- .../azure/lib/deployment_logic/kubernetes.js | 50 ++++++++++++++++--- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-etcd-node-template.yml b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-etcd-node-template.yml index 72b52737f8b..484da417df9 100644 --- a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-etcd-node-template.yml +++ b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-etcd-node-template.yml @@ -39,7 +39,6 @@ coreos: Environment=ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001 Environment=ETCD_ADVERTISE_CLIENT_URLS=http://%H:2379,http://%H:4001 Environment=ETCD_INITIAL_CLUSTER_STATE=new - Environment=ETCD_INITIAL_CLUSTER=etcd-00=http://etcd-00:2380,etcd-01=http://etcd-01:2380,etcd-02=http://etcd-02:2380 ExecStart=/opt/bin/etcd2 Restart=always RestartSec=10 diff --git a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml index 9537978da28..123f3f27140 100644 --- a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml +++ b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml @@ -244,7 +244,7 @@ coreos: ExecStart=/opt/kubernetes/server/bin/kube-apiserver \ --address=0.0.0.0 \ --port=8080 \ - --etcd_servers=http://etcd-00:4001,http://etcd-01:4001,http://etcd-02:4001 \ + $ETCD_SERVERS \ --portal_net=10.1.0.0/16 \ --cloud_provider=vagrant \ --logtostderr=true --v=3 diff --git a/docs/getting-started-guides/coreos/azure/lib/deployment_logic/kubernetes.js b/docs/getting-started-guides/coreos/azure/lib/deployment_logic/kubernetes.js index d13f3701cd7..e497a55708d 100644 --- a/docs/getting-started-guides/coreos/azure/lib/deployment_logic/kubernetes.js +++ b/docs/getting-started-guides/coreos/azure/lib/deployment_logic/kubernetes.js @@ -5,15 +5,51 @@ var util = require('../util.js'); var cloud_config = require('../cloud_config.js'); +etcd_initial_cluster_conf_self = function (conf) { + var port = '2380'; + + var data = { + nodes: _(conf.nodes.etcd).times(function (n) { + var host = util.hostname(n, 'etcd'); + return [host, [host, port].join(':')].join('=http://'); + }), + }; + + return { + 'name': 'etcd2.service', + 'drop-ins': [{ + 'name': '50-etcd-initial-cluster.conf', + 'content': _.template("[Service]\nEnvironment=ETCD_INITIAL_CLUSTER=<%= nodes.join(',') %>\n")(data), + }], + }; +}; + +etcd_initial_cluster_conf_kube = function (conf) { + var port = '4001'; + + var data = { + nodes: _(conf.nodes.etcd).times(function (n) { + var host = util.hostname(n, 'etcd'); + return 'http://' + [host, port].join(':'); + }), + }; + + return { + 'name': 'apiserver.service', + 'drop-ins': [{ + 'name': '50-etcd-initial-cluster.conf', + 'content': _.template("[Service]\nEnvironment=ETCD_SERVERS=--etcd_servers=<%= nodes.join(',') %>\n")(data), + }], + }; +}; + exports.create_etcd_cloud_config = function (node_count, conf) { var input_file = './cloud_config_templates/kubernetes-cluster-etcd-node-template.yml'; + var output_file = util.join_output_file_path('kubernetes-cluster-etcd-nodes', 'generated.yml'); - return _(node_count).times(function (n) { - var output_file = util.join_output_file_path('kubernetes-cluster-etcd-node-' + n, 'generated.yml'); - - return cloud_config.process_template(input_file, output_file, function(data) { - return data; - }); + return cloud_config.process_template(input_file, output_file, function(data) { + data.coreos.units.push(etcd_initial_cluster_conf_self(conf)); + return data; }); }; @@ -31,8 +67,10 @@ exports.create_node_cloud_config = function (node_count, conf) { bridge_address_cidr: util.ipv4([10, 2, n, 1], 24), }); }; + return cloud_config.process_template(input_file, output_file, function(data) { data.write_files = data.write_files.concat(_(node_count).times(make_node_config)); + data.coreos.units.push(etcd_initial_cluster_conf_kube(conf)); return data; }); }; From 9ac9670022567e4ff12a2a7463673982d3a951d7 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Thu, 23 Apr 2015 00:30:18 +0100 Subject: [PATCH 6/9] coreos/azure: Registration before proxy & kublet - it seems a little odd, sometimes we get an error that node had already been registered... it's unclear whether kubelet somehow does self-registration? --- .../kubernetes-cluster-main-nodes-template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml index 123f3f27140..28a3af282f6 100644 --- a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml +++ b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml @@ -345,6 +345,8 @@ coreos: content: | [Unit] After=download-kubernetes.service + Before=proxy.service + Before=kubelet.service ConditionFileIsExecutable=/opt/kubernetes/server/bin/kubectl ConditionFileIsExecutable=/opt/bin/register_minion.sh Description=Kubernetes Create Minion From fc1feebcf5ecf67f4a1c4ac352c54e3ee4118855 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Thu, 23 Apr 2015 11:43:09 +0100 Subject: [PATCH 7/9] coreos/azure: Parametrise registration label --- .../kubernetes-cluster-main-nodes-template.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml index 28a3af282f6..c83cccf630b 100644 --- a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml +++ b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml @@ -10,12 +10,18 @@ write_files: #!/bin/sh -xe minion_id="${1}" master_url="${2}" + env_label="${3}" until healthcheck=$(curl --fail --silent "${master_url}/healthz") do sleep 2 done test -n "${healthcheck}" test "${healthcheck}" = "ok" - printf '{ "id": "%s", "kind": "Minion", "apiVersion": "v1beta1", "labels": { "environment": "production" } }' "${minion_id}" \ + printf '{ + "id": "%s", + "kind": "Minion", + "apiVersion": "v1beta1", + "labels": { "environment": "%s" } + }' "${minion_id}" "${env_label}" \ | /opt/bin/kubectl create -s "${master_url}" -f - - path: /etc/kubernetes/manifests/fluentd.manifest @@ -169,6 +175,7 @@ coreos: Requires=docker.service Requires=install-weave.service [Service] + TimeoutStartSec=0 EnvironmentFile=/etc/weave.%H.env ExecStartPre=/opt/bin/weave setup ExecStartPre=/opt/bin/weave launch $WEAVE_PEERS @@ -354,7 +361,7 @@ coreos: Wants=download-kubernetes.service ConditionHost=!kube-00 [Service] - ExecStart=/opt/bin/register_minion.sh %H http://kube-00:8080 + ExecStart=/opt/bin/register_minion.sh %H http://kube-00:8080 production Type=oneshot [Install] WantedBy=kubernetes-minion.target From 652972b31adc82f64a04b7f43abab63780a8538e Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Thu, 23 Apr 2015 12:48:00 +0100 Subject: [PATCH 8/9] coreos/azure: Update the docs --- docs/getting-started-guides/coreos/azure/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started-guides/coreos/azure/README.md b/docs/getting-started-guides/coreos/azure/README.md index 01ce4195ea7..6bde3c817f6 100644 --- a/docs/getting-started-guides/coreos/azure/README.md +++ b/docs/getting-started-guides/coreos/azure/README.md @@ -60,7 +60,7 @@ kube-02 environment=production Ready Let's follow the Guestbook example now: ``` cd guestbook-example -kubectl create -f redis-master.json +kubectl create -f redis-master-controller.json kubectl create -f redis-master-service.json kubectl create -f redis-slave-controller.json kubectl create -f redis-slave-service.json From c8b06a9a851a9f2fb117e2f1570dfab5a3b6513a Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Thu, 23 Apr 2015 12:53:21 +0100 Subject: [PATCH 9/9] coreos/azure: Use drop-in to fix the interwebz - It turns out `network-online.target` doesn't imply that there is access to the Internet and simply using `curl --retry 10` doesn't work and hangs forever, even if we can already reach github.com... --- .../kubernetes-cluster-main-nodes-template.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml index c83cccf630b..1941978eb4b 100644 --- a/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml +++ b/docs/getting-started-guides/coreos/azure/cloud_config_templates/kubernetes-cluster-main-nodes-template.yml @@ -56,6 +56,14 @@ coreos: group: stable reboot-strategy: off units: + - name: systemd-networkd-wait-online.service + drop-ins: + - name: 50-check-github-is-reachable.conf + content: | + [Service] + ExecStart=/bin/sh -x -c \ + 'until curl --silent --fail https://status.github.com/api/status.json | grep -q \"good\"; do sleep 2; done' + - name: docker.service drop-ins: - name: 50-weave-kubernetes.conf