Merge pull request #15370 from mbruzek/ci-update

Auto commit by PR queue bot

cluster/juju/charms/trusty/kubernetes-master/Makefile

@@ -6,7 +6,7 @@ virtualenv:
 	.venv/bin/pip install -q -r requirements.txt
 
 lint: virtualenv
-	@.venv/bin/flake8 hooks unit_tests --exclude=charmhelpers
+	@.venv/bin/flake8 hooks --exclude=charmhelpers --ignore=W391
 	@.venv/bin/charm proof
 
 test: virtualenv
@@ -27,3 +27,4 @@ endif
 clean:
 	rm -rf .venv
 	find -name *.pyc -delete
+	rm -rf unit_tests/.cache

cluster/juju/charms/trusty/kubernetes-master/README.md

@@ -67,7 +67,7 @@ The charm store version of the Kubernetes bundle can be deployed as follows:
 
 Alternately you could deploy a Kubernetes bundle straight from github or a file:
 
-    juju quickstart -i https://raw.githubusercontent.com/whitmo/bundle-kubernetes/master/bundles.yaml
+    juju quickstart https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/juju/bundles/local.yaml
 
 The command above does few things for you:
 

cluster/juju/charms/trusty/kubernetes-master/actions.yaml

@@ -1,3 +0,0 @@
-petstore:
-    description: Deploy the Kubernetes Petstore app
-

cluster/juju/charms/trusty/kubernetes-master/actions/lib/parse_get_pods

@@ -1,17 +0,0 @@
-#!/usr/bin/env python
-
-import json
-import sys
-
-
-def parse_pod_output():
-    out = sys.stdin.readlines()
-    foo = json.loads(' '.join(out))
-    for item in foo['items']:
-        name_slugs = set(item['metadata']['name'].split('-'))
-        if 'fectrl' in name_slugs:
-            if item['status']['phase'] == "Running":
-                print item['status']['podIP']
-
-if __name__ == "__main__":
-        parse_pod_output()

cluster/juju/charms/trusty/kubernetes-master/actions/petstore

@@ -1,40 +0,0 @@
-#!/bin/bash
-
-set -eux
-
-export KUBE_MASTER_IP=0.0.0.0
-export KUBERNETES_MASTER=http://$KUBE_MASTER_IP
-
-# The test is a bit spammy with files, head over to /tmp to discard when done
-cd /tmp
-
-# Need to update the script with the proposed IP of the web-head pod
-# Currently its set to 10.1.4.89
-
-sed -i 's/pass_http=0/exit 0/' /opt/kubernetes/examples/k8petstore/k8petstore.sh
-/opt/kubernetes/examples/k8petstore/k8petstore.sh
-
-# Loop until the daemon has come online
-counter=0
-KUBE_POD_STATUS="Pending"
-while [ "$KUBE_POD_STATUS" == "Pending" ]
-do
-    if [[ counter -eq 200 ]]; then
-	echo "Pod failed to come online. Timeout reached"
-	exit 1
-    fi
-    CUR_STATUS=$(kubectl get pods -o json | $CHARM_DIR/actions/lib/parse_get_pods)
-    if [ ! -z "$CUR_STATUS" ]; then
-	KUBE_POD_STATUS=$CUR_STATUS
-    fi
-    sleep 1
-    counter+=1
-done
-
-sed -i 's/exit 0/pass_http=0/' /opt/kubernetes/examples/k8petstore/k8petstore.sh
-sed -i "s/PUBLIC_IP=\"10.1.4.89\"/PUBLIC_IP=\"$KUBE_POD_STATUS\"/"  /opt/kubernetes/examples/k8petstore/k8petstore.sh
-/opt/kubernetes/examples/k8petstore/k8petstore.sh
-
-# Return execution to the CHARM_DIR for further actions
-cd $CHARM_DIR
-

cluster/juju/charms/trusty/kubernetes-master/hooks/hooks.py

@@ -97,7 +97,7 @@ def config_changed():
 
     if config.changed('username') or config.changed('password'):
         hookenv.log('Username or password changed, creating authentication.')
-        basic_auth(config['username'], config['username'], config['password'])
+        basic_auth(username, username, password)
         if host.service_running('apiserver'):
             host.service_restart('apiserver')
 

cluster/juju/charms/trusty/kubernetes-master/hooks/install.py

@@ -35,7 +35,7 @@ def install():
         'export GOROOT=/usr/local/go\n',
         'export PATH=$PATH:$GOROOT/bin\n',
         'export KUBERNETES_MASTER=http://{0}:8080\n'.format(address),
-        ]
+    ]
     update_rc_files(strings)
     hookenv.log('Downloading kubernetes code')
     clone_repository()
@@ -91,6 +91,7 @@ def install_packages():
     # Create the list of packages to install.
     apt_packages = ['apache2-utils',
                     'build-essential',
+                    'docker.io',
                     'git',
                     'make',
                     'nginx',
@@ -98,12 +99,14 @@ def install_packages():
     fetch.apt_install(fetch.filter_installed_packages(apt_packages))
 
 
-def update_rc_files(strings):
+def update_rc_files(strings, rc_files=None):
     """
     Preseed the bash environment for ubuntu and root with K8's env vars to
     make interfacing with the api easier. (see: kubectrl docs)
     """
-    rc_files = [Path('/home/ubuntu/.bashrc'), Path('/root/.bashrc')]
+    if not rc_files:
+        rc_files = [Path('/home/ubuntu/.bashrc'), Path('/root/.bashrc')]
+
     for rc_file in rc_files:
         lines = rc_file.lines()
         for string in strings:

cluster/juju/charms/trusty/kubernetes-master/metadata.yaml

@@ -3,7 +3,7 @@ summary: Container Cluster Management Master
 description: |
   Provides a kubernetes api endpoint, scheduler for managing containers.
 maintainers:
-    - Matt Bruzek <matt.bruzek@canonical.com>
+    - Matt Bruzek <matthew.bruzek@canonical.com>
     - Whit Morriss <whit.morriss@canonical.com>
     - Charles Butler <charles.butler@canonical.com>
 tags:

cluster/juju/charms/trusty/kubernetes-master/unit_tests/kubernetes_installer_test.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 from mock import patch
+from mock import ANY
 from path import Path
 import pytest
 import subprocess
@@ -79,15 +80,28 @@ class TestKubernetesInstaller():
         ki.build("master")
         # TODO: run is called many times but mock only remembers last one.
         rmock.assert_called_with('git reset --hard origin/master')
+
         # TODO: call is complex and hard to verify with mock, fix that.
-        cmock.assert_called_once()
+        # this is not doing what we think it should be doing, magic mock
+        # makes this tricky.
+        # list['foo', 'baz'], env = ANY
+        make_args = ['make', 'all', 'WHAT=cmd/kube-apiserver cmd/kubectl cmd/kube-controller-manager plugin/cmd/kube-scheduler cmd/kubelet cmd/kube-proxy']  # noqa
+        cmock.assert_called_once_with(make_args, env=ANY)
+
+    @patch('kubernetes_installer.run')
+    @patch('kubernetes_installer.subprocess.call')
+    def test_schenanigans(self, cmock, rmock):
+        """ Test the build method with master and non-master branches. """
+        directory = Path('/tmp/kubernetes_installer_test/build')
+        ki = self.makeone('amd64', 'v99.00.11', directory)
+        assert not directory.exists(), 'The %s directory exists!' % directory
 
         # Call the build method with something other than "master" branch.
         ki.build("branch")
         # TODO: run is called many times, but mock only remembers last one.
         rmock.assert_called_with('git checkout -b v99.00.11 branch')
         # TODO: call is complex and hard to verify with mock, fix that.
-        cmock.assert_called_once()
+        assert cmock.called
 
         directory.rmtree_p()
 

cluster/juju/charms/trusty/kubernetes-master/unit_tests/test_install.py

@@ -26,15 +26,16 @@ sys.path.insert(0, d.abspath())
 # Import the modules from the hook
 import install
 
+
 class TestInstallHook():
 
-    @patch('install.path')
+    @patch('install.Path')
     def test_update_rc_files(self, pmock):
         """
         Test happy path on updating env files. Assuming everything
         exists and is in place.
         """
-        pmock.return_value.lines.return_value =  ['line1', 'line2']
+        pmock.return_value.lines.return_value = ['line1', 'line2']
         install.update_rc_files(['test1', 'test2'])
         pmock.return_value.write_lines.assert_called_with(['line1', 'line2',
                                                            'test1', 'test2'])
@@ -43,8 +44,9 @@ class TestInstallHook():
         """
         Test an unhappy path if the bashrc/users do not exist.
         """
+        p = [Path('/home/deadbeefdoesnotexist/.bashrc')]
         with pytest.raises(OSError) as exinfo:
-            install.update_rc_files(['test1','test2'])
+            install.update_rc_files(['test1', 'test2'], rc_files=p)
 
     @patch('install.fetch')
     @patch('install.hookenv')
@@ -53,8 +55,13 @@ class TestInstallHook():
         Verify we are calling the known essentials to build and syndicate
         kubes.
         """
-        pkgs = ['build-essential', 'git',
+        pkgs = ['apache2-utils',
-                'make', 'nginx', 'python-pip']
+                'build-essential',
+                'docker.io',
+                'git',
+                'make',
+                'nginx',
+                'python-pip',]
         install.install_packages()
         hemock.log.assert_called_with('Installing Debian packages')
         ftmock.filter_installed_packages.assert_called_with(pkgs)
@@ -62,22 +69,22 @@ class TestInstallHook():
     @patch('install.archiveurl.ArchiveUrlFetchHandler')
     def test_go_download(self, aumock):
         """
-            Test that we are actually handing off to charm-helpers to
+        Test that we are actually handing off to charm-helpers to
-            download a specific archive of Go. This is non-configurable so
+        download a specific archive of Go. This is non-configurable so
-            its reasonably safe to assume we're going to always do this,
+        its reasonably safe to assume we're going to always do this,
-            and when it changes we shall curse the brittleness of this test.
+        and when it changes we shall curse the brittleness of this test.
         """
         ins_mock = aumock.return_value.install
         install.download_go()
-        url = 'https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz'
+        url = 'https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz'  # noqa
-        sha1='5020af94b52b65cc9b6f11d50a67e4bae07b0aff'
+        sha1 = '5020af94b52b65cc9b6f11d50a67e4bae07b0aff'
         ins_mock.assert_called_with(url, '/usr/local', sha1, 'sha1')
 
     @patch('install.subprocess')
     def test_clone_repository(self, spmock):
         """
-         We're not using a unit-tested git library - so ensure our subprocess
+        We're not using a unit-tested git library - so ensure our subprocess
-         call is consistent. If we change this, we want to know we've broken it.
+        call is consistent. If we change this, we want to know we've broken it.
         """
         install.clone_repository()
         repo = 'https://github.com/kubernetes/kubernetes.git'
@@ -88,21 +95,22 @@ class TestInstallHook():
     @patch('install.download_go')
     @patch('install.clone_repository')
     @patch('install.update_rc_files')
+    @patch('install.Path')
     @patch('install.hookenv')
-    def test_install_main(self, hemock, urmock, crmock, dgmock, ipmock):
+    def test_install_main(self, hemock, pmock, urmock, crmock, dgmock, ipmock):
         """
         Ensure the driver/main method is calling all the supporting methods.
         """
-        strings = [
-        'export GOROOT=/usr/local/go\n',
-        'export PATH=$PATH:$GOROOT/bin\n',
-        'export KUBE_MASTER_IP=0.0.0.0\n',
-        'export KUBERNETES_MASTER=http://$KUBE_MASTER_IP\n',
-        ]
-
         install.install()
-        crmock.assert_called_once()
+
-        dgmock.assert_called_once()
+        assert(ipmock.called)
-        crmock.assert_called_once()
+        assert(dgmock.called)
-        urmock.assert_called_with(strings)
+        assert(crmock.called)
-        hemock.open_port.assert_called_with(8080)
+        assert(urmock.called)
+
+        assert(pmock.called)
+        pmock.assert_called_with('/srv/kubernetes')
+
+        hemock.open_port.assert_any_call(443)
+        hemock.open_port.assert_any_call(8080)
+        hemock.open_port.assert_any_call(6443)

cluster/juju/charms/trusty/kubernetes/Makefile

@@ -6,7 +6,7 @@ virtualenv:
 	.venv/bin/pip install -q -r requirements.txt
 
 lint: virtualenv
-	@.venv/bin/flake8 hooks unit_tests --exclude=charmhelpers
+	@.venv/bin/flake8 hooks --exclude=charmhelpers --ignore=W391
 	@.venv/bin/charm proof
 
 test: virtualenv 
@@ -27,3 +27,4 @@ endif
 clean:
 	rm -rf .venv
 	find -name *.pyc -delete
+	rm -rf unit_tests/.cache

cluster/juju/charms/trusty/kubernetes/README.md

@@ -67,7 +67,7 @@ The charm store version of the Kubernetes bundle can be deployed as follows:
 
 Alternately you could deploy a Kubernetes bundle straight from github or a file:
 
-    juju quickstart -i https://raw.githubusercontent.com/whitmo/bundle-kubernetes/master/bundles.yaml
+    juju quickstart https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/juju/bundles/local.yaml
 
 The command above does few things for you:
 

cluster/juju/charms/trusty/kubernetes/hooks/hooks.py

@@ -212,9 +212,14 @@ def register_machine(apiserver, retry=False):
     registration_request.data['spec']['externalID'] = private_address
     registration_request.data['status']['hostIP'] = private_address
 
-    response, result = registration_request.register(parsed.hostname,
+    try:
-                                                     parsed.port,
+        response, result = registration_request.register(parsed.hostname,
-                                                     '/api/v1/nodes')
+                                                         parsed.port,
+                                                         '/api/v1/nodes')
+    except socket.error:
+        hookenv.status_set('blocked',
+                           'Error communicating with Kubenetes Master')
+        return
 
     print(response)
 

cluster/juju/charms/trusty/kubernetes/hooks/lib/registrator.py

@@ -22,24 +22,24 @@ import time
 class Registrator:
 
     def __init__(self):
-        self.ds ={
+        self.ds = {
-          "creationTimestamp": "",
+            "creationTimestamp": "",
-          "kind": "Node",
+            "kind": "Node",
-          "name": "", # private_address
+            "name": "",  # private_address
-          "metadata": {
+            "metadata": {
-            "name": "", #private_address,
+                "name": "",  # private_address,
-          },
+            },
-          "spec": {
+            "spec": {
-            "externalID": "", #private_address
+                "externalID": "",  # private_address
-            "capacity": {
+                "capacity": {
-                "mem": "",  # mem + ' K',
+                    "mem": "",  # mem + ' K',
-                "cpu": "",  # cpus
+                    "cpu": "",  # cpus
+                }
+            },
+            "status": {
+                "conditions": [],
+                "hostIP": "",  # private_address
             }
-          },
-          "status": {
-            "conditions": [],
-            "hostIP": "", #private_address
-          }
         }
 
     @property
@@ -51,15 +51,15 @@ class Registrator:
         ''' Contact the API Server for a new registration '''
         headers = {"Content-type": "application/json",
                    "Accept": "application/json"}
-        connection = httplib.HTTPConnection(hostname, port)
+        connection = httplib.HTTPConnection(hostname, port, timeout=12)
         print 'CONN {}'.format(connection)
         connection.request("POST", api_path, json.dumps(self.data), headers)
         response = connection.getresponse()
         body = response.read()
         print(body)
         result = json.loads(body)
-        print("Response status:%s reason:%s body:%s" % \
+        print("Response status:%s reason:%s body:%s" %
-             (response.status, response.reason, result))
+              (response.status, response.reason, result))
         return response, result
 
     def update(self):
@@ -74,19 +74,17 @@ class Registrator:
         pass
 
     def command_succeeded(self, response, result):
-        ''' Evaluate response data to determine if the command was successful '''
+        ''' Evaluate response data to determine if the command successful '''
-        if response.status in [200, 201]:
+        if response.status in [200, 201, 409]:
+            # The 409 response is when a unit is already registered. We do not
+            # have an update method above, so for now, accept whats in etcd and
+            # assume registered.
             print("Registered")
             return True
-        elif response.status in [409,]:
-            print("Status Conflict")
-            # Suggested return a PUT instead of a POST with this response
-            # code, this predicates use of the UPDATE method
-            # TODO
         elif response.status in (500,) and result.get(
-            'message', '').startswith('The requested resource does not exist'):
+            'message', '').startswith('The requested resource does not exist'):  # noqa
-            # There's something fishy in the kube api here (0.4 dev), first time we
+            # There is something fishy in the kube api here (0.4 dev), first
-            # go to register a new minion, we always seem to get this error.
+            # time to register a new node, we always seem to get this error.
             # http://issue.k8s.io/1995
             time.sleep(1)
             print("Retrying registration...")

cluster/juju/charms/trusty/kubernetes/unit_tests/lib/test_registrator.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 import json
-from mock import MagicMock, patch, call
+from mock import MagicMock, patch
 from path import Path
 import pytest
 import sys
@@ -25,6 +25,7 @@ sys.path.insert(0, d.abspath())
 
 from lib.registrator import Registrator
 
+
 class TestRegistrator():
 
     def setup_method(self, method):
@@ -37,25 +38,23 @@ class TestRegistrator():
     @patch('json.loads')
     @patch('httplib.HTTPConnection')
     def test_register(self, httplibmock, jsonmock):
-        result = self.r.register('foo', 80, '/v1/test')
+        self.r.register('foo', 80, '/v1/test')
 
-        httplibmock.assert_called_with('foo', 80)
+        httplibmock.assert_called_with('foo', 80, timeout=12)
         requestmock = httplibmock().request
         requestmock.assert_called_with(
-                "POST", "/v1/test",
+            "POST", "/v1/test",
-                json.dumps(self.r.data),
+            json.dumps(self.r.data),
-                {"Content-type": "application/json",
+            {"Content-type": "application/json",
-                 "Accept": "application/json"})
+                "Accept": "application/json"})
-
 
     def test_command_succeeded(self):
         response = MagicMock()
-        result = json.loads('{"status": "Failure", "kind": "Status", "code": 409, "apiVersion": "v1", "reason": "AlreadyExists", "details": {"kind": "node", "name": "10.200.147.200"}, "message": "node \\"10.200.147.200\\" already exists", "creationTimestamp": null}')
+        result = json.loads('{"status": "Failure", "kind": "Status", "code": 409, "apiVersion": "v1", "reason": "AlreadyExists", "details": {"kind": "node", "name": "10.200.147.200"}, "message": "node \\"10.200.147.200\\" already exists", "creationTimestamp": null}')  # noqa
         response.status = 200
         self.r.command_succeeded(response, result)
+        response.status = 409
+        self.r.command_succeeded(response, result)
         response.status = 500
         with pytest.raises(RuntimeError):
             self.r.command_succeeded(response, result)
-        response.status = 409
-        with pytest.raises(ValueError):
-            self.r.command_succeeded(response, result)