Merge pull request #2291 from ijc/kubernetes

Kubernetes bump to 1.6.7 and improved functionality on Linux platform
2025-07-19 09:16:29 +00:00 · 2017-07-26 10:38:20 +01:00 · 2017-07-26 10:38:20 +01:00 · 09dc4c1c88
commit 09dc4c1c88
parent 52c3ef9406 828ac913e8
11 changed files with 183 additions and 61 deletions
--- a/projects/kubernetes/README.md
+++ b/projects/kubernetes/README.md
@ -11,7 +11,7 @@ Build OS images:
 make build-vm-images
 ```

-Boot Kubernetes master OS image using `hyperkit` on macOS:
+Boot Kubernetes master OS image using `hyperkit` on macOS: or `qemu` on Linux:
 ```
 ./boot.sh
 ```
@ -45,3 +45,45 @@ shell1> ./boot.sh 1 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443
 shell2> ./boot.sh 2 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443
 shell3> ./boot.sh 3 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443
 ```
+
+## Platform specific information
+
+### MacOS
+
+The above instructions should work as is.
+
+### Linux
+
+By default `linuxkit run` uses user mode networking which does not
+support access from the host. To workaround this you can use port
+forwarding e.g.
+
+    KUBE_RUN_ARGS="-publish 2222:22" ./boot.sh
+
+    ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2222 root@localhost
+
+However you will not be able to run worker nodes since individual
+instances cannot see each other.
+
+To enable networking between instance unfortunately requires `root`
+privileges to configure a bridge and setup the bridge mode privileged
+helper.
+
+See http://wiki.qemu.org/Features/HelperNetworking for details in
+brief you will need:
+
+- To setup and configure a bridge (including e.g. DHCP etc) on the
+  host. (You can reuse a bridge created by e.g. `virt-mananger`)
+- To set the `qemu-bridge-helper` setuid root. The location differs by
+  distro, it could be `/usr/lib/qemu/qemu-bridge-helper` or
+  `/usr/local/libexec/qemu-bridge-helper` or elsewhere. You need to
+  `chmod u+s «PATH»`.
+- List the bridge created in the first step in `/etc/qemu/bridge.conf`
+  with a line like `allow br0` (if your bridge is called `br0`).
+
+## Configuration
+
+The `boot.sh` script has various configuration variables at the top
+which can be overridden via the environment e.g.
+
+    KUBE_VCPUS=4 ./boot.sh
--- a/projects/kubernetes/boot.sh
+++ b/projects/kubernetes/boot.sh
@ -1,8 +1,12 @@
 #!/bin/bash -eu
 : ${KUBE_PORT_BASE:=2222}
+: ${KUBE_VCPUS:=2}
+: ${KUBE_MEM:=4096}
+: ${KUBE_DISK:=4G}
+: ${KUBE_NETWORKING:=default}
+: ${KUBE_RUN_ARGS:=}
 if [ $# -eq 0 ] ; then
    img="kube-master"
-    port=${KUBE_PORT_BASE}
    data=""
    state="kube-master-state"
 elif [ $# -gt 1 ] ; then
@ -19,7 +23,6 @@ elif [ $# -gt 1 ] ; then
    esac
    img="kube-node"
    name="node-${1}"
-    port=$((${KUBE_PORT_BASE} + $1))
    shift
    data="${*}"
    state="kube-${name}-state"
@ -33,4 +36,4 @@ else
 fi
 set -x
 rm -rf "${state}"
-../../bin/linuxkit run -publish $port:22 -cpus 2 -mem 4096 -state "${state}" -disk size=4G -data "${data}" "${img}"
+../../bin/linuxkit run ${KUBE_RUN_ARGS} -networking ${KUBE_NETWORKING} -cpus ${KUBE_VCPUS} -mem ${KUBE_MEM} -state "${state}" -disk size=${KUBE_DISK} -data "${data}" "${img}"
--- a/projects/kubernetes/image-cache/Makefile
+++ b/projects/kubernetes/image-cache/Makefile
@ -1,16 +1,13 @@
 default: push

 COMMON_IMAGES := \
-  kube-proxy-amd64\:v1.6.1@sha256\:243f2120171330a26c2418a4367fb0f3cc3e92683b00d16e3cf8c7f92e25bf14 \
-  k8s-dns-sidecar-amd64\:1.14.1@sha256\:d33a91a5d65c223f410891001cd379ac734d036429e033865d700a4176e944b0 \
-  k8s-dns-kube-dns-amd64\:1.14.1@sha256\:33914315e600dfb756e550828307dfa2b21fb6db24fe3fe495e33d1022f9245d \
-  k8s-dns-dnsmasq-nanny-amd64\:1.14.1@sha256\:89c9a1d3cfbf370a9c1a949f39f92c1dc2dbe8c3e6cc1802b7f2b48e4dfe9a9e \
+  kube-proxy-amd64\:v1.6.7@sha256\:652ca0ef7cdf05341fafb590ced1b737126641829c70f5d23f9b714bc61c8607 \
  pause-amd64\:3.0@sha256\:163ac025575b775d1c0f9bf0bdd0f086883171eb475b5068e7defa4ca9e76516

 CONTROL_PLANE_IMAGES := \
-  kube-apiserver-amd64\:v1.6.1@sha256\:d4387dff51b1f9c94cd1cfac3a4694347970b90e911159ac6fe2d090c96a6184 \
-  kube-controller-manager-amd64\:v1.6.1@sha256\:4bb17ede2e012898169d988facd08d5039d2dcb31532661d4dcdeb161d097d69 \
-  kube-scheduler-amd64\:v1.6.1@sha256\:d3e661bf7bcfb10753e32e1a41615e60fbcddff63232f914e9326a2d1665ce33 \
+  kube-apiserver-amd64\:v1.6.7@sha256\:57e482529b95d32730d1bcd2e374199f27eab4abcf1ff49c5db2a7a7e2231cc8 \
+  kube-controller-manager-amd64\:v1.6.7@sha256\:884f609895fa715d66806681a6bf6f9851a911202ad3484b768fead8b7c78b39 \
+  kube-scheduler-amd64\:v1.6.7@sha256\:a1a498a0ca5ab23c228724d93c3f3d9457b31a046d9025471d98e1096422452c \
  etcd-amd64\:3.0.17@sha256\:d83d3545e06fb035db8512e33bd44afb55dea007a3abd7b17742d3ac6d235940

 dl/%.tar:
--- a/projects/kubernetes/kube-master.yml
+++ b/projects/kubernetes/kube-master.yml
@ -56,11 +56,11 @@ services:
    rootfsPropagation: shared
    command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"]
  - name: kubernetes-image-cache-common
-    image: linuxkitprojects/kubernetes-image-cache-common:d49a861bde872e6e975153a98a2c482834a30ef9
+    image: linuxkitprojects/kubernetes-image-cache-common:6fccda74ea301f9a62cdcfc2fe4952cff2c8c97b
  - name: kubernetes-image-cache-control-plane
-    image: linuxkitprojects/kubernetes-image-cache-control-plane:d49a861bde872e6e975153a98a2c482834a30ef9
+    image: linuxkitprojects/kubernetes-image-cache-control-plane:6fccda74ea301f9a62cdcfc2fe4952cff2c8c97b
  - name: kubelet
-    image: linuxkitprojects/kubernetes:4f8c61254ff6243e93d5bb6315386ac66e94ed14
+    image: linuxkitprojects/kubernetes:d4d722823b1265a57355ae8a309d4953e293fd58
 files:
  - path: root/.ssh/authorized_keys
    source: ~/.ssh/id_rsa.pub
--- a/projects/kubernetes/kube-node.yml
+++ b/projects/kubernetes/kube-node.yml
@ -56,9 +56,9 @@ services:
    rootfsPropagation: shared
    command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"]
  - name: kubernetes-image-cache-common
-    image: linuxkitprojects/kubernetes-image-cache-common:d49a861bde872e6e975153a98a2c482834a30ef9
+    image: linuxkitprojects/kubernetes-image-cache-common:6fccda74ea301f9a62cdcfc2fe4952cff2c8c97b
  - name: kubelet
-    image: linuxkitprojects/kubernetes:4f8c61254ff6243e93d5bb6315386ac66e94ed14
+    image: linuxkitprojects/kubernetes:d4d722823b1265a57355ae8a309d4953e293fd58
 files:
  - path: root/.ssh/authorized_keys
    source: ~/.ssh/id_rsa.pub
--- a/projects/kubernetes/kubernetes/Dockerfile
+++ b/projects/kubernetes/kubernetes/Dockerfile
@ -2,9 +2,9 @@
 # XXX needs ebtables ethtool iproute2 libc6-compat socat
 FROM alpine:3.6 AS build

-ENV kubernetes_version v1.6.1
-ENV weave_version      v1.9.4
-ENV cni_version        0799f5732f2a11b329d9e3d51b9c8f2e3759f2ff
+ENV kubernetes_version v1.6.7
+ENV weave_version      v2.0.1
+ENV cni_version        v0.5.2

 ENV kube_release_artefacts "https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64"

@ -30,16 +30,17 @@ RUN apk add --no-cache --initdb -p /out \
 # Remove apk residuals. We have a read-only rootfs, so apk is of no use.
 RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache

-RUN curl -fSL -o /tmp/cni.tgz https://dl.k8s.io/network-plugins/cni-amd64-${cni_version}.tar.gz && \
-    mkdir -p /out/opt/cni /out/etc/cni/net.d && \
-    tar -xzf /tmp/cni.tgz -C /out/opt/cni
+RUN curl -fSL -o /tmp/cni.tgz https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-${cni_version}.tgz && \
+    mkdir -p /out/opt/cni/bin /out/etc/cni/net.d && \
+    tar -xzf /tmp/cni.tgz -C /out/opt/cni/bin
 RUN curl -fSL -o /out/etc/weave.yaml https://cloud.weave.works/k8s/v1.6/net?v=${weave_version} 
 RUN curl -fSL -o /out/usr/bin/kubelet https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubelet && chmod 0755 /out/usr/bin/kubelet
 RUN curl -fSL -o /out/usr/bin/kubeadm https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubeadm && chmod 0755 /out/usr/bin/kubeadm
 RUN curl -fSL -o /out/usr/bin/kubectl https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubectl && chmod 0755 /out/usr/bin/kubectl

 ADD kubelet.sh /out/usr/bin/kubelet.sh
-ADD kubeadm-init.sh /out/usr/bin/kubeadm-init.sh
+ADD kubeadm-init.sh /kubeadm-init.sh
+RUN sed -e "s/@KUBERNETES_VERSION@/${kubernetes_version}/g" </kubeadm-init.sh >/out/usr/bin/kubeadm-init.sh && chmod +x /out/usr/bin/kubeadm-init.sh

 FROM scratch
 WORKDIR /
--- a/projects/kubernetes/kubernetes/kubeadm-init.sh
+++ b/projects/kubernetes/kubernetes/kubeadm-init.sh
@ -1,4 +1,4 @@
 #!/bin/sh
 set -e
-kubeadm init --skip-preflight-checks --kubernetes-version v1.6.1
+kubeadm init --skip-preflight-checks --kubernetes-version @KUBERNETES_VERSION@
 kubectl create -n kube-system -f /etc/weave.yaml
--- a/projects/kubernetes/ssh.sh
+++ b/projects/kubernetes/ssh.sh
@ -1,13 +0,0 @@
-#!/bin/bash -eux
-docker run \
-  --rm \
-  -ti \
-  -v ~/.ssh/:/root/.ssh \
-    jdeathe/centos-ssh \
-      ssh \
-        -o Compression=yes \
-	-o LogLevel=FATAL \
-	-o StrictHostKeyChecking=no \
-	-o UserKnownHostsFile=/dev/null \
-	-o IdentitiesOnly=yes \
-	  "$@"
--- a/projects/kubernetes/ssh_into_kubelet.sh
+++ b/projects/kubernetes/ssh_into_kubelet.sh
@ -1,2 +1,18 @@
-#!/bin/bash -eux
-./ssh.sh -t root@"$1" nsenter --mount --target 1 runc exec --tty kubelet ash -l
+#!/bin/bash -eu
+
+sshopts="-o LogLevel=FATAL \
+	 -o StrictHostKeyChecking=no \
+	 -o UserKnownHostsFile=/dev/null \
+	 -o IdentitiesOnly=yes"
+
+case $(uname -s) in
+    Linux)
+	ssh=ssh
+	;;
+    *)
+	ssh="docker run --rm -ti \
+	  -v $HOME/.ssh/:/root/.ssh \
+	    ijc25/alpine-ssh"
+	;;
+esac
+$ssh $sshopts -t root@"$1" ctr exec --tty --exec-id ssh kubelet ash -l
--- a/src/cmd/linuxkit/run_hyperkit.go
+++ b/src/cmd/linuxkit/run_hyperkit.go
@ -17,10 +17,11 @@ import (
 )

 const (
-	networkingNone         string = "none"
-	networkingDockerForMac        = "docker-for-mac"
-	networkingVPNKit              = "vpnkit"
-	networkingVMNet               = "vmnet"
+	hyperkitNetworkingNone         string = "none"
+	hyperkitNetworkingDockerForMac        = "docker-for-mac"
+	hyperkitNetworkingVPNKit              = "vpnkit"
+	hyperkitNetworkingVMNet               = "vmnet"
+	hyperkitNetworkingDefault             = hyperkitNetworkingDockerForMac
 )

 // Process the run arguments and execute run
@ -43,7 +44,7 @@ func runHyperKit(args []string) {
 	ipStr := flags.String("ip", "", "IP address for the VM")
 	state := flags.String("state", "", "Path to directory to keep VM state in")
 	vsockports := flags.String("vsock-ports", "", "List of vsock ports to forward from the guest on startup (comma separated). A unix domain socket for each port will be created in the state directory")
-	networking := flags.String("networking", networkingDockerForMac, "Networking mode. Valid options are 'docker-for-mac', 'vpnkit[,socket-path]', 'vmnet' and 'none'. 'docker-for-mac' connects to the network used by Docker for Mac. 'vpnkit' connects to the VPNKit socket specified. If socket-path is omitted a new VPNKit instance will be started and 'vpnkit_eth.sock' will be created in the state directory. 'vmnet' uses the Apple vmnet framework, requires root/sudo. 'none' disables networking.`")
+	networking := flags.String("networking", hyperkitNetworkingDefault, "Networking mode. Valid options are 'default', 'docker-for-mac', 'vpnkit[,socket-path]', 'vmnet' and 'none'. 'docker-for-mac' connects to the network used by Docker for Mac. 'vpnkit' connects to the VPNKit socket specified. If socket-path is omitted a new VPNKit instance will be started and 'vpnkit_eth.sock' will be created in the state directory. 'vmnet' uses the Apple vmnet framework, requires root/sudo. 'none' disables networking.`")

 	if err := flags.Parse(args); err != nil {
 		log.Fatal("Unable to parse args")
@ -130,12 +131,16 @@ func runHyperKit(args []string) {

 	// Select network mode
 	var vpnKitProcess *os.Process
+	if *networking == "" || *networking == "default" {
+		dflt := hyperkitNetworkingDefault
+		networking = &dflt
+	}
 	netMode := strings.SplitN(*networking, ",", 2)

 	switch netMode[0] {
-	case networkingDockerForMac:
+	case hyperkitNetworkingDockerForMac:
 		h.VPNKitSock = filepath.Join(os.Getenv("HOME"), "Library/Containers/com.docker.docker/Data/s50")
-	case networkingVPNKit:
+	case hyperkitNetworkingVPNKit:
 		if len(netMode) > 1 {
 			// Socket path specified, try to use existing VPNKit instance
 			h.VPNKitSock = netMode[1]
@ -161,10 +166,10 @@ func runHyperKit(args []string) {
 			// VSOCK port 62373 is used to pass traffic from host->guest
 			h.VSockPorts = append(h.VSockPorts, 62373)
 		}
-	case networkingVMNet:
+	case hyperkitNetworkingVMNet:
 		h.VPNKitSock = ""
 		h.VMNet = true
-	case networkingNone:
+	case hyperkitNetworkingNone:
 		h.VPNKitSock = ""
 	default:
 		log.Fatalf("Invalid networking mode: %s", netMode[0])
--- a/src/cmd/linuxkit/run_qemu.go
+++ b/src/cmd/linuxkit/run_qemu.go
@ -1,9 +1,11 @@
 package main

 import (
+	"crypto/rand"
 	"flag"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@ -11,6 +13,7 @@ import (
 	"strings"

 	log "github.com/Sirupsen/logrus"
+	"github.com/satori/go.uuid"
 	"golang.org/x/crypto/ssh/terminal"
 )

@ -35,9 +38,18 @@ type QemuConfig struct {
 	QemuBinPath    string
 	QemuImgPath    string
 	PublishedPorts []string
-	TapDevice      string
+	NetdevConfig   string
+	UUID           uuid.UUID
 }

+const (
+	qemuNetworkingNone    string = "none"
+	qemuNetworkingUser           = "user"
+	qemuNetworkingTap            = "tap"
+	qemuNetworkingBridge         = "bridge"
+	qemuNetworkingDefault        = qemuNetworkingUser
+)
+
 func haveKVM() bool {
 	_, err := os.Stat("/dev/kvm")
 	return !os.IsNotExist(err)
@ -56,6 +68,20 @@ func envOverrideBool(env string, b *bool) {
 	}
 }

+func generateMAC() net.HardwareAddr {
+	mac := make([]byte, 6)
+	n, err := rand.Read(mac)
+	if err != nil {
+		log.WithError(err).Fatal("failed to generate random mac address")
+	}
+	if n != 6 {
+		log.WithError(err).Fatal("generated %d bytes for random mac address", n)
+	}
+	mac[0] &^= 0x01 // Clear multicast bit
+	mac[0] |= 0x2   // Set locally administered bit
+	return net.HardwareAddr(mac)
+}
+
 func runQemu(args []string) {
 	invoked := filepath.Base(os.Args[0])
 	flags := flag.NewFlagSet("qemu", flag.ExitOnError)
@ -65,6 +91,10 @@ func runQemu(args []string) {
 		fmt.Printf("\n")
 		fmt.Printf("Options:\n")
 		flags.PrintDefaults()
+		fmt.Printf("\n")
+		fmt.Printf("If not running as root note that '-networking bridge,br0' requires a\n")
+		fmt.Printf("setuid network helper and appropriate host configuration, see\n")
+		fmt.Printf("http://wiki.qemu.org/Features/HelperNetworking.\n")
 	}

 	// Display flags
@ -95,9 +125,14 @@ func runQemu(args []string) {
 	// Backend configuration
 	qemuContainerized := flags.Bool("containerized", false, "Run qemu in a container")

+	// Generate UUID, so that /sys/class/dmi/id/product_uuid is populated
+	vmUUID := uuid.NewV4()
+
+	// Networking
+	networking := flags.String("networking", qemuNetworkingDefault, "Networking mode. Valid options are 'default', 'user', 'bridge[,name]', tap[,name] and 'none'. 'user' uses QEMUs userspace networking. 'bridge' connects to a preexisting bridge. 'tap' uses a prexisting tap device. 'none' disables networking.`")
+
 	publishFlags := multipleFlag{}
 	flags.Var(&publishFlags, "publish", "Publish a vm's port(s) to the host (default [])")
-	tapDevice := flags.String("tap-device", "", "Tap device to use as eth0 (optional)")

 	if err := flags.Parse(args); err != nil {
 		log.Fatal("Unable to parse args")
@ -201,6 +236,40 @@ func runQemu(args []string) {
 	if *isoBoot && isoPath != "" {
 		log.Fatalf("metadata and ISO boot currently cannot coexist")
 	}
+	if *networking == "" || *networking == "default" {
+		dflt := qemuNetworkingDefault
+		networking = &dflt
+	}
+	netMode := strings.SplitN(*networking, ",", 2)
+
+	var netdevConfig string
+	switch netMode[0] {
+	case qemuNetworkingUser:
+		netdevConfig = "user"
+	case qemuNetworkingTap:
+		if len(netMode) != 2 {
+			log.Fatalf("Not enough arugments for %q networking mode", qemuNetworkingTap)
+		}
+		if len(publishFlags) != 0 {
+			log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser)
+		}
+		netdevConfig = fmt.Sprintf("tap,ifname=%s,script=no,downscript=no", netMode[1])
+	case qemuNetworkingBridge:
+		if len(netMode) != 2 {
+			log.Fatalf("Not enough arugments for %q networking mode", qemuNetworkingBridge)
+		}
+		if len(publishFlags) != 0 {
+			log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser)
+		}
+		netdevConfig = fmt.Sprintf("bridge,br=%s", netMode[1])
+	case qemuNetworkingNone:
+		if len(publishFlags) != 0 {
+			log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser)
+		}
+		netdevConfig = ""
+	default:
+		log.Fatalf("Invalid networking mode: %s", netMode[0])
+	}

 	config := QemuConfig{
 		Path:           path,
@ -217,7 +286,8 @@ func runQemu(args []string) {
 		KVM:            *enableKVM,
 		Containerized:  *qemuContainerized,
 		PublishedPorts: publishFlags,
-		TapDevice:      *tapDevice,
+		NetdevConfig:   netdevConfig,
+		UUID:           vmUUID,
 	}

 	config = discoverBackend(config)
@ -380,6 +450,7 @@ func buildQemuCmdline(config QemuConfig) (QemuConfig, []string) {
 	qemuArgs = append(qemuArgs, "-device", "virtio-rng-pci")
 	qemuArgs = append(qemuArgs, "-smp", config.CPUs)
 	qemuArgs = append(qemuArgs, "-m", config.Memory)
+	qemuArgs = append(qemuArgs, "-uuid", config.UUID.String())
 	// Need to specify the vcpu type when running qemu on arm64 platform, for security reason,
 	// the vcpu should be "host" instead of other names such as "cortex-a53"...
 	if config.Arch == "aarch64" {
@ -441,19 +512,16 @@ func buildQemuCmdline(config QemuConfig) (QemuConfig, []string) {
 		}
 	}

-	if config.PublishedPorts != nil && len(config.PublishedPorts) > 0 {
+	if config.NetdevConfig == "" {
+		qemuArgs = append(qemuArgs, "-net", "none")
+	} else {
+		mac := generateMAC()
+		qemuArgs = append(qemuArgs, "-net", "nic,model=virtio,macaddr="+mac.String())
 		forwardings, err := buildQemuForwardings(config.PublishedPorts, config.Containerized)
 		if err != nil {
 			log.Error(err)
 		}
-		qemuArgs = append(qemuArgs, "-net", forwardings)
-		qemuArgs = append(qemuArgs, "-net", "nic")
-	}
-
-	if config.TapDevice != "" {
-		qemuArgs = append(qemuArgs, "-net", "nic,model=virtio")
-		tapArg := fmt.Sprintf("tap,ifname=%s,script=no,downscript=no", config.TapDevice)
-		qemuArgs = append(qemuArgs, "-net", tapArg)
+		qemuArgs = append(qemuArgs, "-net", config.NetdevConfig+forwardings)
 	}

 	if config.GUI != true {
@ -547,7 +615,10 @@ func splitPublish(publish string) (publishedPorts, error) {
 }

 func buildQemuForwardings(publishFlags multipleFlag, containerized bool) (string, error) {
-	forwardings := "user"
+	if len(publishFlags) == 0 {
+		return "", nil
+	}
+	var forwardings string
 	for _, publish := range publishFlags {
 		p, err := splitPublish(publish)
 		if err != nil {