From cb86cdb02712ffd7cc4ee57c34c9ab136213c0af Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 25 Jul 2017 13:29:06 +0100 Subject: [PATCH 01/13] qemu: Generate a random uuid and pass via -uuid This is the same behaviour as the LinuxKit backend. This populates /sys/class/dmi/id/product_uuid, which newer version of weave-net appears to require. Signed-off-by: Ian Campbell --- src/cmd/linuxkit/run_qemu.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/cmd/linuxkit/run_qemu.go b/src/cmd/linuxkit/run_qemu.go index 618d84e84..05f6bf298 100644 --- a/src/cmd/linuxkit/run_qemu.go +++ b/src/cmd/linuxkit/run_qemu.go @@ -11,6 +11,7 @@ import ( "strings" log "github.com/Sirupsen/logrus" + "github.com/satori/go.uuid" "golang.org/x/crypto/ssh/terminal" ) @@ -36,6 +37,7 @@ type QemuConfig struct { QemuImgPath string PublishedPorts []string TapDevice string + UUID uuid.UUID } func haveKVM() bool { @@ -95,6 +97,9 @@ func runQemu(args []string) { // Backend configuration qemuContainerized := flags.Bool("containerized", false, "Run qemu in a container") + // Generate UUID, so that /sys/class/dmi/id/product_uuid is populated + vmUUID := uuid.NewV4() + publishFlags := multipleFlag{} flags.Var(&publishFlags, "publish", "Publish a vm's port(s) to the host (default [])") tapDevice := flags.String("tap-device", "", "Tap device to use as eth0 (optional)") @@ -218,6 +223,7 @@ func runQemu(args []string) { Containerized: *qemuContainerized, PublishedPorts: publishFlags, TapDevice: *tapDevice, + UUID: vmUUID, } config = discoverBackend(config) @@ -380,6 +386,7 @@ func buildQemuCmdline(config QemuConfig) (QemuConfig, []string) { qemuArgs = append(qemuArgs, "-device", "virtio-rng-pci") qemuArgs = append(qemuArgs, "-smp", config.CPUs) qemuArgs = append(qemuArgs, "-m", config.Memory) + qemuArgs = append(qemuArgs, "-uuid", config.UUID.String()) // Need to specify the vcpu type when running qemu on arm64 platform, for security reason, // the vcpu should be "host" instead of other names such as "cortex-a53"... if config.Arch == "aarch64" { From 096aec0a197192a2cb7263af6c419de561b287a4 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 25 Jul 2017 15:26:25 +0100 Subject: [PATCH 02/13] qemu: Add -networking option, with various new alternatives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This follows the model in the hyperkit runner, although the options are different. The options are: - `user`: the existing user mode networking (the default). - `tap,«device»`: replaces the previous `-tap-device «device»` option. - `bridge,«name»`: tap device on (preexisting) named bridge. - `none`: No networking at all. If not running as root then `bridge` mode requires host configuration http://wiki.qemu.org/Features/HelperNetworking. TL;DR: you need to `chmod u+s` the `qemu-bridge-helper` and to whitelist specific bridges in `/etc/qemu/bridge.conf`. Pass an explicit virtio nic and configure a random MAC since QEMU seems to use the same one by default. In the hyperkit runner the various `networking*` constants become `hyperkitNetworking*` to avoid namespace clashes (e.g. for `None`). The QEMU equivalents are `qemuNetworking*`. Both hyperkit and qemu now support an explicit `-networking default` or `-networking ''` to make scripting easier. Signed-off-by: Ian Campbell --- src/cmd/linuxkit/run_hyperkit.go | 23 ++++---- src/cmd/linuxkit/run_qemu.go | 90 +++++++++++++++++++++++++++----- 2 files changed, 91 insertions(+), 22 deletions(-) diff --git a/src/cmd/linuxkit/run_hyperkit.go b/src/cmd/linuxkit/run_hyperkit.go index f0f3466bc..d13a3cef2 100644 --- a/src/cmd/linuxkit/run_hyperkit.go +++ b/src/cmd/linuxkit/run_hyperkit.go @@ -17,10 +17,11 @@ import ( ) const ( - networkingNone string = "none" - networkingDockerForMac = "docker-for-mac" - networkingVPNKit = "vpnkit" - networkingVMNet = "vmnet" + hyperkitNetworkingNone string = "none" + hyperkitNetworkingDockerForMac = "docker-for-mac" + hyperkitNetworkingVPNKit = "vpnkit" + hyperkitNetworkingVMNet = "vmnet" + hyperkitNetworkingDefault = hyperkitNetworkingDockerForMac ) // Process the run arguments and execute run @@ -43,7 +44,7 @@ func runHyperKit(args []string) { ipStr := flags.String("ip", "", "IP address for the VM") state := flags.String("state", "", "Path to directory to keep VM state in") vsockports := flags.String("vsock-ports", "", "List of vsock ports to forward from the guest on startup (comma separated). A unix domain socket for each port will be created in the state directory") - networking := flags.String("networking", networkingDockerForMac, "Networking mode. Valid options are 'docker-for-mac', 'vpnkit[,socket-path]', 'vmnet' and 'none'. 'docker-for-mac' connects to the network used by Docker for Mac. 'vpnkit' connects to the VPNKit socket specified. If socket-path is omitted a new VPNKit instance will be started and 'vpnkit_eth.sock' will be created in the state directory. 'vmnet' uses the Apple vmnet framework, requires root/sudo. 'none' disables networking.`") + networking := flags.String("networking", hyperkitNetworkingDefault, "Networking mode. Valid options are 'default', 'docker-for-mac', 'vpnkit[,socket-path]', 'vmnet' and 'none'. 'docker-for-mac' connects to the network used by Docker for Mac. 'vpnkit' connects to the VPNKit socket specified. If socket-path is omitted a new VPNKit instance will be started and 'vpnkit_eth.sock' will be created in the state directory. 'vmnet' uses the Apple vmnet framework, requires root/sudo. 'none' disables networking.`") if err := flags.Parse(args); err != nil { log.Fatal("Unable to parse args") @@ -130,12 +131,16 @@ func runHyperKit(args []string) { // Select network mode var vpnKitProcess *os.Process + if *networking == "" || *networking == "default" { + dflt := hyperkitNetworkingDefault + networking = &dflt + } netMode := strings.SplitN(*networking, ",", 2) switch netMode[0] { - case networkingDockerForMac: + case hyperkitNetworkingDockerForMac: h.VPNKitSock = filepath.Join(os.Getenv("HOME"), "Library/Containers/com.docker.docker/Data/s50") - case networkingVPNKit: + case hyperkitNetworkingVPNKit: if len(netMode) > 1 { // Socket path specified, try to use existing VPNKit instance h.VPNKitSock = netMode[1] @@ -161,10 +166,10 @@ func runHyperKit(args []string) { // VSOCK port 62373 is used to pass traffic from host->guest h.VSockPorts = append(h.VSockPorts, 62373) } - case networkingVMNet: + case hyperkitNetworkingVMNet: h.VPNKitSock = "" h.VMNet = true - case networkingNone: + case hyperkitNetworkingNone: h.VPNKitSock = "" default: log.Fatalf("Invalid networking mode: %s", netMode[0]) diff --git a/src/cmd/linuxkit/run_qemu.go b/src/cmd/linuxkit/run_qemu.go index 05f6bf298..59fd1fa46 100644 --- a/src/cmd/linuxkit/run_qemu.go +++ b/src/cmd/linuxkit/run_qemu.go @@ -1,9 +1,11 @@ package main import ( + "crypto/rand" "flag" "fmt" "io/ioutil" + "net" "os" "os/exec" "path/filepath" @@ -36,10 +38,18 @@ type QemuConfig struct { QemuBinPath string QemuImgPath string PublishedPorts []string - TapDevice string + NetdevConfig string UUID uuid.UUID } +const ( + qemuNetworkingNone string = "none" + qemuNetworkingUser = "user" + qemuNetworkingTap = "tap" + qemuNetworkingBridge = "bridge" + qemuNetworkingDefault = qemuNetworkingUser +) + func haveKVM() bool { _, err := os.Stat("/dev/kvm") return !os.IsNotExist(err) @@ -58,6 +68,20 @@ func envOverrideBool(env string, b *bool) { } } +func generateMAC() net.HardwareAddr { + mac := make([]byte, 6) + n, err := rand.Read(mac) + if err != nil { + log.WithError(err).Fatal("failed to generate random mac address") + } + if n != 6 { + log.WithError(err).Fatal("generated %d bytes for random mac address", n) + } + mac[0] &^= 0x01 // Clear multicast bit + mac[0] |= 0x2 // Set locally administered bit + return net.HardwareAddr(mac) +} + func runQemu(args []string) { invoked := filepath.Base(os.Args[0]) flags := flag.NewFlagSet("qemu", flag.ExitOnError) @@ -67,6 +91,10 @@ func runQemu(args []string) { fmt.Printf("\n") fmt.Printf("Options:\n") flags.PrintDefaults() + fmt.Printf("\n") + fmt.Printf("If not running as root note that '-networking bridge,br0' requires a\n") + fmt.Printf("setuid network helper and appropriate host configuration, see\n") + fmt.Printf("http://wiki.qemu.org/Features/HelperNetworking.\n") } // Display flags @@ -100,9 +128,11 @@ func runQemu(args []string) { // Generate UUID, so that /sys/class/dmi/id/product_uuid is populated vmUUID := uuid.NewV4() + // Networking + networking := flags.String("networking", qemuNetworkingDefault, "Networking mode. Valid options are 'default', 'user', 'bridge[,name]', tap[,name] and 'none'. 'user' uses QEMUs userspace networking. 'bridge' connects to a preexisting bridge. 'tap' uses a prexisting tap device. 'none' disables networking.`") + publishFlags := multipleFlag{} flags.Var(&publishFlags, "publish", "Publish a vm's port(s) to the host (default [])") - tapDevice := flags.String("tap-device", "", "Tap device to use as eth0 (optional)") if err := flags.Parse(args); err != nil { log.Fatal("Unable to parse args") @@ -206,6 +236,40 @@ func runQemu(args []string) { if *isoBoot && isoPath != "" { log.Fatalf("metadata and ISO boot currently cannot coexist") } + if *networking == "" || *networking == "default" { + dflt := qemuNetworkingDefault + networking = &dflt + } + netMode := strings.SplitN(*networking, ",", 2) + + var netdevConfig string + switch netMode[0] { + case qemuNetworkingUser: + netdevConfig = "user" + case qemuNetworkingTap: + if len(netMode) != 2 { + log.Fatalf("Not enough arugments for %q networking mode", qemuNetworkingTap) + } + if len(publishFlags) != 0 { + log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser) + } + netdevConfig = fmt.Sprintf("tap,ifname=%s,script=no,downscript=no", netMode[1]) + case qemuNetworkingBridge: + if len(netMode) != 2 { + log.Fatalf("Not enough arugments for %q networking mode", qemuNetworkingBridge) + } + if len(publishFlags) != 0 { + log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser) + } + netdevConfig = fmt.Sprintf("bridge,br=%s", netMode[1]) + case qemuNetworkingNone: + if len(publishFlags) != 0 { + log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser) + } + netdevConfig = "" + default: + log.Fatalf("Invalid networking mode: %s", netMode[0]) + } config := QemuConfig{ Path: path, @@ -222,7 +286,7 @@ func runQemu(args []string) { KVM: *enableKVM, Containerized: *qemuContainerized, PublishedPorts: publishFlags, - TapDevice: *tapDevice, + NetdevConfig: netdevConfig, UUID: vmUUID, } @@ -448,19 +512,16 @@ func buildQemuCmdline(config QemuConfig) (QemuConfig, []string) { } } - if config.PublishedPorts != nil && len(config.PublishedPorts) > 0 { + if config.NetdevConfig == "" { + qemuArgs = append(qemuArgs, "-net", "none") + } else { + mac := generateMAC() + qemuArgs = append(qemuArgs, "-net", "nic,model=virtio,macaddr="+mac.String()) forwardings, err := buildQemuForwardings(config.PublishedPorts, config.Containerized) if err != nil { log.Error(err) } - qemuArgs = append(qemuArgs, "-net", forwardings) - qemuArgs = append(qemuArgs, "-net", "nic") - } - - if config.TapDevice != "" { - qemuArgs = append(qemuArgs, "-net", "nic,model=virtio") - tapArg := fmt.Sprintf("tap,ifname=%s,script=no,downscript=no", config.TapDevice) - qemuArgs = append(qemuArgs, "-net", tapArg) + qemuArgs = append(qemuArgs, "-net", config.NetdevConfig+forwardings) } if config.GUI != true { @@ -554,7 +615,10 @@ func splitPublish(publish string) (publishedPorts, error) { } func buildQemuForwardings(publishFlags multipleFlag, containerized bool) (string, error) { - forwardings := "user" + if len(publishFlags) == 0 { + return "", nil + } + var forwardings string for _, publish := range publishFlags { p, err := splitPublish(publish) if err != nil { From 09609bc6ce07759d092d563122313cb6455f5d4c Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 18 Jul 2017 15:51:13 +0100 Subject: [PATCH 03/13] kubernetes: update ssh_into_kubelet.sh to use ctr Signed-off-by: Ian Campbell --- projects/kubernetes/ssh_into_kubelet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/kubernetes/ssh_into_kubelet.sh b/projects/kubernetes/ssh_into_kubelet.sh index 4c87e5eb1..74ba0ad6f 100755 --- a/projects/kubernetes/ssh_into_kubelet.sh +++ b/projects/kubernetes/ssh_into_kubelet.sh @@ -1,2 +1,2 @@ #!/bin/bash -eux -./ssh.sh -t root@"$1" nsenter --mount --target 1 runc exec --tty kubelet ash -l +./ssh.sh -t root@"$1" ctr exec --tty --exec-id ssh kubelet ash -l From 54ddde0d434485b989fe41306d5d588c5789061d Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Mon, 24 Jul 2017 12:23:46 +0100 Subject: [PATCH 04/13] kubernetes: avoid OS specifics in boot.sh Remove `-publish` (which is currently Linux/QEMU specific) and replace with a generic $KUBE_RUN_ARGS envvar. Usage: KUBE_RUN_ARGS="-publish 2222:22" ./boot.sh KUBE_PORT_BASE is thus obsolete and removed. Signed-off-by: Ian Campbell --- projects/kubernetes/boot.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/projects/kubernetes/boot.sh b/projects/kubernetes/boot.sh index 2f69fc990..8c1dfc053 100755 --- a/projects/kubernetes/boot.sh +++ b/projects/kubernetes/boot.sh @@ -1,8 +1,7 @@ #!/bin/bash -eu -: ${KUBE_PORT_BASE:=2222} +: ${KUBE_RUN_ARGS:=} if [ $# -eq 0 ] ; then img="kube-master" - port=${KUBE_PORT_BASE} data="" state="kube-master-state" elif [ $# -gt 1 ] ; then @@ -19,7 +18,6 @@ elif [ $# -gt 1 ] ; then esac img="kube-node" name="node-${1}" - port=$((${KUBE_PORT_BASE} + $1)) shift data="${*}" state="kube-${name}-state" @@ -33,4 +31,4 @@ else fi set -x rm -rf "${state}" -../../bin/linuxkit run -publish $port:22 -cpus 2 -mem 4096 -state "${state}" -disk size=4G -data "${data}" "${img}" +../../bin/linuxkit run ${KUBE_RUN_ARGS} -cpus 2 -mem 4096 -state "${state}" -disk size=4G -data "${data}" "${img}" From 62aa9248a406bbc52f2ee79d9c10dd5f99a5750d Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Mon, 24 Jul 2017 12:26:02 +0100 Subject: [PATCH 05/13] kubernetes: Expose VM parameters as envvars in boot.sh Allows users to override. I debated separate master and node options but decided not for now. Signed-off-by: Ian Campbell --- projects/kubernetes/boot.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/kubernetes/boot.sh b/projects/kubernetes/boot.sh index 8c1dfc053..57acfccfa 100755 --- a/projects/kubernetes/boot.sh +++ b/projects/kubernetes/boot.sh @@ -1,4 +1,8 @@ #!/bin/bash -eu +: ${KUBE_PORT_BASE:=2222} +: ${KUBE_VCPUS:=2} +: ${KUBE_MEM:=4096} +: ${KUBE_DISK:=4G} : ${KUBE_RUN_ARGS:=} if [ $# -eq 0 ] ; then img="kube-master" @@ -31,4 +35,4 @@ else fi set -x rm -rf "${state}" -../../bin/linuxkit run ${KUBE_RUN_ARGS} -cpus 2 -mem 4096 -state "${state}" -disk size=4G -data "${data}" "${img}" +../../bin/linuxkit run ${KUBE_RUN_ARGS} -cpus ${KUBE_VCPUS} -mem ${KUBE_MEM} -state "${state}" -disk size=${KUBE_DISK} -data "${data}" "${img}" From 1dbec1ef30b9d3c61e09e817d813ce5db12434d3 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Mon, 24 Jul 2017 12:28:45 +0100 Subject: [PATCH 06/13] kubernetes: inline ssh.sh into ssh_into_kubelet.sh Signed-off-by: Ian Campbell --- projects/kubernetes/ssh.sh | 13 ------------- projects/kubernetes/ssh_into_kubelet.sh | 9 ++++++++- 2 files changed, 8 insertions(+), 14 deletions(-) delete mode 100755 projects/kubernetes/ssh.sh diff --git a/projects/kubernetes/ssh.sh b/projects/kubernetes/ssh.sh deleted file mode 100755 index 2a29393c6..000000000 --- a/projects/kubernetes/ssh.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -eux -docker run \ - --rm \ - -ti \ - -v ~/.ssh/:/root/.ssh \ - jdeathe/centos-ssh \ - ssh \ - -o Compression=yes \ - -o LogLevel=FATAL \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o IdentitiesOnly=yes \ - "$@" diff --git a/projects/kubernetes/ssh_into_kubelet.sh b/projects/kubernetes/ssh_into_kubelet.sh index 74ba0ad6f..17da1a8a2 100755 --- a/projects/kubernetes/ssh_into_kubelet.sh +++ b/projects/kubernetes/ssh_into_kubelet.sh @@ -1,2 +1,9 @@ #!/bin/bash -eux -./ssh.sh -t root@"$1" ctr exec --tty --exec-id ssh kubelet ash -l +ssh="docker run --rm -ti \ + -v $HOME/.ssh/:/root/.ssh \ + jdeathe/centos-ssh \ + -o LogLevel=FATAL \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o IdentitiesOnly=yes" +$ssh -t root@"$1" ctr exec --tty --exec-id ssh kubelet ash -l From 9f04b403e46c6520fdd8b2dbba559c14223c7cd0 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Mon, 24 Jul 2017 14:14:41 +0100 Subject: [PATCH 07/13] kubernetes: Use ijc25/alpine-ssh for ssh_into_kubelet.sh Much smaller than the CentOS based one. Note that ijc25/alpine-ssh has entrypoint==ssh. Drop Compression=yes, this is used for local ssh so no point compressing (just uses CPU). Signed-off-by: Ian Campbell --- projects/kubernetes/ssh_into_kubelet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/kubernetes/ssh_into_kubelet.sh b/projects/kubernetes/ssh_into_kubelet.sh index 17da1a8a2..6afff40aa 100755 --- a/projects/kubernetes/ssh_into_kubelet.sh +++ b/projects/kubernetes/ssh_into_kubelet.sh @@ -1,7 +1,7 @@ #!/bin/bash -eux ssh="docker run --rm -ti \ -v $HOME/.ssh/:/root/.ssh \ - jdeathe/centos-ssh \ + ijc25/alpine-ssh \ -o LogLevel=FATAL \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ From 7d7001c573da823d84fdbb452b1799f3f82ca7d2 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 25 Jul 2017 16:05:23 +0100 Subject: [PATCH 08/13] kubernetes: ssh_into_kubelet.sh for Linux Only with networking in bridge (or probably tap) modes, not user mode. Signed-off-by: Ian Campbell --- projects/kubernetes/ssh_into_kubelet.sh | 27 ++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/projects/kubernetes/ssh_into_kubelet.sh b/projects/kubernetes/ssh_into_kubelet.sh index 6afff40aa..0130dd945 100755 --- a/projects/kubernetes/ssh_into_kubelet.sh +++ b/projects/kubernetes/ssh_into_kubelet.sh @@ -1,9 +1,18 @@ -#!/bin/bash -eux -ssh="docker run --rm -ti \ - -v $HOME/.ssh/:/root/.ssh \ - ijc25/alpine-ssh \ - -o LogLevel=FATAL \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o IdentitiesOnly=yes" -$ssh -t root@"$1" ctr exec --tty --exec-id ssh kubelet ash -l +#!/bin/bash -eu + +sshopts="-o LogLevel=FATAL \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o IdentitiesOnly=yes" + +case $(uname -s) in + Linux) + ssh=ssh + ;; + *) + ssh="docker run --rm -ti \ + -v $HOME/.ssh/:/root/.ssh \ + ijc25/alpine-ssh" + ;; +esac +$ssh $sshopts -t root@"$1" ctr exec --tty --exec-id ssh kubelet ash -l From d5bcb62419f8a9f633cdd9e258f6038fcc00a021 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 25 Jul 2017 14:09:00 +0100 Subject: [PATCH 09/13] kubernetes: Bump to 1.6.7 and other updates Bumps kubernetes and associated tools and images to v1.6.7 (from v1.6.1). Updates weave from v1.9.4 to v2.0.1 Updates cni from a snapshot to v0.5.2. Note that the download location has changed and the tarball no longer includes the `bin` subdirectory, so adjust build to compensate. Signed-off-by: Ian Campbell --- projects/kubernetes/image-cache/Makefile | 14 +++++++------- projects/kubernetes/kubernetes/Dockerfile | 15 ++++++++------- projects/kubernetes/kubernetes/kubeadm-init.sh | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/projects/kubernetes/image-cache/Makefile b/projects/kubernetes/image-cache/Makefile index 3f2c459af..ae1985f2a 100644 --- a/projects/kubernetes/image-cache/Makefile +++ b/projects/kubernetes/image-cache/Makefile @@ -1,16 +1,16 @@ default: push COMMON_IMAGES := \ - kube-proxy-amd64\:v1.6.1@sha256\:243f2120171330a26c2418a4367fb0f3cc3e92683b00d16e3cf8c7f92e25bf14 \ - k8s-dns-sidecar-amd64\:1.14.1@sha256\:d33a91a5d65c223f410891001cd379ac734d036429e033865d700a4176e944b0 \ - k8s-dns-kube-dns-amd64\:1.14.1@sha256\:33914315e600dfb756e550828307dfa2b21fb6db24fe3fe495e33d1022f9245d \ - k8s-dns-dnsmasq-nanny-amd64\:1.14.1@sha256\:89c9a1d3cfbf370a9c1a949f39f92c1dc2dbe8c3e6cc1802b7f2b48e4dfe9a9e \ + kube-proxy-amd64\:v1.6.7@sha256\:652ca0ef7cdf05341fafb590ced1b737126641829c70f5d23f9b714bc61c8607 \ + k8s-dns-sidecar-amd64\:1.14.4@sha256\:d33a91a5d65c223f410891001cd379ac734d036429e033865d700a4176e944b0 \ + k8s-dns-kube-dns-amd64\:1.14.4@sha256\:33914315e600dfb756e550828307dfa2b21fb6db24fe3fe495e33d1022f9245d \ + k8s-dns-dnsmasq-nanny-amd64\:1.14.4@sha256\:89c9a1d3cfbf370a9c1a949f39f92c1dc2dbe8c3e6cc1802b7f2b48e4dfe9a9e \ pause-amd64\:3.0@sha256\:163ac025575b775d1c0f9bf0bdd0f086883171eb475b5068e7defa4ca9e76516 CONTROL_PLANE_IMAGES := \ - kube-apiserver-amd64\:v1.6.1@sha256\:d4387dff51b1f9c94cd1cfac3a4694347970b90e911159ac6fe2d090c96a6184 \ - kube-controller-manager-amd64\:v1.6.1@sha256\:4bb17ede2e012898169d988facd08d5039d2dcb31532661d4dcdeb161d097d69 \ - kube-scheduler-amd64\:v1.6.1@sha256\:d3e661bf7bcfb10753e32e1a41615e60fbcddff63232f914e9326a2d1665ce33 \ + kube-apiserver-amd64\:v1.6.7@sha256\:57e482529b95d32730d1bcd2e374199f27eab4abcf1ff49c5db2a7a7e2231cc8 \ + kube-controller-manager-amd64\:v1.6.7@sha256\:884f609895fa715d66806681a6bf6f9851a911202ad3484b768fead8b7c78b39 \ + kube-scheduler-amd64\:v1.6.7@sha256\:a1a498a0ca5ab23c228724d93c3f3d9457b31a046d9025471d98e1096422452c \ etcd-amd64\:3.0.17@sha256\:d83d3545e06fb035db8512e33bd44afb55dea007a3abd7b17742d3ac6d235940 dl/%.tar: diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile index 6a39d6910..911372f9b 100644 --- a/projects/kubernetes/kubernetes/Dockerfile +++ b/projects/kubernetes/kubernetes/Dockerfile @@ -2,9 +2,9 @@ # XXX needs ebtables ethtool iproute2 libc6-compat socat FROM alpine:3.6 AS build -ENV kubernetes_version v1.6.1 -ENV weave_version v1.9.4 -ENV cni_version 0799f5732f2a11b329d9e3d51b9c8f2e3759f2ff +ENV kubernetes_version v1.6.7 +ENV weave_version v2.0.1 +ENV cni_version v0.5.2 ENV kube_release_artefacts "https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64" @@ -30,16 +30,17 @@ RUN apk add --no-cache --initdb -p /out \ # Remove apk residuals. We have a read-only rootfs, so apk is of no use. RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache -RUN curl -fSL -o /tmp/cni.tgz https://dl.k8s.io/network-plugins/cni-amd64-${cni_version}.tar.gz && \ - mkdir -p /out/opt/cni /out/etc/cni/net.d && \ - tar -xzf /tmp/cni.tgz -C /out/opt/cni +RUN curl -fSL -o /tmp/cni.tgz https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-${cni_version}.tgz && \ + mkdir -p /out/opt/cni/bin /out/etc/cni/net.d && \ + tar -xzf /tmp/cni.tgz -C /out/opt/cni/bin RUN curl -fSL -o /out/etc/weave.yaml https://cloud.weave.works/k8s/v1.6/net?v=${weave_version} RUN curl -fSL -o /out/usr/bin/kubelet https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubelet && chmod 0755 /out/usr/bin/kubelet RUN curl -fSL -o /out/usr/bin/kubeadm https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubeadm && chmod 0755 /out/usr/bin/kubeadm RUN curl -fSL -o /out/usr/bin/kubectl https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubectl && chmod 0755 /out/usr/bin/kubectl ADD kubelet.sh /out/usr/bin/kubelet.sh -ADD kubeadm-init.sh /out/usr/bin/kubeadm-init.sh +ADD kubeadm-init.sh /kubeadm-init.sh +RUN sed -e "s/@KUBERNETES_VERSION@/${kubernetes_version}/g" /out/usr/bin/kubeadm-init.sh && chmod +x /out/usr/bin/kubeadm-init.sh FROM scratch WORKDIR / diff --git a/projects/kubernetes/kubernetes/kubeadm-init.sh b/projects/kubernetes/kubernetes/kubeadm-init.sh index 5e953229d..1229dae64 100755 --- a/projects/kubernetes/kubernetes/kubeadm-init.sh +++ b/projects/kubernetes/kubernetes/kubeadm-init.sh @@ -1,4 +1,4 @@ #!/bin/sh set -e -kubeadm init --skip-preflight-checks --kubernetes-version v1.6.1 +kubeadm init --skip-preflight-checks --kubernetes-version @KUBERNETES_VERSION@ kubectl create -n kube-system -f /etc/weave.yaml From 8acecf1b62b4ef27437a7872121ae0ecab0aa899 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 25 Jul 2017 14:11:13 +0100 Subject: [PATCH 10/13] kubernetes: Drop k8s-dns-* from image cache These contribute ~140M to the common image cache but do not appear to be used by either the base system nor the sock-shop demo. They can/will still be pulled on demands as necessary. Signed-off-by: Ian Campbell --- projects/kubernetes/image-cache/Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/projects/kubernetes/image-cache/Makefile b/projects/kubernetes/image-cache/Makefile index ae1985f2a..f9e0c93c5 100644 --- a/projects/kubernetes/image-cache/Makefile +++ b/projects/kubernetes/image-cache/Makefile @@ -2,9 +2,6 @@ default: push COMMON_IMAGES := \ kube-proxy-amd64\:v1.6.7@sha256\:652ca0ef7cdf05341fafb590ced1b737126641829c70f5d23f9b714bc61c8607 \ - k8s-dns-sidecar-amd64\:1.14.4@sha256\:d33a91a5d65c223f410891001cd379ac734d036429e033865d700a4176e944b0 \ - k8s-dns-kube-dns-amd64\:1.14.4@sha256\:33914315e600dfb756e550828307dfa2b21fb6db24fe3fe495e33d1022f9245d \ - k8s-dns-dnsmasq-nanny-amd64\:1.14.4@sha256\:89c9a1d3cfbf370a9c1a949f39f92c1dc2dbe8c3e6cc1802b7f2b48e4dfe9a9e \ pause-amd64\:3.0@sha256\:163ac025575b775d1c0f9bf0bdd0f086883171eb475b5068e7defa4ca9e76516 CONTROL_PLANE_IMAGES := \ From 6c3dd4e54a0067dba010000becf09f4aa11daf70 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 25 Jul 2017 16:10:51 +0100 Subject: [PATCH 11/13] kubernetes: allow configuration of networking Signed-off-by: Ian Campbell --- projects/kubernetes/boot.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/kubernetes/boot.sh b/projects/kubernetes/boot.sh index 57acfccfa..51aa3f9d6 100755 --- a/projects/kubernetes/boot.sh +++ b/projects/kubernetes/boot.sh @@ -3,6 +3,7 @@ : ${KUBE_VCPUS:=2} : ${KUBE_MEM:=4096} : ${KUBE_DISK:=4G} +: ${KUBE_NETWORKING:=default} : ${KUBE_RUN_ARGS:=} if [ $# -eq 0 ] ; then img="kube-master" @@ -35,4 +36,4 @@ else fi set -x rm -rf "${state}" -../../bin/linuxkit run ${KUBE_RUN_ARGS} -cpus ${KUBE_VCPUS} -mem ${KUBE_MEM} -state "${state}" -disk size=${KUBE_DISK} -data "${data}" "${img}" +../../bin/linuxkit run ${KUBE_RUN_ARGS} -networking ${KUBE_NETWORKING} -cpus ${KUBE_VCPUS} -mem ${KUBE_MEM} -state "${state}" -disk size=${KUBE_DISK} -data "${data}" "${img}" From d5a53968fec74691a9ef58c7e411754643ef8334 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 25 Jul 2017 17:20:14 +0100 Subject: [PATCH 12/13] kubernetes: Documentation updates for Linux platform Signed-off-by: Ian Campbell --- projects/kubernetes/README.md | 44 ++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/projects/kubernetes/README.md b/projects/kubernetes/README.md index 1d5c57b21..f7a3c1b21 100644 --- a/projects/kubernetes/README.md +++ b/projects/kubernetes/README.md @@ -11,7 +11,7 @@ Build OS images: make build-vm-images ``` -Boot Kubernetes master OS image using `hyperkit` on macOS: +Boot Kubernetes master OS image using `hyperkit` on macOS: or `qemu` on Linux: ``` ./boot.sh ``` @@ -45,3 +45,45 @@ shell1> ./boot.sh 1 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443 shell2> ./boot.sh 2 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443 shell3> ./boot.sh 3 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443 ``` + +## Platform specific information + +### MacOS + +The above instructions should work as is. + +### Linux + +By default `linuxkit run` uses user mode networking which does not +support access from the host. To workaround this you can use port +forwarding e.g. + + KUBE_RUN_ARGS="-publish 2222:22" ./boot.sh + + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2222 root@localhost + +However you will not be able to run worker nodes since individual +instances cannot see each other. + +To enable networking between instance unfortunately requires `root` +privileges to configure a bridge and setup the bridge mode privileged +helper. + +See http://wiki.qemu.org/Features/HelperNetworking for details in +brief you will need: + +- To setup and configure a bridge (including e.g. DHCP etc) on the + host. (You can reuse a bridge created by e.g. `virt-mananger`) +- To set the `qemu-bridge-helper` setuid root. The location differs by + distro, it could be `/usr/lib/qemu/qemu-bridge-helper` or + `/usr/local/libexec/qemu-bridge-helper` or elsewhere. You need to + `chmod u+s «PATH»`. +- List the bridge created in the first step in `/etc/qemu/bridge.conf` + with a line like `allow br0` (if your bridge is called `br0`). + +## Configuration + +The `boot.sh` script has various configuration variables at the top +which can be overridden via the environment e.g. + + KUBE_VCPUS=4 ./boot.sh From 828ac913e878269f49fb6e9e8096d6e56c73f1a2 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 25 Jul 2017 17:25:08 +0100 Subject: [PATCH 13/13] kubernetes: Update yml files Signed-off-by: Ian Campbell --- projects/kubernetes/kube-master.yml | 6 +++--- projects/kubernetes/kube-node.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index 40b39a50e..ff7a66351 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -56,11 +56,11 @@ services: rootfsPropagation: shared command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"] - name: kubernetes-image-cache-common - image: linuxkitprojects/kubernetes-image-cache-common:d49a861bde872e6e975153a98a2c482834a30ef9 + image: linuxkitprojects/kubernetes-image-cache-common:6fccda74ea301f9a62cdcfc2fe4952cff2c8c97b - name: kubernetes-image-cache-control-plane - image: linuxkitprojects/kubernetes-image-cache-control-plane:d49a861bde872e6e975153a98a2c482834a30ef9 + image: linuxkitprojects/kubernetes-image-cache-control-plane:6fccda74ea301f9a62cdcfc2fe4952cff2c8c97b - name: kubelet - image: linuxkitprojects/kubernetes:4f8c61254ff6243e93d5bb6315386ac66e94ed14 + image: linuxkitprojects/kubernetes:d4d722823b1265a57355ae8a309d4953e293fd58 files: - path: root/.ssh/authorized_keys source: ~/.ssh/id_rsa.pub diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml index a7ac93d2f..eb5a1bce6 100644 --- a/projects/kubernetes/kube-node.yml +++ b/projects/kubernetes/kube-node.yml @@ -56,9 +56,9 @@ services: rootfsPropagation: shared command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"] - name: kubernetes-image-cache-common - image: linuxkitprojects/kubernetes-image-cache-common:d49a861bde872e6e975153a98a2c482834a30ef9 + image: linuxkitprojects/kubernetes-image-cache-common:6fccda74ea301f9a62cdcfc2fe4952cff2c8c97b - name: kubelet - image: linuxkitprojects/kubernetes:4f8c61254ff6243e93d5bb6315386ac66e94ed14 + image: linuxkitprojects/kubernetes:d4d722823b1265a57355ae8a309d4953e293fd58 files: - path: root/.ssh/authorized_keys source: ~/.ssh/id_rsa.pub