diff --git a/projects/kubernetes/README.md b/projects/kubernetes/README.md index 1d5c57b21..f7a3c1b21 100644 --- a/projects/kubernetes/README.md +++ b/projects/kubernetes/README.md @@ -11,7 +11,7 @@ Build OS images: make build-vm-images ``` -Boot Kubernetes master OS image using `hyperkit` on macOS: +Boot Kubernetes master OS image using `hyperkit` on macOS: or `qemu` on Linux: ``` ./boot.sh ``` @@ -45,3 +45,45 @@ shell1> ./boot.sh 1 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443 shell2> ./boot.sh 2 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443 shell3> ./boot.sh 3 --token bb38c6.117e66eabbbce07d 192.168.65.22:6443 ``` + +## Platform specific information + +### MacOS + +The above instructions should work as is. + +### Linux + +By default `linuxkit run` uses user mode networking which does not +support access from the host. To workaround this you can use port +forwarding e.g. + + KUBE_RUN_ARGS="-publish 2222:22" ./boot.sh + + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 2222 root@localhost + +However you will not be able to run worker nodes since individual +instances cannot see each other. + +To enable networking between instance unfortunately requires `root` +privileges to configure a bridge and setup the bridge mode privileged +helper. + +See http://wiki.qemu.org/Features/HelperNetworking for details in +brief you will need: + +- To setup and configure a bridge (including e.g. DHCP etc) on the + host. (You can reuse a bridge created by e.g. `virt-mananger`) +- To set the `qemu-bridge-helper` setuid root. The location differs by + distro, it could be `/usr/lib/qemu/qemu-bridge-helper` or + `/usr/local/libexec/qemu-bridge-helper` or elsewhere. You need to + `chmod u+s «PATH»`. +- List the bridge created in the first step in `/etc/qemu/bridge.conf` + with a line like `allow br0` (if your bridge is called `br0`). + +## Configuration + +The `boot.sh` script has various configuration variables at the top +which can be overridden via the environment e.g. + + KUBE_VCPUS=4 ./boot.sh diff --git a/projects/kubernetes/boot.sh b/projects/kubernetes/boot.sh index 2f69fc990..51aa3f9d6 100755 --- a/projects/kubernetes/boot.sh +++ b/projects/kubernetes/boot.sh @@ -1,8 +1,12 @@ #!/bin/bash -eu : ${KUBE_PORT_BASE:=2222} +: ${KUBE_VCPUS:=2} +: ${KUBE_MEM:=4096} +: ${KUBE_DISK:=4G} +: ${KUBE_NETWORKING:=default} +: ${KUBE_RUN_ARGS:=} if [ $# -eq 0 ] ; then img="kube-master" - port=${KUBE_PORT_BASE} data="" state="kube-master-state" elif [ $# -gt 1 ] ; then @@ -19,7 +23,6 @@ elif [ $# -gt 1 ] ; then esac img="kube-node" name="node-${1}" - port=$((${KUBE_PORT_BASE} + $1)) shift data="${*}" state="kube-${name}-state" @@ -33,4 +36,4 @@ else fi set -x rm -rf "${state}" -../../bin/linuxkit run -publish $port:22 -cpus 2 -mem 4096 -state "${state}" -disk size=4G -data "${data}" "${img}" +../../bin/linuxkit run ${KUBE_RUN_ARGS} -networking ${KUBE_NETWORKING} -cpus ${KUBE_VCPUS} -mem ${KUBE_MEM} -state "${state}" -disk size=${KUBE_DISK} -data "${data}" "${img}" diff --git a/projects/kubernetes/image-cache/Makefile b/projects/kubernetes/image-cache/Makefile index 3f2c459af..f9e0c93c5 100644 --- a/projects/kubernetes/image-cache/Makefile +++ b/projects/kubernetes/image-cache/Makefile @@ -1,16 +1,13 @@ default: push COMMON_IMAGES := \ - kube-proxy-amd64\:v1.6.1@sha256\:243f2120171330a26c2418a4367fb0f3cc3e92683b00d16e3cf8c7f92e25bf14 \ - k8s-dns-sidecar-amd64\:1.14.1@sha256\:d33a91a5d65c223f410891001cd379ac734d036429e033865d700a4176e944b0 \ - k8s-dns-kube-dns-amd64\:1.14.1@sha256\:33914315e600dfb756e550828307dfa2b21fb6db24fe3fe495e33d1022f9245d \ - k8s-dns-dnsmasq-nanny-amd64\:1.14.1@sha256\:89c9a1d3cfbf370a9c1a949f39f92c1dc2dbe8c3e6cc1802b7f2b48e4dfe9a9e \ + kube-proxy-amd64\:v1.6.7@sha256\:652ca0ef7cdf05341fafb590ced1b737126641829c70f5d23f9b714bc61c8607 \ pause-amd64\:3.0@sha256\:163ac025575b775d1c0f9bf0bdd0f086883171eb475b5068e7defa4ca9e76516 CONTROL_PLANE_IMAGES := \ - kube-apiserver-amd64\:v1.6.1@sha256\:d4387dff51b1f9c94cd1cfac3a4694347970b90e911159ac6fe2d090c96a6184 \ - kube-controller-manager-amd64\:v1.6.1@sha256\:4bb17ede2e012898169d988facd08d5039d2dcb31532661d4dcdeb161d097d69 \ - kube-scheduler-amd64\:v1.6.1@sha256\:d3e661bf7bcfb10753e32e1a41615e60fbcddff63232f914e9326a2d1665ce33 \ + kube-apiserver-amd64\:v1.6.7@sha256\:57e482529b95d32730d1bcd2e374199f27eab4abcf1ff49c5db2a7a7e2231cc8 \ + kube-controller-manager-amd64\:v1.6.7@sha256\:884f609895fa715d66806681a6bf6f9851a911202ad3484b768fead8b7c78b39 \ + kube-scheduler-amd64\:v1.6.7@sha256\:a1a498a0ca5ab23c228724d93c3f3d9457b31a046d9025471d98e1096422452c \ etcd-amd64\:3.0.17@sha256\:d83d3545e06fb035db8512e33bd44afb55dea007a3abd7b17742d3ac6d235940 dl/%.tar: diff --git a/projects/kubernetes/kube-master.yml b/projects/kubernetes/kube-master.yml index 01ac9b568..8d501668b 100644 --- a/projects/kubernetes/kube-master.yml +++ b/projects/kubernetes/kube-master.yml @@ -56,11 +56,11 @@ services: rootfsPropagation: shared command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"] - name: kubernetes-image-cache-common - image: linuxkitprojects/kubernetes-image-cache-common:d49a861bde872e6e975153a98a2c482834a30ef9 + image: linuxkitprojects/kubernetes-image-cache-common:6fccda74ea301f9a62cdcfc2fe4952cff2c8c97b - name: kubernetes-image-cache-control-plane - image: linuxkitprojects/kubernetes-image-cache-control-plane:d49a861bde872e6e975153a98a2c482834a30ef9 + image: linuxkitprojects/kubernetes-image-cache-control-plane:6fccda74ea301f9a62cdcfc2fe4952cff2c8c97b - name: kubelet - image: linuxkitprojects/kubernetes:4f8c61254ff6243e93d5bb6315386ac66e94ed14 + image: linuxkitprojects/kubernetes:d4d722823b1265a57355ae8a309d4953e293fd58 files: - path: root/.ssh/authorized_keys source: ~/.ssh/id_rsa.pub diff --git a/projects/kubernetes/kube-node.yml b/projects/kubernetes/kube-node.yml index ec2c5290b..0eb8dcf09 100644 --- a/projects/kubernetes/kube-node.yml +++ b/projects/kubernetes/kube-node.yml @@ -56,9 +56,9 @@ services: rootfsPropagation: shared command: ["/usr/local/bin/docker-init", "/usr/local/bin/dockerd"] - name: kubernetes-image-cache-common - image: linuxkitprojects/kubernetes-image-cache-common:d49a861bde872e6e975153a98a2c482834a30ef9 + image: linuxkitprojects/kubernetes-image-cache-common:6fccda74ea301f9a62cdcfc2fe4952cff2c8c97b - name: kubelet - image: linuxkitprojects/kubernetes:4f8c61254ff6243e93d5bb6315386ac66e94ed14 + image: linuxkitprojects/kubernetes:d4d722823b1265a57355ae8a309d4953e293fd58 files: - path: root/.ssh/authorized_keys source: ~/.ssh/id_rsa.pub diff --git a/projects/kubernetes/kubernetes/Dockerfile b/projects/kubernetes/kubernetes/Dockerfile index 6a39d6910..911372f9b 100644 --- a/projects/kubernetes/kubernetes/Dockerfile +++ b/projects/kubernetes/kubernetes/Dockerfile @@ -2,9 +2,9 @@ # XXX needs ebtables ethtool iproute2 libc6-compat socat FROM alpine:3.6 AS build -ENV kubernetes_version v1.6.1 -ENV weave_version v1.9.4 -ENV cni_version 0799f5732f2a11b329d9e3d51b9c8f2e3759f2ff +ENV kubernetes_version v1.6.7 +ENV weave_version v2.0.1 +ENV cni_version v0.5.2 ENV kube_release_artefacts "https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64" @@ -30,16 +30,17 @@ RUN apk add --no-cache --initdb -p /out \ # Remove apk residuals. We have a read-only rootfs, so apk is of no use. RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache -RUN curl -fSL -o /tmp/cni.tgz https://dl.k8s.io/network-plugins/cni-amd64-${cni_version}.tar.gz && \ - mkdir -p /out/opt/cni /out/etc/cni/net.d && \ - tar -xzf /tmp/cni.tgz -C /out/opt/cni +RUN curl -fSL -o /tmp/cni.tgz https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-${cni_version}.tgz && \ + mkdir -p /out/opt/cni/bin /out/etc/cni/net.d && \ + tar -xzf /tmp/cni.tgz -C /out/opt/cni/bin RUN curl -fSL -o /out/etc/weave.yaml https://cloud.weave.works/k8s/v1.6/net?v=${weave_version} RUN curl -fSL -o /out/usr/bin/kubelet https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubelet && chmod 0755 /out/usr/bin/kubelet RUN curl -fSL -o /out/usr/bin/kubeadm https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubeadm && chmod 0755 /out/usr/bin/kubeadm RUN curl -fSL -o /out/usr/bin/kubectl https://dl.k8s.io/${kubernetes_version}/bin/linux/amd64/kubectl && chmod 0755 /out/usr/bin/kubectl ADD kubelet.sh /out/usr/bin/kubelet.sh -ADD kubeadm-init.sh /out/usr/bin/kubeadm-init.sh +ADD kubeadm-init.sh /kubeadm-init.sh +RUN sed -e "s/@KUBERNETES_VERSION@/${kubernetes_version}/g" /out/usr/bin/kubeadm-init.sh && chmod +x /out/usr/bin/kubeadm-init.sh FROM scratch WORKDIR / diff --git a/projects/kubernetes/kubernetes/kubeadm-init.sh b/projects/kubernetes/kubernetes/kubeadm-init.sh index 5e953229d..1229dae64 100755 --- a/projects/kubernetes/kubernetes/kubeadm-init.sh +++ b/projects/kubernetes/kubernetes/kubeadm-init.sh @@ -1,4 +1,4 @@ #!/bin/sh set -e -kubeadm init --skip-preflight-checks --kubernetes-version v1.6.1 +kubeadm init --skip-preflight-checks --kubernetes-version @KUBERNETES_VERSION@ kubectl create -n kube-system -f /etc/weave.yaml diff --git a/projects/kubernetes/ssh.sh b/projects/kubernetes/ssh.sh deleted file mode 100755 index 2a29393c6..000000000 --- a/projects/kubernetes/ssh.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -eux -docker run \ - --rm \ - -ti \ - -v ~/.ssh/:/root/.ssh \ - jdeathe/centos-ssh \ - ssh \ - -o Compression=yes \ - -o LogLevel=FATAL \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o IdentitiesOnly=yes \ - "$@" diff --git a/projects/kubernetes/ssh_into_kubelet.sh b/projects/kubernetes/ssh_into_kubelet.sh index 4c87e5eb1..0130dd945 100755 --- a/projects/kubernetes/ssh_into_kubelet.sh +++ b/projects/kubernetes/ssh_into_kubelet.sh @@ -1,2 +1,18 @@ -#!/bin/bash -eux -./ssh.sh -t root@"$1" nsenter --mount --target 1 runc exec --tty kubelet ash -l +#!/bin/bash -eu + +sshopts="-o LogLevel=FATAL \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o IdentitiesOnly=yes" + +case $(uname -s) in + Linux) + ssh=ssh + ;; + *) + ssh="docker run --rm -ti \ + -v $HOME/.ssh/:/root/.ssh \ + ijc25/alpine-ssh" + ;; +esac +$ssh $sshopts -t root@"$1" ctr exec --tty --exec-id ssh kubelet ash -l diff --git a/src/cmd/linuxkit/run_hyperkit.go b/src/cmd/linuxkit/run_hyperkit.go index f0f3466bc..d13a3cef2 100644 --- a/src/cmd/linuxkit/run_hyperkit.go +++ b/src/cmd/linuxkit/run_hyperkit.go @@ -17,10 +17,11 @@ import ( ) const ( - networkingNone string = "none" - networkingDockerForMac = "docker-for-mac" - networkingVPNKit = "vpnkit" - networkingVMNet = "vmnet" + hyperkitNetworkingNone string = "none" + hyperkitNetworkingDockerForMac = "docker-for-mac" + hyperkitNetworkingVPNKit = "vpnkit" + hyperkitNetworkingVMNet = "vmnet" + hyperkitNetworkingDefault = hyperkitNetworkingDockerForMac ) // Process the run arguments and execute run @@ -43,7 +44,7 @@ func runHyperKit(args []string) { ipStr := flags.String("ip", "", "IP address for the VM") state := flags.String("state", "", "Path to directory to keep VM state in") vsockports := flags.String("vsock-ports", "", "List of vsock ports to forward from the guest on startup (comma separated). A unix domain socket for each port will be created in the state directory") - networking := flags.String("networking", networkingDockerForMac, "Networking mode. Valid options are 'docker-for-mac', 'vpnkit[,socket-path]', 'vmnet' and 'none'. 'docker-for-mac' connects to the network used by Docker for Mac. 'vpnkit' connects to the VPNKit socket specified. If socket-path is omitted a new VPNKit instance will be started and 'vpnkit_eth.sock' will be created in the state directory. 'vmnet' uses the Apple vmnet framework, requires root/sudo. 'none' disables networking.`") + networking := flags.String("networking", hyperkitNetworkingDefault, "Networking mode. Valid options are 'default', 'docker-for-mac', 'vpnkit[,socket-path]', 'vmnet' and 'none'. 'docker-for-mac' connects to the network used by Docker for Mac. 'vpnkit' connects to the VPNKit socket specified. If socket-path is omitted a new VPNKit instance will be started and 'vpnkit_eth.sock' will be created in the state directory. 'vmnet' uses the Apple vmnet framework, requires root/sudo. 'none' disables networking.`") if err := flags.Parse(args); err != nil { log.Fatal("Unable to parse args") @@ -130,12 +131,16 @@ func runHyperKit(args []string) { // Select network mode var vpnKitProcess *os.Process + if *networking == "" || *networking == "default" { + dflt := hyperkitNetworkingDefault + networking = &dflt + } netMode := strings.SplitN(*networking, ",", 2) switch netMode[0] { - case networkingDockerForMac: + case hyperkitNetworkingDockerForMac: h.VPNKitSock = filepath.Join(os.Getenv("HOME"), "Library/Containers/com.docker.docker/Data/s50") - case networkingVPNKit: + case hyperkitNetworkingVPNKit: if len(netMode) > 1 { // Socket path specified, try to use existing VPNKit instance h.VPNKitSock = netMode[1] @@ -161,10 +166,10 @@ func runHyperKit(args []string) { // VSOCK port 62373 is used to pass traffic from host->guest h.VSockPorts = append(h.VSockPorts, 62373) } - case networkingVMNet: + case hyperkitNetworkingVMNet: h.VPNKitSock = "" h.VMNet = true - case networkingNone: + case hyperkitNetworkingNone: h.VPNKitSock = "" default: log.Fatalf("Invalid networking mode: %s", netMode[0]) diff --git a/src/cmd/linuxkit/run_qemu.go b/src/cmd/linuxkit/run_qemu.go index 618d84e84..59fd1fa46 100644 --- a/src/cmd/linuxkit/run_qemu.go +++ b/src/cmd/linuxkit/run_qemu.go @@ -1,9 +1,11 @@ package main import ( + "crypto/rand" "flag" "fmt" "io/ioutil" + "net" "os" "os/exec" "path/filepath" @@ -11,6 +13,7 @@ import ( "strings" log "github.com/Sirupsen/logrus" + "github.com/satori/go.uuid" "golang.org/x/crypto/ssh/terminal" ) @@ -35,9 +38,18 @@ type QemuConfig struct { QemuBinPath string QemuImgPath string PublishedPorts []string - TapDevice string + NetdevConfig string + UUID uuid.UUID } +const ( + qemuNetworkingNone string = "none" + qemuNetworkingUser = "user" + qemuNetworkingTap = "tap" + qemuNetworkingBridge = "bridge" + qemuNetworkingDefault = qemuNetworkingUser +) + func haveKVM() bool { _, err := os.Stat("/dev/kvm") return !os.IsNotExist(err) @@ -56,6 +68,20 @@ func envOverrideBool(env string, b *bool) { } } +func generateMAC() net.HardwareAddr { + mac := make([]byte, 6) + n, err := rand.Read(mac) + if err != nil { + log.WithError(err).Fatal("failed to generate random mac address") + } + if n != 6 { + log.WithError(err).Fatal("generated %d bytes for random mac address", n) + } + mac[0] &^= 0x01 // Clear multicast bit + mac[0] |= 0x2 // Set locally administered bit + return net.HardwareAddr(mac) +} + func runQemu(args []string) { invoked := filepath.Base(os.Args[0]) flags := flag.NewFlagSet("qemu", flag.ExitOnError) @@ -65,6 +91,10 @@ func runQemu(args []string) { fmt.Printf("\n") fmt.Printf("Options:\n") flags.PrintDefaults() + fmt.Printf("\n") + fmt.Printf("If not running as root note that '-networking bridge,br0' requires a\n") + fmt.Printf("setuid network helper and appropriate host configuration, see\n") + fmt.Printf("http://wiki.qemu.org/Features/HelperNetworking.\n") } // Display flags @@ -95,9 +125,14 @@ func runQemu(args []string) { // Backend configuration qemuContainerized := flags.Bool("containerized", false, "Run qemu in a container") + // Generate UUID, so that /sys/class/dmi/id/product_uuid is populated + vmUUID := uuid.NewV4() + + // Networking + networking := flags.String("networking", qemuNetworkingDefault, "Networking mode. Valid options are 'default', 'user', 'bridge[,name]', tap[,name] and 'none'. 'user' uses QEMUs userspace networking. 'bridge' connects to a preexisting bridge. 'tap' uses a prexisting tap device. 'none' disables networking.`") + publishFlags := multipleFlag{} flags.Var(&publishFlags, "publish", "Publish a vm's port(s) to the host (default [])") - tapDevice := flags.String("tap-device", "", "Tap device to use as eth0 (optional)") if err := flags.Parse(args); err != nil { log.Fatal("Unable to parse args") @@ -201,6 +236,40 @@ func runQemu(args []string) { if *isoBoot && isoPath != "" { log.Fatalf("metadata and ISO boot currently cannot coexist") } + if *networking == "" || *networking == "default" { + dflt := qemuNetworkingDefault + networking = &dflt + } + netMode := strings.SplitN(*networking, ",", 2) + + var netdevConfig string + switch netMode[0] { + case qemuNetworkingUser: + netdevConfig = "user" + case qemuNetworkingTap: + if len(netMode) != 2 { + log.Fatalf("Not enough arugments for %q networking mode", qemuNetworkingTap) + } + if len(publishFlags) != 0 { + log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser) + } + netdevConfig = fmt.Sprintf("tap,ifname=%s,script=no,downscript=no", netMode[1]) + case qemuNetworkingBridge: + if len(netMode) != 2 { + log.Fatalf("Not enough arugments for %q networking mode", qemuNetworkingBridge) + } + if len(publishFlags) != 0 { + log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser) + } + netdevConfig = fmt.Sprintf("bridge,br=%s", netMode[1]) + case qemuNetworkingNone: + if len(publishFlags) != 0 { + log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser) + } + netdevConfig = "" + default: + log.Fatalf("Invalid networking mode: %s", netMode[0]) + } config := QemuConfig{ Path: path, @@ -217,7 +286,8 @@ func runQemu(args []string) { KVM: *enableKVM, Containerized: *qemuContainerized, PublishedPorts: publishFlags, - TapDevice: *tapDevice, + NetdevConfig: netdevConfig, + UUID: vmUUID, } config = discoverBackend(config) @@ -380,6 +450,7 @@ func buildQemuCmdline(config QemuConfig) (QemuConfig, []string) { qemuArgs = append(qemuArgs, "-device", "virtio-rng-pci") qemuArgs = append(qemuArgs, "-smp", config.CPUs) qemuArgs = append(qemuArgs, "-m", config.Memory) + qemuArgs = append(qemuArgs, "-uuid", config.UUID.String()) // Need to specify the vcpu type when running qemu on arm64 platform, for security reason, // the vcpu should be "host" instead of other names such as "cortex-a53"... if config.Arch == "aarch64" { @@ -441,19 +512,16 @@ func buildQemuCmdline(config QemuConfig) (QemuConfig, []string) { } } - if config.PublishedPorts != nil && len(config.PublishedPorts) > 0 { + if config.NetdevConfig == "" { + qemuArgs = append(qemuArgs, "-net", "none") + } else { + mac := generateMAC() + qemuArgs = append(qemuArgs, "-net", "nic,model=virtio,macaddr="+mac.String()) forwardings, err := buildQemuForwardings(config.PublishedPorts, config.Containerized) if err != nil { log.Error(err) } - qemuArgs = append(qemuArgs, "-net", forwardings) - qemuArgs = append(qemuArgs, "-net", "nic") - } - - if config.TapDevice != "" { - qemuArgs = append(qemuArgs, "-net", "nic,model=virtio") - tapArg := fmt.Sprintf("tap,ifname=%s,script=no,downscript=no", config.TapDevice) - qemuArgs = append(qemuArgs, "-net", tapArg) + qemuArgs = append(qemuArgs, "-net", config.NetdevConfig+forwardings) } if config.GUI != true { @@ -547,7 +615,10 @@ func splitPublish(publish string) (publishedPorts, error) { } func buildQemuForwardings(publishFlags multipleFlag, containerized bool) (string, error) { - forwardings := "user" + if len(publishFlags) == 0 { + return "", nil + } + var forwardings string for _, publish := range publishFlags { p, err := splitPublish(publish) if err != nil {