github/linuxkit
1
0
Fork 0
mirror of https://github.com/linuxkit/linuxkit.git synced 2025-07-21 18:11:35 +00:00
Code Issues Projects Releases Wiki Activity

Kubernetes with Weave CNI networking

Browse Source 
For the time being I've decided to exploit shared mounts to make
`/opt/cni` and `/etc/cni` work as expected. We need these directories
to appear writable on the host, and allow Weave Net pod to bind-mount
out them in order to install plugin binaries, and allow for vanilla
CNI plugins to be also accessible to kubelet.

Signed-off-by: Ilya Dmitrichenko <errordeveloper@gmail.com>
This commit is contained in:
Ilya Dmitrichenko 2017-04-11 17:37:56 +01:00
parent f603d37638
commit 340e97ae53
14 changed files with 208 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
projects/kubernetes/Boxfile
View File

@ -1,94 +0,0 @@
@image_name = "mobylinux/kubernetes"
@versions = {
kubernetes: "v1.6.1",
weave: "v1.9.4",
tini: "v0.14.0",
}
from "gcr.io/google_containers/hyperkube-amd64:#{@versions[:kubernetes]}"
def install_packages pkgs
cmds = [
%(apt-get update -q),
%(apt-get upgrade -qy),
%(apt-get install -qy #{pkgs}),
]
cmds.each { |cmd| run cmd }
end
def install_dependencies
prepare = [
'curl --silent "https://packages.cloud.google.com/apt/doc/apt-key.gpg" | apt-key add -',
'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list',
]
dependencies = %(curl apt-transport-https)
official_packages = %(kubernetes-cni)
kube_release_artefacts = "https://dl.k8s.io/#{@versions[:kubernetes]}/bin/linux/amd64"
weave_launcher = "https://cloud.weave.works/k8s/v1.6/net.yaml?v=#{@versions[:weave]}"
download_files = [
"/etc/weave.yaml" => {
url: weave_launcher,
mode: '0644',
},
"/usr/bin/kubeadm" => {
url: "#{kube_release_artefacts}/kubeadm",
mode: '0755',
},
"/usr/bin/tini" => {
url: "https://github.com/krallin/tini/releases/download/#{@versions[:tini]}/tini",
mode: '0755',
},
]
install_packages dependencies
prepare.each { |cmd| run cmd }
install_packages official_packages
download_files.each do |file|
file.each do |dest,info|
run %(curl --output "#{dest}" --fail --silent --location "#{info[:url]}")
run %(chmod "#{info[:mode]}" "#{dest}")
end
end
end
def kubelet_cmd
%w(
/hyperkube kubelet
--kubeconfig=/var/lib/kubeadm/kubelet.conf --require-kubeconfig=true
--pod-manifest-path=/var/lib/kubeadm/manifests --allow-privileged=true
--cluster-dns=10.96.0.10 --cluster-domain=cluster.local
--cgroups-per-qos=false --enforce-node-allocatable=""
)
#--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin
#--node-ip="192.168.65.2"
end
def create_shell_wrapper script, path
run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}"
end
install_dependencies
# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume
create_shell_wrapper "until #{kubelet_cmd.join(' ')} ; do [ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config)) ; done", '/usr/bin/kubelet.sh'
create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]}", '/usr/bin/kubeadm-init.sh'
flatten
env KUBECONFIG: "/etc/kubernetes/admin.conf"
set_exec entrypoint: %w(tini -s --), cmd: %w(kubelet.sh)
tag "#{@image_name}:latest"

23
projects/kubernetes/Makefile
View File

@ -1,14 +1,19 @@
all: build-container-images build-vm-images
build-container-image: Boxfile
docker run --rm -ti \
-v $(PWD):$(PWD) \
-v /var/run/docker.sock:/var/run/docker.sock \
-w $(PWD) \
boxbuilder/box:master Boxfile
BOX_PLANS = kubernetes.rb mounts.rb
push-container-images: build-container-image cache-images
build-container-images: $(BOX_PLANS)
for plan in $(BOX_PLANS) ; do \
docker run --rm -ti \
-v $(PWD):$(PWD) \
-v /var/run/docker.sock:/var/run/docker.sock \
-w $(PWD) \
boxbuilder/box:master $$plan \
; done
push-container-images: build-container-images cache-images
docker image push mobylinux/kubernetes:latest
docker image push mobylinux/kubernetes:latest-mounts
docker image push mobylinux/kubernetes:latest-image-cache-common
docker image push mobylinux/kubernetes:latest-image-cache-control-plane
@ -47,10 +52,10 @@ cache-images:
for image in $(COMMON_IMAGES) ; \
do $(MAKE) "image-cache/common/$${image}.tar" \
; done
cp image-cache/Dockerfile image-cache/common
cp image-cache/Dockerfile image-cache/.dockerignore image-cache/common
docker image build -t mobylinux/kubernetes:latest-image-cache-common image-cache/common
for image in $(CONTROL_PLANE_IMAGES) ; \
do $(MAKE) "image-cache/control-plane/$${image}.tar" \
; done
cp image-cache/Dockerfile image-cache/control-plane
cp image-cache/Dockerfile image-cache/.dockerignore image-cache/control-plane
docker image build -t mobylinux/kubernetes:latest-image-cache-control-plane image-cache/control-plane

18
projects/kubernetes/README.md
View File

@ -4,9 +4,11 @@ This project aims to demonstrate how one can create minimal and immutable Kubern
Make sure to `cd projects/kubernetes` first.
Build container & OS images:
Edit `kube-master.yml` and add your public SSH key to `files` section.
Build OS images:
```
make
make build-vm-images
```
Boot Kubernetes master OS image using `hyperkit` on macOS:
@ -14,9 +16,19 @@ Boot Kubernetes master OS image using `hyperkit` on macOS:
./boot-master.sh
```
Get IP address of the master:
```
ip addr show dev eth0
```
Login to the kubelet container:
```
./ssh_into_kubelet.sh <master-ip>
```
Manually initialise master with `kubeadm`:
```
runc exec kubelet kubeadm-init.sh
kubeadm-init.sh
```
Once `kubeadm` exits, make sure to copy the `kubeadm join` arguments,

2
projects/kubernetes/boot-master.sh
View File

@ -2,4 +2,4 @@
disk="kube-master-disk.img"
set -x
rm -f "${disk}"
../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 kube-master
../../bin/moby run hyperkit -cpus 2 -mem 6144 -disk-size 6144 kube-master

2
projects/kubernetes/boot-node.sh
View File

@ -5,4 +5,4 @@ shift
disk="kube-${name}-disk.img"
set -x
rm -f "${disk}"
../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 -disk "${disk}" -data "${*}" kube-node
../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 1024 -disk "${disk}" -data "${*}" kube-node

54
projects/kubernetes/common.rb Normal file
View File

@ -0,0 +1,54 @@
@image_name = "mobylinux/kubernetes"
@versions = {
kubernetes: "v1.6.1",
weave: "v1.9.4",
tini: "v0.14.0",
}
def install_packages pkgs
cmds = [
%(apt-get update -q),
%(apt-get upgrade -qy),
%(apt-get install -qy #{pkgs}),
]
cmds.each { |cmd| run cmd }
end
def setup_apt_config
prepare = [
'curl --silent "https://packages.cloud.google.com/apt/doc/apt-key.gpg" | apt-key add -',
'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list',
]
dependencies = %(curl apt-transport-https)
install_packages dependencies
prepare.each { |cmd| run cmd }
end
def create_shell_wrapper script, path
run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}"
end
def mount_bind src, dst
"mount --bind #{src} #{dst}"
end
def mount_bind_hostns_self mnt
"nsenter --mount=/proc/1/ns/mnt mount -- --bind #{mnt} #{mnt}"
end
def mount_make_hostns_rshared mnt
"nsenter --mount=/proc/1/ns/mnt mount -- --make-rshared #{mnt}"
end
def mount_persistent_disk mnt
"/mount.sh #{mnt}"
end
def mkdir_p dir
"mkdir -p #{dir}"
end

1
projects/kubernetes/image-cache/.dockerignore Normal file
View File

@ -0,0 +1 @@
Dockerfile

4
projects/kubernetes/image-cache/Dockerfile
View File

@ -1,6 +1,4 @@
FROM mobylinux/docker-ce:741bf21513328f674e0cdcaa55492b0b75974e08
ADD . /images
ENTRYPOINT [ "/bin/sh", "-c" ]
CMD [ "for image in /images/*.tar ; do docker image load -i $image ; done" ]
CMD [ "for image in /images/*.tar ; do docker image load -i $image && rm -f $image ; done" ]

25
projects/kubernetes/kube-master.yml
View File

@ -29,20 +29,15 @@ onboot:
capabilities:
- CAP_SYS_ADMIN
- CAP_MKNOD
- name: mount
image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
- name: mounts
image: "mobylinux/kubernetes:latest-mounts"
capabilities:
- all
pid: host
rootfsPropagation: shared
binds:
- /dev:/dev
- /var:/var:rshared,rbind
capabilities:
- CAP_SYS_ADMIN
rootfsPropagation: shared
command: ["/mount.sh", "/var/lib"]
- name: mount
image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
binds:
- /var:/var
command: ["mkdir", "-p", "/var/lib/kubeadm"]
services:
- name: rngd
image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92"
@ -92,6 +87,8 @@ services:
- /lib/modules:/lib/modules
- /var:/var:rshared,rbind
- /var/lib/kubeadm:/etc/kubernetes
- /etc/cni:/etc/cni:rshared,rbind
- /opt/cni:/opt/cni:rshared,rbind
rootfsPropagation: shared
- name: kubernetes-image-cache-common
image: "mobylinux/kubernetes:latest-image-cache-common"
@ -114,11 +111,13 @@ services:
- /dev:/dev
- /var:/var:rshared,rbind
- /var/lib/kubeadm:/etc/kubernetes
#- /var/log:/var/log #/var/log/containers
# TODO /{etc,opt}/cni & /var/lib/kubelet:rw,rshared
- /etc/cni:/rootfs/etc/cni:rshared,rbind
- /opt/cni:/rootfs/opt/cni:rshared,rbind
rootfsPropagation: shared
files:
- path: root/.ssh/authorized_keys
contents: '# Your ssh key goes here'
- {path: etc/cni, directory: true}
- {path: opt/cni, directory: true}
outputs:
- format: kernel+initrd

25
projects/kubernetes/kube-node.yml
View File

@ -29,20 +29,15 @@ onboot:
capabilities:
- CAP_SYS_ADMIN
- CAP_MKNOD
- name: mount
image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
- name: mounts
image: "mobylinux/kubernetes:latest-mounts"
capabilities:
- all
pid: host
rootfsPropagation: shared
binds:
- /dev:/dev
- /var:/var:rshared,rbind
capabilities:
- CAP_SYS_ADMIN
rootfsPropagation: shared
command: ["/mount.sh", "/var/lib"]
- name: mount
image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
binds:
- /var:/var
command: ["mkdir", "-p", "/var/lib/kubeadm"]
services:
- name: rngd
image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92"
@ -92,6 +87,8 @@ services:
- /lib/modules:/lib/modules
- /var:/var:rshared,rbind
- /var/lib/kubeadm:/etc/kubernetes
- /etc/cni:/etc/cni:rshared,rbind
- /opt/cni:/opt/cni:rshared,rbind
rootfsPropagation: shared
- name: kubernetes-image-cache-common
image: "mobylinux/kubernetes:latest-image-cache-common"
@ -110,11 +107,13 @@ services:
- /dev:/dev
- /var:/var:rshared,rbind
- /var/lib/kubeadm:/etc/kubernetes
#- /var/log:/var/log #/var/log/containers
# TODO /{etc,opt}/cni & /var/lib/kubelet:rw,rshared
- /etc/cni:/rootfs/etc/cni:rshared,rbind
- /opt/cni:/rootfs/opt/cni:rshared,rbind
rootfsPropagation: shared
files:
- path: root/.ssh/authorized_keys
contents: '# Your ssh key goes here'
- {path: etc/cni, directory: true}
- {path: opt/cni, directory: true}
outputs:
- format: kernel+initrd

68
projects/kubernetes/kubernetes.rb Normal file
View File

@ -0,0 +1,68 @@
import 'common.rb'
from "gcr.io/google_containers/hyperkube-amd64:#{@versions[:kubernetes]}"
def install_node_dependencies
kube_release_artefacts = "https://dl.k8s.io/#{@versions[:kubernetes]}/bin/linux/amd64"
weave_launcher = "https://frontend.dev.weave.works/k8s/v1.6/net?v=#{@versions[:weave]}"
download_files = [
"/etc/weave.yaml" => {
url: weave_launcher,
mode: '0644',
},
"/usr/bin/kubeadm" => {
url: "#{kube_release_artefacts}/kubeadm",
mode: '0755',
},
"/usr/bin/tini" => {
url: "https://github.com/krallin/tini/releases/download/#{@versions[:tini]}/tini",
mode: '0755',
},
]
download_files.each do |file|
file.each do |dest,info|
run %(curl --insecure --output "#{dest}" --fail --silent --location "#{info[:url]}")
run %(chmod "#{info[:mode]}" "#{dest}")
end
end
end
def kubelet_cmd
%w(
/hyperkube kubelet
--kubeconfig=/var/lib/kubeadm/kubelet.conf --require-kubeconfig=true
--pod-manifest-path=/var/lib/kubeadm/manifests --allow-privileged=true
--cluster-dns=10.96.0.10 --cluster-domain=cluster.local
--cgroups-per-qos=false --enforce-node-allocatable=""
--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin
)
#--node-ip="192.168.65.2"
end
setup_apt_config
run "rm -f /etc/cni/net.d/10-containernet.conf"
install_packages 'kubernetes-cni'
install_node_dependencies
# Exploit shared mounts, give CNI paths back to the host
mount_cni_dirs = [
mount_bind("/opt/cni", "/rootfs/opt/cni"),
mount_bind("/etc/cni", "/rootfs/etc/cni"),
]
# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume
wait_for_node_metadata_or_sleep_until_master_init = "[ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config))"
create_shell_wrapper "#{mount_cni_dirs.join(' && ')} && until #{kubelet_cmd.join(' ')} ; do #{wait_for_node_metadata_or_sleep_until_master_init} ; done", '/usr/bin/kubelet.sh'
create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]} && kubectl create -n kube-system -f /etc/weave.yaml", '/usr/bin/kubeadm-init.sh'
flatten
env KUBECONFIG: "/etc/kubernetes/admin.conf"
set_exec entrypoint: %w(tini -s --), cmd: %w(kubelet.sh)
tag "#{@image_name}:latest"

15
projects/kubernetes/mounts.rb Normal file
View File

@ -0,0 +1,15 @@
import 'common.rb'
from "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
script = [
mount_bind_hostns_self("/etc/cni"), mount_make_hostns_rshared("/etc/cni"),
mount_bind_hostns_self("/opt/cni"), mount_make_hostns_rshared("/opt/cni"),
mount_persistent_disk("/var/lib"),
mkdir_p("/var/lib/kubeadm"),
]
create_shell_wrapper script.join(' && '), '/usr/bin/kube-mounts.sh'
set_exec cmd: [ '/usr/bin/kube-mounts.sh' ]
tag "#{@image_name}:latest-mounts"

12
projects/kubernetes/ssh.sh Executable file
View File

@ -0,0 +1,12 @@
#!/bin/bash -eux
docker run \
-ti \
-v ~/.ssh/:/root/.ssh \
jdeathe/centos-ssh \
ssh \
-o Compression=yes \
-o LogLevel=FATAL \
-o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null \
-o IdentitiesOnly=yes \
"$@"

2
projects/kubernetes/ssh_into_kubelet.sh Executable file
View File

@ -0,0 +1,2 @@
#!/bin/bash -eux
./ssh.sh -t "$1" nsenter --mount --target 1 runc exec --tty kubelet bash -l