mirror of
https://github.com/linuxkit/linuxkit.git
synced 2025-10-08 05:14:18 +00:00
Merge pull request #1587 from errordeveloper/kubernetes
Kubernetes with Weave CNI networking
This commit is contained in:
Justin Cormack
GitHub
@@ -1,94 +0,0 @@
|
||||
@image_name = "mobylinux/kubernetes"
|
||||
|
||||
@versions = {
|
||||
kubernetes: "v1.6.1",
|
||||
weave: "v1.9.4",
|
||||
tini: "v0.14.0",
|
||||
}
|
||||
|
||||
from "gcr.io/google_containers/hyperkube-amd64:#{@versions[:kubernetes]}"
|
||||
|
||||
def install_packages pkgs
|
||||
cmds = [
|
||||
%(apt-get update -q),
|
||||
%(apt-get upgrade -qy),
|
||||
%(apt-get install -qy #{pkgs}),
|
||||
]
|
||||
|
||||
cmds.each { |cmd| run cmd }
|
||||
end
|
||||
|
||||
def install_dependencies
|
||||
|
||||
prepare = [
|
||||
'curl --silent "https://packages.cloud.google.com/apt/doc/apt-key.gpg" | apt-key add -',
|
||||
'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list',
|
||||
]
|
||||
|
||||
dependencies = %(curl apt-transport-https)
|
||||
|
||||
official_packages = %(kubernetes-cni)
|
||||
|
||||
kube_release_artefacts = "https://dl.k8s.io/#{@versions[:kubernetes]}/bin/linux/amd64"
|
||||
weave_launcher = "https://cloud.weave.works/k8s/v1.6/net.yaml?v=#{@versions[:weave]}"
|
||||
|
||||
download_files = [
|
||||
"/etc/weave.yaml" => {
|
||||
url: weave_launcher,
|
||||
mode: '0644',
|
||||
},
|
||||
"/usr/bin/kubeadm" => {
|
||||
url: "#{kube_release_artefacts}/kubeadm",
|
||||
mode: '0755',
|
||||
},
|
||||
"/usr/bin/tini" => {
|
||||
url: "https://github.com/krallin/tini/releases/download/#{@versions[:tini]}/tini",
|
||||
mode: '0755',
|
||||
},
|
||||
]
|
||||
|
||||
install_packages dependencies
|
||||
|
||||
prepare.each { |cmd| run cmd }
|
||||
|
||||
install_packages official_packages
|
||||
|
||||
download_files.each do |file|
|
||||
file.each do |dest,info|
|
||||
run %(curl --output "#{dest}" --fail --silent --location "#{info[:url]}")
|
||||
run %(chmod "#{info[:mode]}" "#{dest}")
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
def kubelet_cmd
|
||||
%w(
|
||||
/hyperkube kubelet
|
||||
--kubeconfig=/var/lib/kubeadm/kubelet.conf --require-kubeconfig=true
|
||||
--pod-manifest-path=/var/lib/kubeadm/manifests --allow-privileged=true
|
||||
--cluster-dns=10.96.0.10 --cluster-domain=cluster.local
|
||||
--cgroups-per-qos=false --enforce-node-allocatable=""
|
||||
)
|
||||
#--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin
|
||||
#--node-ip="192.168.65.2"
|
||||
end
|
||||
|
||||
def create_shell_wrapper script, path
|
||||
run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}"
|
||||
end
|
||||
|
||||
install_dependencies
|
||||
|
||||
# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume
|
||||
create_shell_wrapper "until #{kubelet_cmd.join(' ')} ; do [ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config)) ; done", '/usr/bin/kubelet.sh'
|
||||
|
||||
create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]}", '/usr/bin/kubeadm-init.sh'
|
||||
|
||||
flatten
|
||||
|
||||
env KUBECONFIG: "/etc/kubernetes/admin.conf"
|
||||
|
||||
set_exec entrypoint: %w(tini -s --), cmd: %w(kubelet.sh)
|
||||
|
||||
tag "#{@image_name}:latest"
|
||||
@@ -1,14 +1,19 @@
|
||||
all: build-container-images build-vm-images
|
||||
|
||||
build-container-image: Boxfile
|
||||
docker run --rm -ti \
|
||||
-v $(PWD):$(PWD) \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-w $(PWD) \
|
||||
boxbuilder/box:master Boxfile
|
||||
BOX_PLANS = kubernetes.rb mounts.rb
|
||||
|
||||
push-container-images: build-container-image cache-images
|
||||
build-container-images: $(BOX_PLANS)
|
||||
for plan in $(BOX_PLANS) ; do \
|
||||
docker run --rm -ti \
|
||||
-v $(PWD):$(PWD) \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-w $(PWD) \
|
||||
boxbuilder/box:master $$plan \
|
||||
; done
|
||||
|
||||
push-container-images: build-container-images cache-images
|
||||
docker image push mobylinux/kubernetes:latest
|
||||
docker image push mobylinux/kubernetes:latest-mounts
|
||||
docker image push mobylinux/kubernetes:latest-image-cache-common
|
||||
docker image push mobylinux/kubernetes:latest-image-cache-control-plane
|
||||
|
||||
@@ -47,10 +52,10 @@ cache-images:
|
||||
for image in $(COMMON_IMAGES) ; \
|
||||
do $(MAKE) "image-cache/common/$${image}.tar" \
|
||||
; done
|
||||
cp image-cache/Dockerfile image-cache/common
|
||||
cp image-cache/Dockerfile image-cache/.dockerignore image-cache/common
|
||||
docker image build -t mobylinux/kubernetes:latest-image-cache-common image-cache/common
|
||||
for image in $(CONTROL_PLANE_IMAGES) ; \
|
||||
do $(MAKE) "image-cache/control-plane/$${image}.tar" \
|
||||
; done
|
||||
cp image-cache/Dockerfile image-cache/control-plane
|
||||
cp image-cache/Dockerfile image-cache/.dockerignore image-cache/control-plane
|
||||
docker image build -t mobylinux/kubernetes:latest-image-cache-control-plane image-cache/control-plane
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
# Kubernetes
|
||||
# Kubernetes and LinuxKit
|
||||
|
||||
This project aims to demonstrate how one can create minimal and immutable Kubernetes OS images with Moby.
|
||||
This project aims to demonstrate how one can create minimal and immutable Kubernetes OS images with LinuxKit.
|
||||
|
||||
Make sure to `cd projects/kubernetes` first.
|
||||
|
||||
Build container & OS images:
|
||||
Edit `kube-master.yml` and add your public SSH key to `files` section.
|
||||
|
||||
Build OS images:
|
||||
```
|
||||
make
|
||||
make build-vm-images
|
||||
```
|
||||
|
||||
Boot Kubernetes master OS image using `hyperkit` on macOS:
|
||||
@@ -14,9 +16,19 @@ Boot Kubernetes master OS image using `hyperkit` on macOS:
|
||||
./boot-master.sh
|
||||
```
|
||||
|
||||
Get IP address of the master:
|
||||
```
|
||||
ip addr show dev eth0
|
||||
```
|
||||
|
||||
Login to the kubelet container:
|
||||
```
|
||||
./ssh_into_kubelet.sh <master-ip>
|
||||
```
|
||||
|
||||
Manually initialise master with `kubeadm`:
|
||||
```
|
||||
runc exec kubelet kubeadm-init.sh
|
||||
kubeadm-init.sh
|
||||
```
|
||||
|
||||
Once `kubeadm` exits, make sure to copy the `kubeadm join` arguments,
|
||||
|
||||
@@ -2,4 +2,4 @@
|
||||
disk="kube-master-disk.img"
|
||||
set -x
|
||||
rm -f "${disk}"
|
||||
../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 kube-master
|
||||
../../bin/moby run hyperkit -cpus 2 -mem 6144 -disk-size 6144 kube-master
|
||||
|
||||
@@ -5,4 +5,4 @@ shift
|
||||
disk="kube-${name}-disk.img"
|
||||
set -x
|
||||
rm -f "${disk}"
|
||||
../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 2048 -disk "${disk}" -data "${*}" kube-node
|
||||
../../bin/moby run hyperkit -cpus 2 -mem 4096 -disk-size 1024 -disk "${disk}" -data "${*}" kube-node
|
||||
|
||||
54
projects/kubernetes/common.rb
Normal file
54
projects/kubernetes/common.rb
Normal file
@@ -0,0 +1,54 @@
|
||||
@image_name = "mobylinux/kubernetes"
|
||||
|
||||
@versions = {
|
||||
kubernetes: "v1.6.1",
|
||||
weave: "v1.9.4",
|
||||
tini: "v0.14.0",
|
||||
}
|
||||
|
||||
def install_packages pkgs
|
||||
cmds = [
|
||||
%(apt-get update -q),
|
||||
%(apt-get upgrade -qy),
|
||||
%(apt-get install -qy #{pkgs}),
|
||||
]
|
||||
|
||||
cmds.each { |cmd| run cmd }
|
||||
end
|
||||
|
||||
def setup_apt_config
|
||||
prepare = [
|
||||
'curl --silent "https://packages.cloud.google.com/apt/doc/apt-key.gpg" | apt-key add -',
|
||||
'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list',
|
||||
]
|
||||
|
||||
dependencies = %(curl apt-transport-https)
|
||||
|
||||
install_packages dependencies
|
||||
|
||||
prepare.each { |cmd| run cmd }
|
||||
end
|
||||
|
||||
def create_shell_wrapper script, path
|
||||
run "echo \"#!/bin/sh\n#{script}\n\" > #{path} && chmod 0755 #{path}"
|
||||
end
|
||||
|
||||
def mount_bind src, dst
|
||||
"mount --bind #{src} #{dst}"
|
||||
end
|
||||
|
||||
def mount_bind_hostns_self mnt
|
||||
"nsenter --mount=/proc/1/ns/mnt mount -- --bind #{mnt} #{mnt}"
|
||||
end
|
||||
|
||||
def mount_make_hostns_rshared mnt
|
||||
"nsenter --mount=/proc/1/ns/mnt mount -- --make-rshared #{mnt}"
|
||||
end
|
||||
|
||||
def mount_persistent_disk mnt
|
||||
"/mount.sh #{mnt}"
|
||||
end
|
||||
|
||||
def mkdir_p dir
|
||||
"mkdir -p #{dir}"
|
||||
end
|
||||
1
projects/kubernetes/image-cache/.dockerignore
Normal file
1
projects/kubernetes/image-cache/.dockerignore
Normal file
@@ -0,0 +1 @@
|
||||
Dockerfile
|
||||
@@ -1,6 +1,4 @@
|
||||
FROM mobylinux/docker-ce:741bf21513328f674e0cdcaa55492b0b75974e08
|
||||
|
||||
ADD . /images
|
||||
|
||||
ENTRYPOINT [ "/bin/sh", "-c" ]
|
||||
CMD [ "for image in /images/*.tar ; do docker image load -i $image ; done" ]
|
||||
CMD [ "for image in /images/*.tar ; do docker image load -i $image && rm -f $image ; done" ]
|
||||
|
||||
@@ -29,20 +29,15 @@ onboot:
|
||||
capabilities:
|
||||
- CAP_SYS_ADMIN
|
||||
- CAP_MKNOD
|
||||
- name: mount
|
||||
image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
|
||||
- name: mounts
|
||||
image: "mobylinux/kubernetes:latest-mounts"
|
||||
capabilities:
|
||||
- all
|
||||
pid: host
|
||||
rootfsPropagation: shared
|
||||
binds:
|
||||
- /dev:/dev
|
||||
- /var:/var:rshared,rbind
|
||||
capabilities:
|
||||
- CAP_SYS_ADMIN
|
||||
rootfsPropagation: shared
|
||||
command: ["/mount.sh", "/var/lib"]
|
||||
- name: mount
|
||||
image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
|
||||
binds:
|
||||
- /var:/var
|
||||
command: ["mkdir", "-p", "/var/lib/kubeadm"]
|
||||
services:
|
||||
- name: rngd
|
||||
image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92"
|
||||
@@ -92,6 +87,8 @@ services:
|
||||
- /lib/modules:/lib/modules
|
||||
- /var:/var:rshared,rbind
|
||||
- /var/lib/kubeadm:/etc/kubernetes
|
||||
- /etc/cni:/etc/cni:rshared,rbind
|
||||
- /opt/cni:/opt/cni:rshared,rbind
|
||||
rootfsPropagation: shared
|
||||
- name: kubernetes-image-cache-common
|
||||
image: "mobylinux/kubernetes:latest-image-cache-common"
|
||||
@@ -114,11 +111,13 @@ services:
|
||||
- /dev:/dev
|
||||
- /var:/var:rshared,rbind
|
||||
- /var/lib/kubeadm:/etc/kubernetes
|
||||
#- /var/log:/var/log #/var/log/containers
|
||||
# TODO /{etc,opt}/cni & /var/lib/kubelet:rw,rshared
|
||||
- /etc/cni:/rootfs/etc/cni:rshared,rbind
|
||||
- /opt/cni:/rootfs/opt/cni:rshared,rbind
|
||||
rootfsPropagation: shared
|
||||
files:
|
||||
- path: root/.ssh/authorized_keys
|
||||
contents: '# Your ssh key goes here'
|
||||
- {path: etc/cni, directory: true}
|
||||
- {path: opt/cni, directory: true}
|
||||
outputs:
|
||||
- format: kernel+initrd
|
||||
|
||||
@@ -29,20 +29,15 @@ onboot:
|
||||
capabilities:
|
||||
- CAP_SYS_ADMIN
|
||||
- CAP_MKNOD
|
||||
- name: mount
|
||||
image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
|
||||
- name: mounts
|
||||
image: "mobylinux/kubernetes:latest-mounts"
|
||||
capabilities:
|
||||
- all
|
||||
pid: host
|
||||
rootfsPropagation: shared
|
||||
binds:
|
||||
- /dev:/dev
|
||||
- /var:/var:rshared,rbind
|
||||
capabilities:
|
||||
- CAP_SYS_ADMIN
|
||||
rootfsPropagation: shared
|
||||
command: ["/mount.sh", "/var/lib"]
|
||||
- name: mount
|
||||
image: "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
|
||||
binds:
|
||||
- /var:/var
|
||||
command: ["mkdir", "-p", "/var/lib/kubeadm"]
|
||||
services:
|
||||
- name: rngd
|
||||
image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92"
|
||||
@@ -92,6 +87,8 @@ services:
|
||||
- /lib/modules:/lib/modules
|
||||
- /var:/var:rshared,rbind
|
||||
- /var/lib/kubeadm:/etc/kubernetes
|
||||
- /etc/cni:/etc/cni:rshared,rbind
|
||||
- /opt/cni:/opt/cni:rshared,rbind
|
||||
rootfsPropagation: shared
|
||||
- name: kubernetes-image-cache-common
|
||||
image: "mobylinux/kubernetes:latest-image-cache-common"
|
||||
@@ -110,11 +107,13 @@ services:
|
||||
- /dev:/dev
|
||||
- /var:/var:rshared,rbind
|
||||
- /var/lib/kubeadm:/etc/kubernetes
|
||||
#- /var/log:/var/log #/var/log/containers
|
||||
# TODO /{etc,opt}/cni & /var/lib/kubelet:rw,rshared
|
||||
- /etc/cni:/rootfs/etc/cni:rshared,rbind
|
||||
- /opt/cni:/rootfs/opt/cni:rshared,rbind
|
||||
rootfsPropagation: shared
|
||||
files:
|
||||
- path: root/.ssh/authorized_keys
|
||||
contents: '# Your ssh key goes here'
|
||||
- {path: etc/cni, directory: true}
|
||||
- {path: opt/cni, directory: true}
|
||||
outputs:
|
||||
- format: kernel+initrd
|
||||
|
||||
68
projects/kubernetes/kubernetes.rb
Normal file
68
projects/kubernetes/kubernetes.rb
Normal file
@@ -0,0 +1,68 @@
|
||||
import 'common.rb'
|
||||
|
||||
from "gcr.io/google_containers/hyperkube-amd64:#{@versions[:kubernetes]}"
|
||||
|
||||
def install_node_dependencies
|
||||
kube_release_artefacts = "https://dl.k8s.io/#{@versions[:kubernetes]}/bin/linux/amd64"
|
||||
weave_launcher = "https://frontend.dev.weave.works/k8s/v1.6/net?v=#{@versions[:weave]}"
|
||||
|
||||
download_files = [
|
||||
"/etc/weave.yaml" => {
|
||||
url: weave_launcher,
|
||||
mode: '0644',
|
||||
},
|
||||
"/usr/bin/kubeadm" => {
|
||||
url: "#{kube_release_artefacts}/kubeadm",
|
||||
mode: '0755',
|
||||
},
|
||||
"/usr/bin/tini" => {
|
||||
url: "https://github.com/krallin/tini/releases/download/#{@versions[:tini]}/tini",
|
||||
mode: '0755',
|
||||
},
|
||||
]
|
||||
|
||||
download_files.each do |file|
|
||||
file.each do |dest,info|
|
||||
run %(curl --insecure --output "#{dest}" --fail --silent --location "#{info[:url]}")
|
||||
run %(chmod "#{info[:mode]}" "#{dest}")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def kubelet_cmd
|
||||
%w(
|
||||
/hyperkube kubelet
|
||||
--kubeconfig=/var/lib/kubeadm/kubelet.conf --require-kubeconfig=true
|
||||
--pod-manifest-path=/var/lib/kubeadm/manifests --allow-privileged=true
|
||||
--cluster-dns=10.96.0.10 --cluster-domain=cluster.local
|
||||
--cgroups-per-qos=false --enforce-node-allocatable=""
|
||||
--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin
|
||||
)
|
||||
#--node-ip="192.168.65.2"
|
||||
end
|
||||
|
||||
setup_apt_config
|
||||
run "rm -f /etc/cni/net.d/10-containernet.conf"
|
||||
install_packages 'kubernetes-cni'
|
||||
install_node_dependencies
|
||||
|
||||
# Exploit shared mounts, give CNI paths back to the host
|
||||
mount_cni_dirs = [
|
||||
mount_bind("/opt/cni", "/rootfs/opt/cni"),
|
||||
mount_bind("/etc/cni", "/rootfs/etc/cni"),
|
||||
]
|
||||
|
||||
# At the moment we trigger `kubeadm init` manually on the master, then start nodes which expect `kubeadm join` args in metadata volume
|
||||
wait_for_node_metadata_or_sleep_until_master_init = "[ ! -e /dev/sr0 ] && sleep 1 || (mount -o ro /dev/sr0 /mnt && kubeadm join --skip-preflight-checks \\\$(cat /mnt/config))"
|
||||
|
||||
create_shell_wrapper "#{mount_cni_dirs.join(' && ')} && until #{kubelet_cmd.join(' ')} ; do #{wait_for_node_metadata_or_sleep_until_master_init} ; done", '/usr/bin/kubelet.sh'
|
||||
|
||||
create_shell_wrapper "kubeadm init --skip-preflight-checks --kubernetes-version #{@versions[:kubernetes]} && kubectl create -n kube-system -f /etc/weave.yaml", '/usr/bin/kubeadm-init.sh'
|
||||
|
||||
flatten
|
||||
|
||||
env KUBECONFIG: "/etc/kubernetes/admin.conf"
|
||||
|
||||
set_exec entrypoint: %w(tini -s --), cmd: %w(kubelet.sh)
|
||||
|
||||
tag "#{@image_name}:latest"
|
||||
15
projects/kubernetes/mounts.rb
Normal file
15
projects/kubernetes/mounts.rb
Normal file
@@ -0,0 +1,15 @@
|
||||
import 'common.rb'
|
||||
|
||||
from "mobylinux/mount:d2669e7c8ddda99fa0618a414d44261eba6e299a"
|
||||
|
||||
script = [
|
||||
mount_bind_hostns_self("/etc/cni"), mount_make_hostns_rshared("/etc/cni"),
|
||||
mount_bind_hostns_self("/opt/cni"), mount_make_hostns_rshared("/opt/cni"),
|
||||
mount_persistent_disk("/var/lib"),
|
||||
mkdir_p("/var/lib/kubeadm"),
|
||||
]
|
||||
|
||||
create_shell_wrapper script.join(' && '), '/usr/bin/kube-mounts.sh'
|
||||
set_exec cmd: [ '/usr/bin/kube-mounts.sh' ]
|
||||
|
||||
tag "#{@image_name}:latest-mounts"
|
||||
12
projects/kubernetes/ssh.sh
Executable file
12
projects/kubernetes/ssh.sh
Executable file
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash -eux
|
||||
docker run \
|
||||
-ti \
|
||||
-v ~/.ssh/:/root/.ssh \
|
||||
jdeathe/centos-ssh \
|
||||
ssh \
|
||||
-o Compression=yes \
|
||||
-o LogLevel=FATAL \
|
||||
-o StrictHostKeyChecking=no \
|
||||
-o UserKnownHostsFile=/dev/null \
|
||||
-o IdentitiesOnly=yes \
|
||||
"$@"
|
||||
2
projects/kubernetes/ssh_into_kubelet.sh
Executable file
2
projects/kubernetes/ssh_into_kubelet.sh
Executable file
@@ -0,0 +1,2 @@
|
||||
#!/bin/bash -eux
|
||||
./ssh.sh -t "$1" nsenter --mount --target 1 runc exec --tty kubelet bash -l
|
||||
Reference in New Issue
Block a user