mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-27 05:27:21 +00:00
Refactor hyperkube, remove unnecessary packages, optimize layers, bump cni version, add new features, run kube-proxy in a daemonset
This commit is contained in:
Lucas Käldström
parent
c958d3d4fd
commit
d711fd8764
@ -24,10 +24,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -y \
|
||||
ethtool \
|
||||
ca-certificates \
|
||||
conntrack \
|
||||
file \
|
||||
util-linux \
|
||||
socat \
|
||||
curl \
|
||||
git \
|
||||
nfs-common \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
|
||||
@ -40,34 +38,28 @@ RUN cp /usr/bin/nsenter /nsenter
|
||||
COPY hyperkube /hyperkube
|
||||
|
||||
# Manifests for the docker guide
|
||||
COPY static-pods/master.json /etc/kubernetes/manifests/
|
||||
COPY static-pods/etcd.json /etc/kubernetes/manifests/
|
||||
COPY static-pods/addon-manager.json /etc/kubernetes/manifests/
|
||||
|
||||
# TODO: Move out kube-proxy to a DaemonSet again
|
||||
COPY static-pods/kube-proxy.json /etc/kubernetes/manifests/
|
||||
COPY static-pods/master.json \
|
||||
static-pods/etcd.json \
|
||||
static-pods/addon-manager-singlenode.json \
|
||||
static-pods/kube-proxy.json \
|
||||
/etc/kubernetes/manifests/
|
||||
|
||||
# Manifests for the docker-multinode guide
|
||||
COPY static-pods/master-multi.json /etc/kubernetes/manifests-multi/
|
||||
COPY static-pods/addon-manager.json /etc/kubernetes/manifests-multi/
|
||||
|
||||
# TODO: Move out kube-proxy to a DaemonSet again
|
||||
COPY static-pods/kube-proxy.json /etc/kubernetes/manifests-multi/
|
||||
COPY static-pods/master-multi.json \
|
||||
static-pods/addon-manager-multinode.json \
|
||||
/etc/kubernetes/manifests-multi/
|
||||
|
||||
# Copy over all addons
|
||||
COPY addons /etc/kubernetes/addons
|
||||
|
||||
# Other required scripts for the setup
|
||||
COPY safe_format_and_mount /usr/share/google/safe_format_and_mount
|
||||
COPY setup-files.sh /setup-files.sh
|
||||
COPY make-ca-cert.sh /make-ca-cert.sh
|
||||
COPY copy-addons.sh /copy-addons.sh
|
||||
# Copy other required scripts for the setup
|
||||
COPY setup-files.sh make-ca-cert.sh copy-addons.sh /
|
||||
|
||||
# easy-rsa package required by make-ca-cert
|
||||
ADD https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz /root/kube/
|
||||
|
||||
# Copy the cni folder into /opt/
|
||||
COPY cni /opt/cni
|
||||
# Copy the the cni-bin folder into /opt/cni/bin
|
||||
COPY cni-bin /opt/cni/bin
|
||||
|
||||
# Copy overlay configuration to default directory
|
||||
COPY cni-conf /etc/cni/net.d
|
||||
|
||||
@ -15,12 +15,12 @@
|
||||
# Build the hyperkube image.
|
||||
#
|
||||
# Usage:
|
||||
# [ARCH=amd64] [REGISTRY="gcr.io/google_containers"] make (build|push) VERSION={some_version_number e.g. v1.2.0}
|
||||
# [ARCH=amd64] [REGISTRY="gcr.io/google_containers"] make (build|push) VERSION={some_released_version_of_kubernetes}
|
||||
|
||||
REGISTRY?=gcr.io/google_containers
|
||||
ARCH?=amd64
|
||||
TEMP_DIR:=$(shell mktemp -d)
|
||||
CNI_RELEASE=8a936732094c0941e1543ef5d292a1f4fffa1ac5
|
||||
CNI_RELEASE=9d5e6e60e79491207834ae8439e80c943db65a69
|
||||
|
||||
UNAME_S:=$(shell uname -s)
|
||||
ifeq ($(UNAME_S),Darwin)
|
||||
@ -54,33 +54,35 @@ ifndef VERSION
|
||||
$(error VERSION is undefined)
|
||||
endif
|
||||
cp -r ./* ${TEMP_DIR}
|
||||
mkdir -p ${TEMP_DIR}/cni ${TEMP_DIR}/addons
|
||||
cp ../../saltbase/salt/helpers/safe_format_and_mount ${TEMP_DIR}
|
||||
mkdir -p ${TEMP_DIR}/cni-bin ${TEMP_DIR}/addons ${TEMP_DIR}/addons/singlenode ${TEMP_DIR}/addons/multinode
|
||||
cp ../../saltbase/salt/generate-cert/make-ca-cert.sh ${TEMP_DIR}
|
||||
cp ../../addons/dns/skydns-rc.yaml.base ${TEMP_DIR}/addons/skydns-rc.yaml
|
||||
cp ../../addons/dns/skydns-svc.yaml.base ${TEMP_DIR}/addons/skydns-svc.yaml
|
||||
cp ../../addons/dashboard/dashboard-controller.yaml ${TEMP_DIR}/addons
|
||||
cp ../../addons/dashboard/dashboard-service.yaml ${TEMP_DIR}/addons
|
||||
|
||||
# TODO: Move out kube-proxy to a DaemonSet again
|
||||
#cp kube-proxy-ds.yaml ${TEMP_DIR}/addons/kube-proxy.yaml
|
||||
# Singlenode addons
|
||||
cp ../../addons/dns/skydns-rc.yaml.base ${TEMP_DIR}/addons/singlenode/skydns-rc.yaml
|
||||
cp ../../addons/dns/skydns-svc.yaml.base ${TEMP_DIR}/addons/singlenode/skydns-svc.yaml
|
||||
cp ../../addons/dashboard/dashboard-controller.yaml ${TEMP_DIR}/addons/singlenode/
|
||||
cp ../../addons/dashboard/dashboard-service.yaml ${TEMP_DIR}/addons/singlenode/
|
||||
|
||||
# Multinode addons; all singlenode addons plus kube-proxy (and soon flannel)
|
||||
cp ${TEMP_DIR}/addons/singlenode/*.yaml ${TEMP_DIR}/addons/multinode/
|
||||
cp kube-proxy-ds.yaml ${TEMP_DIR}/addons/multinode/kube-proxy.yaml
|
||||
|
||||
cp ../../../_output/dockerized/bin/linux/${ARCH}/hyperkube ${TEMP_DIR}
|
||||
|
||||
cd ${TEMP_DIR} && sed -i.back "s|VERSION|${VERSION}|g" addons/*.yaml static-pods/*.json
|
||||
cd ${TEMP_DIR} && sed -i.back "s|REGISTRY|${REGISTRY}|g" addons/*.yaml static-pods/*.json
|
||||
cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" addons/*.yaml static-pods/*.json
|
||||
cd ${TEMP_DIR} && sed -i.back "s|VERSION|${VERSION}|g" addons/singlenode/*.yaml addons/multinode/*.yaml static-pods/*.json
|
||||
cd ${TEMP_DIR} && sed -i.back "s|REGISTRY|${REGISTRY}|g" addons/singlenode/*.yaml addons/multinode/*.yaml static-pods/*.json
|
||||
cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" addons/singlenode/*.yaml addons/multinode/*.yaml static-pods/*.json
|
||||
cd ${TEMP_DIR} && sed -i.back "s|ARCH|${QEMUARCH}|g" Dockerfile
|
||||
cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
|
||||
cd ${TEMP_DIR} && sed -i.back "s|-amd64|-${ARCH}|g" addons/*.yaml
|
||||
cd ${TEMP_DIR} && sed -i.back "s|__PILLAR__DNS__REPLICAS__|1|g;s|__PILLAR__DNS__SERVER__|10.0.0.10|g;" addons/skydns*.yaml
|
||||
cd ${TEMP_DIR} && sed -i.back "s|__PILLAR__DNS__DOMAIN__|cluster.local|g;s|__PILLAR__FEDERATIONS__DOMAIN__MAP__||g;" addons/skydns*.yaml
|
||||
rm ${TEMP_DIR}/addons/*.back
|
||||
cd ${TEMP_DIR} && sed -i.back "s|-amd64|-${ARCH}|g" addons/singlenode/*.yaml addons/multinode/*.yaml
|
||||
cd ${TEMP_DIR} && sed -i.back "s|__PILLAR__DNS__REPLICAS__|1|g;s|__PILLAR__DNS__SERVER__|10.0.0.10|g;" addons/singlenode/skydns*.yaml addons/multinode/skydns*.yaml
|
||||
cd ${TEMP_DIR} && sed -i.back "s|__PILLAR__DNS__DOMAIN__|cluster.local|g;s|__PILLAR__FEDERATIONS__DOMAIN__MAP__||g;" addons/singlenode/skydns*.yaml addons/multinode/skydns*.yaml
|
||||
cd ${TEMP_DIR} && rm -f addons/singlenode/*.back addons/multinode/*.back static-pods/*.back
|
||||
|
||||
# Make scripts executable before they are copied into the Docker image. If we make them executable later, in another layer
|
||||
# they'll take up twice the space because the new executable binary differs from the old one, but everything is cached in layers.
|
||||
cd ${TEMP_DIR} && chmod a+rx \
|
||||
hyperkube \
|
||||
safe_format_and_mount \
|
||||
setup-files.sh \
|
||||
make-ca-cert.sh \
|
||||
copy-addons.sh
|
||||
@ -88,8 +90,6 @@ endif
|
||||
ifeq ($(ARCH),amd64)
|
||||
# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
|
||||
cd ${TEMP_DIR} && ${SED_CMD} "/CROSS_BUILD_/d" Dockerfile
|
||||
# Download CNI
|
||||
curl -sSL --retry 5 https://storage.googleapis.com/kubernetes-release/network-plugins/cni-${CNI_RELEASE}.tar.gz | tar -xz -C ${TEMP_DIR}/cni
|
||||
else
|
||||
cd ${TEMP_DIR} && ${SED_CMD} "s/CROSS_BUILD_//g" Dockerfile
|
||||
|
||||
@ -97,17 +97,16 @@ else
|
||||
# Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel
|
||||
docker run --rm --privileged multiarch/qemu-user-static:register --reset
|
||||
curl -sSL --retry 5 https://github.com/multiarch/qemu-user-static/releases/download/v2.5.0/x86_64_qemu-${QEMUARCH}-static.tar.xz | tar -xJ -C ${TEMP_DIR}
|
||||
endif
|
||||
|
||||
# This cross-compiles cni for the other architectures, until CNI releases binaries for all arches: https://github.com/containernetworking/cni/pull/241
|
||||
docker run -it -v ${TEMP_DIR}/cni:/cnibin golang:1.6 /bin/bash -c "\
|
||||
# This cross-compiles cni for all architectures
|
||||
# TODO(freehan): Push the latest cni for all arches to storage.googleapis.com so we may just download the binaries
|
||||
docker run -it -v ${TEMP_DIR}/cni-bin:/cnibin golang:1.6 /bin/bash -c "\
|
||||
git clone https://github.com/containernetworking/cni \
|
||||
&& cd cni \
|
||||
&& git checkout $(CNI_RELEASE) \
|
||||
&& curl -sSL https://patch-diff.githubusercontent.com/raw/containernetworking/cni/pull/241.patch > multiarch.patch \
|
||||
&& git apply --exclude=.travis.yml --exclude=scripts/release-with-rkt.sh < multiarch.patch \
|
||||
&& GOARCH=$(ARCH) ./build \
|
||||
&& cp bin/* /cnibin"
|
||||
endif
|
||||
|
||||
docker build -t ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${TEMP_DIR}
|
||||
rm -rf "${TEMP_DIR}"
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
"delegate": {
|
||||
"bridge": "cni0",
|
||||
"mtu": 1450,
|
||||
"isDefaultGateway": true
|
||||
"isDefaultGateway": true,
|
||||
"forceAddress": true
|
||||
}
|
||||
}
|
||||
|
||||
@ -13,16 +13,19 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# Now we're running in the sidecar container
|
||||
# /etc/kubernetes/addons holds the data in the hyperkube container
|
||||
# /srv/kubernetes is an emptyDir that maps to /etc/kubernetes in the addon-manager container
|
||||
# This way we're using the latest manifests from hyperkube without updating
|
||||
# kube-addon-manager which is used for other deployments too
|
||||
|
||||
# Inside /etc/kubernetes/addons, there are two directories: singlenode and multinode
|
||||
# If "singlenode" is passed to this script; those manifests are copied to the addon-manager and vice versa with "multinode"
|
||||
SUBDIRECTORY=$1
|
||||
|
||||
# While there is no data copied over to the emptyDir, try to copy it.
|
||||
while [[ ! -d /srv/kubernetes/addons ]]; do
|
||||
cp -r /etc/kubernetes/* /srv/kubernetes/
|
||||
while [[ -z $(ls /srv/kubernetes/addons) ]]; do
|
||||
cp -r /etc/kubernetes/addons/${SUBDIRECTORY}/* /srv/kubernetes/addons/
|
||||
done
|
||||
|
||||
# Then sleep forever
|
||||
|
||||
@ -12,12 +12,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# TODO: Move out kube-proxy to a DaemonSet again
|
||||
# This is disabled for the v1.3 release, due to bootstrapping complexities
|
||||
apiVersion: extensions/v1beta1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: k8s-proxy
|
||||
name: k8s-proxy-v1
|
||||
namespace: kube-system
|
||||
labels:
|
||||
k8s-app: k8s-proxy
|
||||
@ -38,8 +36,16 @@ spec:
|
||||
command:
|
||||
- /hyperkube
|
||||
- proxy
|
||||
- --master=http://127.0.0.1:8080
|
||||
- --kubeconfig=/var/lib/kubelet/kubeconfig/kubeconfig.yaml
|
||||
- --v=2
|
||||
- --resource-container=""
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: kubeconfig
|
||||
mountPath: /var/lib/kubelet/kubeconfig
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: kubeconfig
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/kubeconfig
|
||||
|
||||
@ -30,39 +30,40 @@ create_token() {
|
||||
# list of Subject Alternative Names of the server TLS certificate
|
||||
# Should contain internal IP, i.e. IP:10.0.0.1 for 10.0.0.0/24 cluster IP range
|
||||
EXTRA_SANS=$1
|
||||
DATA_DIR=/srv/kubernetes
|
||||
|
||||
# Files in /data are persistent across reboots, so we don't want to re-create the files if they already
|
||||
# exist, because the state is persistent in etcd too, and we don't want a conflict between "old" data in
|
||||
# etcd and "new" data that this script would create for apiserver. Therefore, if the file exist, skip it.
|
||||
if [[ ! -f /data/ca.crt ]]; then
|
||||
if [[ ! -f ${DATA_DIR}/ca.crt ]]; then
|
||||
|
||||
# Create HTTPS certificates
|
||||
groupadd -f -r kube-cert-test
|
||||
groupadd -f -r kube-cert
|
||||
|
||||
# hostname -I gets the ip of the node
|
||||
CERT_DIR=/data CERT_GROUP=kube-cert-test /make-ca-cert.sh $(hostname -I | awk '{print $1}') ${EXTRA_SANS}
|
||||
/make-ca-cert.sh $(hostname -I | awk '{print $1}') ${EXTRA_SANS}
|
||||
|
||||
echo "Certificates created $(date)"
|
||||
else
|
||||
echo "Certificates already found, not recreating."
|
||||
fi
|
||||
|
||||
if [[ ! -f /data/basic_auth.csv ]]; then
|
||||
if [[ ! -f ${DATA_DIR}/basic_auth.csv ]]; then
|
||||
|
||||
# Create basic token authorization
|
||||
echo "admin,admin,admin" > /data/basic_auth.csv
|
||||
echo "admin,admin,admin" > ${DATA_DIR}/basic_auth.csv
|
||||
|
||||
echo "basic_auth.csv created $(date)"
|
||||
else
|
||||
echo "basic_auth.csv already found, not recreating."
|
||||
fi
|
||||
|
||||
if [[ ! -f /data/known_tokens.csv ]]; then
|
||||
if [[ ! -f ${DATA_DIR}/known_tokens.csv ]]; then
|
||||
|
||||
# Create known tokens for service accounts
|
||||
echo "$(create_token),admin,admin" >> /data/known_tokens.csv
|
||||
echo "$(create_token),kubelet,kubelet" >> /data/known_tokens.csv
|
||||
echo "$(create_token),kube_proxy,kube_proxy" >> /data/known_tokens.csv
|
||||
echo "$(create_token),admin,admin" >> ${DATA_DIR}/known_tokens.csv
|
||||
echo "$(create_token),kubelet,kubelet" >> ${DATA_DIR}/known_tokens.csv
|
||||
echo "$(create_token),kube_proxy,kube_proxy" >> ${DATA_DIR}/known_tokens.csv
|
||||
|
||||
echo "known_tokens.csv created $(date)"
|
||||
else
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
"containers": [
|
||||
{
|
||||
"name": "kube-addon-manager",
|
||||
"image": "gcr.io/google-containers/kube-addon-manager-ARCH:v5.1",
|
||||
"image": "REGISTRY/kube-addon-manager-ARCH:v5.1",
|
||||
"resources": {
|
||||
"requests": {
|
||||
"cpu": "5m",
|
||||
@ -21,7 +21,7 @@
|
||||
"volumeMounts": [
|
||||
{
|
||||
"name": "addons",
|
||||
"mountPath": "/etc/kubernetes/",
|
||||
"mountPath": "/etc/kubernetes/addons",
|
||||
"readOnly": true
|
||||
}
|
||||
]
|
||||
@ -30,12 +30,13 @@
|
||||
"name": "kube-addon-manager-data",
|
||||
"image": "REGISTRY/hyperkube-ARCH:VERSION",
|
||||
"command": [
|
||||
"/copy-addons.sh"
|
||||
"/copy-addons.sh",
|
||||
"multinode"
|
||||
],
|
||||
"volumeMounts": [
|
||||
{
|
||||
"name": "addons",
|
||||
"mountPath": "/srv/kubernetes/",
|
||||
"mountPath": "/srv/kubernetes/addons",
|
||||
"readOnly": false
|
||||
}
|
||||
]
|
||||
@ -0,0 +1,52 @@
|
||||
{
|
||||
"apiVersion": "v1",
|
||||
"kind": "Pod",
|
||||
"metadata": {
|
||||
"name": "kube-addon-manager",
|
||||
"namespace": "kube-system",
|
||||
"version": "v1"
|
||||
},
|
||||
"spec": {
|
||||
"hostNetwork": true,
|
||||
"containers": [
|
||||
{
|
||||
"name": "kube-addon-manager",
|
||||
"image": "REGISTRY/kube-addon-manager-ARCH:v5.1",
|
||||
"resources": {
|
||||
"requests": {
|
||||
"cpu": "5m",
|
||||
"memory": "50Mi"
|
||||
}
|
||||
},
|
||||
"volumeMounts": [
|
||||
{
|
||||
"name": "addons",
|
||||
"mountPath": "/etc/kubernetes/addons",
|
||||
"readOnly": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "kube-addon-manager-data",
|
||||
"image": "REGISTRY/hyperkube-ARCH:VERSION",
|
||||
"command": [
|
||||
"/copy-addons.sh",
|
||||
"singlenode"
|
||||
],
|
||||
"volumeMounts": [
|
||||
{
|
||||
"name": "addons",
|
||||
"mountPath": "/srv/kubernetes/addons",
|
||||
"readOnly": false
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"volumes":[
|
||||
{
|
||||
"name": "addons",
|
||||
"emptyDir": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@ -13,8 +13,8 @@
|
||||
"image": "gcr.io/google_containers/etcd-ARCH:3.0.4",
|
||||
"command": [
|
||||
"/usr/local/bin/etcd",
|
||||
"--listen-client-urls=http://127.0.0.1:2379",
|
||||
"--advertise-client-urls=http://127.0.0.1:2379",
|
||||
"--listen-client-urls=http://127.0.0.1:2379,http://127.0.0.1:4001",
|
||||
"--advertise-client-urls=http://127.0.0.1:2379,http://127.0.0.1:4001",
|
||||
"--data-dir=/var/etcd/data"
|
||||
],
|
||||
"volumeMounts": [
|
||||
|
||||
@ -18,6 +18,8 @@
|
||||
"--service-account-private-key-file=/srv/kubernetes/server.key",
|
||||
"--root-ca-file=/srv/kubernetes/ca.crt",
|
||||
"--min-resync-period=3m",
|
||||
"--leader-elect=true",
|
||||
"--cluster-cidr=10.1.0.0/16",
|
||||
"--v=2"
|
||||
],
|
||||
"volumeMounts": [
|
||||
@ -60,6 +62,7 @@
|
||||
"/hyperkube",
|
||||
"scheduler",
|
||||
"--master=127.0.0.1:8080",
|
||||
"--leader-elect=true",
|
||||
"--v=2"
|
||||
]
|
||||
},
|
||||
@ -73,7 +76,7 @@
|
||||
"volumeMounts": [
|
||||
{
|
||||
"name": "data",
|
||||
"mountPath": "/data"
|
||||
"mountPath": "/srv/kubernetes"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -18,6 +18,7 @@
|
||||
"--service-account-private-key-file=/srv/kubernetes/server.key",
|
||||
"--root-ca-file=/srv/kubernetes/ca.crt",
|
||||
"--min-resync-period=3m",
|
||||
"--leader-elect=true",
|
||||
"--v=2"
|
||||
],
|
||||
"volumeMounts": [
|
||||
@ -60,6 +61,7 @@
|
||||
"/hyperkube",
|
||||
"scheduler",
|
||||
"--master=127.0.0.1:8080",
|
||||
"--leader-elect=true",
|
||||
"--v=2"
|
||||
]
|
||||
},
|
||||
@ -73,7 +75,7 @@
|
||||
"volumeMounts": [
|
||||
{
|
||||
"name": "data",
|
||||
"mountPath": "/data"
|
||||
"mountPath": "/srv/kubernetes"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user