diff --git a/cluster/openstack-heat/config-default.sh b/cluster/openstack-heat/config-default.sh index c5c076e6056..03442d97307 100644 --- a/cluster/openstack-heat/config-default.sh +++ b/cluster/openstack-heat/config-default.sh @@ -38,6 +38,8 @@ EXTERNAL_NETWORK=${EXTERNAL_NETWORK:-public} LBAAS_VERSION=${LBAAS_VERSION:-} FIXED_NETWORK_CIDR=${FIXED_NETWORK_CIDR:-10.0.0.0/24} +SERVICE_CLUSTER_IP_RANGE=${SERVICE_CLUSTER_IP_RANGE:-10.0.0.0/16} +CLUSTER_IP_RANGE=${CLUSTER_IP_RANGE:-10.244.0.0/16} SWIFT_SERVER_URL=${SWIFT_SERVER_URL:-} diff --git a/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml b/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml index 59abb9b4a27..e86fb5d0142 100644 --- a/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml +++ b/cluster/openstack-heat/kubernetes-heat/fragments/configure-salt.yaml @@ -12,6 +12,7 @@ write_files: content: | grains: node_ip: $MASTER_IP + cbr-cidr: $MASTER_IP_RANGE publicAddressOverride: $MASTER_IP network_mode: openvswitch networkInterfaceName: eth0 @@ -21,7 +22,7 @@ write_files: roles: - $role runtime_config: "" - docker_opts: "" + docker_opts: "--bridge=cbr0 --iptables=false --ip-masq=false" master_extra_sans: "DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local,DNS:kubernetes-master" keep_host_etcd: true kube_user: $KUBE_USER @@ -37,8 +38,11 @@ write_files: lb-version=$LBAAS_VERSION subnet-id=$SUBNET_ID floating-network-id=$FLOATING_NETWORK_ID + [Route] + router-id=$router_id - path: /srv/salt-overlay/pillar/cluster-params.sls content: | + allocate_node_cidrs: "true" service_cluster_ip_range: 10.246.0.0/16 cert_ip: 10.246.0.1 enable_cluster_monitoring: influxdb @@ -56,6 +60,7 @@ write_files: admission_control: NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,DefaultStorageClass,ResourceQuota enable_cpu_cfs_quota: "true" network_provider: none + cluster_cidr: "$cluster_cidr" opencontrail_tag: R2.20 opencontrail_kubernetes_tag: master opencontrail_public_subnet: 10.1.0.0/16 diff --git a/cluster/openstack-heat/kubernetes-heat/fragments/provision-network-master.sh b/cluster/openstack-heat/kubernetes-heat/fragments/provision-network-master.sh index 895575a8ccd..ad75cc2a774 100644 --- a/cluster/openstack-heat/kubernetes-heat/fragments/provision-network-master.sh +++ b/cluster/openstack-heat/kubernetes-heat/fragments/provision-network-master.sh @@ -20,47 +20,4 @@ set -o pipefail . /etc/sysconfig/heat-params -FLANNEL_ETCD_URL="http://${MASTER_IP}:4379" - -# Install etcd for flannel data -if ! which etcd > /dev/null 2>&1; then - yum install -y etcd -fi - -cat < /etc/etcd/etcd.conf -ETCD_NAME=flannel -ETCD_DATA_DIR="/var/lib/etcd/flannel.etcd" -ETCD_LISTEN_PEER_URLS="http://${MASTER_IP}:4380" -ETCD_LISTEN_CLIENT_URLS="http://${MASTER_IP}:4379" -ETCD_INITIAL_ADVERTISE_PEER_URLS="http://${MASTER_IP}:4380" -ETCD_INITIAL_CLUSTER="flannel=http://${MASTER_IP}:4380" -ETCD_ADVERTISE_CLIENT_URLS="${FLANNEL_ETCD_URL}" -EOF -systemctl enable etcd -systemctl restart etcd - -# Install flannel for overlay -if ! which flanneld > /dev/null 2>&1; then - yum install -y flannel -fi - -cat < /etc/flannel-config.json -{ - "Network": "${CONTAINER_SUBNET}", - "SubnetLen": 24, - "Backend": { - "Type": "host-gw" - } -} -EOF - -etcdctl -C ${FLANNEL_ETCD_URL} set /coreos.com/network/config < /etc/flannel-config.json - -cat < /etc/sysconfig/flanneld -FLANNEL_ETCD="${FLANNEL_ETCD_URL}" -FLANNEL_ETCD_KEY="/coreos.com/network" -FLANNEL_OPTIONS="-iface=eth0 --ip-masq" -EOF - -systemctl enable flanneld -systemctl restart flanneld +# nothing to do diff --git a/cluster/openstack-heat/kubernetes-heat/fragments/provision-network-node.sh b/cluster/openstack-heat/kubernetes-heat/fragments/provision-network-node.sh index eff3fb7b698..1b30d29b092 100644 --- a/cluster/openstack-heat/kubernetes-heat/fragments/provision-network-node.sh +++ b/cluster/openstack-heat/kubernetes-heat/fragments/provision-network-node.sh @@ -18,24 +18,6 @@ set -o errexit set -o nounset set -o pipefail -. /etc/sysconfig/heat-params - -FLANNEL_ETCD_URL="http://${MASTER_IP}:4379" - -# Install flannel for overlay -if ! which flanneld >/dev/null 2>&1; then - yum install -y flannel -fi - -cat </etc/sysconfig/flanneld -FLANNEL_ETCD="${FLANNEL_ETCD_URL}" -FLANNEL_ETCD_KEY="/coreos.com/network" -FLANNEL_OPTIONS="-iface=eth0 --ip-masq" -EOF - -systemctl enable flanneld -systemctl restart flanneld - # Kubernetes node shoud be able to resolve its hostname. # In some cloud providers, myhostname is not enabled by default. grep '^hosts:.*myhostname' /etc/nsswitch.conf || ( diff --git a/cluster/openstack-heat/kubernetes-heat/kubecluster.yaml b/cluster/openstack-heat/kubernetes-heat/kubecluster.yaml index bae3cf83b96..9aa07c54477 100644 --- a/cluster/openstack-heat/kubernetes-heat/kubecluster.yaml +++ b/cluster/openstack-heat/kubernetes-heat/kubecluster.yaml @@ -52,6 +52,23 @@ parameters: description: network range for fixed ip network default: 10.0.0.0/24 + cluster_cidr: + type: string + description: network range for pod IPs + default: 10.244.0.0/16 + + service_cluster_cidr: + type: string + description: network range for service IPs + default: 10.10.0.0/16 + + master_pod_cidr: + type: string + description: >- + network range for master pod IPs (ignored, but must not conflict + with other subnets) + default: 10.245.1.0/24 + kubernetes_server_url: type: string description: URL of kubernetes server binary. Must be tar.gz. @@ -301,6 +318,9 @@ resources: "$SUBNET_ID": {get_resource: fixed_subnet} "$FLOATING_NETWORK_ID": {get_attr: [kube_master_floating, floating_network_id]} "$role": "kubernetes-master" + "$router_id": {get_resource: extrouter} + "$cluster_cidr": {get_param: cluster_cidr} + "$MASTER_IP_RANGE": {get_param: master_pod_cidr} run_salt: type: OS::Heat::SoftwareConfig @@ -390,6 +410,7 @@ resources: token_kube_proxy: {get_param: token_kube_proxy} fixed_network: {get_resource: fixed_network} fixed_subnet: {get_resource: fixed_subnet} + cluster_cidr: {get_param: cluster_cidr} kube_master_ip: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} external_network: {get_param: external_network} wait_condition_timeout: {get_param: wait_condition_timeout} diff --git a/cluster/openstack-heat/kubernetes-heat/kubeminion.yaml b/cluster/openstack-heat/kubernetes-heat/kubeminion.yaml index a51aab2a22b..5c3d2125bc0 100644 --- a/cluster/openstack-heat/kubernetes-heat/kubeminion.yaml +++ b/cluster/openstack-heat/kubernetes-heat/kubeminion.yaml @@ -106,6 +106,9 @@ parameters: fixed_subnet: type: string description: Subnet from which to allocate fixed addresses. + cluster_cidr: + type: string + description: Subnet from which to allocate pod subnets. wait_condition_timeout: type: number description : > @@ -215,6 +218,7 @@ resources: "$OS_REGION_NAME": {get_param: os_region_name} "$OS_TENANT_ID": {get_param: os_tenant_id} "$role": "kubernetes-pool" + "$cluster_cidr": {get_param: cluster_cidr} run_salt: type: OS::Heat::SoftwareConfig diff --git a/cluster/saltbase/salt/docker/init.sls b/cluster/saltbase/salt/docker/init.sls index e14f0e35930..f11420e2bde 100644 --- a/cluster/saltbase/salt/docker/init.sls +++ b/cluster/saltbase/salt/docker/init.sls @@ -18,6 +18,27 @@ bridge-utils: - mode: 644 - makedirs: true +{% if grains.cloud is defined and grains.cloud == 'openstack' %} + +cbr0: + # workaround https://github.com/saltstack/salt/issues/20570 + kmod.present: + - name: bridge + + network.managed: + - enabled: True + - type: bridge + - proto: none + - ports: none + - bridge: cbr0 + - delay: 0 + - bypassfirewall: True + - require_in: + - service: docker + - require: + - kmod: cbr0 +{% endif %} + {% if (grains.os == 'Fedora' and grains.osrelease_info[0] >= 22) or (grains.os == 'CentOS' and grains.osrelease_info[0] >= 7) %} docker: @@ -512,4 +533,3 @@ docker: - cmd: fix-service-docker {% endif %} {% endif %} # end grains.os_family != 'RedHat' -