diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 1b455f65c6e..0bf402b3ead 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -25,4 +25,5 @@ If the matter is security related, please disclose it privately via https://kube - OS (e.g: `cat /etc/os-release`): - Kernel (e.g. `uname -a`): - Install tools: +- Network plugin and version (if this is a network-related bug): - Others: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c99222977cc..245aa98617c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,7 +2,7 @@ 1. If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide 2. Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here: -https://git.k8s.io/community/contributors/devel/release.md#issue-kind-label +https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label 3. Ensure you have added or ran the appropriate tests for your PR: https://git.k8s.io/community/contributors/devel/testing.md 4. If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews 5. Follow the instructions for writing a release note: https://git.k8s.io/community/contributors/guide/release-notes.md diff --git a/CHANGELOG-1.12.md b/CHANGELOG-1.12.md index ce04fceab39..61128f607be 100644 --- a/CHANGELOG-1.12.md +++ b/CHANGELOG-1.12.md @@ -1,59 +1,66 @@ -- [v1.12.7](#v1127) - - [Downloads for v1.12.7](#downloads-for-v1127) +- [v1.12.8](#v1128) + - [Downloads for v1.12.8](#downloads-for-v1128) - [Client Binaries](#client-binaries) - [Server Binaries](#server-binaries) - [Node Binaries](#node-binaries) - - [Changelog since v1.12.6](#changelog-since-v1126) + - [Changelog since v1.12.7](#changelog-since-v1127) - [Other notable changes](#other-notable-changes) -- [v1.12.6](#v1126) - - [Downloads for v1.12.6](#downloads-for-v1126) +- [v1.12.7](#v1127) + - [Downloads for v1.12.7](#downloads-for-v1127) - [Client Binaries](#client-binaries-1) - [Server Binaries](#server-binaries-1) - [Node Binaries](#node-binaries-1) - - [Changelog since v1.12.5](#changelog-since-v1125) + - [Changelog since v1.12.6](#changelog-since-v1126) - [Other notable changes](#other-notable-changes-1) -- [v1.12.5](#v1125) - - [Downloads for v1.12.5](#downloads-for-v1125) +- [v1.12.6](#v1126) + - [Downloads for v1.12.6](#downloads-for-v1126) - [Client Binaries](#client-binaries-2) - [Server Binaries](#server-binaries-2) - [Node Binaries](#node-binaries-2) - - [Changelog since v1.12.4](#changelog-since-v1124) + - [Changelog since v1.12.5](#changelog-since-v1125) - [Other notable changes](#other-notable-changes-2) -- [v1.12.4](#v1124) - - [Downloads for v1.12.4](#downloads-for-v1124) +- [v1.12.5](#v1125) + - [Downloads for v1.12.5](#downloads-for-v1125) - [Client Binaries](#client-binaries-3) - [Server Binaries](#server-binaries-3) - [Node Binaries](#node-binaries-3) - - [Changelog since v1.12.3](#changelog-since-v1123) - - [Action Required](#action-required) + - [Changelog since v1.12.4](#changelog-since-v1124) - [Other notable changes](#other-notable-changes-3) -- [v1.12.3](#v1123) - - [Downloads for v1.12.3](#downloads-for-v1123) +- [v1.12.4](#v1124) + - [Downloads for v1.12.4](#downloads-for-v1124) - [Client Binaries](#client-binaries-4) - [Server Binaries](#server-binaries-4) - [Node Binaries](#node-binaries-4) - - [Changelog since v1.12.2](#changelog-since-v1122) + - [Changelog since v1.12.3](#changelog-since-v1123) + - [Action Required](#action-required) - [Other notable changes](#other-notable-changes-4) -- [v1.12.2](#v1122) - - [Downloads for v1.12.2](#downloads-for-v1122) +- [v1.12.3](#v1123) + - [Downloads for v1.12.3](#downloads-for-v1123) - [Client Binaries](#client-binaries-5) - [Server Binaries](#server-binaries-5) - [Node Binaries](#node-binaries-5) - - [Changelog since v1.12.1](#changelog-since-v1121) + - [Changelog since v1.12.2](#changelog-since-v1122) - [Other notable changes](#other-notable-changes-5) -- [v1.12.1](#v1121) - - [Downloads for v1.12.1](#downloads-for-v1121) +- [v1.12.2](#v1122) + - [Downloads for v1.12.2](#downloads-for-v1122) - [Client Binaries](#client-binaries-6) - [Server Binaries](#server-binaries-6) - [Node Binaries](#node-binaries-6) - - [Changelog since v1.12.0](#changelog-since-v1120) + - [Changelog since v1.12.1](#changelog-since-v1121) - [Other notable changes](#other-notable-changes-6) -- [v1.12.0](#v1120) - - [Downloads for v1.12.0](#downloads-for-v1120) +- [v1.12.1](#v1121) + - [Downloads for v1.12.1](#downloads-for-v1121) - [Client Binaries](#client-binaries-7) - [Server Binaries](#server-binaries-7) - [Node Binaries](#node-binaries-7) + - [Changelog since v1.12.0](#changelog-since-v1120) + - [Other notable changes](#other-notable-changes-7) +- [v1.12.0](#v1120) + - [Downloads for v1.12.0](#downloads-for-v1120) + - [Client Binaries](#client-binaries-8) + - [Server Binaries](#server-binaries-8) + - [Node Binaries](#node-binaries-8) - [Known Issues](#known-issues) - [Major Themes](#major-themes) - [SIG API Machinery](#sig-api-machinery) @@ -75,7 +82,7 @@ - [Deprecations and removals](#deprecations-and-removals) - [New Features](#new-features) - [API Changes](#api-changes) - - [Other Notable Changes](#other-notable-changes-7) + - [Other Notable Changes](#other-notable-changes-8) - [SIG API Machinery](#sig-api-machinery-1) - [SIG Apps](#sig-apps) - [SIG Auth](#sig-auth) @@ -94,54 +101,127 @@ - [SIG Storage](#sig-storage-1) - [SIG VMWare](#sig-vmware-1) - [SIG Windows](#sig-windows-1) - - [Other Notable Changes](#other-notable-changes-8) + - [Other Notable Changes](#other-notable-changes-9) - [Bug Fixes](#bug-fixes) - [Not Very Notable (that is, non-user-facing)](#not-very-notable-that-is-non-user-facing) - [External Dependencies](#external-dependencies) - [v1.12.0-rc.2](#v1120-rc2) - [Downloads for v1.12.0-rc.2](#downloads-for-v1120-rc2) - - [Client Binaries](#client-binaries-8) - - [Server Binaries](#server-binaries-8) - - [Node Binaries](#node-binaries-8) - - [Changelog since v1.12.0-rc.1](#changelog-since-v1120-rc1) - - [Other notable changes](#other-notable-changes-9) -- [v1.12.0-rc.1](#v1120-rc1) - - [Downloads for v1.12.0-rc.1](#downloads-for-v1120-rc1) - [Client Binaries](#client-binaries-9) - [Server Binaries](#server-binaries-9) - [Node Binaries](#node-binaries-9) - - [Changelog since v1.12.0-beta.2](#changelog-since-v1120-beta2) - - [Action Required](#action-required-2) + - [Changelog since v1.12.0-rc.1](#changelog-since-v1120-rc1) - [Other notable changes](#other-notable-changes-10) -- [v1.12.0-beta.2](#v1120-beta2) - - [Downloads for v1.12.0-beta.2](#downloads-for-v1120-beta2) +- [v1.12.0-rc.1](#v1120-rc1) + - [Downloads for v1.12.0-rc.1](#downloads-for-v1120-rc1) - [Client Binaries](#client-binaries-10) - [Server Binaries](#server-binaries-10) - [Node Binaries](#node-binaries-10) - - [Changelog since v1.12.0-beta.1](#changelog-since-v1120-beta1) - - [Action Required](#action-required-3) + - [Changelog since v1.12.0-beta.2](#changelog-since-v1120-beta2) + - [Action Required](#action-required-2) - [Other notable changes](#other-notable-changes-11) -- [v1.12.0-beta.1](#v1120-beta1) - - [Downloads for v1.12.0-beta.1](#downloads-for-v1120-beta1) +- [v1.12.0-beta.2](#v1120-beta2) + - [Downloads for v1.12.0-beta.2](#downloads-for-v1120-beta2) - [Client Binaries](#client-binaries-11) - [Server Binaries](#server-binaries-11) - [Node Binaries](#node-binaries-11) - - [Changelog since v1.12.0-alpha.1](#changelog-since-v1120-alpha1) - - [Action Required](#action-required-4) + - [Changelog since v1.12.0-beta.1](#changelog-since-v1120-beta1) + - [Action Required](#action-required-3) - [Other notable changes](#other-notable-changes-12) -- [v1.12.0-alpha.1](#v1120-alpha1) - - [Downloads for v1.12.0-alpha.1](#downloads-for-v1120-alpha1) +- [v1.12.0-beta.1](#v1120-beta1) + - [Downloads for v1.12.0-beta.1](#downloads-for-v1120-beta1) - [Client Binaries](#client-binaries-12) - [Server Binaries](#server-binaries-12) - [Node Binaries](#node-binaries-12) + - [Changelog since v1.12.0-alpha.1](#changelog-since-v1120-alpha1) + - [Action Required](#action-required-4) + - [Other notable changes](#other-notable-changes-13) +- [v1.12.0-alpha.1](#v1120-alpha1) + - [Downloads for v1.12.0-alpha.1](#downloads-for-v1120-alpha1) + - [Client Binaries](#client-binaries-13) + - [Server Binaries](#server-binaries-13) + - [Node Binaries](#node-binaries-13) - [Changelog since v1.11.0](#changelog-since-v1110) - [Action Required](#action-required-5) - - [Other notable changes](#other-notable-changes-13) + - [Other notable changes](#other-notable-changes-14) +# v1.12.8 + +[Documentation](https://docs.k8s.io) + +## Downloads for v1.12.8 + + +filename | sha512 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes.tar.gz) | `0f14f54bcd3ef8260e424ccb9be4d1d7ad8d03e15d00d081fdf564adc319ca4040d404f37466a2342650d08d5c41b1d411f172ff78e611b05fca8fd5404590d9` +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-src.tar.gz) | `10b6ce78a906effbb38600d8e496c49e9739fffaba8d44eff54d298b0f899481b9e4cc60eb918586f3d1055f4db44880fd2b42ad40a391aadfd8a53c584c8c1c` + +### Client Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-client-darwin-386.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-client-darwin-386.tar.gz) | `bfb3680c47f674773c50c446577eb3f10468a6fd367a2ee7f851d299f4ff04071757962ddff10659b185ab80e4fc474f10354273560803101b66c9c939279e08` +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-client-darwin-amd64.tar.gz) | `b6543d97975add3a27f75ff6fcc7c3caeb8749ac88967cb79a6688ba4ba1837fda3582a0f5588073a855a2da43c9b353b565974b7a29f619709f862d8ce1e0b3` +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-client-linux-386.tar.gz) | `57358d71b4c19d826e4979b1ef3f33b5b1e05c50ba257d6bbfa8d76f15849ebcba389c55f1be50fdc77a311935a0e7ecc827a3f35ee5896a6ceda7580d8b4680` +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-client-linux-amd64.tar.gz) | `55e6c2ec67aa3283e3b6904418b35845fa14f5faaed0cf503a7adb4e52842f7c3aaa5fbbfdbcf508794c784d93bec48e27e598879e89302c48f54eebdef69d3d` +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-client-linux-arm.tar.gz) | `0a7b54f8846ddf9d6ef6df863a0211ab448dfbdeeaf78ec163b4e46fa4d7f92611f71ac757bb00d6dfee6314c78ac12cf50020d8d6c9b1dbac550425ccb53743` +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-client-linux-arm64.tar.gz) | `ad68df3f56c2622a01f54a8575c7cec3b9f508c1332bd16cf3f39b9e3f66dae3b495fc1dce3d69504f18b0feb281268fed306538db538d01e74210be45bafb97` +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-client-linux-ppc64le.tar.gz) | `1452011ed3f37984ff9493df0d490eefb8a5c0d84c2f87d9ff47ffe9924a14d918c5dfa755494c05975a10b191d75173d0d30be3449e36cffff4b0495f22efa8` +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-client-linux-s390x.tar.gz) | `edca658d8f91dd4939c6eba444b2b56a30304d3d0c42607e823acf64dace852cc66a8f14d4bf2fc2bdf0c99bbbe4a9625c86f38535d5b24e8c3b95e76193e530` +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-client-windows-386.tar.gz) | `14e5daaf4623d11380b552fc3fa5ad6bf98488dcf365c8cfa8d7f1d26fe73b317e5cfeb3e46f4e9d582e2a04cd70bc2ed3dfb915c88aacd997324ca8c2582d52` +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-client-windows-amd64.tar.gz) | `1a1d4457620daf2f54e11b2ef790f30890bca71502f86a3c0163a4e6a5afb701c3d60511b944eb4b80c9418e7ee6864d44aba26deacd44a717a5c8c4850794af` + +### Server Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-server-linux-amd64.tar.gz) | `8d1a70cfa9012282f679d876ae070f7830aad11ef64f437b90320ccae5253a3f527df0abb56f34004ccb2113e195638b6ca69aad9aff85f9dcb588aacca81d55` +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-server-linux-arm.tar.gz) | `67022706b4bf98aba305fd3759940ce396e35474814ced4152152b4cc536d79e1b4e3a4027e45af3637ea006fbabadca34d8ecc6874138300230b2b0bcd2dcb5` +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-server-linux-arm64.tar.gz) | `00d7e79fa71f4265b8ba5cc2e62c2ab4b5d1076bddc8155a3b7a5e589c34446860c25571b972566e694709230d32de763ac3ba0a97a2cc2cbc6c7b431b30a1e0` +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-server-linux-ppc64le.tar.gz) | `8add81c5f767dbdd04ac39f07aa4855be86c91f848c2e331d40734e85d0d6c7ea5cd0c575ab49b101c1e6ba5224eb1762f8f73f39610c773b76f7d1ebffc86cc` +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-server-linux-s390x.tar.gz) | `63e58ea49072ac058e74b989f9a74887b27c52d56923f44a7d53cb384915f4a2425e65d6e9f6642d4fbac102bb9a45baad901a32a5414989c0b2d2cc57ffa59f` + +### Node Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-node-linux-amd64.tar.gz) | `6677af5330149f39c6d84722e5418bf35caf4d431fb97fd0df102373a5faaf4a8344921bc2a51290abe521620f6543c482a54720692d245ff36142019fcc0c19` +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-node-linux-arm.tar.gz) | `aa0f2abaec8ac765acffe1c6ff00c01cd74befba98a5c7afb30f716bd37f9094e1c314df7f3b7c8361c86e6c78f9aa246623e5f1934d60148663851810aa4815` +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-node-linux-arm64.tar.gz) | `c4bb230afcf78414461b32cedd0564a58e02e82b0a679ea42f977e3bc501cc4857694774dad423b4a76542a4698929c8f5429b9737f1e324d21e39afbc5be48f` +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-node-linux-ppc64le.tar.gz) | `31d4bdc1528ee8d4ab4ee16e3da08c1e8c026eaafd88950836f19e10bf3e87d12876a25e2c90a81529a9d26f8465cf50e8997e8c14fb555f21d69323a885f2eb` +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-node-linux-s390x.tar.gz) | `d7b8a81b14a12578ca6273dc32703f906627e244ed00639436fb3cb38d4b4aa55d7a857f9a844844bc2d463619b890329043b78c9ec8ff0f5b38dc55b572cd71` +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.12.8/kubernetes-node-windows-amd64.tar.gz) | `9ca9ef41a42d8cb5c15533a10848170fa0f7c1e4eccbc8d6269ce085ab7670e446473e4e240b3bb1905dda3078725976b0506a27d67e6e3a2fd546aaa6678d84` + +## Changelog since v1.12.7 + +### Other notable changes + +* Connections from Pods to Services with 0 endpoints will now ICMP reject immediately, rather than blackhole and timeout. ([#72534](https://github.com/kubernetes/kubernetes/pull/72534), [@thockin](https://github.com/thockin)) +* Services of type=LoadBalancer which have no endpoints will now immediately ICMP reject connections, rather than time out. ([#74394](https://github.com/kubernetes/kubernetes/pull/74394), [@thockin](https://github.com/thockin)) +* Ensure the backend pools are set correctly for Azure SLB with multiple backend pools (e.g. outbound rules) ([#76691](https://github.com/kubernetes/kubernetes/pull/76691), [@feiskyer](https://github.com/feiskyer)) +* Connections from Pods to Services with 0 endpoints will now ICMP reject immediately, rather than blackhole and timeout. ([#72534](https://github.com/kubernetes/kubernetes/pull/72534), [@thockin](https://github.com/thockin)) +* Services of type=LoadBalancer which have no endpoints will now immediately ICMP reject connections, rather than time out. ([#74394](https://github.com/kubernetes/kubernetes/pull/74394), [@thockin](https://github.com/thockin)) +* fix race condition issue for smb mount on windows ([#75371](https://github.com/kubernetes/kubernetes/pull/75371), [@andyzhangx](https://github.com/andyzhangx)) +* fix smb unmount issue on Windows ([#75087](https://github.com/kubernetes/kubernetes/pull/75087), [@andyzhangx](https://github.com/andyzhangx)) +* Increase Azure default maximumLoadBalancerRuleCount to 250. ([#72621](https://github.com/kubernetes/kubernetes/pull/72621), [@feiskyer](https://github.com/feiskyer)) +* Fixes bug in DaemonSetController causing it to stop processing some DaemonSets for 5 minutes after node removal. ([#76060](https://github.com/kubernetes/kubernetes/pull/76060), [@krzysztof-jastrzebski](https://github.com/krzysztof-jastrzebski)) +* Fixes a NPD bug on GCI, so that it disables glog writing to files for log-counter ([#76211](https://github.com/kubernetes/kubernetes/pull/76211), [@wangzhen127](https://github.com/wangzhen127)) +* [stackdriver addon] Bump prometheus-to-sd to v0.5.0 to pick up security fixes. ([#75362](https://github.com/kubernetes/kubernetes/pull/75362), [@serathius](https://github.com/serathius)) + * [fluentd-gcp addon] Bump fluentd-gcp-scaler to v0.5.1 to pick up security fixes. + * [fluentd-gcp addon] Bump event-exporter to v0.2.4 to pick up security fixes. + * [fluentd-gcp addon] Bump prometheus-to-sd to v0.5.0 to pick up security fixes. + * [metatada-proxy addon] Bump prometheus-to-sd v0.5.0 to pick up security fixes. +* Fixed parsing of fsType in AWS StorageClass parameters ([#75944](https://github.com/kubernetes/kubernetes/pull/75944), [@jsafrane](https://github.com/jsafrane)) +* Node-Problem-Detector configuration is now decoupled from the Kubernetes release on GKE/GCE. ([#73288](https://github.com/kubernetes/kubernetes/pull/73288), [@wangzhen127](https://github.com/wangzhen127)) +* [IPVS] Allow for transparent kube-proxy restarts ([#75283](https://github.com/kubernetes/kubernetes/pull/75283), [@lbernail](https://github.com/lbernail)) + + + # v1.12.7 [Documentation](https://docs.k8s.io) diff --git a/Godeps/LICENSES b/Godeps/LICENSES index ba5c5553d6c..2024eb466af 100644 --- a/Godeps/LICENSES +++ b/Godeps/LICENSES @@ -427,7 +427,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2014 Google Inc. + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -441,7 +441,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. See the License for the specific language governing permissions and limitations under the License. -= vendor/cloud.google.com/go/LICENSE a873c5645c184d51e0f9b34e1d7cf559 += vendor/cloud.google.com/go/LICENSE 3b83ef96387f14655fc854ddc3c6bd57 ================================================================================ @@ -11507,6 +11507,35 @@ under a dual license of LGPLv3+ or GPLv2. ================================================================================ +================================================================================ += vendor/github.com/hpcloud/tail licensed under: = + +# The MIT License (MIT) + +# © Copyright 2015 Hewlett Packard Enterprise Development LP +Copyright (c) 2014 ActiveState + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + += vendor/github.com/hpcloud/tail/LICENSE.txt 0bdce43b16cd5c587124d6f274632c87 +================================================================================ + + ================================================================================ = vendor/github.com/imdario/mergo licensed under: = @@ -16617,191 +16646,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ -================================================================================ -= vendor/github.com/rancher/go-rancher licensed under: = - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - -= vendor/github.com/rancher/go-rancher/LICENSE 2ee41112a44fe7014dce33e26468ba93 -================================================================================ - - ================================================================================ = vendor/github.com/robfig/cron licensed under: = @@ -19051,7 +18895,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ = vendor/golang.org/x/oauth2 licensed under: = -Copyright (c) 2009 The oauth2 Authors. All rights reserved. +Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -19079,7 +18923,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -= vendor/golang.org/x/oauth2/LICENSE 704b1e0c436dbf193e7dcbd4cf06ec81 += vendor/golang.org/x/oauth2/LICENSE 5d4950ecb7b26d2c5e4e7b4e0dd74707 ================================================================================ @@ -19918,6 +19762,42 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================================================ +================================================================================ += vendor/gopkg.in/fsnotify.v1 licensed under: = + +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2012 fsnotify Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/gopkg.in/fsnotify.v1/LICENSE c38914c9a7ab03bb2b96d4baaee10769 +================================================================================ + + ================================================================================ = vendor/gopkg.in/gcfg.v1 licensed under: = @@ -20228,6 +20108,43 @@ SOFTWARE. ================================================================================ +================================================================================ += vendor/gopkg.in/tomb.v1 licensed under: = + +tomb - support for clean goroutine termination in Go. + +Copyright (c) 2010-2011 - Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/gopkg.in/tomb.v1/LICENSE 95d4102f39f26da9b66fee5d05ac597b +================================================================================ + + ================================================================================ = vendor/gopkg.in/warnings.v0 licensed under: = diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 9705b3f4b00..e623e74af02 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -345,9 +345,11 @@ aliases: # api-reviewers targeted by sig area # see https://git.k8s.io/community/sig-architecture/api-review-process.md#training-reviews - # sig-api-machinery-api-reviewers: - # - - # - + sig-api-machinery-api-reviewers: + - caesarxuchao + - deads2k + - jpbetz + - sttts # sig-apps-api-reviewers: # - @@ -357,10 +359,11 @@ aliases: # - # - - # sig-cli-api-reviewers: - # - - # - + sig-cli-api-reviewers: + - pwittrock + - soltysh + sig-cloud-provider-api-reviewers: - andrewsykim - cheftako @@ -381,13 +384,15 @@ aliases: - bsalamat - k82cn - # sig-storage-api-reviewers: - # - - # - + sig-storage-api-reviewers: + - saad-ali + - msau42 + - jsafrane + - # sig-windows-api-reviewers: - # - - # - + sig-windows-api-reviewers: + - patricklang + - michmike dep-approvers: - apelisse diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 65106418a9b..516bb748971 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -9337,7 +9337,7 @@ "type": "string" }, "runtimeClassName": { - "description": "RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. If unset or empty, the \"legacy\" RuntimeClass will be used, which is an implicit class with an empty definition that uses the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md This is an alpha feature and may change in the future.", + "description": "RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. If unset or empty, the \"legacy\" RuntimeClass will be used, which is an implicit class with an empty definition that uses the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md This is a beta feature as of Kubernetes v1.14.", "type": "string" }, "schedulerName": { @@ -11122,7 +11122,7 @@ "type": "string" }, "subPathExpr": { - "description": "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive. This field is alpha in 1.14.", + "description": "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive. This field is beta in 1.15.", "type": "string" } }, diff --git a/build/build-image/cross/Dockerfile b/build/build-image/cross/Dockerfile index b0cd8ed3fe9..f0d0cd1f482 100644 --- a/build/build-image/cross/Dockerfile +++ b/build/build-image/cross/Dockerfile @@ -15,7 +15,7 @@ # This file creates a standard build environment for building cross # platform go binary for the architecture kubernetes cares about. -FROM golang:1.12.1 +FROM golang:1.12.4 ENV GOARM 7 ENV KUBE_DYNAMIC_CROSSPLATFORMS \ diff --git a/build/build-image/cross/VERSION b/build/build-image/cross/VERSION index c167785a48d..3ff2cc2b66f 100644 --- a/build/build-image/cross/VERSION +++ b/build/build-image/cross/VERSION @@ -1 +1 @@ -v1.12.1-2 +v1.12.4-1 diff --git a/build/common.sh b/build/common.sh index 8aa2a62746d..5bfb754671f 100755 --- a/build/common.sh +++ b/build/common.sh @@ -89,8 +89,8 @@ readonly KUBE_CONTAINER_RSYNC_PORT=8730 # $1 - server architecture kube::build::get_docker_wrapped_binaries() { local arch=$1 - local debian_base_version=0.4.1 - local debian_iptables_version=v11.0.1 + local debian_base_version=v1.0.0 + local debian_iptables_version=v11.0.2 ### If you change any of these lists, please also update DOCKERIZED_BINARIES ### in build/BUILD. And kube::golang::server_image_targets local targets=( diff --git a/build/root/.bazelrc b/build/root/.bazelrc index f08b09c8e08..f596e7083fb 100644 --- a/build/root/.bazelrc +++ b/build/root/.bazelrc @@ -39,3 +39,32 @@ build:cross:linux_arm --config=repo_infra_crosstool --platforms=@io_bazel_rules_ build:cross:linux_arm64 --config=repo_infra_crosstool --platforms=@io_bazel_rules_go//go/toolchain:linux_arm64 --cpu=arm64 build:cross:linux_ppc64le --config=repo_infra_crosstool --platforms=@io_bazel_rules_go//go/toolchain:linux_ppc64le --cpu=ppc64le build:cross:linux_s390x --config=repo_infra_crosstool --platforms=@io_bazel_rules_go//go/toolchain:linux_s390x --cpu=s390x + +# --config=remote-cache enables a remote bazel cache +# Note needs a --remote_instance_name=projects/PROJ/instances/default_instance flag +build:remote-cache --remote_cache=remotebuildexecution.googleapis.com +build:remote-cache --tls_enabled=true +build:remote-cache --remote_timeout=3600 +build:remote-cache --auth_enabled=true + +# --config=remote adds remote execution to the --config=remote-cache +# Note needs a --remote_instance_name=projects/PROJ/instances/default_instance flag +build:remote --config=remote-cache +build:remote --remote_executor=remotebuildexecution.googleapis.com +build:remote --jobs=500 +build:remote --host_javabase=@rbe_default//java:jdk +build:remote --javabase=@rbe_default//java:jdk +build:remote --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 +build:remote --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 +build:remote --crosstool_top=@rbe_default//cc:toolchain +build:remote --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 +build:remote --extra_toolchains=@rbe_default//config:cc-toolchain +build:remote --extra_execution_platforms=:rbe_with_network +build:remote --host_platform=:rbe_with_network +build:remote --platforms=:rbe_with_network +build:remote --spawn_strategy=remote +build:remote --strategy=Javac=remote +build:remote --strategy=Closure=remote +build:remote --strategy=Genrule=remote +build:remote --define=EXECUTOR=remote + diff --git a/build/root/BUILD.root b/build/root/BUILD.root index 3469dfab6d0..41b76e68ba5 100644 --- a/build/root/BUILD.root +++ b/build/root/BUILD.root @@ -106,3 +106,20 @@ genrule( cmd = "grep ^STABLE_BUILD_SCM_REVISION bazel-out/stable-status.txt | awk '{print $$2}' >$@", stamp = 1, ) + +platform( + name = "rbe_with_network", + parents = ["@rbe_default//config:platform"], + # https://cloud.google.com/remote-build-execution/docs/remote-execution-environment#remote_execution_properties + remote_execution_properties = """ + properties: { + name: "dockerNetwork" + value: "standard" + } + properties: { + name: "dockerPrivileged" + value: "true" + } + {PARENT_REMOTE_EXECUTION_PROPERTIES} + """, +) diff --git a/build/root/WORKSPACE b/build/root/WORKSPACE index b0a1501f6b7..3b88a4a65c7 100644 --- a/build/root/WORKSPACE +++ b/build/root/WORKSPACE @@ -1,6 +1,20 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") load("//build:workspace_mirror.bzl", "mirror") +http_archive( + name = "bazel_toolchains", + sha256 = "8d43844d1d4447be2a108834771d617a1ad2a107f1680190bfe44925e7bf530e", + strip_prefix = "bazel-toolchains-4c003ad45e8a2d829ffc40e3aecfb6b8577a9406", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/4c003ad45e8a2d829ffc40e3aecfb6b8577a9406.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/archive/4c003ad45e8a2d829ffc40e3aecfb6b8577a9406.tar.gz", + ], +) + +load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig") + +rbe_autoconfig(name = "rbe_default") + http_archive( name = "bazel_skylib", sha256 = "eb5c57e4c12e68c0c20bc774bfbc60a568e800d025557bc4ea022c6479acc867", @@ -21,8 +35,8 @@ http_archive( http_archive( name = "io_bazel_rules_go", - sha256 = "6433336b4c5feb54e2f45df4c1c84ea4385b2dc0b6f274ec2cd5d745045eae1f", - urls = mirror("https://github.com/bazelbuild/rules_go/releases/download/0.17.2/rules_go-0.17.2.tar.gz"), + sha256 = "91b79f4758fd16f2c6426279ce00c1d2d8577d61c519db39675ed84657e1a95e", + urls = mirror("https://github.com/bazelbuild/rules_go/releases/download/0.17.4/rules_go-0.17.4.tar.gz"), ) load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") @@ -30,7 +44,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe go_rules_dependencies() go_register_toolchains( - go_version = "1.12.1", + go_version = "1.12.4", ) http_archive( @@ -51,7 +65,7 @@ load("@io_bazel_rules_docker//container:container.bzl", "container_pull") container_pull( name = "official_busybox", - digest = "sha256:cb63aa0641a885f54de20f61d152187419e8f6b159ed11a251a09d115fdff9bd", + digest = "sha256:5e8e0509e829bb8f990249135a36e81a3ecbe94294e7a185cc14616e5fad96bd", registry = "index.docker.io", repository = "library/busybox", tag = "latest", # ignored, but kept here for documentation diff --git a/build/workspace.bzl b/build/workspace.bzl index b4cd5ab205f..05e18f01bac 100644 --- a/build/workspace.bzl +++ b/build/workspace.bzl @@ -42,13 +42,6 @@ _ETCD_TARBALL_ARCH_SHA256 = { "ppc64le": "148fe96f0ec1813c5db9916199e96a913174304546bc8447a2d2f9fee4b8f6c2", } -# Note that these are digests for the manifest list. We resolve the manifest -# list to each of its platform-specific images in -# debian_image_dependencies(). -_DEBIAN_BASE_DIGEST = "sha256:6966a0aedd7592c18ff2dd803c08bd85780ee19f5e3a2e7cf908a4cd837afcde" # 0.4.1 -_DEBIAN_IPTABLES_DIGEST = "sha256:656e45c00083359107b1d6ae0411ff3894ba23011a8533e229937a71be84e063" # v11.0.1 -_DEBIAN_HYPERKUBE_BASE_DIGEST = "sha256:8cabe02be6e86685d8860b7ace7c7addc9591a339728703027a4854677f1c772" # 0.12.1 - # Dependencies needed for a Kubernetes "release", e.g. building docker images, # debs, RPMs, or tarballs. def release_dependencies(): @@ -75,30 +68,67 @@ def cri_tarballs(): urls = mirror("https://github.com/kubernetes-incubator/cri-tools/releases/download/v%s/crictl-v%s-linux-%s.tar.gz" % (CRI_TOOLS_VERSION, CRI_TOOLS_VERSION, arch)), ) +# Use go get -u github.com/estesp/manifest-tool to find these values +_DEBIAN_BASE_DIGEST = { + "manifest": "sha256:6966a0aedd7592c18ff2dd803c08bd85780ee19f5e3a2e7cf908a4cd837afcde", + "amd64": "sha256:8ccb65cd2dd7e0c24193d0742a20e4a673dbd11af5a33f16fcd471a31486866c", + "arm": "sha256:3432b41de3f6dfffdc1386fce961cfd1f9f8e208b3a35070e10ef3e2a733cb17", + "arm64": "sha256:9189251e1d1eb4126d6e6add2e272338f9c8a6a3db38863044625bca4b667f31", + "ppc64le": "sha256:50aa659e1e75e4231ee8293c3b4115e5755bb0517142b9b4bddbc134bf4354db", + "s390x": "sha256:bbb8ee3a2aaca738c00809f450233d98029fea4e319d8faaa30aa94c8b17a806", +} + +_DEBIAN_IPTABLES_DIGEST = { + "manifest": "sha256:b522b0035dba3ac2d5c0dbaaf8217bd66248e790332ccfdf653e0f943a280dcf", + "amd64": "sha256:adc40e9ec817c15d35b26d1d6aa4d0f8096fba4c99e26a026159bb0bc98c6a89", + "arm": "sha256:58e8a1d3b187eed2d8d3664cd1c9723e5029698714a24dfca4b6ef42ea27a9d4", + "arm64": "sha256:1a63fdd216fe7b84561d40ab1ebaa0daae1fc73e4232a6caffbd8353d9a14cea", + "ppc64le": "sha256:9f90adbc7513cc96d92fcec7633c4b29e766dd31cf876af03c0b54374e22fa9c", + "s390x": "sha256:4f147708deff2a0163ee49b6980cc95423514bec5f4091612d65773b898fbdae", +} + +_DEBIAN_HYPERKUBE_BASE_DIGEST = { + "manifest": "sha256:8cabe02be6e86685d8860b7ace7c7addc9591a339728703027a4854677f1c772", + "amd64": "sha256:5d4ea2fb5fbe9a9a9da74f67cf2faefc881968bc39f2ac5d62d9167e575812a1", + "arm": "sha256:73260814af61522ff6aa48291df457d3bb0a91c4bf72e7cfa51fbaf03eb65fae", + "arm64": "sha256:78eeb1a31eef7c16f954444d64636d939d89307e752964ad6d9d06966c722da3", + "ppc64le": "sha256:92857d647abe8d9c7b4d7160cd5699112afc12fde369082a8ed00688b17928a9", + "s390x": "sha256:c11d74fa0538c67238576c247bfaddf95ebaa90cd03cb4d2f2ac3c6ebe0441e2", +} + +def _digest(d, arch): + if arch not in d: + print("WARNING: %s not found in %r" % (arch, d)) + return d["manifest"] + return d[arch] + def debian_image_dependencies(): for arch in SERVER_PLATFORMS["linux"]: container_pull( name = "debian-base-" + arch, architecture = arch, - digest = _DEBIAN_BASE_DIGEST, + digest = _digest(_DEBIAN_BASE_DIGEST, arch), registry = "k8s.gcr.io", repository = "debian-base", + tag = "0.4.1", # ignored, but kept here for documentation ) container_pull( name = "debian-iptables-" + arch, architecture = arch, - digest = _DEBIAN_IPTABLES_DIGEST, + digest = _digest(_DEBIAN_IPTABLES_DIGEST, arch), registry = "k8s.gcr.io", repository = "debian-iptables", + tag = "v11.0.2", # ignored, but kept here for documentation ) container_pull( name = "debian-hyperkube-base-" + arch, architecture = arch, - digest = _DEBIAN_HYPERKUBE_BASE_DIGEST, + digest = _digest(_DEBIAN_HYPERKUBE_BASE_DIGEST, arch), registry = "k8s.gcr.io", repository = "debian-hyperkube-base", + tag = "0.12.1", # ignored, but kept here for documentation ) def etcd_tarballs(): diff --git a/cluster/addons/dns/coredns/coredns.yaml.base b/cluster/addons/dns/coredns/coredns.yaml.base index 6936dc79215..efc946b144a 100644 --- a/cluster/addons/dns/coredns/coredns.yaml.base +++ b/cluster/addons/dns/coredns/coredns.yaml.base @@ -68,6 +68,7 @@ data: pods insecure upstream fallthrough in-addr.arpa ip6.arpa + ttl 30 } prometheus :9153 forward . /etc/resolv.conf diff --git a/cluster/addons/dns/coredns/coredns.yaml.in b/cluster/addons/dns/coredns/coredns.yaml.in index 0821820fd2a..17bb9de0320 100644 --- a/cluster/addons/dns/coredns/coredns.yaml.in +++ b/cluster/addons/dns/coredns/coredns.yaml.in @@ -68,6 +68,7 @@ data: pods insecure upstream fallthrough in-addr.arpa ip6.arpa + ttl 30 } prometheus :9153 forward . /etc/resolv.conf diff --git a/cluster/addons/dns/coredns/coredns.yaml.sed b/cluster/addons/dns/coredns/coredns.yaml.sed index 38688ced306..53910bb6507 100644 --- a/cluster/addons/dns/coredns/coredns.yaml.sed +++ b/cluster/addons/dns/coredns/coredns.yaml.sed @@ -68,6 +68,7 @@ data: pods insecure upstream fallthrough in-addr.arpa ip6.arpa + ttl 30 } prometheus :9153 forward . /etc/resolv.conf diff --git a/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml b/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml index 1ddd9b4e03f..9ed27376559 100644 --- a/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml +++ b/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml @@ -111,7 +111,7 @@ spec: operator: "Exists" containers: - name: node-cache - image: k8s.gcr.io/k8s-dns-node-cache:1.15.0 + image: k8s.gcr.io/k8s-dns-node-cache:1.15.1 resources: limits: memory: 30Mi diff --git a/cluster/addons/metrics-server/README.md b/cluster/addons/metrics-server/README.md index f5f063bb7a7..67df8590def 100644 --- a/cluster/addons/metrics-server/README.md +++ b/cluster/addons/metrics-server/README.md @@ -3,7 +3,7 @@ [Metrics Server](https://github.com/kubernetes-incubator/metrics-server) exposes core Kubernetes metrics via metrics API. -More details can be found in [Core metrics pipeline documentation](https://kubernetes.io/docs/tasks/debug-application-cluster/core-metrics-pipeline/). +More details can be found in [Core metrics pipeline documentation](https://kubernetes.io/docs/tasks/debug-application-cluster/resource-metrics-pipeline/). ## Troubleshooting diff --git a/cluster/addons/metrics-server/metrics-server-deployment.yaml b/cluster/addons/metrics-server/metrics-server-deployment.yaml index 0ab7bef2e8f..5862f0679fb 100644 --- a/cluster/addons/metrics-server/metrics-server-deployment.yaml +++ b/cluster/addons/metrics-server/metrics-server-deployment.yaml @@ -57,6 +57,7 @@ spec: # Remove these lines for non-GKE clusters, and when GKE supports token-based auth. - --kubelet-port=10255 - --deprecated-kubelet-completely-insecure=true + - --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP ports: - containerPort: 443 name: https diff --git a/cluster/centos/.gitignore b/cluster/centos/.gitignore deleted file mode 100644 index 8452b51087b..00000000000 --- a/cluster/centos/.gitignore +++ /dev/null @@ -1,14 +0,0 @@ -binaries -ca-cert -etcd-cert - -master/bin/etcd -master/bin/etcdctl -master/bin/kube* - -node/bin/docker -node/bin/etcd -node/bin/etcdctl -node/bin/flanneld -node/bin/kube* -local-test.sh diff --git a/cluster/centos/OWNERS b/cluster/centos/OWNERS deleted file mode 100644 index 3a91eff1417..00000000000 --- a/cluster/centos/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: - - zouyee diff --git a/cluster/centos/build.sh b/cluster/centos/build.sh deleted file mode 100755 index 1f8c0795df9..00000000000 --- a/cluster/centos/build.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Download the flannel, etcd, docker, bridge-utils and K8s binaries automatically -# and store into binaries directory. -# Run as sudoers only - -# author @kevin-wangzefeng - -set -o errexit -set -o nounset -set -o pipefail - -readonly ROOT=$(dirname "${BASH_SOURCE[0]}") -source "${ROOT}/config-build.sh" - -# ensure $RELEASES_DIR is an absolute file path -mkdir -p "${RELEASES_DIR}" -RELEASES_DIR=$(cd "${RELEASES_DIR}"; pwd) - -# get absolute file path of binaries -BINARY_DIR=$(cd "${ROOT}"; pwd)/binaries - -function clean-up() { - rm -rf "${RELEASES_DIR}" - rm -rf "${BINARY_DIR}" -} - -function download-releases() { - rm -rf "${RELEASES_DIR}" - mkdir -p "${RELEASES_DIR}" - - echo "Download flannel release v${FLANNEL_VERSION} ..." - curl -L "${FLANNEL_DOWNLOAD_URL}" -o "${RELEASES_DIR}/flannel.tar.gz" - - echo "Download etcd release v${ETCD_VERSION} ..." - curl -L "${ETCD_DOWNLOAD_URL}" -o "${RELEASES_DIR}/etcd.tar.gz" - - echo "Download kubernetes release v${K8S_VERSION} ..." - curl -L "${K8S_CLIENT_DOWNLOAD_URL}" -o "${RELEASES_DIR}/kubernetes-client-linux-amd64.tar.gz" - curl -L "${K8S_SERVER_DOWNLOAD_URL}" -o "${RELEASES_DIR}/kubernetes-server-linux-amd64.tar.gz" - - echo "Download docker release v${DOCKER_VERSION} ..." - curl -L "${DOCKER_DOWNLOAD_URL}" -o "${RELEASES_DIR}/docker.tar.gz" -} - -function unpack-releases() { - rm -rf "${BINARY_DIR}" - mkdir -p "${BINARY_DIR}/master/bin" - mkdir -p "${BINARY_DIR}/node/bin" - - # flannel - if [[ -f "${RELEASES_DIR}/flannel.tar.gz" ]] ; then - tar xzf "${RELEASES_DIR}/flannel.tar.gz" -C "${RELEASES_DIR}" - cp "${RELEASES_DIR}/flanneld" "${BINARY_DIR}/master/bin" - cp "${RELEASES_DIR}/flanneld" "${BINARY_DIR}/node/bin" - fi - - # etcd - if [[ -f "${RELEASES_DIR}/etcd.tar.gz" ]] ; then - tar xzf "${RELEASES_DIR}/etcd.tar.gz" -C "${RELEASES_DIR}" - ETCD="etcd-v${ETCD_VERSION}-linux-amd64" - cp "${RELEASES_DIR}/${ETCD}/etcd" \ - "${RELEASES_DIR}/${ETCD}/etcdctl" "${BINARY_DIR}/master/bin" - cp "${RELEASES_DIR}/${ETCD}/etcd" \ - "${RELEASES_DIR}/${ETCD}/etcdctl" "${BINARY_DIR}/node/bin" - fi - - # k8s - if [[ -f "${RELEASES_DIR}/kubernetes-client-linux-amd64.tar.gz" ]] ; then - tar xzf "${RELEASES_DIR}/kubernetes-client-linux-amd64.tar.gz" -C "${RELEASES_DIR}" - cp "${RELEASES_DIR}/kubernetes/client/bin/kubectl" "${BINARY_DIR}" - fi - - if [[ -f "${RELEASES_DIR}/kubernetes-server-linux-amd64.tar.gz" ]] ; then - tar xzf "${RELEASES_DIR}/kubernetes-server-linux-amd64.tar.gz" -C "${RELEASES_DIR}" - cp "${RELEASES_DIR}/kubernetes/server/bin/kube-apiserver" \ - "${RELEASES_DIR}/kubernetes/server/bin/kube-controller-manager" \ - "${RELEASES_DIR}/kubernetes/server/bin/kube-scheduler" "${BINARY_DIR}/master/bin" - cp "${RELEASES_DIR}/kubernetes/server/bin/kubelet" \ - "${RELEASES_DIR}/kubernetes/server/bin/kube-proxy" "${BINARY_DIR}/node/bin" - fi - - # docker - if [[ -f "${RELEASES_DIR}/docker.tar.gz" ]]; then - tar xzf "${RELEASES_DIR}/docker.tar.gz" -C "${RELEASES_DIR}" - - cp "${RELEASES_DIR}/docker/docker*" "${BINARY_DIR}/node/bin" - fi - - chmod -R +x "${BINARY_DIR}" - echo "Done! All binaries are stored in ${BINARY_DIR}" -} - -function parse-opt() { - local opt=${1-} - - case $opt in - download) - download-releases - ;; - unpack) - unpack-releases - ;; - clean) - clean-up - ;; - all) - download-releases - unpack-releases - ;; - *) - echo "Usage: " - echo " build.sh " - echo "Commands:" - echo " clean Clean up downloaded releases and unpacked binaries." - echo " download Download releases to \"${RELEASES_DIR}\"." - echo " unpack Unpack releases downloaded in \"${RELEASES_DIR}\", and copy binaries to \"${BINARY_DIR}\"." - echo " all Download releases and unpack them." - ;; - esac -} - -parse-opt "${@}" diff --git a/cluster/centos/config-build.sh b/cluster/centos/config-build.sh deleted file mode 100755 index 9ed75ae7974..00000000000 --- a/cluster/centos/config-build.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## Contains configuration values for the Binaries downloading and unpacking. - -# Directory to store release packages that will be downloaded. -RELEASES_DIR=${RELEASES_DIR:-/tmp/downloads} - -# Define docker version to use. -DOCKER_VERSION=${DOCKER_VERSION:-"1.12.1"} - -# Define flannel version to use. -FLANNEL_VERSION=${FLANNEL_VERSION:-"0.6.1"} - -# Define etcd version to use. -ETCD_VERSION=${ETCD_VERSION:-"3.0.9"} - -# Define k8s version to use. -K8S_VERSION=${K8S_VERSION:-"1.3.7"} - -# shellcheck disable=2034 # Variables sourced in other scripts executed from the same shell -DOCKER_DOWNLOAD_URL=\ -"https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz" - -# shellcheck disable=2034 # Variables sourced in other scripts executed from the same shell -FLANNEL_DOWNLOAD_URL=\ -"https://github.com/coreos/flannel/releases/download/v${FLANNEL_VERSION}/flannel-v${FLANNEL_VERSION}-linux-amd64.tar.gz" - -# shellcheck disable=2034 # Variables sourced in other scripts executed from the same shell -ETCD_DOWNLOAD_URL=\ -"https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/etcd-v${ETCD_VERSION}-linux-amd64.tar.gz" - -# shellcheck disable=2034 # Variables sourced in other scripts executed from the same shell -K8S_CLIENT_DOWNLOAD_URL=\ -"https://dl.k8s.io/v${K8S_VERSION}/kubernetes-client-linux-amd64.tar.gz" - -# shellcheck disable=2034 # Variables sourced in other scripts executed from the same shell -K8S_SERVER_DOWNLOAD_URL=\ -"https://dl.k8s.io/v${K8S_VERSION}/kubernetes-server-linux-amd64.tar.gz" diff --git a/cluster/centos/config-default.sh b/cluster/centos/config-default.sh deleted file mode 100755 index a08ff2fed92..00000000000 --- a/cluster/centos/config-default.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -readonly root=$(dirname "${BASH_SOURCE[0]}") - -## Contains configuration values for the CentOS cluster -# The user should have sudo privilege -export MASTER=${MASTER:-"centos@172.10.0.11"} -export MASTER_IP=${MASTER#*@} - -# Define all your master nodes, -# And separated with blank space like . -# The user should have sudo privilege -export MASTERS="${MASTERS:-$MASTER}" - -# length-of -# Get the length of specific arg0, could be a space-separate string or array. -function length-of() { - local len=0 - # shellcheck disable=SC2034 # Unused variables left for readability - for part in $1; do - ((++len)) - done - echo $len -} -# Number of nodes in your cluster. -export NUM_MASTERS="${NUM_MASTERS:-$(length-of "$MASTERS")}" - -# Get default master advertise address: first master node. -function default-advertise-address() { - # get the first master node - local masters_array=("${MASTERS}") - local master=${masters_array[0]} - echo "${master#*@}" -} - -# Define advertise address of masters, could be a load balancer address. -# If not provided, the default is ip of first master node. -export MASTER_ADVERTISE_ADDRESS="${MASTER_ADVERTISE_ADDRESS:-$(default-advertise-address)}" -export MASTER_ADVERTISE_IP="${MASTER_ADVERTISE_IP:-$(getent hosts "${MASTER_ADVERTISE_ADDRESS}" | awk '{print $1; exit}')}" - -# Define all your minion nodes, -# And separated with blank space like . -# The user should have sudo privilege -export NODES="${NODES:-"centos@172.10.0.12 centos@172.10.0.13"}" - -# Number of nodes in your cluster. -export NUM_NODES="${NUM_NODES:-$(length-of "$NODES")}" - -# Should be removed when NUM_NODES is deprecated in validate-cluster.sh -export NUM_NODES="${NUM_NODES}" - -# By default, the cluster will use the etcd installed on master. -function concat-etcd-servers() { - local etcd_servers="" - for master in ${MASTERS}; do - local master_ip=${master#*@} - local prefix="" - if [ -n "$etcd_servers" ]; then - prefix="${etcd_servers}," - fi - etcd_servers="${prefix}https://${master_ip}:2379" - done - - echo "$etcd_servers" -} -ETCD_SERVERS="$(concat-etcd-servers)" -export ETCD_SERVERS - -# By default, etcd cluster will use runtime configuration -# https://coreos.com/etcd/docs/latest/v2/runtime-configuration.html -# Get etc initial cluster and store in ETCD_INITIAL_CLUSTER -function concat-etcd-initial-cluster() { - local etcd_initial_cluster="" - local num_infra=0 - for master in ${MASTERS}; do - local master_ip="${master#*@}" - if [ -n "$etcd_initial_cluster" ]; then - etcd_initial_cluster+="," - fi - etcd_initial_cluster+="infra${num_infra}=https://${master_ip}:2380" - ((++num_infra)) - done - - echo "$etcd_initial_cluster" -} -ETCD_INITIAL_CLUSTER="$(concat-etcd-initial-cluster)" -export ETCD_INITIAL_CLUSTER - -CERT_DIR="${CERT_DIR:-${root}/ca-cert}" -mkdir -p "${CERT_DIR}" -# CERT_DIR path must be absolute. -CERT_DIR="$(cd "${CERT_DIR}" && pwd)" -export CERT_DIR - -# define the IP range used for service cluster IPs. -# according to rfc 1918 ref: https://tools.ietf.org/html/rfc1918 choose a private ip range here. -export SERVICE_CLUSTER_IP_RANGE=${SERVICE_CLUSTER_IP_RANGE:-"192.168.3.0/24"} - -# Optional: Install cluster DNS. -ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}" -export ENABLE_CLUSTER_DNS -# DNS_SERVER_IP must be a IP in SERVICE_CLUSTER_IP_RANGE -DNS_SERVER_IP=${DNS_SERVER_IP:-"192.168.3.100"} -DNS_DOMAIN=${DNS_DOMAIN:-"cluster.local"} - -# Optional: Install Kubernetes UI -ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" -export ENABLE_CLUSTER_UI - -# define the IP range used for flannel overlay network, should not conflict with above SERVICE_CLUSTER_IP_RANGE -export FLANNEL_NET=${FLANNEL_NET:-"172.16.0.0/16"} - -# Admission Controllers to invoke prior to persisting objects in cluster. -# MutatingAdmissionWebhook should be the last controller that modifies the -# request object, otherwise users will be confused if the mutating webhooks' -# modification is overwritten. -# If we included ResourceQuota, we should keep it at the end of the list to -# prevent incrementing quota usage prematurely. -export ADMISSION_CONTROL=${ADMISSION_CONTROL:-"NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeClaimResize,DefaultTolerationSeconds,Priority,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"} - -# Extra options to set on the Docker command line. -# This is useful for setting --insecure-registry for local registries. -export DOCKER_OPTS=${DOCKER_OPTS:-""} - - -# Timeouts for process checking on master and minion -export PROCESS_CHECK_TIMEOUT=${PROCESS_CHECK_TIMEOUT:-180} # seconds. - -unset -f default-advertise-address concat-etcd-servers length-of concat-etcd-initial-cluster diff --git a/cluster/centos/config-test.sh b/cluster/centos/config-test.sh deleted file mode 100644 index 45dd75c0da3..00000000000 --- a/cluster/centos/config-test.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## for CentOS/Fedora/RHEL cluster in test mode -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. -source "${KUBE_ROOT}/cluster/centos/config-default.sh" diff --git a/cluster/centos/deployAddons.sh b/cluster/centos/deployAddons.sh deleted file mode 100755 index fce75750f18..00000000000 --- a/cluster/centos/deployAddons.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# deploy the add-on services after the cluster is available - -set -e - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. -source "${KUBE_ROOT}/cluster/centos/config-default.sh" -KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" -export KUBECTL_PATH="${KUBE_ROOT}/cluster/centos/binaries/kubectl" -export KUBE_CONFIG_FILE=${KUBE_CONFIG_FILE:-${KUBE_ROOT}/cluster/centos/config-default.sh} - -function deploy_dns { - echo "Deploying DNS on Kubernetes" - cp "${KUBE_ROOT}/cluster/addons/dns/kube-dns/kube-dns.yaml.sed" kube-dns.yaml - sed -i -e "s/\\\$DNS_DOMAIN/${DNS_DOMAIN}/g" kube-dns.yaml - sed -i -e "s/\\\$DNS_SERVER_IP/${DNS_SERVER_IP}/g" kube-dns.yaml - - KUBEDNS=$("${KUBECTL} get services --namespace=kube-system | grep kube-dns | cat") - - if [ ! "$KUBEDNS" ]; then - # use kubectl to create kube-dns addon - ${KUBECTL} --namespace=kube-system create -f kube-dns.yaml - - echo "Kube-dns addon is successfully deployed." - else - echo "Kube-dns addon is already deployed. Skipping." - fi - - echo -} - -function deploy_dashboard { - echo "Deploying Kubernetes Dashboard" - - ${KUBECTL} apply -f "${KUBE_ROOT}/cluster/addons/dashboard/dashboard-secret.yaml" - ${KUBECTL} apply -f "${KUBE_ROOT}/cluster/addons/dashboard/dashboard-configmap.yaml" - ${KUBECTL} apply -f "${KUBE_ROOT}/cluster/addons/dashboard/dashboard-rbac.yaml" - ${KUBECTL} apply -f "${KUBE_ROOT}/cluster/addons/dashboard/dashboard-controller.yaml" - ${KUBECTL} apply -f "${KUBE_ROOT}/cluster/addons/dashboard/dashboard-service.yaml" - - echo -} - - -if [ "${ENABLE_CLUSTER_DNS}" == true ]; then - deploy_dns -fi - -if [ "${ENABLE_CLUSTER_UI}" == true ]; then - deploy_dashboard -fi diff --git a/cluster/centos/make-ca-cert.sh b/cluster/centos/make-ca-cert.sh deleted file mode 100755 index 8206e45f637..00000000000 --- a/cluster/centos/make-ca-cert.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -DEBUG="${DEBUG:-false}" - -if [ "${DEBUG}" == "true" ]; then - set -x -fi - -cert_ip=$1 -extra_sans=${2:-} -cert_dir=${CERT_DIR:-/srv/kubernetes} -cert_group=${CERT_GROUP:-kube-cert} - -mkdir -p "$cert_dir" - -use_cn=false - -sans="IP:${cert_ip}" -if [[ -n "${extra_sans}" ]]; then - sans="${sans},${extra_sans}" -fi - -tmpdir=$(mktemp -d -t kubernetes_cacert.XXXXXX) -trap 'rm -rf "${tmpdir}"' EXIT -cd "${tmpdir}" - -# TODO: For now, this is a patched tool that makes subject-alt-name work, when -# the fix is upstream move back to the upstream easyrsa. This is cached in GCS -# but is originally taken from: -# https://github.com/brendandburns/easy-rsa/archive/master.tar.gz -# -# To update, do the following: -# curl -o easy-rsa.tar.gz https://github.com/brendandburns/easy-rsa/archive/master.tar.gz -# gsutil cp easy-rsa.tar.gz gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz -# gsutil acl ch -R -g all:R gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz -# -# Due to GCS caching of public objects, it may take time for this to be widely -# distributed. -# -# Use ~/kube/easy-rsa.tar.gz if it exists, so that it can be -# pre-pushed in cases where an outgoing connection is not allowed. -if [ -f ~/kube/easy-rsa.tar.gz ]; then - ln -s ~/kube/easy-rsa.tar.gz . -else - curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz > /dev/null 2>&1 -fi -tar xzf easy-rsa.tar.gz > /dev/null 2>&1 - -cd easy-rsa-master/easyrsa3 -./easyrsa init-pki > /dev/null 2>&1 -./easyrsa --batch "--req-cn=${cert_ip}@$(date +%s)" build-ca nopass > /dev/null 2>&1 -if [ $use_cn = "true" ]; then - ./easyrsa build-server-full "${cert_ip}" nopass > /dev/null 2>&1 - cp -p "pki/issued/${cert_ip}.crt" "${cert_dir}/server.cert" > /dev/null 2>&1 - cp -p "pki/private/${cert_ip}.key" "${cert_dir}/server.key" > /dev/null 2>&1 -else - ./easyrsa --subject-alt-name="${sans}" build-server-full kubernetes-master nopass > /dev/null 2>&1 - cp -p pki/issued/kubernetes-master.crt "${cert_dir}/server.cert" > /dev/null 2>&1 - cp -p pki/private/kubernetes-master.key "${cert_dir}/server.key" > /dev/null 2>&1 -fi -# Make a superuser client cert with subject "O=system:masters, CN=kubecfg" -./easyrsa --dn-mode=org \ - --req-cn=kubecfg --req-org=system:masters \ - --req-c= --req-st= --req-city= --req-email= --req-ou= \ - build-client-full kubecfg nopass > /dev/null 2>&1 -cp -p pki/ca.crt "${cert_dir}/ca.crt" -cp -p pki/issued/kubecfg.crt "${cert_dir}/kubecfg.crt" -cp -p pki/private/kubecfg.key "${cert_dir}/kubecfg.key" -# Make server certs accessible to apiserver. -chgrp "${cert_group}" "${cert_dir}/server.key" "${cert_dir}/server.cert" "${cert_dir}/ca.crt" -chmod 660 "${cert_dir}/server.key" "${cert_dir}/server.cert" "${cert_dir}/ca.crt" diff --git a/cluster/centos/master/scripts/apiserver.sh b/cluster/centos/master/scripts/apiserver.sh deleted file mode 100755 index be62549ce47..00000000000 --- a/cluster/centos/master/scripts/apiserver.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -MASTER_ADDRESS=${1:-"8.8.8.18"} -ETCD_SERVERS=${2:-"https://8.8.8.18:2379"} -SERVICE_CLUSTER_IP_RANGE=${3:-"10.10.10.0/24"} -ADMISSION_CONTROL=${4:-""} - -cat </opt/kubernetes/cfg/kube-apiserver -# --logtostderr=true: log to standard error instead of files -KUBE_LOGTOSTDERR="--logtostderr=true" - -# --v=0: log level for V logs -KUBE_LOG_LEVEL="--v=4" - -# --etcd-servers=[]: List of etcd servers to watch (http://ip:port), -# comma separated. Mutually exclusive with -etcd-config -KUBE_ETCD_SERVERS="--etcd-servers=${ETCD_SERVERS}" - -# --etcd-cafile="": SSL Certificate Authority file used to secure etcd communication. -KUBE_ETCD_CAFILE="--etcd-cafile=/srv/kubernetes/etcd/ca.pem" - -# --etcd-certfile="": SSL certification file used to secure etcd communication. -KUBE_ETCD_CERTFILE="--etcd-certfile=/srv/kubernetes/etcd/client.pem" - -# --etcd-keyfile="": key file used to secure etcd communication. -KUBE_ETCD_KEYFILE="--etcd-keyfile=/srv/kubernetes/etcd/client-key.pem" - -# --insecure-bind-address=127.0.0.1: The IP address on which to serve the --insecure-port. -KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0" - -# --insecure-port=8080: The port on which to serve unsecured, unauthenticated access. -KUBE_API_PORT="--insecure-port=8080" - -# --kubelet-port=10250: Kubelet port -NODE_PORT="--kubelet-port=10250" - -# --advertise-address=: The IP address on which to advertise -# the apiserver to members of the cluster. -KUBE_ADVERTISE_ADDR="--advertise-address=${MASTER_ADDRESS}" - -# --allow-privileged=false: If true, allow privileged containers. -KUBE_ALLOW_PRIV="--allow-privileged=false" - -# --service-cluster-ip-range=: A CIDR notation IP range from which to assign service cluster IPs. -# This must not overlap with any IP ranges assigned to nodes for pods. -KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" - -# --admission-control="AlwaysAdmit": Ordered list of plug-ins -# to do admission control of resources into cluster. -# Comma-delimited list of: -# LimitRanger, AlwaysDeny, SecurityContextDeny, NamespaceExists, -# NamespaceLifecycle, NamespaceAutoProvision, AlwaysAdmit, -# ServiceAccount, DefaultStorageClass, DefaultTolerationSeconds, ResourceQuota -# Mark Deprecated. Use --enable-admission-plugins or --disable-admission-plugins instead since v1.10. -# It will be removed in a future version. -KUBE_ADMISSION_CONTROL="--admission-control=${ADMISSION_CONTROL}" - -# --client-ca-file="": If set, any request presenting a client certificate signed -# by one of the authorities in the client-ca-file is authenticated with an identity -# corresponding to the CommonName of the client certificate. -KUBE_API_CLIENT_CA_FILE="--client-ca-file=/srv/kubernetes/ca.crt" - -# --tls-cert-file="": File containing x509 Certificate for HTTPS. (CA cert, if any, -# concatenated after server cert). If HTTPS serving is enabled, and --tls-cert-file -# and --tls-private-key-file are not provided, a self-signed certificate and key are -# generated for the public address and saved to /var/run/kubernetes. -KUBE_API_TLS_CERT_FILE="--tls-cert-file=/srv/kubernetes/server.cert" - -# --tls-private-key-file="": File containing x509 private key matching --tls-cert-file. -KUBE_API_TLS_PRIVATE_KEY_FILE="--tls-private-key-file=/srv/kubernetes/server.key" -EOF - -KUBE_APISERVER_OPTS=" \${KUBE_LOGTOSTDERR} \\ - \${KUBE_LOG_LEVEL} \\ - \${KUBE_ETCD_SERVERS} \\ - \${KUBE_ETCD_CAFILE} \\ - \${KUBE_ETCD_CERTFILE} \\ - \${KUBE_ETCD_KEYFILE} \\ - \${KUBE_API_ADDRESS} \\ - \${KUBE_API_PORT} \\ - \${NODE_PORT} \\ - \${KUBE_ADVERTISE_ADDR} \\ - \${KUBE_ALLOW_PRIV} \\ - \${KUBE_SERVICE_ADDRESSES} \\ - \${KUBE_ADMISSION_CONTROL} \\ - \${KUBE_API_CLIENT_CA_FILE} \\ - \${KUBE_API_TLS_CERT_FILE} \\ - \${KUBE_API_TLS_PRIVATE_KEY_FILE}" - - -cat </usr/lib/systemd/system/kube-apiserver.service -[Unit] -Description=Kubernetes API Server -Documentation=https://github.com/kubernetes/kubernetes - -[Service] -EnvironmentFile=-/opt/kubernetes/cfg/kube-apiserver -ExecStart=/opt/kubernetes/bin/kube-apiserver ${KUBE_APISERVER_OPTS} -Restart=on-failure - -[Install] -WantedBy=multi-user.target -EOF - -systemctl daemon-reload -systemctl enable kube-apiserver -systemctl restart kube-apiserver diff --git a/cluster/centos/master/scripts/controller-manager.sh b/cluster/centos/master/scripts/controller-manager.sh deleted file mode 100755 index 684825a3e46..00000000000 --- a/cluster/centos/master/scripts/controller-manager.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -MASTER_ADDRESS=${1:-"8.8.8.18"} - -cat </opt/kubernetes/cfg/kube-controller-manager -KUBE_LOGTOSTDERR="--logtostderr=true" -KUBE_LOG_LEVEL="--v=4" -KUBE_MASTER="--master=${MASTER_ADDRESS}:8080" - -# --root-ca-file="": If set, this root certificate authority will be included in -# service account's token secret. This must be a valid PEM-encoded CA bundle. -KUBE_CONTROLLER_MANAGER_ROOT_CA_FILE="--root-ca-file=/srv/kubernetes/ca.crt" - -# --service-account-private-key-file="": Filename containing a PEM-encoded private -# RSA key used to sign service account tokens. -KUBE_CONTROLLER_MANAGER_SERVICE_ACCOUNT_PRIVATE_KEY_FILE="--service-account-private-key-file=/srv/kubernetes/server.key" - -# --leader-elect: Start a leader election client and gain leadership before -# executing the main loop. Enable this when running replicated components for high availability. -KUBE_LEADER_ELECT="--leader-elect" -EOF - -KUBE_CONTROLLER_MANAGER_OPTS=" \${KUBE_LOGTOSTDERR} \\ - \${KUBE_LOG_LEVEL} \\ - \${KUBE_MASTER} \\ - \${KUBE_CONTROLLER_MANAGER_ROOT_CA_FILE} \\ - \${KUBE_CONTROLLER_MANAGER_SERVICE_ACCOUNT_PRIVATE_KEY_FILE}\\ - \${KUBE_LEADER_ELECT}" - -cat </usr/lib/systemd/system/kube-controller-manager.service -[Unit] -Description=Kubernetes Controller Manager -Documentation=https://github.com/kubernetes/kubernetes - -[Service] -EnvironmentFile=-/opt/kubernetes/cfg/kube-controller-manager -ExecStart=/opt/kubernetes/bin/kube-controller-manager ${KUBE_CONTROLLER_MANAGER_OPTS} -Restart=on-failure - -[Install] -WantedBy=multi-user.target -EOF - -systemctl daemon-reload -systemctl enable kube-controller-manager -systemctl restart kube-controller-manager diff --git a/cluster/centos/master/scripts/etcd.sh b/cluster/centos/master/scripts/etcd.sh deleted file mode 100755 index aa73b57b490..00000000000 --- a/cluster/centos/master/scripts/etcd.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## Create etcd.conf, etcd.service, and start etcd service. - - -etcd_data_dir=/var/lib/etcd -mkdir -p ${etcd_data_dir} - -ETCD_NAME=${1:-"default"} -ETCD_LISTEN_IP=${2:-"0.0.0.0"} -ETCD_INITIAL_CLUSTER=${3:-} - -cat </opt/kubernetes/cfg/etcd.conf -# [member] -ETCD_NAME="${ETCD_NAME}" -ETCD_DATA_DIR="${etcd_data_dir}/default.etcd" -#ETCD_SNAPSHOT_COUNTER="10000" -#ETCD_HEARTBEAT_INTERVAL="100" -#ETCD_ELECTION_TIMEOUT="1000" -ETCD_LISTEN_PEER_URLS="https://${ETCD_LISTEN_IP}:2380" -ETCD_LISTEN_CLIENT_URLS="https://${ETCD_LISTEN_IP}:2379,https://127.0.0.1:2379" -#ETCD_MAX_SNAPSHOTS="5" -#ETCD_MAX_WALS="5" -#ETCD_CORS="" -# -#[cluster] -ETCD_INITIAL_ADVERTISE_PEER_URLS="https://${ETCD_LISTEN_IP}:2380" -# if you use different ETCD_NAME (e.g. test), -# set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..." -ETCD_INITIAL_CLUSTER="${ETCD_INITIAL_CLUSTER}" -ETCD_INITIAL_CLUSTER_STATE="new" -ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster" -ETCD_ADVERTISE_CLIENT_URLS="https://${ETCD_LISTEN_IP}:2379" -#ETCD_DISCOVERY="" -#ETCD_DISCOVERY_SRV="" -#ETCD_DISCOVERY_FALLBACK="proxy" -#ETCD_DISCOVERY_PROXY="" -# -#[proxy] -#ETCD_PROXY="off" -# -#[security] -CLIENT_CERT_AUTH="true" -ETCD_CA_FILE="/srv/kubernetes/etcd/ca.pem" -ETCD_CERT_FILE="/srv/kubernetes/etcd/server-${ETCD_NAME}.pem" -ETCD_KEY_FILE="/srv/kubernetes/etcd/server-${ETCD_NAME}-key.pem" -PEER_CLIENT_CERT_AUTH="true" -ETCD_PEER_CA_FILE="/srv/kubernetes/etcd/ca.pem" -ETCD_PEER_CERT_FILE="/srv/kubernetes/etcd/peer-${ETCD_NAME}.pem" -ETCD_PEER_KEY_FILE="/srv/kubernetes/etcd/peer-${ETCD_NAME}-key.pem" -EOF - -cat <//usr/lib/systemd/system/etcd.service -[Unit] -Description=Etcd Server -After=network.target - -[Service] -Type=simple -WorkingDirectory=${etcd_data_dir} -EnvironmentFile=-/opt/kubernetes/cfg/etcd.conf -# set GOMAXPROCS to number of processors -ExecStart=/bin/bash -c "GOMAXPROCS=\$(nproc) /opt/kubernetes/bin/etcd" -Type=notify - -[Install] -WantedBy=multi-user.target -EOF - -systemctl daemon-reload -systemctl enable etcd -systemctl restart etcd diff --git a/cluster/centos/master/scripts/flannel.sh b/cluster/centos/master/scripts/flannel.sh deleted file mode 100644 index 83a7de5b58a..00000000000 --- a/cluster/centos/master/scripts/flannel.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -ETCD_SERVERS=${1:-"http://8.8.8.18:4001"} -FLANNEL_NET=${2:-"172.16.0.0/16"} - -CA_FILE="/srv/kubernetes/etcd/ca.pem" -CERT_FILE="/srv/kubernetes/etcd/client.pem" -KEY_FILE="/srv/kubernetes/etcd/client-key.pem" - -cat </opt/kubernetes/cfg/flannel -FLANNEL_ETCD="-etcd-endpoints=${ETCD_SERVERS}" -FLANNEL_ETCD_KEY="-etcd-prefix=/coreos.com/network" -FLANNEL_ETCD_CAFILE="--etcd-cafile=${CA_FILE}" -FLANNEL_ETCD_CERTFILE="--etcd-certfile=${CERT_FILE}" -FLANNEL_ETCD_KEYFILE="--etcd-keyfile=${KEY_FILE}" -EOF - -cat </usr/lib/systemd/system/flannel.service -[Unit] -Description=Flanneld overlay address etcd agent -After=network.target - -[Service] -EnvironmentFile=-/opt/kubernetes/cfg/flannel -ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \${FLANNEL_ETCD} \${FLANNEL_ETCD_KEY} \${FLANNEL_ETCD_CAFILE} \${FLANNEL_ETCD_CERTFILE} \${FLANNEL_ETCD_KEYFILE} - -Type=notify - -[Install] -WantedBy=multi-user.target -EOF - -# Store FLANNEL_NET to etcd. -attempt=0 -while true; do - if /opt/kubernetes/bin/etcdctl --ca-file ${CA_FILE} --cert-file ${CERT_FILE} --key-file ${KEY_FILE} \ - --no-sync -C "${ETCD_SERVERS}" \ - get /coreos.com/network/config >/dev/null 2>&1; then - break - else - if (( attempt > 600 )); then - echo "timeout for waiting network config" > ~/kube/err.log - exit 2 - fi - - /opt/kubernetes/bin/etcdctl --ca-file ${CA_FILE} --cert-file ${CERT_FILE} --key-file ${KEY_FILE} \ - --no-sync -C "${ETCD_SERVERS}" \ - mk /coreos.com/network/config "{\"Network\":\"${FLANNEL_NET}\"}" >/dev/null 2>&1 - attempt=$((attempt+1)) - sleep 3 - fi -done -wait - -systemctl enable flannel -systemctl daemon-reload -systemctl restart flannel diff --git a/cluster/centos/master/scripts/post-etcd.sh b/cluster/centos/master/scripts/post-etcd.sh deleted file mode 100644 index 6bdfd53094e..00000000000 --- a/cluster/centos/master/scripts/post-etcd.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## Set initial-cluster-state to existing, and restart etcd service. - -sed -i 's/ETCD_INITIAL_CLUSTER_STATE="new"/ETCD_INITIAL_CLUSTER_STATE="existing"/' /opt/kubernetes/cfg/etcd.conf - -systemctl daemon-reload -systemctl enable etcd -systemctl restart etcd diff --git a/cluster/centos/master/scripts/scheduler.sh b/cluster/centos/master/scripts/scheduler.sh deleted file mode 100755 index 22d66650190..00000000000 --- a/cluster/centos/master/scripts/scheduler.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -MASTER_ADDRESS=${1:-"8.8.8.18"} - -cat </opt/kubernetes/cfg/kube-scheduler -### -# kubernetes scheduler config - -# --logtostderr=true: log to standard error instead of files -KUBE_LOGTOSTDERR="--logtostderr=true" - -# --v=0: log level for V logs -KUBE_LOG_LEVEL="--v=4" - -# --master: The address of the Kubernetes API server (overrides any value in kubeconfig). -KUBE_MASTER="--master=${MASTER_ADDRESS}:8080" - -# --leader-elect: Start a leader election client and gain leadership before -# executing the main loop. Enable this when running replicated components for high availability. -KUBE_LEADER_ELECT="--leader-elect" - -# Add your own! -KUBE_SCHEDULER_ARGS="" - -EOF - -KUBE_SCHEDULER_OPTS=" \${KUBE_LOGTOSTDERR} \\ - \${KUBE_LOG_LEVEL} \\ - \${KUBE_MASTER} \\ - \${KUBE_LEADER_ELECT} \\ - \$KUBE_SCHEDULER_ARGS" - -cat </usr/lib/systemd/system/kube-scheduler.service -[Unit] -Description=Kubernetes Scheduler -Documentation=https://github.com/kubernetes/kubernetes - -[Service] -EnvironmentFile=-/opt/kubernetes/cfg/kube-scheduler -ExecStart=/opt/kubernetes/bin/kube-scheduler ${KUBE_SCHEDULER_OPTS} -Restart=on-failure - -[Install] -WantedBy=multi-user.target -EOF - -systemctl daemon-reload -systemctl enable kube-scheduler -systemctl restart kube-scheduler diff --git a/cluster/centos/node/bin/mk-docker-opts.sh b/cluster/centos/node/bin/mk-docker-opts.sh deleted file mode 100755 index 22a459f5134..00000000000 --- a/cluster/centos/node/bin/mk-docker-opts.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Generate Docker daemon options based on flannel env file. - -# exit on any error -set -e - -usage() { - echo "$0 [-f FLANNEL-ENV-FILE] [-d DOCKER-ENV-FILE] [-i] [-c] [-m] [-k COMBINED-KEY] - -Generate Docker daemon options based on flannel env file -OPTIONS: - -f Path to flannel env file. Defaults to /run/flannel/subnet.env - -d Path to Docker env file to write to. Defaults to /run/docker_opts.env - -i Output each Docker option as individual var. e.g. DOCKER_OPT_MTU=1500 - -c Output combined Docker options into DOCKER_OPTS var - -k Set the combined options key to this value (default DOCKER_OPTS=) - -m Do not output --ip-masq (useful for older Docker version) -" >/dev/stderr - exit 1 -} - -flannel_env="/run/flannel/subnet.env" -docker_env="/run/docker_opts.env" -combined_opts_key="DOCKER_OPTS" -indiv_opts=false -combined_opts=false -ipmasq=true -val="" - -while getopts "f:d:icmk:" opt; do - case $opt in - f) - flannel_env=$OPTARG - ;; - d) - docker_env=$OPTARG - ;; - i) - indiv_opts=true - ;; - c) - combined_opts=true - ;; - m) - ipmasq=false - ;; - k) - combined_opts_key=$OPTARG - ;; - \?) - usage - ;; - esac -done - -if [[ $indiv_opts = false ]] && [[ $combined_opts = false ]]; then - indiv_opts=true - combined_opts=true -fi - -if [[ -f "${flannel_env}" ]]; then - source "${flannel_env}" -fi - -if [[ -n "$FLANNEL_SUBNET" ]]; then - # shellcheck disable=SC2034 # Variable name referenced in OPT_LOOP below - DOCKER_OPT_BIP="--bip=$FLANNEL_SUBNET" -fi - -if [[ -n "$FLANNEL_MTU" ]]; then - # shellcheck disable=SC2034 # Variable name referenced in OPT_LOOP below - DOCKER_OPT_MTU="--mtu=$FLANNEL_MTU" -fi - -if [[ "$FLANNEL_IPMASQ" = true ]] && [[ $ipmasq = true ]]; then - # shellcheck disable=SC2034 # Variable name referenced in OPT_LOOP below - DOCKER_OPT_IPMASQ="--ip-masq=false" -fi - -eval docker_opts="\$${combined_opts_key}" -docker_opts+=" " - -echo -n "" >"${docker_env}" - -# OPT_LOOP -for opt in $(compgen -v DOCKER_OPT_); do - eval val=\$"${opt}" - - if [[ "$indiv_opts" = true ]]; then - echo "$opt=\"$val\"" >>"${docker_env}" - fi - - docker_opts+="$val " -done - -if [[ "$combined_opts" = true ]]; then - echo "${combined_opts_key}=\"${docker_opts}\"" >>"${docker_env}" -fi diff --git a/cluster/centos/node/bin/remove-docker0.sh b/cluster/centos/node/bin/remove-docker0.sh deleted file mode 100755 index 8c862fbce31..00000000000 --- a/cluster/centos/node/bin/remove-docker0.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Delete default docker bridge, so that docker can start with flannel network. - -# exit on any error -set -e - -rc=0 -ip link show docker0 >/dev/null 2>&1 || rc="$?" -if [[ "$rc" -eq "0" ]]; then - ip link set dev docker0 down - ip link delete docker0 -fi diff --git a/cluster/centos/node/scripts/docker.sh b/cluster/centos/node/scripts/docker.sh deleted file mode 100755 index 8b878b24c9b..00000000000 --- a/cluster/centos/node/scripts/docker.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -DOCKER_OPTS=${1:-""} - -DOCKER_CONFIG=/opt/kubernetes/cfg/docker - -cat <$DOCKER_CONFIG -DOCKER_OPTS="-H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock -s overlay --selinux-enabled=false ${DOCKER_OPTS}" -EOF - -cat </usr/lib/systemd/system/docker.service -[Unit] -Description=Docker Application Container Engine -Documentation=http://docs.docker.com -After=network.target flannel.service -Requires=flannel.service - -[Service] -Type=notify -EnvironmentFile=-/run/flannel/docker -EnvironmentFile=-/opt/kubernetes/cfg/docker -WorkingDirectory=/opt/kubernetes/bin -ExecStart=/opt/kubernetes/bin/dockerd \$DOCKER_OPT_BIP \$DOCKER_OPT_MTU \$DOCKER_OPTS -LimitNOFILE=1048576 -LimitNPROC=1048576 - -[Install] -WantedBy=multi-user.target -EOF - -systemctl daemon-reload -systemctl enable docker -systemctl restart docker diff --git a/cluster/centos/node/scripts/flannel.sh b/cluster/centos/node/scripts/flannel.sh deleted file mode 100755 index 41ec84c5375..00000000000 --- a/cluster/centos/node/scripts/flannel.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -ETCD_SERVERS=${1:-"https://8.8.8.18:2379"} -FLANNEL_NET=${2:-"172.16.0.0/16"} - -CA_FILE="/srv/kubernetes/etcd/ca.pem" -CERT_FILE="/srv/kubernetes/etcd/client.pem" -KEY_FILE="/srv/kubernetes/etcd/client-key.pem" - -cat </opt/kubernetes/cfg/flannel -FLANNEL_ETCD="-etcd-endpoints=${ETCD_SERVERS}" -FLANNEL_ETCD_KEY="-etcd-prefix=/coreos.com/network" -FLANNEL_ETCD_CAFILE="--etcd-cafile=${CA_FILE}" -FLANNEL_ETCD_CERTFILE="--etcd-certfile=${CERT_FILE}" -FLANNEL_ETCD_KEYFILE="--etcd-keyfile=${KEY_FILE}" -EOF - -cat </usr/lib/systemd/system/flannel.service -[Unit] -Description=Flanneld overlay address etcd agent -After=network.target -Before=docker.service - -[Service] -EnvironmentFile=-/opt/kubernetes/cfg/flannel -ExecStartPre=/opt/kubernetes/bin/remove-docker0.sh -ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \${FLANNEL_ETCD} \${FLANNEL_ETCD_KEY} \${FLANNEL_ETCD_CAFILE} \${FLANNEL_ETCD_CERTFILE} \${FLANNEL_ETCD_KEYFILE} -ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -d /run/flannel/docker - -Type=notify - -[Install] -WantedBy=multi-user.target -RequiredBy=docker.service -EOF - -# Store FLANNEL_NET to etcd. -attempt=0 -while true; do - if /opt/kubernetes/bin/etcdctl --ca-file ${CA_FILE} --cert-file ${CERT_FILE} --key-file ${KEY_FILE} \ - --no-sync -C "${ETCD_SERVERS}" \ - get /coreos.com/network/config >/dev/null 2>&1; then - break - else - if (( attempt > 600 )); then - echo "timeout for waiting network config" > ~/kube/err.log - exit 2 - fi - - /opt/kubernetes/bin/etcdctl --ca-file ${CA_FILE} --cert-file ${CERT_FILE} --key-file ${KEY_FILE} \ - --no-sync -C "${ETCD_SERVERS}" \ - mk /coreos.com/network/config "{\"Network\":\"${FLANNEL_NET}\"}" >/dev/null 2>&1 - attempt=$((attempt+1)) - sleep 3 - fi -done -wait - -systemctl daemon-reload diff --git a/cluster/centos/node/scripts/kubelet.sh b/cluster/centos/node/scripts/kubelet.sh deleted file mode 100755 index 88d6b149efe..00000000000 --- a/cluster/centos/node/scripts/kubelet.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MASTER_ADDRESS=${1:-"8.8.8.18"} -NODE_ADDRESS=${2:-"8.8.8.20"} -DNS_SERVER_IP=${3:-"192.168.3.100"} -DNS_DOMAIN=${4:-"cluster.local"} -KUBECONFIG_DIR=${KUBECONFIG_DIR:-/opt/kubernetes/cfg} - -# Generate a kubeconfig file -cat < "${KUBECONFIG_DIR}/kubelet.kubeconfig" -apiVersion: v1 -kind: Config -clusters: - - cluster: - server: http://${MASTER_ADDRESS}:8080/ - name: local -contexts: - - context: - cluster: local - name: local -current-context: local -EOF - -cat </opt/kubernetes/cfg/kubelet -# --logtostderr=true: log to standard error instead of files -KUBE_LOGTOSTDERR="--logtostderr=true" - -# --v=0: log level for V logs -KUBE_LOG_LEVEL="--v=4" - -# --address=0.0.0.0: The IP address for the Kubelet to serve on (set to 0.0.0.0 for all interfaces) -NODE_ADDRESS="--address=${NODE_ADDRESS}" - -# --port=10250: The port for the Kubelet to serve on. Note that "kubectl logs" will not work if you set this flag. -NODE_PORT="--port=10250" - -# --hostname-override="": If non-empty, will use this string as identification instead of the actual hostname. -NODE_HOSTNAME="--hostname-override=${NODE_ADDRESS}" - -# Path to a kubeconfig file, specifying how to connect to the API server. -KUBELET_KUBECONFIG="--kubeconfig=${KUBECONFIG_DIR}/kubelet.kubeconfig" - -# --allow-privileged=false: If true, allow containers to request privileged mode. [default=false] -KUBE_ALLOW_PRIV="--allow-privileged=false" - -# DNS info -KUBELET__DNS_IP="--cluster-dns=${DNS_SERVER_IP}" -KUBELET_DNS_DOMAIN="--cluster-domain=${DNS_DOMAIN}" - -# Add your own! -KUBELET_ARGS="" -EOF - -KUBELET_OPTS=" \${KUBE_LOGTOSTDERR} \\ - \${KUBE_LOG_LEVEL} \\ - \${NODE_ADDRESS} \\ - \${NODE_PORT} \\ - \${NODE_HOSTNAME} \\ - \${KUBELET_KUBECONFIG} \\ - \${KUBE_ALLOW_PRIV} \\ - \${KUBELET__DNS_IP} \\ - \${KUBELET_DNS_DOMAIN} \\ - \$KUBELET_ARGS" - -cat </usr/lib/systemd/system/kubelet.service -[Unit] -Description=Kubernetes Kubelet -After=docker.service -Requires=docker.service - -[Service] -EnvironmentFile=-/opt/kubernetes/cfg/kubelet -ExecStart=/opt/kubernetes/bin/kubelet ${KUBELET_OPTS} -Restart=on-failure -KillMode=process -RestartSec=15s - -[Install] -WantedBy=multi-user.target -EOF - -systemctl daemon-reload -systemctl enable kubelet -systemctl restart kubelet diff --git a/cluster/centos/node/scripts/proxy.sh b/cluster/centos/node/scripts/proxy.sh deleted file mode 100755 index 455084866ac..00000000000 --- a/cluster/centos/node/scripts/proxy.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -MASTER_ADDRESS=${1:-"8.8.8.18"} -NODE_ADDRESS=${2:-"8.8.8.20"} - -cat </opt/kubernetes/cfg/kube-proxy -# --logtostderr=true: log to standard error instead of files -KUBE_LOGTOSTDERR="--logtostderr=true" - -# --v=0: log level for V logs -KUBE_LOG_LEVEL="--v=4" - -# --hostname-override="": If non-empty, will use this string as identification instead of the actual hostname. -NODE_HOSTNAME="--hostname-override=${NODE_ADDRESS}" - -# --master="": The address of the Kubernetes API server (overrides any value in kubeconfig) -KUBE_MASTER="--master=http://${MASTER_ADDRESS}:8080" -EOF - -KUBE_PROXY_OPTS=" \${KUBE_LOGTOSTDERR} \\ - \${KUBE_LOG_LEVEL} \\ - \${NODE_HOSTNAME} \\ - \${KUBE_MASTER}" - -cat </usr/lib/systemd/system/kube-proxy.service -[Unit] -Description=Kubernetes Proxy -After=network.target - -[Service] -EnvironmentFile=-/opt/kubernetes/cfg/kube-proxy -ExecStart=/opt/kubernetes/bin/kube-proxy ${KUBE_PROXY_OPTS} -Restart=on-failure - -[Install] -WantedBy=multi-user.target -EOF - -systemctl daemon-reload -systemctl enable kube-proxy -systemctl restart kube-proxy diff --git a/cluster/centos/util.sh b/cluster/centos/util.sh deleted file mode 100755 index d8cae8cf17a..00000000000 --- a/cluster/centos/util.sh +++ /dev/null @@ -1,388 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A library of helper functions that each provider hosting Kubernetes must implement to use cluster/kube-*.sh scripts. - -# exit on any error -set -e - -SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oLogLevel=ERROR -C" - -# Use the config file specified in $KUBE_CONFIG_FILE, or default to -# config-default.sh. -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. -readonly ROOT=$(dirname "${BASH_SOURCE[0]}") -source "${ROOT}/${KUBE_CONFIG_FILE:-"config-default.sh"}" -source "$KUBE_ROOT/cluster/common.sh" - -# shellcheck disable=SC2034 # Can't tell if this is still needed or not -KUBECTL_PATH=${KUBE_ROOT}/cluster/centos/binaries/kubectl - -# Directory to be used for master and node provisioning. -KUBE_TEMP="${HOME}/kube_temp" - - -# Get master IP addresses and store in KUBE_MASTER_IP_ADDRESSES[] -# Must ensure that the following ENV vars are set: -# MASTERS -function detect-masters() { - KUBE_MASTER_IP_ADDRESSES=() - for master in ${MASTERS}; do - KUBE_MASTER_IP_ADDRESSES+=("${master#*@}") - done - echo "KUBE_MASTERS: ${MASTERS}" 1>&2 - echo "KUBE_MASTER_IP_ADDRESSES: [${KUBE_MASTER_IP_ADDRESSES[*]}]" 1>&2 -} - -# Get node IP addresses and store in KUBE_NODE_IP_ADDRESSES[] -function detect-nodes() { - KUBE_NODE_IP_ADDRESSES=() - for node in ${NODES}; do - KUBE_NODE_IP_ADDRESSES+=("${node#*@}") - done - echo "KUBE_NODE_IP_ADDRESSES: [${KUBE_NODE_IP_ADDRESSES[*]}]" 1>&2 -} - -# Verify prereqs on host machine -function verify-prereqs() { - local rc - rc=0 - ssh-add -L 1> /dev/null 2> /dev/null || rc="$?" - # "Could not open a connection to your authentication agent." - if [[ "${rc}" -eq 2 ]]; then - eval "$(ssh-agent)" > /dev/null - trap-add "kill ${SSH_AGENT_PID}" EXIT - fi - rc=0 - ssh-add -L 1> /dev/null 2> /dev/null || rc="$?" - # "The agent has no identities." - if [[ "${rc}" -eq 1 ]]; then - # Try adding one of the default identities, with or without passphrase. - ssh-add || true - fi - rc=0 - # Expect at least one identity to be available. - if ! ssh-add -L 1> /dev/null 2> /dev/null; then - echo "Could not find or add an SSH identity." - echo "Please start ssh-agent, add your identity, and retry." - exit 1 - fi -} - -# Install handler for signal trap -function trap-add { - local handler="$1" - local signal="${2-EXIT}" - local cur - - cur="$(eval "sh -c 'echo \$3' -- $(trap -p "${signal}")")" - if [[ -n "${cur}" ]]; then - handler="${cur}; ${handler}" - fi - - # shellcheck disable=SC2064 # Early expansion is intentional here. - trap "${handler}" "${signal}" -} - -# Validate a kubernetes cluster -function validate-cluster() { - # by default call the generic validate-cluster.sh script, customizable by - # any cluster provider if this does not fit. - set +e - if ! "${KUBE_ROOT}/cluster/validate-cluster.sh"; then - for master in ${MASTERS}; do - troubleshoot-master "${master}" - done - for node in ${NODES}; do - troubleshoot-node "${node}" - done - exit 1 - fi - set -e -} - -# Instantiate a kubernetes cluster -function kube-up() { - make-ca-cert - - local num_infra=0 - for master in ${MASTERS}; do - provision-master "${master}" "infra${num_infra}" - ((++num_infra)) - done - - for master in ${MASTERS}; do - post-provision-master "${master}" - done - - for node in ${NODES}; do - provision-node "${node}" - done - - detect-masters - - # set CONTEXT and KUBE_SERVER values for create-kubeconfig() and get-password() - export CONTEXT="centos" - export KUBE_SERVER="http://${MASTER_ADVERTISE_ADDRESS}:8080" - source "${KUBE_ROOT}/cluster/common.sh" - - # set kubernetes user and password - get-password - create-kubeconfig -} - -# Delete a kubernetes cluster -function kube-down() { - for master in ${MASTERS}; do - tear-down-master "${master}" - done - - for node in ${NODES}; do - tear-down-node "${node}" - done -} - -function troubleshoot-master() { - # Troubleshooting on master if all required daemons are active. - echo "[INFO] Troubleshooting on master $1" - local -a required_daemon=("kube-apiserver" "kube-controller-manager" "kube-scheduler") - local daemon - local daemon_status - printf "%-24s %-10s \n" "PROCESS" "STATUS" - for daemon in "${required_daemon[@]}"; do - local rc=0 - kube-ssh "${1}" "sudo systemctl is-active ${daemon}" >/dev/null 2>&1 || rc="$?" - if [[ "${rc}" -ne "0" ]]; then - daemon_status="inactive" - else - daemon_status="active" - fi - printf "%-24s %s\n" "${daemon}" ${daemon_status} - done - printf "\n" -} - -function troubleshoot-node() { - # Troubleshooting on node if all required daemons are active. - echo "[INFO] Troubleshooting on node ${1}" - local -a required_daemon=("kube-proxy" "kubelet" "docker" "flannel") - local daemon - local daemon_status - printf "%-24s %-10s \n" "PROCESS" "STATUS" - for daemon in "${required_daemon[@]}"; do - local rc=0 - kube-ssh "${1}" "sudo systemctl is-active ${daemon}" >/dev/null 2>&1 || rc="$?" - if [[ "${rc}" -ne "0" ]]; then - daemon_status="inactive" - else - daemon_status="active" - fi - printf "%-24s %s\n" "${daemon}" ${daemon_status} - done - printf "\n" -} - -# Clean up on master -function tear-down-master() { -echo "[INFO] tear-down-master on $1" - for service_name in etcd kube-apiserver kube-controller-manager kube-scheduler ; do - service_file="/usr/lib/systemd/system/${service_name}.service" - kube-ssh "$1" " \ - if [[ -f $service_file ]]; then \ - sudo systemctl stop $service_name; \ - sudo systemctl disable $service_name; \ - sudo rm -f $service_file; \ - fi" - done - kube-ssh "${1}" "sudo rm -rf /opt/kubernetes" - kube-ssh "${1}" "sudo rm -rf /srv/kubernetes" - kube-ssh "${1}" "sudo rm -rf ${KUBE_TEMP}" - kube-ssh "${1}" "sudo rm -rf /var/lib/etcd" -} - -# Clean up on node -function tear-down-node() { -echo "[INFO] tear-down-node on $1" - for service_name in kube-proxy kubelet docker flannel ; do - service_file="/usr/lib/systemd/system/${service_name}.service" - kube-ssh "$1" " \ - if [[ -f $service_file ]]; then \ - sudo systemctl stop $service_name; \ - sudo systemctl disable $service_name; \ - sudo rm -f $service_file; \ - fi" - done - kube-ssh "$1" "sudo rm -rf /run/flannel" - kube-ssh "$1" "sudo rm -rf /opt/kubernetes" - kube-ssh "$1" "sudo rm -rf /srv/kubernetes" - kube-ssh "$1" "sudo rm -rf ${KUBE_TEMP}" -} - -# Generate the CA certificates for k8s components -function make-ca-cert() { - echo "[INFO] make-ca-cert" - bash "${ROOT}/make-ca-cert.sh" "${MASTER_ADVERTISE_IP}" "IP:${MASTER_ADVERTISE_IP},IP:${SERVICE_CLUSTER_IP_RANGE%.*}.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local" -} - -# Provision master -# -# Assumed vars: -# $1 (master) -# $2 (etcd_name) -# KUBE_TEMP -# ETCD_SERVERS -# ETCD_INITIAL_CLUSTER -# SERVICE_CLUSTER_IP_RANGE -# MASTER_ADVERTISE_ADDRESS -function provision-master() { - echo "[INFO] Provision master on $1" - local master="$1" - local master_ip="${master#*@}" - local etcd_name="$2" - ensure-setup-dir "${master}" - ensure-etcd-cert "${etcd_name}" "${master_ip}" - - kube-scp "${master}" "${ROOT}/ca-cert ${ROOT}/binaries/master ${ROOT}/master ${ROOT}/config-default.sh ${ROOT}/util.sh" "${KUBE_TEMP}" - kube-scp "${master}" "${ROOT}/etcd-cert/ca.pem \ - ${ROOT}/etcd-cert/client.pem \ - ${ROOT}/etcd-cert/client-key.pem \ - ${ROOT}/etcd-cert/server-${etcd_name}.pem \ - ${ROOT}/etcd-cert/server-${etcd_name}-key.pem \ - ${ROOT}/etcd-cert/peer-${etcd_name}.pem \ - ${ROOT}/etcd-cert/peer-${etcd_name}-key.pem" "${KUBE_TEMP}/etcd-cert" - kube-ssh "${master}" " \ - sudo rm -rf /opt/kubernetes/bin; \ - sudo cp -r ${KUBE_TEMP}/master/bin /opt/kubernetes; \ - sudo mkdir -p /srv/kubernetes/; sudo cp -f ${KUBE_TEMP}/ca-cert/* /srv/kubernetes/; \ - sudo mkdir -p /srv/kubernetes/etcd; sudo cp -f ${KUBE_TEMP}/etcd-cert/* /srv/kubernetes/etcd/; \ - sudo chmod -R +x /opt/kubernetes/bin; \ - sudo ln -sf /opt/kubernetes/bin/* /usr/local/bin/; \ - sudo bash ${KUBE_TEMP}/master/scripts/etcd.sh ${etcd_name} ${master_ip} ${ETCD_INITIAL_CLUSTER}; \ - sudo bash ${KUBE_TEMP}/master/scripts/apiserver.sh ${master_ip} ${ETCD_SERVERS} ${SERVICE_CLUSTER_IP_RANGE} ${ADMISSION_CONTROL}; \ - sudo bash ${KUBE_TEMP}/master/scripts/controller-manager.sh ${MASTER_ADVERTISE_ADDRESS}; \ - sudo bash ${KUBE_TEMP}/master/scripts/scheduler.sh ${MASTER_ADVERTISE_ADDRESS}" -} - -# Post-provision master, run after all masters were provisioned -# -# Assumed vars: -# $1 (master) -# KUBE_TEMP -# ETCD_SERVERS -# FLANNEL_NET -function post-provision-master() { - echo "[INFO] Post provision master on $1" - local master=$1 - kube-ssh "${master}" " \ - sudo bash ${KUBE_TEMP}/master/scripts/flannel.sh ${ETCD_SERVERS} ${FLANNEL_NET}; \ - sudo bash ${KUBE_TEMP}/master/scripts/post-etcd.sh" -} - -# Provision node -# -# Assumed vars: -# $1 (node) -# KUBE_TEMP -# ETCD_SERVERS -# FLANNEL_NET -# MASTER_ADVERTISE_ADDRESS -# DOCKER_OPTS -# DNS_SERVER_IP -# DNS_DOMAIN -function provision-node() { - echo "[INFO] Provision node on $1" - local node=$1 - local node_ip=${node#*@} - local dns_ip=${DNS_SERVER_IP#*@} - # shellcheck disable=SC2153 # DNS_DOMAIN sourced from external file - local dns_domain=${DNS_DOMAIN#*@} - ensure-setup-dir "${node}" - - kube-scp "${node}" "${ROOT}/binaries/node ${ROOT}/node ${ROOT}/config-default.sh ${ROOT}/util.sh" "${KUBE_TEMP}" - kube-scp "${node}" "${ROOT}/etcd-cert/ca.pem \ - ${ROOT}/etcd-cert/client.pem \ - ${ROOT}/etcd-cert/client-key.pem" "${KUBE_TEMP}/etcd-cert" - kube-ssh "${node}" " \ - rm -rf /opt/kubernetes/bin; \ - sudo cp -r ${KUBE_TEMP}/node/bin /opt/kubernetes; \ - sudo chmod -R +x /opt/kubernetes/bin; \ - sudo mkdir -p /srv/kubernetes/etcd; sudo cp -f ${KUBE_TEMP}/etcd-cert/* /srv/kubernetes/etcd/; \ - sudo ln -s /opt/kubernetes/bin/* /usr/local/bin/; \ - sudo mkdir -p /srv/kubernetes/etcd; sudo cp -f ${KUBE_TEMP}/etcd-cert/* /srv/kubernetes/etcd/; \ - sudo bash ${KUBE_TEMP}/node/scripts/flannel.sh ${ETCD_SERVERS} ${FLANNEL_NET}; \ - sudo bash ${KUBE_TEMP}/node/scripts/docker.sh \"${DOCKER_OPTS}\"; \ - sudo bash ${KUBE_TEMP}/node/scripts/kubelet.sh ${MASTER_ADVERTISE_ADDRESS} ${node_ip} ${dns_ip} ${dns_domain}; \ - sudo bash ${KUBE_TEMP}/node/scripts/proxy.sh ${MASTER_ADVERTISE_ADDRESS}" -} - -# Create dirs that'll be used during setup on target machine. -# -# Assumed vars: -# KUBE_TEMP -function ensure-setup-dir() { - kube-ssh "${1}" "mkdir -p ${KUBE_TEMP}; \ - mkdir -p ${KUBE_TEMP}/etcd-cert; \ - sudo mkdir -p /opt/kubernetes/bin; \ - sudo mkdir -p /opt/kubernetes/cfg" -} - -# Generate certificates for etcd cluster -# -# Assumed vars: -# $1 (etcd member name) -# $2 (master ip) -function ensure-etcd-cert() { - local etcd_name="$1" - local master_ip="$2" - local cert_dir="${ROOT}/etcd-cert" - - if [[ ! -r "${cert_dir}/client.pem" || ! -r "${cert_dir}/client-key.pem" ]]; then - generate-etcd-cert "${cert_dir}" "${master_ip}" "client" "client" - fi - - generate-etcd-cert "${cert_dir}" "${master_ip}" "server" "server-${etcd_name}" - generate-etcd-cert "${cert_dir}" "${master_ip}" "peer" "peer-${etcd_name}" -} - -# Run command over ssh -function kube-ssh() { - local host="$1" - shift - ssh "${SSH_OPTS}" -t "${host}" "$@" >/dev/null 2>&1 -} - -# Copy file recursively over ssh -function kube-scp() { - local host="$1" - local src=("$2") - local dst="$3" - scp -r "${SSH_OPTS}" "${src[*]}" "${host}:${dst}" -} - -# Ensure that we have a password created for validating to the master. Will -# read from kubeconfig if available. -# -# Vars set: -# KUBE_USER -# KUBE_PASSWORD -function get-password { - load-or-gen-kube-basicauth - if [[ -z "${KUBE_USER}" || -z "${KUBE_PASSWORD}" ]]; then - KUBE_USER="admin" - KUBE_PASSWORD=$(python -c 'import string,random; '\ - 'print("".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)))') - fi -} diff --git a/cluster/clientbin.sh b/cluster/clientbin.sh index 078729eb9d1..dc4f673f17c 100755 --- a/cluster/clientbin.sh +++ b/cluster/clientbin.sh @@ -18,7 +18,7 @@ set -o errexit set -o nounset set -o pipefail -KUBE_ROOT=${KUBE_ROOT:-$(dirname "${BASH_SOURCE}")/..} +KUBE_ROOT=${KUBE_ROOT:-$(dirname "${BASH_SOURCE[0]}")/..} # Detect the OS name/arch so that we can find our binary case "$(uname -s)" in diff --git a/cluster/common.sh b/cluster/common.sh index 2acc6b26002..d7b19b01cf6 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -20,7 +20,7 @@ set -o errexit set -o nounset set -o pipefail -KUBE_ROOT=$(cd $(dirname "${BASH_SOURCE}")/.. && pwd) +KUBE_ROOT=$(cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd) DEFAULT_KUBECONFIG="${HOME:-.}/.kube/config" diff --git a/cluster/gce/OWNERS b/cluster/gce/OWNERS index eef8316838b..8dffe265b8e 100644 --- a/cluster/gce/OWNERS +++ b/cluster/gce/OWNERS @@ -2,6 +2,7 @@ reviewers: - bowei + - cjcullen - gmarek - jszczepkowski - vishh @@ -9,8 +10,10 @@ reviewers: - MaciekPytel - jingax10 - yujuhong + - zmerlynn approvers: - bowei + - cjcullen - gmarek - jszczepkowski - vishh @@ -18,3 +21,4 @@ approvers: - MaciekPytel - jingax10 - yujuhong + - zmerlynn diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index c916f1f4f3b..70ebda6e04e 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1593,6 +1593,10 @@ function start-kube-apiserver { params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}" fi params+=" --secure-port=443" + if [[ "${ENABLE_APISERVER_INSECURE_PORT:-true}" != "true" ]]; then + # Default is :8080 + params+=" --insecure-port=0" + fi params+=" --tls-cert-file=${APISERVER_SERVER_CERT_PATH}" params+=" --tls-private-key-file=${APISERVER_SERVER_KEY_PATH}" params+=" --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname" @@ -1878,7 +1882,6 @@ function start-kube-apiserver { sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}" sed -i -e "s@{{liveness_probe_initial_delay}}@${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${src_file}" sed -i -e "s@{{secure_port}}@443@g" "${src_file}" - sed -i -e "s@{{secure_port}}@8080@g" "${src_file}" sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}" sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}" sed -i -e "s@{{webhook_authn_config_mount}}@${webhook_authn_config_mount}@g" "${src_file}" diff --git a/cluster/gce/gci/mounter/stage-upload.sh b/cluster/gce/gci/mounter/stage-upload.sh index f3359447c39..ae1657e283a 100755 --- a/cluster/gce/gci/mounter/stage-upload.sh +++ b/cluster/gce/gci/mounter/stage-upload.sh @@ -39,29 +39,30 @@ ACI_DIR=${STAGING_DIR}/gci-mounter CWD=${PWD} # Cleanup the temporary directories -function cleanup { - rm -rf ${DOWNLOAD_DIR} - rm -rf ${STAGING_DIR} - cd ${CWD} +cleanup() { + rm -rf "${DOWNLOAD_DIR}" + rm -rf "${STAGING_DIR}" + cd "${CWD}" } # Delete temporary directories on exit trap cleanup EXIT -mkdir ${ACI_DIR} +mkdir "${ACI_DIR}" # Convert docker image to aci and stage it echo "Downloading docker2aci ${DOCKER2ACI_VERSION}" -wget "https://github.com/appc/docker2aci/releases/download/${DOCKER2ACI_VERSION}/docker2aci-${DOCKER2ACI_VERSION}.tar.gz" &> /dev/null +wget "https://github.com/appc/docker2aci/releases/download/${DOCKER2ACI_VERSION}/docker2aci-${DOCKER2ACI_VERSION}.tar.gz" >/dev/null 2>&1 echo "Extracting docker2aci ${DOCKER2ACI_VERSION}" tar xzf docker2aci-${DOCKER2ACI_VERSION}.tar.gz -ACI_IMAGE=$(${DOWNLOAD_DIR}/docker2aci-${DOCKER2ACI_VERSION}/docker2aci ${DOCKER_IMAGE} 2>/dev/null | tail -n 1) -cp ${ACI_IMAGE} ${ACI_DIR}/${MOUNTER_ACI_IMAGE} +ACI_IMAGE=$("${DOWNLOAD_DIR}/docker2aci-${DOCKER2ACI_VERSION}/docker2aci" "${DOCKER_IMAGE}" 2>/dev/null | tail -n 1) +cp "${ACI_IMAGE}" "${ACI_DIR}/${MOUNTER_ACI_IMAGE}" # Upload the contents to gcs echo "Uploading gci mounter ACI in ${ACI_DIR} to ${MOUNTER_GCS_DIR}" -gsutil cp ${ACI_DIR}/${MOUNTER_ACI_IMAGE} ${MOUNTER_GCS_DIR} +gsutil cp "${ACI_DIR}/${MOUNTER_ACI_IMAGE}" "${MOUNTER_GCS_DIR}" echo "Upload completed" echo "Updated gci-mounter ACI version and SHA1 in cluster/gce/gci/configure.sh" -echo "${MOUNTER_ACI_IMAGE} hash: $(sha1sum ${ACI_DIR}/${MOUNTER_ACI_IMAGE})" +ACI_HASH=$(sha1sum "${ACI_DIR}/${MOUNTER_ACI_IMAGE}") +echo "${MOUNTER_ACI_IMAGE} hash: ${ACI_HASH}" diff --git a/cluster/gce/gci/shutdown.sh b/cluster/gce/gci/shutdown.sh index dab0d34c62f..e50d4efefeb 100755 --- a/cluster/gce/gci/shutdown.sh +++ b/cluster/gce/gci/shutdown.sh @@ -16,7 +16,7 @@ # A script that let's gci preemptible nodes gracefully terminate in the event of a VM shutdown. preemptible=$(curl "http://metadata.google.internal/computeMetadata/v1/instance/scheduling/preemptible" -H "Metadata-Flavor: Google") -if [ ${preemptible} == "TRUE" ]; then +if [ "${preemptible}" == "TRUE" ]; then echo "Shutting down! Sleeping for a minute to let the node gracefully terminate" # https://cloud.google.com/compute/docs/instances/stopping-or-deleting-an-instance#delete_timeout sleep 30 diff --git a/cluster/gce/list-resources.sh b/cluster/gce/list-resources.sh index 3e33075db8d..aca87c57650 100755 --- a/cluster/gce/list-resources.sh +++ b/cluster/gce/list-resources.sh @@ -49,20 +49,20 @@ function gcloud-list() { local attempt=1 local result="" while true; do - if result=$(gcloud ${group} ${resource} list --project=${PROJECT} ${filter:+--filter="$filter"} ${@:4}); then - if [[ ! -z "${GREP_REGEX}" ]]; then + if result=$(gcloud "${group}" "${resource}" list --project="${PROJECT}" ${filter:+--filter="$filter"} "${@:4}"); then + if [[ -n "${GREP_REGEX:-}" ]]; then result=$(echo "${result}" | grep "${GREP_REGEX}" || true) fi echo "${result}" return fi echo -e "Attempt ${attempt} failed to list ${resource}. Retrying." >&2 - attempt=$(($attempt+1)) + attempt=$((attempt + 1)) if [[ ${attempt} -gt 5 ]]; then echo -e "List ${resource} failed!" >&2 exit 2 fi - sleep $((5*${attempt})) + sleep $((5 * attempt)) done } diff --git a/cluster/gce/manifests/kube-apiserver.manifest b/cluster/gce/manifests/kube-apiserver.manifest index d045c844c47..acbdcee0a55 100644 --- a/cluster/gce/manifests/kube-apiserver.manifest +++ b/cluster/gce/manifests/kube-apiserver.manifest @@ -32,8 +32,9 @@ {{container_env}} "livenessProbe": { "httpGet": { + "scheme": "HTTPS", "host": "127.0.0.1", - "port": 8080, + "port": {{secure_port}}, "path": "/healthz?exclude=etcd" }, "initialDelaySeconds": {{liveness_probe_initial_delay}}, @@ -41,8 +42,9 @@ }, "readinessProbe": { "httpGet": { + "scheme": "HTTPS", "host": "127.0.0.1", - "port": 8080, + "port": {{secure_port}}, "path": "/healthz" }, "periodSeconds": 1, diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 2175a89b627..2eea8085dc2 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -2031,7 +2031,9 @@ function create-node-template() { if [[ "${os}" == 'linux' ]]; then node_image_flags="--image-project ${NODE_IMAGE_PROJECT} --image ${NODE_IMAGE}" elif [[ "${os}" == 'windows' ]]; then - node_image_flags="--image-project ${WINDOWS_NODE_IMAGE_PROJECT} --image-family ${WINDOWS_NODE_IMAGE_FAMILY}" + # TODO(pjh): revert back to using WINDOWS_NODE_IMAGE_FAMILY instead of + # pinning to the v20190312 image once #76666 is resolved. + node_image_flags="--image-project ${WINDOWS_NODE_IMAGE_PROJECT} --image=windows-server-1809-dc-core-for-containers-v20190312" else echo "Unknown OS ${os}" >&2 exit 1 diff --git a/cluster/gce/windows/configure.ps1 b/cluster/gce/windows/configure.ps1 index a3d9d896be8..0235cb9c618 100644 --- a/cluster/gce/windows/configure.ps1 +++ b/cluster/gce/windows/configure.ps1 @@ -112,10 +112,10 @@ try { Set-EnvironmentVars Create-Directories Download-HelperScripts - # Disable Stackdrver logging until issue is fixed. - # InstallAndStart-LoggingAgent + InstallAndStart-LoggingAgent Create-DockerRegistryKey + Configure-Dockerd DownloadAndInstall-KubernetesBinaries Create-NodePki Create-KubeletKubeconfig diff --git a/cluster/gce/windows/k8s-node-setup.psm1 b/cluster/gce/windows/k8s-node-setup.psm1 index 9abd5bab4d0..6f20f65d383 100644 --- a/cluster/gce/windows/k8s-node-setup.psm1 +++ b/cluster/gce/windows/k8s-node-setup.psm1 @@ -270,11 +270,13 @@ function Disable-WindowsDefender { # Creates directories where other functions in this module will read and write # data. # Note: C:\tmp is required for running certain kubernetes tests. +# C:\var\log is used by kubelet to stored container logs and also +# hard-coded in the fluentd/stackdriver config for log collection. function Create-Directories { Log-Output "Creating ${env:K8S_DIR} and its subdirectories." ForEach ($dir in ("${env:K8S_DIR}", "${env:NODE_DIR}", "${env:LOGS_DIR}", "${env:CNI_DIR}", "${env:CNI_CONFIG_DIR}", "${env:MANIFESTS_DIR}", - "${env:PKI_DIR}"), "C:\tmp") { + "${env:PKI_DIR}"), "C:\tmp", "C:\var\log") { mkdir -Force $dir } } @@ -1056,10 +1058,25 @@ function Create-DockerRegistryKey { Remove-Item -Force -Recurse ${tmp_dir} } +# Configure Docker daemon and restart the service. +function Configure-Dockerd { + Set-Content "C:\ProgramData\docker\config\daemon.json" @' +{ + "log-driver": "json-file", + "log-opts": { + "max-size": "1m", + "max-file": "5" + } +} +'@ + + Restart-Service Docker +} + # TODO(pjh): move the Stackdriver logging agent code below into a separate # module; it was put here temporarily to avoid disrupting the file layout in # the K8s release machinery. -$STACKDRIVER_VERSION = 'v1-8' +$STACKDRIVER_VERSION = 'v1-9' $STACKDRIVER_ROOT = 'C:\Program Files (x86)\Stackdriver' # Install and start the Stackdriver logging agent according to @@ -1123,9 +1140,6 @@ function InstallAndStart-LoggingAgent { Remove-Item -Force -Recurse $tmp_dir } -# TODO(yujuhong): -# - Collect kubelet/kube-proxy logs. -# - Add tag for kubernetes node name. $FLUENTD_CONFIG = @' # This configuration file for Fluentd is used to watch changes to kubernetes # container logs in the directory /var/lib/docker/containers/ and submit the @@ -1184,6 +1198,34 @@ $FLUENTD_CONFIG = @' read_from_head true +# Example: +# I0204 07:32:30.020537 3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537] + + @type tail + format multiline + multiline_flush_interval 5s + format_firstline /^\w\d{4}/ + format1 /^(?\w)(?