diff --git a/build/root/WORKSPACE b/build/root/WORKSPACE index 6f3a3df39ce..c6ea40e4eca 100644 --- a/build/root/WORKSPACE +++ b/build/root/WORKSPACE @@ -1,21 +1,23 @@ +load("//build:workspace_mirror.bzl", "mirror") + http_archive( name = "io_bazel_rules_go", sha256 = "4b2c61795ac2eefcb28f3eb8e1cb2d8fb3c2eafa0f6712473bc5f93728f38758", - urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.10.2/rules_go-0.10.2.tar.gz"], + urls = mirror("https://github.com/bazelbuild/rules_go/releases/download/0.10.2/rules_go-0.10.2.tar.gz"), ) http_archive( name = "io_kubernetes_build", sha256 = "007774f06536059f3f782d1a092bddc625d88c17f20bbe731cea844a52485b11", strip_prefix = "repo-infra-97099dccc8807e9159dc28f374a8f0602cab07e1", - urls = ["https://github.com/kubernetes/repo-infra/archive/97099dccc8807e9159dc28f374a8f0602cab07e1.tar.gz"], + urls = mirror("https://github.com/kubernetes/repo-infra/archive/97099dccc8807e9159dc28f374a8f0602cab07e1.tar.gz"), ) http_archive( name = "bazel_skylib", sha256 = "bbccf674aa441c266df9894182d80de104cabd19be98be002f6d478aaa31574d", strip_prefix = "bazel-skylib-2169ae1c374aab4a09aa90e65efe1a3aad4e279b", - urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"], + urls = mirror("https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"), ) ETCD_VERSION = "3.2.18" @@ -25,14 +27,14 @@ new_http_archive( build_file = "third_party/etcd.BUILD", sha256 = "b729db0732448064271ea6fdcb901773c4fe917763ca07776f22d0e5e0bd4097", strip_prefix = "etcd-v%s-linux-amd64" % ETCD_VERSION, - urls = ["https://github.com/coreos/etcd/releases/download/v%s/etcd-v%s-linux-amd64.tar.gz" % (ETCD_VERSION, ETCD_VERSION)], + urls = mirror("https://github.com/coreos/etcd/releases/download/v%s/etcd-v%s-linux-amd64.tar.gz" % (ETCD_VERSION, ETCD_VERSION)), ) http_archive( name = "io_bazel_rules_docker", sha256 = "c440717ee9b1b2f4a1e9bf5622539feb5aef9db83fc1fa1517818f13c041b0be", strip_prefix = "rules_docker-8bbe2a8abd382641e65ff7127a3700a8530f02ce", - urls = ["https://github.com/bazelbuild/rules_docker/archive/8bbe2a8abd382641e65ff7127a3700a8530f02ce.tar.gz"], + urls = mirror("https://github.com/bazelbuild/rules_docker/archive/8bbe2a8abd382641e65ff7127a3700a8530f02ce.tar.gz"), ) load("@bazel_skylib//:lib.bzl", "versions") @@ -53,7 +55,7 @@ docker_repositories() http_file( name = "kubernetes_cni", sha256 = "f04339a21b8edf76d415e7f17b620e63b8f37a76b2f706671587ab6464411f2d", - url = "https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.6.0.tgz", + urls = mirror("https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.6.0.tgz"), ) docker_pull( @@ -79,3 +81,6 @@ docker_pull( repository = "library/busybox", tag = "latest", # ignored, but kept here for documentation ) + +load("//build:workspace_mirror.bzl", "export_urls") +export_urls("workspace_urls") diff --git a/build/workspace_mirror.bzl b/build/workspace_mirror.bzl new file mode 100644 index 00000000000..bbc02c88a76 --- /dev/null +++ b/build/workspace_mirror.bzl @@ -0,0 +1,53 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +prefix = "https://storage.googleapis.com/k8s-bazel-cache/" + +def mirror(url): + """Try downloading a URL from a GCS mirror first, then from the original. + + Update the GCS bucket using bazel run //hack:update-mirror""" + return [prefix + url, url] + +# This function only gives proper results when executed from WORKSPACE, +# but the data is needed in sh_binary, which can only be in a BUILD file. +# Thus, it is be exported by a repository_rule (which executes in WORKSPACE) +# to be used by the sh_binary. +def mirror_urls(): + urls = [] + for k, v in native.existing_rules().items(): + us = list(v.get('urls', [])) + if 'url' in v: + us.append(v['url']) + for u in us: + if u and not u.startswith(prefix): + urls.append(u) + return sorted(urls) + +def export_urls_impl(repo_ctx): + repo_ctx.file(repo_ctx.path("BUILD.bazel"), """ +exports_files(glob(["**"]), visibility=["//visibility:public"]) +""") + repo_ctx.file(repo_ctx.path("urls.txt"), content="\n".join(repo_ctx.attr.urls)) + +_export_urls = repository_rule( + attrs = { + "urls": attr.string_list(mandatory = True), + }, + local = True, + implementation = export_urls_impl, +) + +def export_urls(name): + return _export_urls(name=name, urls=mirror_urls()) diff --git a/hack/BUILD b/hack/BUILD index d562a44883b..f502e144403 100644 --- a/hack/BUILD +++ b/hack/BUILD @@ -72,3 +72,10 @@ go_library( srcs = ["e2e.go"], importpath = "k8s.io/kubernetes/hack", ) + +sh_binary( + name = "update-mirror", + srcs = ["update-workspace-mirror.sh"], + args = ["$(location @workspace_urls//:urls.txt)"], + data = ["@workspace_urls//:urls.txt"], +) diff --git a/hack/update-workspace-mirror.sh b/hack/update-workspace-mirror.sh new file mode 100755 index 00000000000..9b7c456c994 --- /dev/null +++ b/hack/update-workspace-mirror.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +if [[ $# -ne 1 ]]; then + echo 'use "bazel run //build:update-mirror"' + echo "(usage: $0 )" + exit 1 +fi + +BUCKET="gs://k8s-bazel-cache" + +gsutil acl get "${BUCKET}" > /dev/null + +tmpfile=$(mktemp bazel_workspace_mirror.XXXXXX) +trap "rm ${tmpfile}" EXIT +cat "$1" | while read url; do + echo "${url}" + if gsutil ls "${BUCKET}/${url}" &> /dev/null; then + echo present + else + echo missing + if curl -fLag "${url}" > "${tmpfile}"; then + gsutil cp -a public-read "${tmpfile}" "${BUCKET}/${url}" + fi + fi +done