mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-23 11:50:44 +00:00
Merge pull request #36361 from ivan4th/build-container-rsync-optimizations
Automatic merge from submit-queue (batch tested with PRs 38277, 36361, 38452) Add options for build container rsync optimization KUBE_RSYNC_COMPRESS env var sets rsync compression level. KUBE_RSYNC_GENERATED_TO_BUILD_CONTAINER env var disables rsyncing generated files to build containers. Why KUBE_RSYNC_COMPRESS is needed -- from rsync manual on `--compress` option (implied by non-zero `--compress-level`): > Note that this option typically achieves better compression ratios than can be achieved by using a compressing remote shell or a compressing transport because it takes advantage of the implicit information in the matching data blocks that are not explicitly sent over the connection. Use case for `KUBE_RSYNC_GENERATED_TO_BUILD_CONTAINER`: when you sometimes build stuff locally (e.g. `make WHAT=cmd/kubectl`) and sometimes do it on remote docker (`build-tools/run.sh make WHAT=cmd/hyperkube`), local builds touch generated files which causes them to be rsynced to the build data container, which may slow down the builds. Still, I'm not sure whether local->remote rsync of generated files is useful (e.g. someone may want to edit generated files for debugging purposes?), so I made not rsyncing these files an option instead of forcing such behavior.
This commit is contained in:
commit
48cae78257
@ -656,6 +656,22 @@ function kube::build::stop_rsyncd_container() {
|
|||||||
kube::build::destroy_container "${KUBE_RSYNC_CONTAINER_NAME}"
|
kube::build::destroy_container "${KUBE_RSYNC_CONTAINER_NAME}"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function kube::build::rsync {
|
||||||
|
local -a rsync_opts=(
|
||||||
|
--archive
|
||||||
|
--prune-empty-dirs
|
||||||
|
--password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password"
|
||||||
|
)
|
||||||
|
if (( ${KUBE_VERBOSE} >= 6 )); then
|
||||||
|
rsync_opts+=("-iv")
|
||||||
|
fi
|
||||||
|
if (( ${KUBE_RSYNC_COMPRESS} > 0 )); then
|
||||||
|
rsync_opts+=("--compress-level=${KUBE_RSYNC_COMPRESS}")
|
||||||
|
fi
|
||||||
|
V=3 kube::log::status "Running rsync"
|
||||||
|
rsync "${rsync_opts[@]}" "$@"
|
||||||
|
}
|
||||||
|
|
||||||
# This will launch rsyncd in a container and then sync the source tree to the
|
# This will launch rsyncd in a container and then sync the source tree to the
|
||||||
# container over the local network.
|
# container over the local network.
|
||||||
function kube::build::sync_to_container() {
|
function kube::build::sync_to_container() {
|
||||||
@ -663,26 +679,22 @@ function kube::build::sync_to_container() {
|
|||||||
|
|
||||||
kube::build::start_rsyncd_container
|
kube::build::start_rsyncd_container
|
||||||
|
|
||||||
local rsync_extra=""
|
|
||||||
if (( ${KUBE_VERBOSE} >= 6 )); then
|
|
||||||
rsync_extra="-iv"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# rsync filters are a bit confusing. Here we are syncing everything except
|
# rsync filters are a bit confusing. Here we are syncing everything except
|
||||||
# output only directories and things that are not necessary like the git
|
# output only directories and things that are not necessary like the git
|
||||||
# directory. The '- /' filter prevents rsync from trying to set the
|
# directory and generated files. The '- /' filter prevents rsync
|
||||||
# uid/gid/perms on the root of the sync tree.
|
# from trying to set the uid/gid/perms on the root of the sync tree.
|
||||||
V=3 kube::log::status "Running rsync"
|
# As an exception, we need to sync generated files in staging/, because
|
||||||
rsync ${rsync_extra} \
|
# they will not be re-generated by 'make'.
|
||||||
--archive \
|
kube::build::rsync \
|
||||||
--delete \
|
--delete \
|
||||||
--prune-empty-dirs \
|
--filter='+ /staging/**' \
|
||||||
--password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password" \
|
|
||||||
--filter='- /.git/' \
|
--filter='- /.git/' \
|
||||||
--filter='- /.make/' \
|
--filter='- /.make/' \
|
||||||
--filter='- /_tmp/' \
|
--filter='- /_tmp/' \
|
||||||
--filter='- /_output/' \
|
--filter='- /_output/' \
|
||||||
--filter='- /' \
|
--filter='- /' \
|
||||||
|
--filter='- zz_generated.*' \
|
||||||
|
--filter='- generated.proto' \
|
||||||
"${KUBE_ROOT}/" "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/"
|
"${KUBE_ROOT}/" "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/"
|
||||||
|
|
||||||
kube::build::stop_rsyncd_container
|
kube::build::stop_rsyncd_container
|
||||||
@ -707,11 +719,7 @@ function kube::build::copy_output() {
|
|||||||
#
|
#
|
||||||
# We are looking to copy out all of the built binaries along with various
|
# We are looking to copy out all of the built binaries along with various
|
||||||
# generated files.
|
# generated files.
|
||||||
V=3 kube::log::status "Running rsync"
|
kube::build::rsync \
|
||||||
rsync ${rsync_extra} \
|
|
||||||
--archive \
|
|
||||||
--prune-empty-dirs \
|
|
||||||
--password-file="${LOCAL_OUTPUT_BUILD_CONTEXT}/rsyncd.password" \
|
|
||||||
--filter='- /vendor/' \
|
--filter='- /vendor/' \
|
||||||
--filter='- /_temp/' \
|
--filter='- /_temp/' \
|
||||||
--filter='+ /_output/dockerized/bin/**' \
|
--filter='+ /_output/dockerized/bin/**' \
|
||||||
|
@ -25,6 +25,10 @@ KUBE_OUTPUT_SUBPATH="${KUBE_OUTPUT_SUBPATH:-_output/local}"
|
|||||||
KUBE_OUTPUT="${KUBE_ROOT}/${KUBE_OUTPUT_SUBPATH}"
|
KUBE_OUTPUT="${KUBE_ROOT}/${KUBE_OUTPUT_SUBPATH}"
|
||||||
KUBE_OUTPUT_BINPATH="${KUBE_OUTPUT}/bin"
|
KUBE_OUTPUT_BINPATH="${KUBE_OUTPUT}/bin"
|
||||||
|
|
||||||
|
# This controls rsync compression. Set to a value > 0 to enable rsync
|
||||||
|
# compression for build container
|
||||||
|
KUBE_RSYNC_COMPRESS="${KUBE_RSYNC_COMPRESS:-0}"
|
||||||
|
|
||||||
# Set no_proxy for localhost if behind a proxy, otherwise,
|
# Set no_proxy for localhost if behind a proxy, otherwise,
|
||||||
# the connections to localhost in scripts will time out
|
# the connections to localhost in scripts will time out
|
||||||
export no_proxy=127.0.0.1,localhost
|
export no_proxy=127.0.0.1,localhost
|
||||||
|
Loading…
Reference in New Issue
Block a user