From 96f2588b610d6c2ac240d3697dbfc83d86215fa4 Mon Sep 17 00:00:00 2001 From: Stephen Augustus Date: Tue, 9 Jul 2019 12:03:31 -0400 Subject: [PATCH 1/3] cni: Update CNI download URLs to use new GCS bucket (k8s-artifacts-cni) Signed-off-by: Stephen Augustus --- build/debian-hyperkube-base/Makefile | 3 ++- build/workspace.bzl | 3 ++- cluster/gce/config-default.sh | 6 +++--- cluster/gce/config-test.sh | 6 +++--- cluster/gce/gci/configure.sh | 10 +++++++--- cluster/gce/util.sh | 2 +- cluster/gce/windows/k8s-node-setup.psm1 | 9 ++++----- test/e2e_node/remote/utils.go | 3 ++- 8 files changed, 24 insertions(+), 18 deletions(-) diff --git a/build/debian-hyperkube-base/Makefile b/build/debian-hyperkube-base/Makefile index 57728a4cc60..2358cb0b13e 100644 --- a/build/debian-hyperkube-base/Makefile +++ b/build/debian-hyperkube-base/Makefile @@ -28,6 +28,7 @@ BASEIMAGE=k8s.gcr.io/debian-base-$(ARCH):0.4.1 CNI_VERSION=v0.7.5 TEMP_DIR:=$(shell mktemp -d) +# TODO(justaugustus): Post-0.7.5 CNI needs a "cni-plugins-linux-" prefix CNI_TARBALL=cni-plugins-$(ARCH)-$(CNI_VERSION).tgz # This option is for running docker manifest command @@ -58,7 +59,7 @@ push-manifest: cni-tars/$(CNI_TARBALL): mkdir -p cni-tars/ - cd cni-tars/ && curl -sSLO --retry 5 https://storage.googleapis.com/kubernetes-release/network-plugins/${CNI_TARBALL} + cd cni-tars/ && curl -sSLO --retry 5 https://storage.googleapis.com/k8s-artifacts-cni/release/${CNI_VERSION}/${CNI_TARBALL} clean: rm -rf cni-tars/ diff --git a/build/workspace.bzl b/build/workspace.bzl index d39bdd72ad4..7068eb44e53 100644 --- a/build/workspace.bzl +++ b/build/workspace.bzl @@ -53,13 +53,14 @@ def release_dependencies(): debian_image_dependencies() etcd_tarballs() +# TODO(justaugustus): Post-0.7.5 CNI needs a "cni-plugins-linux-" prefix def cni_tarballs(): for arch, sha in _CNI_TARBALL_ARCH_SHA256.items(): http_file( name = "kubernetes_cni_%s" % arch, downloaded_file_path = "kubernetes_cni.tgz", sha256 = sha, - urls = mirror("https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-%s-v%s.tgz" % (arch, CNI_VERSION)), + urls = mirror("https://storage.googleapis.com/k8s-artifacts-cni/release/v%s/cni-plugins-%s-v%s.tgz" % (CNI_VERSION, arch, CNI_VERSION)), ) def cri_tarballs(): diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 12a23233810..edb13460566 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -290,10 +290,10 @@ NODE_PROBLEM_DETECTOR_TAR_HASH="${NODE_PROBLEM_DETECTOR_TAR_HASH:-}" NODE_PROBLEM_DETECTOR_RELEASE_PATH="${NODE_PROBLEM_DETECTOR_RELEASE_PATH:-}" NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS="${NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS:-}" -CNI_STORAGE_PATH="${CNI_STORAGE_PATH:-https://storage.googleapis.com/kubernetes-release/network-plugins}" -CNI_TAR_PREFIX="${CNI_TAR_PREFIX:-cni-plugins-amd64-}" -CNI_VERSION="${CNI_VERSION:-}" CNI_SHA1="${CNI_SHA1:-}" +# TODO(justaugustus): Post-0.7.5 CNI needs a "cni-plugins-linux-" prefix +CNI_TAR_PREFIX="${CNI_TAR_PREFIX:-cni-plugins-amd64-}" +CNI_STORAGE_URL_BASE="${CNI_STORAGE_URL_BASE:-https://storage.googleapis.com/k8s-artifacts-cni/release}" # Optional: Create autoscaler for cluster's nodes. ENABLE_CLUSTER_AUTOSCALER="${KUBE_ENABLE_CLUSTER_AUTOSCALER:-false}" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index f185b5a683b..4b4be5dda56 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -321,10 +321,10 @@ NODE_PROBLEM_DETECTOR_TAR_HASH="${NODE_PROBLEM_DETECTOR_TAR_HASH:-}" NODE_PROBLEM_DETECTOR_RELEASE_PATH="${NODE_PROBLEM_DETECTOR_RELEASE_PATH:-}" NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS="${NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS:-}" -CNI_STORAGE_PATH="${CNI_STORAGE_PATH:-https://storage.googleapis.com/kubernetes-release/network-plugins}" -CNI_TAR_PREFIX="${CNI_TAR_PREFIX:-cni-plugins-amd64-}" -CNI_VERSION="${CNI_VERSION:-}" CNI_SHA1="${CNI_SHA1:-}" +# TODO(justaugustus): Post-0.7.5 CNI needs a "cni-plugins-linux-" prefix +CNI_TAR_PREFIX="${CNI_TAR_PREFIX:-cni-plugins-amd64-}" +CNI_STORAGE_URL_BASE="${CNI_STORAGE_URL_BASE:-https://storage.googleapis.com/k8s-artifacts-cni/release}" # Optional: Create autoscaler for cluster's nodes. ENABLE_CLUSTER_AUTOSCALER="${KUBE_ENABLE_CLUSTER_AUTOSCALER:-false}" diff --git a/cluster/gce/gci/configure.sh b/cluster/gce/gci/configure.sh index bc476b411e8..0d152a7ac10 100644 --- a/cluster/gce/gci/configure.sh +++ b/cluster/gce/gci/configure.sh @@ -236,19 +236,23 @@ function install-node-problem-detector { function install-cni-binaries { if [[ -n "${CNI_VERSION:-}" ]]; then - local -r cni_tar="${CNI_TAR_PREFIX}${CNI_VERSION}.tgz" + local -r cni_version="${CNI_VERSION}" local -r cni_sha1="${CNI_SHA1}" else - local -r cni_tar="${CNI_TAR_PREFIX}${DEFAULT_CNI_VERSION}.tgz" + local -r cni_version="${DEFAULT_CNI_VERSION}" local -r cni_sha1="${DEFAULT_CNI_SHA1}" fi + + local -r cni_tar="${CNI_TAR_PREFIX}${cni_version}.tgz" + local -r cni_url="${CNI_STORAGE_URL_BASE}/${cni_version}/${cni_tar}" + if is-preloaded "${cni_tar}" "${cni_sha1}"; then echo "${cni_tar} is preloaded." return fi echo "Downloading cni binaries" - download-or-bust "${cni_sha1}" "${CNI_STORAGE_PATH}/${cni_tar}" + download-or-bust "${cni_sha1}" "${cni_url}" local -r cni_dir="${KUBE_HOME}/cni" mkdir -p "${cni_dir}/bin" tar xzf "${KUBE_HOME}/${cni_tar}" -C "${cni_dir}/bin" --overwrite diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 793f2063256..bc7ae41e986 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1142,7 +1142,7 @@ NODE_PROBLEM_DETECTOR_VERSION: $(yaml-quote ${NODE_PROBLEM_DETECTOR_VERSION:-}) NODE_PROBLEM_DETECTOR_TAR_HASH: $(yaml-quote ${NODE_PROBLEM_DETECTOR_TAR_HASH:-}) NODE_PROBLEM_DETECTOR_RELEASE_PATH: $(yaml-quote ${NODE_PROBLEM_DETECTOR_RELEASE_PATH:-}) NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS: $(yaml-quote ${NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS:-}) -CNI_STORAGE_PATH: $(yaml-quote ${CNI_STORAGE_PATH:-}) +CNI_STORAGE_URL_BASE: $(yaml-quote ${CNI_STORAGE_URL_BASE:-}) CNI_TAR_PREFIX: $(yaml-quote ${CNI_TAR_PREFIX:-}) CNI_VERSION: $(yaml-quote ${CNI_VERSION:-}) CNI_SHA1: $(yaml-quote ${CNI_SHA1:-}) diff --git a/cluster/gce/windows/k8s-node-setup.psm1 b/cluster/gce/windows/k8s-node-setup.psm1 index 43ae0120107..5cf37201e08 100644 --- a/cluster/gce/windows/k8s-node-setup.psm1 +++ b/cluster/gce/windows/k8s-node-setup.psm1 @@ -953,18 +953,17 @@ function Configure-CniNetworking { # CLUSTER_IP_RANGE # SERVICE_CLUSTER_IP_RANGE function Configure_Dockerd_CniNetworking { - $CNI_RELEASE_VERSION = 'v0.8.2-gke.0' + $CNI_RELEASE_VERSION = 'v0.8.2' if ((ShouldWrite-File ${env:CNI_DIR}\win-bridge.exe) -or (ShouldWrite-File ${env:CNI_DIR}\host-local.exe)) { $tmp_dir = 'C:\cni_tmp' New-Item $tmp_dir -ItemType 'directory' -Force | Out-Null - $release_url = ('https://www.googleapis.com/storage/v1/b/gke-release/o/cni-plugins%2f' + + $release_url = ('https://storage.googleapis.com/k8s-artifacts-cni/release%2f' + $CNI_RELEASE_VERSION + '%2f') - $sha_url = ($release_url + - "cni-plugins-windows-amd64-$CNI_RELEASE_VERSION.tgz.sha1?alt=media") $tgz_url = ($release_url + - "cni-plugins-windows-amd64-$CNI_RELEASE_VERSION.tgz?alt=media") + "cni-plugins-windows-amd64-$CNI_RELEASE_VERSION.tgz") + $sha_url = ($tgz_url + ".sha1") MustDownload-File -URLs $sha_url -OutFile $tmp_dir\cni-plugins.sha1 $sha1_val = ($(Get-Content $tmp_dir\cni-plugins.sha1) -split ' ',2)[0] MustDownload-File ` diff --git a/test/e2e_node/remote/utils.go b/test/e2e_node/remote/utils.go index e7b1647a8fd..57d487626a7 100644 --- a/test/e2e_node/remote/utils.go +++ b/test/e2e_node/remote/utils.go @@ -31,7 +31,8 @@ const ( cniArch = "amd64" cniDirectory = "cni/bin" // The CNI tarball places binaries under directory under "cni/bin". cniConfDirectory = "cni/net.d" - cniURL = "https://dl.k8s.io/network-plugins/cni-plugins-" + cniArch + "-" + cniVersion + ".tgz" + // TODO(justaugustus): Post-0.7.5 CNI needs a "cni-plugins-linux-" prefix + cniURL = "https://storage.googleapis.com/k8s-artifacts-cni/release/" + cniVersion + "/" + "cni-plugins-" + cniArch + "-" + cniVersion + ".tgz" ) const cniConfig = `{ From b430f022fcea016f26623a487116c022a25d85ca Mon Sep 17 00:00:00 2001 From: Stephen Augustus Date: Mon, 27 Jan 2020 05:44:16 -0500 Subject: [PATCH 2/3] build: Don't attempt to use mirror for CNI plugin downloads Signed-off-by: Stephen Augustus --- build/workspace.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/workspace.bzl b/build/workspace.bzl index 7068eb44e53..2a6b54c62ac 100644 --- a/build/workspace.bzl +++ b/build/workspace.bzl @@ -60,7 +60,7 @@ def cni_tarballs(): name = "kubernetes_cni_%s" % arch, downloaded_file_path = "kubernetes_cni.tgz", sha256 = sha, - urls = mirror("https://storage.googleapis.com/k8s-artifacts-cni/release/v%s/cni-plugins-%s-v%s.tgz" % (CNI_VERSION, arch, CNI_VERSION)), + urls = ["https://storage.googleapis.com/k8s-artifacts-cni/release/v%s/cni-plugins-%s-v%s.tgz" % (CNI_VERSION, arch, CNI_VERSION)], ) def cri_tarballs(): From 1174e6698ece2cc8d370aba8ef77461db7be5c2b Mon Sep 17 00:00:00 2001 From: Stephen Augustus Date: Sun, 9 Jun 2019 03:48:14 -0400 Subject: [PATCH 3/3] cni: Update CNI version to v0.8.5 Signed-off-by: Stephen Augustus --- build/debian-hyperkube-base/Makefile | 7 +++---- build/dependencies.yaml | 4 +++- build/rpms/kubeadm.spec | 2 +- build/rpms/kubelet.spec | 2 +- build/workspace.bzl | 15 +++++++-------- cluster/gce/config-default.sh | 3 +-- cluster/gce/config-test.sh | 3 +-- cluster/gce/gci/configure.sh | 4 ++-- cluster/gce/windows/k8s-node-setup.psm1 | 2 +- test/e2e_node/remote/utils.go | 5 ++--- 10 files changed, 22 insertions(+), 25 deletions(-) diff --git a/build/debian-hyperkube-base/Makefile b/build/debian-hyperkube-base/Makefile index 2358cb0b13e..b237b14535f 100644 --- a/build/debian-hyperkube-base/Makefile +++ b/build/debian-hyperkube-base/Makefile @@ -19,17 +19,16 @@ REGISTRY?=staging-k8s.gcr.io IMAGE?=$(REGISTRY)/debian-hyperkube-base -TAG=0.12.1 +TAG=0.12.2 ARCH?=amd64 ALL_ARCH = amd64 arm arm64 ppc64le s390x CACHEBUST?=1 BASEIMAGE=k8s.gcr.io/debian-base-$(ARCH):0.4.1 -CNI_VERSION=v0.7.5 +CNI_VERSION=v0.8.5 TEMP_DIR:=$(shell mktemp -d) -# TODO(justaugustus): Post-0.7.5 CNI needs a "cni-plugins-linux-" prefix -CNI_TARBALL=cni-plugins-$(ARCH)-$(CNI_VERSION).tgz +CNI_TARBALL=cni-plugins-linux-$(ARCH)-$(CNI_VERSION).tgz # This option is for running docker manifest command export DOCKER_CLI_EXPERIMENTAL := enabled diff --git a/build/dependencies.yaml b/build/dependencies.yaml index a08185f66e9..b960bd73e06 100644 --- a/build/dependencies.yaml +++ b/build/dependencies.yaml @@ -44,7 +44,7 @@ dependencies: - name: "cni" - version: 0.7.5 + version: 0.8.5 refPaths: - path: build/debian-hyperkube-base/Makefile match: CNI_VERSION= @@ -56,6 +56,8 @@ dependencies: match: CNI_VERSION = - path: cluster/gce/gci/configure.sh match: DEFAULT_CNI_VERSION= + - path: cluster/gce/windows/k8s-node-setup.psm1 + match: CNI_RELEASE_VERSION = - path: test/e2e_node/remote/utils.go match: cniVersion[\t\n\f\r ]*= diff --git a/build/rpms/kubeadm.spec b/build/rpms/kubeadm.spec index 6c25ee992f4..80e05d068d5 100644 --- a/build/rpms/kubeadm.spec +++ b/build/rpms/kubeadm.spec @@ -5,7 +5,7 @@ License: ASL 2.0 Summary: Container Cluster Manager - Kubernetes Cluster Bootstrapping Tool Requires: kubelet >= 1.8.0 Requires: kubectl >= 1.8.0 -Requires: kubernetes-cni >= 0.7.5 +Requires: kubernetes-cni >= 0.8.5 Requires: cri-tools >= 1.11.0 URL: https://kubernetes.io diff --git a/build/rpms/kubelet.spec b/build/rpms/kubelet.spec index 2603c4b07c8..8906bb6a7f8 100644 --- a/build/rpms/kubelet.spec +++ b/build/rpms/kubelet.spec @@ -11,7 +11,7 @@ Requires: ebtables Requires: ethtool Requires: iproute Requires: iptables >= 1.4.21 -Requires: kubernetes-cni >= 0.7.5 +Requires: kubernetes-cni >= 0.8.5 Requires: socat Requires: util-linux diff --git a/build/workspace.bzl b/build/workspace.bzl index 2a6b54c62ac..2f73d71169c 100644 --- a/build/workspace.bzl +++ b/build/workspace.bzl @@ -17,13 +17,13 @@ load("//build:workspace_mirror.bzl", "mirror") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") load("@io_bazel_rules_docker//container:container.bzl", "container_pull") -CNI_VERSION = "0.7.5" +CNI_VERSION = "0.8.5" _CNI_TARBALL_ARCH_SHA256 = { - "amd64": "3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64", - "arm": "0eb4a528b5b2e4ce23ebc96e41b2f5280d5a64d41eec8dd8b16c3d66aaa0f6b8", - "arm64": "7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded", - "ppc64le": "9164a26ed8dd398b2fe3b15d9d456271dfa59aa537528d10572ea9fa2cef7679", - "s390x": "415cdcf02c65c22f5b7e55b0ab61208a10f2b95a0c8310176c771d07a9f448cf", + "amd64": "bd682ffcf701e8f83283cdff7281aad0c83b02a56084d6e601216210732833f9", + "arm": "86a868234045837cb3f5d58a0a4468ff42845d50b5e87bd128f050ef393d7495", + "arm64": "a7881ec37e592c897bdfd2a225b4ed74caa981e3c4cdcf8f45574f8d2f111bce", + "ppc64le": "a26cc3734f7cb980ab8fb3aaa64ccf2d67291478130009fa9542355fbdd94aa5", + "s390x": "033ea910a83144609083d5c3fb62bf4a379b0b17729a1a9e829feed9fa7a9d97", } CRI_TOOLS_VERSION = "1.17.0" @@ -53,14 +53,13 @@ def release_dependencies(): debian_image_dependencies() etcd_tarballs() -# TODO(justaugustus): Post-0.7.5 CNI needs a "cni-plugins-linux-" prefix def cni_tarballs(): for arch, sha in _CNI_TARBALL_ARCH_SHA256.items(): http_file( name = "kubernetes_cni_%s" % arch, downloaded_file_path = "kubernetes_cni.tgz", sha256 = sha, - urls = ["https://storage.googleapis.com/k8s-artifacts-cni/release/v%s/cni-plugins-%s-v%s.tgz" % (CNI_VERSION, arch, CNI_VERSION)], + urls = ["https://storage.googleapis.com/k8s-artifacts-cni/release/v%s/cni-plugins-linux-%s-v%s.tgz" % (CNI_VERSION, arch, CNI_VERSION)], ) def cri_tarballs(): diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index edb13460566..821cda404e9 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -291,8 +291,7 @@ NODE_PROBLEM_DETECTOR_RELEASE_PATH="${NODE_PROBLEM_DETECTOR_RELEASE_PATH:-}" NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS="${NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS:-}" CNI_SHA1="${CNI_SHA1:-}" -# TODO(justaugustus): Post-0.7.5 CNI needs a "cni-plugins-linux-" prefix -CNI_TAR_PREFIX="${CNI_TAR_PREFIX:-cni-plugins-amd64-}" +CNI_TAR_PREFIX="${CNI_TAR_PREFIX:-cni-plugins-linux-amd64-}" CNI_STORAGE_URL_BASE="${CNI_STORAGE_URL_BASE:-https://storage.googleapis.com/k8s-artifacts-cni/release}" # Optional: Create autoscaler for cluster's nodes. diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 4b4be5dda56..d161df31799 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -322,8 +322,7 @@ NODE_PROBLEM_DETECTOR_RELEASE_PATH="${NODE_PROBLEM_DETECTOR_RELEASE_PATH:-}" NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS="${NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS:-}" CNI_SHA1="${CNI_SHA1:-}" -# TODO(justaugustus): Post-0.7.5 CNI needs a "cni-plugins-linux-" prefix -CNI_TAR_PREFIX="${CNI_TAR_PREFIX:-cni-plugins-amd64-}" +CNI_TAR_PREFIX="${CNI_TAR_PREFIX:-cni-plugins-linux-amd64-}" CNI_STORAGE_URL_BASE="${CNI_STORAGE_URL_BASE:-https://storage.googleapis.com/k8s-artifacts-cni/release}" # Optional: Create autoscaler for cluster's nodes. diff --git a/cluster/gce/gci/configure.sh b/cluster/gce/gci/configure.sh index 0d152a7ac10..6adb3e6e381 100644 --- a/cluster/gce/gci/configure.sh +++ b/cluster/gce/gci/configure.sh @@ -24,8 +24,8 @@ set -o nounset set -o pipefail ### Hardcoded constants -DEFAULT_CNI_VERSION="v0.7.5" -DEFAULT_CNI_SHA1="52e9d2de8a5f927307d9397308735658ee44ab8d" +DEFAULT_CNI_VERSION="v0.8.5" +DEFAULT_CNI_SHA1="677d218b62c0ef941c1d0b606d6570faa5277ffd" DEFAULT_NPD_VERSION="v0.8.0" DEFAULT_NPD_SHA1="9406c975b1b035995a137029a004622b905b4e7f" DEFAULT_CRICTL_VERSION="v1.17.0" diff --git a/cluster/gce/windows/k8s-node-setup.psm1 b/cluster/gce/windows/k8s-node-setup.psm1 index 5cf37201e08..011b926feba 100644 --- a/cluster/gce/windows/k8s-node-setup.psm1 +++ b/cluster/gce/windows/k8s-node-setup.psm1 @@ -953,7 +953,7 @@ function Configure-CniNetworking { # CLUSTER_IP_RANGE # SERVICE_CLUSTER_IP_RANGE function Configure_Dockerd_CniNetworking { - $CNI_RELEASE_VERSION = 'v0.8.2' + $CNI_RELEASE_VERSION = 'v0.8.5' if ((ShouldWrite-File ${env:CNI_DIR}\win-bridge.exe) -or (ShouldWrite-File ${env:CNI_DIR}\host-local.exe)) { $tmp_dir = 'C:\cni_tmp' diff --git a/test/e2e_node/remote/utils.go b/test/e2e_node/remote/utils.go index 57d487626a7..fcff2349756 100644 --- a/test/e2e_node/remote/utils.go +++ b/test/e2e_node/remote/utils.go @@ -27,12 +27,11 @@ import ( // utils.go contains functions used across test suites. const ( - cniVersion = "v0.7.5" + cniVersion = "v0.8.5" cniArch = "amd64" cniDirectory = "cni/bin" // The CNI tarball places binaries under directory under "cni/bin". cniConfDirectory = "cni/net.d" - // TODO(justaugustus): Post-0.7.5 CNI needs a "cni-plugins-linux-" prefix - cniURL = "https://storage.googleapis.com/k8s-artifacts-cni/release/" + cniVersion + "/" + "cni-plugins-" + cniArch + "-" + cniVersion + ".tgz" + cniURL = "https://storage.googleapis.com/k8s-artifacts-cni/release/" + cniVersion + "/" + "cni-plugins-linux-" + cniArch + "-" + cniVersion + ".tgz" ) const cniConfig = `{