From da8d5ed027f59788c98e41e080db7f67b2c6d135 Mon Sep 17 00:00:00 2001 From: niusmallnan Date: Mon, 16 Apr 2018 14:15:47 +0800 Subject: [PATCH 1/4] Bump system-docker to 17.06-ros4 --- Dockerfile.dapper | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.dapper b/Dockerfile.dapper index a889a93f..33a83105 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -82,7 +82,7 @@ ARG DFS_IMAGE=${OS_REPO}/docker:v${DOCKER_VERSION}-2 ARG OS_BASE_URL_amd64=https://github.com/rancher/os-base/releases/download/v2018.02-3/os-base_amd64.tar.xz ARG OS_BASE_URL_arm64=https://github.com/rancher/os-base/releases/download/v2018.02-3/os-base_arm64.tar.xz -ARG SYSTEM_DOCKER_VERSION=17.06-ros3 +ARG SYSTEM_DOCKER_VERSION=17.06-ros4 ARG SYSTEM_DOCKER_URL_amd64=https://github.com/niusmallnan/os-system-docker/releases/download/${SYSTEM_DOCKER_VERSION}/docker-amd64-${SYSTEM_DOCKER_VERSION}.tgz ARG SYSTEM_DOCKER_URL_arm64=https://github.com/niusmallnan/os-system-docker/releases/download/${SYSTEM_DOCKER_VERSION}/docker-arm64-${SYSTEM_DOCKER_VERSION}.tgz ###################################################### From ba388b6bc6a8d5f822f3a271b7e7b07269d8295a Mon Sep 17 00:00:00 2001 From: niusmallnan Date: Mon, 16 Apr 2018 14:16:08 +0800 Subject: [PATCH 2/4] Build docker-sys bridge via system-docker args and remove cni-glue --- assets/docker/cni/bridge.d/bridge.conf | 12 ------ assets/docker/cni/default.d | 1 - assets/docker/hooks/poststop.d/network.json | 7 ---- assets/docker/hooks/prestart.d/network.json | 6 --- .../metadata/cloudstack/metadata.go | 7 +--- config/schema.go | 1 + config/types.go | 1 + main.go | 6 --- netconf/netconf_linux.go | 40 +++++++++++++------ os-config.tpl.yml | 2 +- 10 files changed, 31 insertions(+), 52 deletions(-) delete mode 100644 assets/docker/cni/bridge.d/bridge.conf delete mode 120000 assets/docker/cni/default.d delete mode 100644 assets/docker/hooks/poststop.d/network.json delete mode 100644 assets/docker/hooks/prestart.d/network.json diff --git a/assets/docker/cni/bridge.d/bridge.conf b/assets/docker/cni/bridge.d/bridge.conf deleted file mode 100644 index a090c5e8..00000000 --- a/assets/docker/cni/bridge.d/bridge.conf +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "bridge", - "type": "bridge", - "bridge": "docker-sys", - "isDefaultGateway": true, - "ipMasq": true, - "hairpinMode": true, - "ipam": { - "type": "host-local", - "subnet": "172.18.42.1/16" - } -} diff --git a/assets/docker/cni/default.d b/assets/docker/cni/default.d deleted file mode 120000 index 505083cd..00000000 --- a/assets/docker/cni/default.d +++ /dev/null @@ -1 +0,0 @@ -bridge.d/ \ No newline at end of file diff --git a/assets/docker/hooks/poststop.d/network.json b/assets/docker/hooks/poststop.d/network.json deleted file mode 100644 index b2eae3cf..00000000 --- a/assets/docker/hooks/poststop.d/network.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "path": "/usr/bin/ros", - "args": [ - "cni-glue", - "poststop" - ] -} diff --git a/assets/docker/hooks/prestart.d/network.json b/assets/docker/hooks/prestart.d/network.json deleted file mode 100644 index a38ada87..00000000 --- a/assets/docker/hooks/prestart.d/network.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "path": "/usr/bin/ros", - "args": [ - "cni-glue" - ] -} diff --git a/config/cloudinit/datasource/metadata/cloudstack/metadata.go b/config/cloudinit/datasource/metadata/cloudstack/metadata.go index 3cb38188..d8261ef7 100644 --- a/config/cloudinit/datasource/metadata/cloudstack/metadata.go +++ b/config/cloudinit/datasource/metadata/cloudstack/metadata.go @@ -25,7 +25,6 @@ import ( "github.com/rancher/os/config/cloudinit/datasource/metadata" "github.com/rancher/os/config/cloudinit/pkg" "github.com/rancher/os/log" - "github.com/vishvananda/netlink" ) const ( @@ -44,14 +43,10 @@ func NewDatasource(root string) []*MetadataService { roots := make([]string, 0, 5) if root == "" { - if links, err := netlink.LinkList(); err == nil { + if links, err := netconf.GetValidLinkList(); err == nil { log.Infof("Checking to see if a cloudstack server-identifier is available") for _, link := range links { linkName := link.Attrs().Name - if linkName == "lo" { - continue - } - log.Infof("searching for cloudstack server %s on %s", serverIdentifier, linkName) lease := netconf.GetDhcpLease(linkName) if server, ok := lease[serverIdentifier]; ok { diff --git a/config/schema.go b/config/schema.go index a147fd4d..12a9aa2d 100644 --- a/config/schema.go +++ b/config/schema.go @@ -122,6 +122,7 @@ var schema = `{ "storage_context": {"type": "string"}, "exec": {"type": ["boolean", "null"]}, "bridge": {"type": "string"}, + "bip": {"type": "string"}, "config_file": {"type": "string"}, "containerd": {"type": "string"}, "debug": {"type": ["boolean", "null"]}, diff --git a/config/types.go b/config/types.go index 1b1fc8f6..64c7bb95 100644 --- a/config/types.go +++ b/config/types.go @@ -148,6 +148,7 @@ type UpgradeConfig struct { type EngineOpts struct { Bridge string `yaml:"bridge,omitempty" opt:"bridge"` + BIP string `yaml:"bip,omitempty" opt:"bip"` ConfigFile string `yaml:"config_file,omitempty" opt:"config-file"` Containerd string `yaml:"containerd,omitempty" opt:"containerd"` Debug *bool `yaml:"debug,omitempty" opt:"debug"` diff --git a/main.go b/main.go index bb6496d9..b554de85 100644 --- a/main.go +++ b/main.go @@ -4,11 +4,8 @@ import ( "fmt" "os" - "github.com/containernetworking/cni/plugins/ipam/host-local" - "github.com/containernetworking/cni/plugins/main/bridge" "github.com/docker/docker/docker" "github.com/docker/docker/pkg/reexec" - "github.com/rancher/cniglue" "github.com/rancher/os/cmd/cloudinitexecute" "github.com/rancher/os/cmd/cloudinitsave" "github.com/rancher/os/cmd/control" @@ -35,9 +32,6 @@ var entrypoints = map[string]func(){ "ros-bootstrap": control.BootstrapMain, "ros-sysinit": sysinit.Main, "wait-for-docker": wait.Main, - "cni-glue": glue.Main, - "bridge": bridge.Main, - "host-local": hostlocal.Main, "respawn": respawn.Main, // Power commands diff --git a/netconf/netconf_linux.go b/netconf/netconf_linux.go index 4f240922..651abf22 100644 --- a/netconf/netconf_linux.go +++ b/netconf/netconf_linux.go @@ -66,7 +66,7 @@ func createInterfaces(netCfg *NetworkConfig) { } func createSlaveInterfaces(netCfg *NetworkConfig) { - links, err := netlink.LinkList() + links, err := GetValidLinkList() if err != nil { log.Errorf("Failed to list links: %v", err) return @@ -167,7 +167,7 @@ func ApplyNetworkConfigs(netCfg *NetworkConfig, userSetHostname, userSetDNS bool createInterfaces(netCfg) createSlaveInterfaces(netCfg) - links, err := netlink.LinkList() + links, err := GetValidLinkList() if err != nil { log.Errorf("error getting LinkList: %s", err) return false, err @@ -177,10 +177,7 @@ func ApplyNetworkConfigs(netCfg *NetworkConfig, userSetHostname, userSetDNS bool //apply network config for _, link := range links { - linkName := link.Attrs().Name - if linkName != "lo" { - applyOuter(link, netCfg, &wg, userSetHostname, userSetDNS) - } + applyOuter(link, netCfg, &wg, userSetHostname, userSetDNS) } wg.Wait() @@ -189,13 +186,11 @@ func ApplyNetworkConfigs(netCfg *NetworkConfig, userSetHostname, userSetDNS bool dnsSet := false for _, link := range links { linkName := link.Attrs().Name - if linkName != "lo" { - log.Infof("dns testing %s", linkName) - lease := GetDhcpLease(linkName) - if _, ok := lease["domain_name_servers"]; ok { - log.Infof("dns was dhcp set for %s", linkName) - dnsSet = true - } + log.Infof("dns testing %s", linkName) + lease := GetDhcpLease(linkName) + if _, ok := lease["domain_name_servers"]; ok { + log.Infof("dns was dhcp set for %s", linkName) + dnsSet = true } } @@ -510,3 +505,22 @@ func runCmds(cmds []string, iface string) { } } } + +func GetValidLinkList() ([]netlink.Link, error) { + var validLinkList []netlink.Link + links, err := netlink.LinkList() + if err != nil { + return validLinkList, err + } + + for _, l := range links { + linkName := l.Attrs().Name + if linkName == "lo" || linkName == "docker-sys" { + continue + } + validLinkList = append(validLinkList, l) + } + + return validLinkList, nil + +} diff --git a/os-config.tpl.yml b/os-config.tpl.yml index f6f42ac9..1f7b5018 100644 --- a/os-config.tpl.yml +++ b/os-config.tpl.yml @@ -368,7 +368,7 @@ rancher: system_docker: exec: true storage_driver: overlay2 - bridge: none + bip: 172.18.42.1/16 restart: false graph: /var/lib/system-docker group: root From 08fbe26c67831a9a77e4116364b6963f4aba1198 Mon Sep 17 00:00:00 2001 From: niusmallnan Date: Mon, 16 Apr 2018 08:23:47 +0000 Subject: [PATCH 3/4] Update trash.conf for removing cni-glue --- trash.conf | 3 --- 1 file changed, 3 deletions(-) diff --git a/trash.conf b/trash.conf index 4ae10015..40a77b07 100644 --- a/trash.conf +++ b/trash.conf @@ -5,8 +5,6 @@ github.com/boltdb/bolt v1.2.0 github.com/cloudfoundry-incubator/candiedyaml 01cbc92901719f599b11f3a7e3b1768d7002b0bb https://github.com/rancher/candiedyaml github.com/cloudfoundry/gosigar 3ed7c74352dae6dc00bdc8c74045375352e3ec05 github.com/codegangsta/cli d2b9ba9c38eb353ba3c6df3f57072348e19cc5c7 https://github.com/rancher/cli-1 -github.com/containernetworking/cni a8e4fa0dffdac6a236f85be91502603ec06957f9 https://github.com/rancher/cni.git -github.com/coreos/go-iptables fbb73372b87f6e89951c2b6b31470c2c9d5cfae3 github.com/coreos/go-systemd v4 github.com/coreos/yaml 6b16a5714269b2f70720a45406b1babd947a17ef github.com/davecgh/go-spew 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d @@ -35,7 +33,6 @@ github.com/opencontainers/specs f955d90e70a98ddfb886bd930ffd076da9b67998 github.com/packethost/packngo v0.1.0 github.com/pkg/errors d62207b3dc916c342cd6a7180fa861d898cf42ee github.com/pmezard/go-difflib d8ed2627bdf02c080bf22230dbb337003b7aba2d -github.com/rancher/cniglue b56bd68e5df113ad3fcc59c58034c22afaede877 github.com/rancher/catalog-service/utils/version 81799167a6ed8666fb1640b3aae12bc407050c4f github.com/rcrowley/go-metrics eeba7bd0dd01ace6e690fa833b3f22aaec29af43 github.com/ryanuber/go-glob 0067a9abd927e50aed5190662702f81231413ae0 From de5a1991f4a4cce9535228680b4a2b459e9ffe0c Mon Sep 17 00:00:00 2001 From: niusmallnan Date: Mon, 16 Apr 2018 08:26:12 +0000 Subject: [PATCH 4/4] Update vendor for removing cni-glue --- .../containernetworking/cni/.gitignore | 3 - .../containernetworking/cni/.travis.yml | 29 -- .../containernetworking/cni/CONTRIBUTING.md | 86 ----- vendor/github.com/containernetworking/cni/DCO | 36 -- .../containernetworking/cni/LICENSE | 202 ----------- .../containernetworking/cni/MAINTAINERS | 5 - .../containernetworking/cni/README.md | 164 --------- .../containernetworking/cni/ROADMAP.md | 33 -- .../containernetworking/cni/SPEC.md | 267 --------------- .../github.com/containernetworking/cni/build | 30 -- .../containernetworking/cni/libcni/api.go | 73 ---- .../containernetworking/cni/libcni/conf.go | 85 ----- .../cni/pkg/invoke/args.go | 76 ----- .../cni/pkg/invoke/delegate.go | 53 --- .../cni/pkg/invoke/exec.go | 75 ---- .../cni/pkg/invoke/find.go | 47 --- .../containernetworking/cni/pkg/ip/cidr.go | 51 --- .../cni/pkg/ip/ipforward.go | 31 -- .../containernetworking/cni/pkg/ip/ipmasq.go | 66 ---- .../containernetworking/cni/pkg/ip/link.go | 153 --------- .../containernetworking/cni/pkg/ip/route.go | 47 --- .../containernetworking/cni/pkg/ipam/ipam.go | 68 ---- .../containernetworking/cni/pkg/ns/README.md | 31 -- .../containernetworking/cni/pkg/ns/ns.go | 315 ----------------- .../containernetworking/cni/pkg/skel/skel.go | 161 --------- .../containernetworking/cni/pkg/types/args.go | 101 ------ .../cni/pkg/types/types.go | 191 ----------- .../cni/pkg/utils/utils.go | 41 --- .../cni/plugins/ipam/host-local/README.md | 86 ----- .../cni/plugins/ipam/host-local/allocator.go | 202 ----------- .../ipam/host-local/backend/disk/backend.go | 107 ------ .../ipam/host-local/backend/disk/lock.go | 50 --- .../plugins/ipam/host-local/backend/store.go | 27 -- .../cni/plugins/ipam/host-local/config.go | 70 ---- .../cni/plugins/ipam/host-local/main.go | 74 ---- .../cni/plugins/main/bridge/bridge.go | 319 ------------------ .../github.com/containernetworking/cni/test | 80 ----- .../github.com/coreos/go-iptables/.travis.yml | 25 -- vendor/github.com/coreos/go-iptables/LICENSE | 191 ----------- .../github.com/coreos/go-iptables/README.md | 9 - vendor/github.com/coreos/go-iptables/build | 22 -- .../coreos/go-iptables/iptables/iptables.go | 295 ---------------- .../coreos/go-iptables/iptables/lock.go | 84 ----- vendor/github.com/coreos/go-iptables/test | 55 --- .../github.com/rancher/cniglue/.dockerignore | 4 - vendor/github.com/rancher/cniglue/.drone.yml | 4 - vendor/github.com/rancher/cniglue/.gitignore | 4 - .../rancher/cniglue/Dockerfile.dapper | 15 - vendor/github.com/rancher/cniglue/LICENSE | 177 ---------- vendor/github.com/rancher/cniglue/Makefile | 23 -- vendor/github.com/rancher/cniglue/README.md | 28 -- vendor/github.com/rancher/cniglue/cni.go | 123 ------- vendor/github.com/rancher/cniglue/conf.go | 43 --- vendor/github.com/rancher/cniglue/glue.go | 46 --- vendor/github.com/rancher/cniglue/hosts.go | 67 ---- vendor/github.com/rancher/cniglue/io.go | 51 --- .../github.com/rancher/cniglue/resolvconf.go | 83 ----- vendor/github.com/rancher/cniglue/trash.conf | 7 - 58 files changed, 4891 deletions(-) delete mode 100644 vendor/github.com/containernetworking/cni/.gitignore delete mode 100644 vendor/github.com/containernetworking/cni/.travis.yml delete mode 100644 vendor/github.com/containernetworking/cni/CONTRIBUTING.md delete mode 100644 vendor/github.com/containernetworking/cni/DCO delete mode 100644 vendor/github.com/containernetworking/cni/LICENSE delete mode 100644 vendor/github.com/containernetworking/cni/MAINTAINERS delete mode 100644 vendor/github.com/containernetworking/cni/README.md delete mode 100644 vendor/github.com/containernetworking/cni/ROADMAP.md delete mode 100644 vendor/github.com/containernetworking/cni/SPEC.md delete mode 100755 vendor/github.com/containernetworking/cni/build delete mode 100644 vendor/github.com/containernetworking/cni/libcni/api.go delete mode 100644 vendor/github.com/containernetworking/cni/libcni/conf.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/invoke/args.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/invoke/delegate.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/invoke/exec.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/invoke/find.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/ip/cidr.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/ip/ipforward.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/ip/ipmasq.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/ip/link.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/ip/route.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/ipam/ipam.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/ns/README.md delete mode 100644 vendor/github.com/containernetworking/cni/pkg/ns/ns.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/skel/skel.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/types/args.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/types/types.go delete mode 100644 vendor/github.com/containernetworking/cni/pkg/utils/utils.go delete mode 100644 vendor/github.com/containernetworking/cni/plugins/ipam/host-local/README.md delete mode 100644 vendor/github.com/containernetworking/cni/plugins/ipam/host-local/allocator.go delete mode 100644 vendor/github.com/containernetworking/cni/plugins/ipam/host-local/backend/disk/backend.go delete mode 100644 vendor/github.com/containernetworking/cni/plugins/ipam/host-local/backend/disk/lock.go delete mode 100644 vendor/github.com/containernetworking/cni/plugins/ipam/host-local/backend/store.go delete mode 100644 vendor/github.com/containernetworking/cni/plugins/ipam/host-local/config.go delete mode 100644 vendor/github.com/containernetworking/cni/plugins/ipam/host-local/main.go delete mode 100644 vendor/github.com/containernetworking/cni/plugins/main/bridge/bridge.go delete mode 100755 vendor/github.com/containernetworking/cni/test delete mode 100644 vendor/github.com/coreos/go-iptables/.travis.yml delete mode 100644 vendor/github.com/coreos/go-iptables/LICENSE delete mode 100644 vendor/github.com/coreos/go-iptables/README.md delete mode 100755 vendor/github.com/coreos/go-iptables/build delete mode 100644 vendor/github.com/coreos/go-iptables/iptables/iptables.go delete mode 100644 vendor/github.com/coreos/go-iptables/iptables/lock.go delete mode 100755 vendor/github.com/coreos/go-iptables/test delete mode 100644 vendor/github.com/rancher/cniglue/.dockerignore delete mode 100644 vendor/github.com/rancher/cniglue/.drone.yml delete mode 100644 vendor/github.com/rancher/cniglue/.gitignore delete mode 100644 vendor/github.com/rancher/cniglue/Dockerfile.dapper delete mode 100644 vendor/github.com/rancher/cniglue/LICENSE delete mode 100644 vendor/github.com/rancher/cniglue/Makefile delete mode 100644 vendor/github.com/rancher/cniglue/README.md delete mode 100644 vendor/github.com/rancher/cniglue/cni.go delete mode 100644 vendor/github.com/rancher/cniglue/conf.go delete mode 100644 vendor/github.com/rancher/cniglue/glue.go delete mode 100644 vendor/github.com/rancher/cniglue/hosts.go delete mode 100644 vendor/github.com/rancher/cniglue/io.go delete mode 100644 vendor/github.com/rancher/cniglue/resolvconf.go delete mode 100644 vendor/github.com/rancher/cniglue/trash.conf diff --git a/vendor/github.com/containernetworking/cni/.gitignore b/vendor/github.com/containernetworking/cni/.gitignore deleted file mode 100644 index 06f78b4b..00000000 --- a/vendor/github.com/containernetworking/cni/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -bin/ -gopath/ -*.sw[ponm] diff --git a/vendor/github.com/containernetworking/cni/.travis.yml b/vendor/github.com/containernetworking/cni/.travis.yml deleted file mode 100644 index fb135c93..00000000 --- a/vendor/github.com/containernetworking/cni/.travis.yml +++ /dev/null @@ -1,29 +0,0 @@ -language: go -sudo: required -dist: trusty - - -matrix: - include: - - go: 1.5.4 - env: GO15VENDOREXPERIMENT=1 - - go: 1.6.2 - - go: tip - allow_failures: - - go: tip - -env: - global: - - TOOLS_CMD=golang.org/x/tools/cmd - - PATH=$GOROOT/bin:$PATH - -install: - - go get ${TOOLS_CMD}/cover - - go get github.com/modocache/gover - - go get github.com/mattn/goveralls - -script: - - ./test - -notifications: - email: false diff --git a/vendor/github.com/containernetworking/cni/CONTRIBUTING.md b/vendor/github.com/containernetworking/cni/CONTRIBUTING.md deleted file mode 100644 index fc637b15..00000000 --- a/vendor/github.com/containernetworking/cni/CONTRIBUTING.md +++ /dev/null @@ -1,86 +0,0 @@ -# How to Contribute - -CNI is [Apache 2.0 licensed](LICENSE) and accepts contributions via GitHub -pull requests. This document outlines some of the conventions on development -workflow, commit message formatting, contact points and other resources to make -it easier to get your contribution accepted. - -We gratefully welcome improvements to documentation as well as to code. - -# Certificate of Origin - -By contributing to this project you agree to the Developer Certificate of -Origin (DCO). This document was created by the Linux Kernel community and is a -simple statement that you, as a contributor, have the legal right to make the -contribution. See the [DCO](DCO) file for details. - -# Email and Chat - -The project uses the the cni-dev email list and IRC chat: -- Email: [cni-dev](https://groups.google.com/forum/#!forum/cni-dev) -- IRC: #[containernetworking](irc://irc.freenode.org:6667/#containernetworking) channel on freenode.org - -Please avoid emailing maintainers found in the MAINTAINERS file directly. They -are very busy and read the mailing lists. - -## Getting Started - -- Fork the repository on GitHub -- Read the [README](README.md) for build and test instructions -- Play with the project, submit bugs, submit pull requests! - -## Contribution workflow - -This is a rough outline of how to prepare a contribution: - -- Create a topic branch from where you want to base your work (usually branched from master). -- Make commits of logical units. -- Make sure your commit messages are in the proper format (see below). -- Push your changes to a topic branch in your fork of the repository. -- If you changed code, make sure the tests pass, and add any new tests as appropriate. -- Make sure any new code files have a license header. -- Submit a pull request to the original repository. - -# Acceptance policy - -These things will make a PR more likely to be accepted: - - * a well-described requirement - * tests for new code - * tests for old code! - * new code follows the conventions in old code - * a good commit message (see below) - -In general, we will merge a PR once two maintainers have endorsed it. -Trivial changes (e.g., corrections to spelling) may get waved through. -For substantial changes, more people may become involved, and you might get asked to resubmit the PR or divide the changes into more than one PR. - -### Format of the Commit Message - -We follow a rough convention for commit messages that is designed to answer two -questions: what changed and why. The subject line should feature the what and -the body of the commit should describe the why. - -``` -scripts: add the test-cluster command - -this uses tmux to setup a test cluster that you can easily kill and -start for debugging. - -Fixes #38 -``` - -The format can be described more formally as follows: - -``` -: - - - -