Merge pull request #26 from dougbtv/grpc-dependency-updates-CVE-2019-9511-4.1.z

Bug 1746664: Updates to grpc (and deps) for cve-2019-9511 (z-stream update for 4.1)

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,28 @@
+---
+name: Bug Report
+about: Report a bug encountered
+
+---
+<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!-->
+
+
+**What happend**:
+
+**What you expected to happen**:
+
+**How to reproduce it (as minimally and precisely as possible)**:
+
+**Anything else we need to know?**:
+
+**Environment**:
+
+- Multus version 
+  image path and image ID (from 'docker images')
+- Kubernetes version (use `kubectl version`):
+- Primary CNI for Kubernetes cluster:
+- OS (e.g. from /etc/os-release):
+- File of '/etc/cni/net.d/'
+- File of '/etc/cni/multus/net.d'
+- NetworkAttachment info (use `kubectl get net-attach-def -o yaml`)
+- Target pod yaml info (with annotation, use `kubectl get pod <podname> -o yaml`)
+- Other log outputs (if you use multus logging)

.github/ISSUE_TEMPLATE/enhancement.md

@@ -0,0 +1,10 @@
+---
+name: Enhancement Request
+about: Suggest an enhancement to multus
+
+---
+<!-- Please only use this template for submitting enhancement requests -->
+
+**What would you like to be added**:
+
+**Why is this needed**:

.github/ISSUE_TEMPLATE/support.md

@@ -0,0 +1,5 @@
+---
+name: Support Request
+about: Support request or question relating to multus-cni
+
+---

.gitignore

@@ -0,0 +1,9 @@
+# Binary output dir
+bin/
+
+# GOPATH created by the build script
+gopath/
+
+# Test outputs
+*.out
+*.test

.travis.yml

@@ -5,20 +5,16 @@ language: go
 dist: trusty
 env:
   global:
-    - REGISTRY_USER=nfvperobot
-    - secure: "LnQV09sy5nfrJd0PKAbxYPdKJ5QtLECofsunYfVk7tFp+ivKyZBXHwi4V4aGFuB2SqCnpauXBRTLet8hrfm5kN9ZZQRqy0WNs/fJHdFC6YKOKwyCQwczFb1by/iTX68dxWc2nK9+Opi6s/81Bh5yb3Oquqzdk+OEgaQHz2KP7BwI4yDrobinBR5laJ4KdxZJYgYx4mP6uUPxj7UZww+HaWqyiGy8cAeK3L81sGjxXJIYTRRfG1J4pifI5A3c3IOJRID0pvifgUIsQXp5MHpx+nxmhRJ7KMBLeNkUKruLTEsufgGCvhY5eWpdBhVN2YefGTqlKBCtKEqRUPlLbP5eJGUdY1PlUMUnQsr+FRWAZz90A1TESOZXZqDs4xR1ox1wX7mBUeelViXvUfLQB9sOD8G86FkXqNTqx/thp3x0Dqgy44pL+12Y3k5xVZmIsWDSpGmmIe1jOCsoL26Fdic+dTO/l3mx3KP1+gPNqbScuJsccLyPsr96uFCBCPJ2mSy7nCqb01KZTbbkIvv6oOCQ+Mfq8MT9lkxf6FJ+K+7vVbcgshOGhqA/l1UO3rKxnGt8Rkj/5XoHkcjXjM6YzT5LvljVWszJGXeTQxGjcsPrK2AscyX7JvNp/AMElII/Hxm6P0NESfV0whrZHyVOaqIRrbhUsK9j4YP8IMFoI4qYp4g="
+    - REGISTRY_USER=${REGISTRY_USER}
+    - REGISTRY_PASS=${REGISTRY_PASS}
+    - secure: "${REGISTRY_SECURE}"
 
 before_install:
   - sudo apt-get update -qq
   - go get github.com/mattn/goveralls
 
 install:
-  # workaround golint install error in https://github.com/golang/lint/issues/288
-  - mkdir -p $GOPATH/src/golang.org/x
-  - pushd $GOPATH/src/golang.org/x
-  - git clone https://github.com/golang/lint.git
-  - go get github.com/golang/lint/golint
-  - popd
+  - go get -u golang.org/x/lint/golint
 
 before_script:
   - golint ./multus/... | grep -v ALL_CAPS | xargs -r false
@@ -42,7 +38,7 @@ before_deploy:
 deploy:
   - provider: releases
     api_key:
-      secure: "iy7eqzXNvb/juc+5eVPQ/pFYDTCqDt8Zjt63n+zEK856Qzr2aEZwwOguMWs78XFDMFXagCs5PRTvtvZz8apoTfHX7Wkss3kRyEziAkuldQbH5yGDvpGyHsGBw78N95hauMoogefE7NuuLG3qRSWPeVz8RAKGhP7ADwEVyyfQKKYdum3Bqrz0D89HqKbCQqs3eZae7ppDIler3lab9WAQGuKNJ2HL6mqREVe48kb8sdsuSr+yV4qwVrBDNhXxQDxAT6LYuMXbknE7qTde2vViP13ZHpptbuZqiZG2ytzReIIs/iC9AWoIQXr3XTXl9z8fqlC3VljPCikBWVcmxDFA2aANYzx3M/7fMOO/DniwNhlZc9+pYfAkUrpoQPfPOWNqf45Qz0jP3wk49xy5hxEqe/rfmo5lipSsqeUsk+j3pT8kjVIAnDLrQpxSx7xwnijPLgtm34UwROVowfwLlOhE/7mUOFCbYlzEo3CKvjDN3Kmn35yHEueuu//Gv5jesVYvgcNPBHqaTKb5AXVTqymNBtA43PchLJ8gCC1mNukzSZifQP996vzbV5c9AxzBLjWbiDJ3lOFIpNhF8Sed0m0C0RylrTXHTX5TSrlMdXXffzYwbjJ96J+cFPBTpJNfSn+3N7hiart1r1k1bSXoPqYW4+94M8E1eZ5LjszoeiZbRrI="
+      secure: "${DEPLOY_SECURE}"
     file_glob: true
     file: "$TRAVIS_BUILD_DIR/dist/*/*.gz"
     skip_cleanup: true
@@ -50,18 +46,29 @@ deploy:
       tags: true
       all_branches: true
       condition: "$TRAVIS_TAG =~ ^v[0-9].*$"
-  # Push images to Dockerhub
+  # Push images to Dockerhub on tag
   - provider: script
     script: >
       bash -c '
       docker tag nfvpe/multus nfvpe/multus:$TRAVIS_TAG;
       docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS"; 
       docker push nfvpe/multus; 
-      docker push nfvpe/multus:$TRAVIS_TAG'
+      docker push nfvpe/multus:$TRAVIS_TAG;
+      echo foo'
     on:
       tags: true
       all_branches: true
       condition: "$TRAVIS_TAG =~ ^v[0-9].*$"
+  # Push images to Dockerhub on merge to master
+  - provider: script
+    on:
+      branch: master
+    script: >
+      bash -c '
+      docker tag nfvpe/multus nfvpe/multus:snapshot;
+      docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS";
+      docker push nfvpe/multus:snapshot; 
+      echo foo'
 
 after_success:
   # put build tgz to bintray

Dockerfile

@@ -1,10 +1,13 @@
+# This Dockerfile is used to build the image available on DockerHub
 FROM centos:centos7
 
 # Add everything
 ADD . /usr/src/multus-cni
 
 ENV INSTALL_PKGS "git golang"
-RUN yum install -y $INSTALL_PKGS && \
+RUN rpm --import https://mirror.go-repo.io/centos/RPM-GPG-KEY-GO-REPO && \
+    curl -s https://mirror.go-repo.io/centos/go-repo.repo | tee /etc/yum.repos.d/go-repo.repo && \
+    yum install -y $INSTALL_PKGS && \
     rpm -V $INSTALL_PKGS && \
     cd /usr/src/multus-cni && \
     ./build && \
@@ -14,14 +17,6 @@ RUN yum install -y $INSTALL_PKGS && \
 
 WORKDIR /
 
-LABEL io.k8s.display-name="Multus CNI" \
-      io.k8s.description="This is a component of OpenShift Container Platform and provides a meta CNI plugin." \
-      io.openshift.tags="openshift" \
-      maintainer="Doug Smith <dosmith@redhat.com>"
-
 ADD ./images/entrypoint.sh /
 
-# does it require a root user?
-# USER 1001
-
-ENTRYPOINT /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

Dockerfile.openshift

@@ -0,0 +1,20 @@
+# This dockerfile is specific to building Multus for OpenShift
+FROM openshift/origin-release:golang-1.10 as builder
+
+ADD . /usr/src/multus-cni
+
+WORKDIR /usr/src/multus-cni
+RUN ./build
+
+FROM openshift/origin-base
+RUN mkdir -p /usr/src/multus-cni/images && mkdir -p /usr/src/multus-cni/bin
+COPY --from=builder /usr/src/multus-cni/images/70-multus.conf /usr/src/multus-cni/images
+COPY --from=builder /usr/src/multus-cni/bin/multus /usr/src/multus-cni/bin
+ADD ./images/entrypoint.sh /
+
+LABEL io.k8s.display-name="Multus CNI" \
+      io.k8s.description="This is a component of OpenShift Container Platform and provides a meta CNI plugin." \
+      io.openshift.tags="openshift" \
+      maintainer="Doug Smith <dosmith@redhat.com>"
+
+ENTRYPOINT ["/entrypoint.sh"]

OWNERS

@@ -0,0 +1,17 @@
+reviewers:
+  - dougbtv
+  - dcbw
+  - squeed
+  - zshi
+  - fepan
+  - s1061123
+  - jeremyeder
+approvers:
+  - dougbtv
+  - dcbw
+  - squeed
+  - zshi
+  - fepan
+  - s1061123
+  - jeremyeder
+

README.md

@@ -1,44 +1,28 @@
+# Multus-CNI
+
 ![multus-cni Logo](https://github.com/intel/multus-cni/blob/master/doc/images/Multus.png)
 
 [![Travis CI](https://travis-ci.org/intel/multus-cni.svg?branch=master)](https://travis-ci.org/intel/multus-cni/builds)[![Go Report Card](https://goreportcard.com/badge/github.com/intel/multus-cni)](https://goreportcard.com/report/github.com/intel/multus-cni)
 
-   * [MULTUS CNI plugin](#multus-cni-plugin)
-      * [Quickstart Guide](#quickstart-guide)
-      * [Multi-Homed pod](#multi-homed-pod)
-      * [Building from source](#building-from-source)
-      * [Work flow](#work-flow)
-      * [Usage with Kubernetes CRD based network objects](#usage-with-kubernetes-crd-based-network-objects)
-         * [Creating "Network" resources in Kubernetes](#creating-network-resources-in-kubernetes)
-            * [<strong>CRD based Network objects</strong>](#crd-based-network-objects)
-         * [Configuring Multus to use the kubeconfig](#configuring-multus-to-use-the-kubeconfig)
-         * [Configuring Multus to use kubeconfig and a default network](#configuring-multus-to-use-kubeconfig-and-a-default-network)
-         * [Configuring Pod to use the CRD network objects](#configuring-pod-to-use-the-crd-network-objects)
-         * [Verifying Pod network interfaces](#verifying-pod-network-interfaces)
-      * [Using with Multus conf file](#using-with-multus-conf-file)
-      * [Logging Options](#logging-options)
-      * [Testing Multus CNI](#testing-multus-cni)
-         * [Multiple flannel networks](#multiple-flannel-networks)
-            * [Configure Kubernetes with CNI](#configure-kubernetes-with-cni)
-            * [Launching workloads in Kubernetes](#launching-workloads-in-kubernetes)
-      * [Multus additional plugins](#multus-additional-plugins)
-      * [NFV based networking in Kubernetes](#nfv-based-networking-in-kubernetes)
-      * [Need help](#need-help)
-      * [Contacts](#contacts)
+Multus CNI enables attaching multiple network interfaces to pods in Kubernetes.
 
-# MULTUS CNI plugin
-- _Multus_ is a latin word for &quot;Multi&quot;
-- As the name suggests, it acts as a Multi plugin in Kubernetes and provides the multiple network interface support in a pod
-- Multus supports all [reference plugins](https://github.com/containernetworking/plugins) (eg. [Flannel](https://github.com/containernetworking/plugins/tree/master/plugins/meta/flannel), [DHCP](https://github.com/containernetworking/plugins/tree/master/plugins/ipam/dhcp), [Macvlan](https://github.com/containernetworking/plugins/tree/master/plugins/main/macvlan)) that implement the CNI specification and all 3rd party plugins (eg. [Calico](https://github.com/projectcalico/cni-plugin), [Weave](https://github.com/weaveworks/weave), [Cilium](https://github.com/cilium/cilium), [Contiv](https://github.com/contiv/netplugin)). In addition to it, Multus supports [SRIOV](https://github.com/hustcat/sriov-cni), [SRIOV-DPDK](https://github.com/Intel-Corp/sriov-cni), [OVS-DPDK &amp; VPP](https://github.com/intel/vhost-user-net-plugin) workloads in Kubernetes with both cloud native and NFV based applications in Kubernetes
-- It is a contact between the container runtime and other plugins, and it doesn&#39;t have any of its own net configuration, it calls other plugins like flannel/calico to do the real net conf job.
-- Multus reuses the concept of invoking delegates as used in flannel by grouping multiple plugins into delegates and invoking them in the sequential order of the CNI configuration file provided in json format
-- The default network gets "eth0" and additional network Pod interface name as “net0”, “net1”,… “netX and so on. Multus also support interface names from the user.
-- Multus is one of the projects in the [Baremetal Container Experience kit](https://networkbuilders.intel.com/network-technologies/container-experience-kits).
+## How it works
 
-Please check the [CNI](https://github.com/containernetworking/cni) documentation for more information on container networking.
+Multus CNI is a container network interface (CNI) plugin for Kubernetes that enables attaching multiple network interfaces to pods. Typically, in Kubernetes each pod only has one network interface (apart from a loopback) -- with Multus you can create a multi-homed pod that has multiple interfaces. This is accomplished by Multus acting as a "meta-plugin", a CNI plugin that can call multiple other CNI plugins.
 
-# Quickstart Guide
+Multus CNI follows the [Kubernetes Network Custom Resource Definition De-facto Standard](https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ/edit) to provide a standardized method by which to specify the configurations for additional network interfaces. This standard is put forward by the Kubernetes [Network Plumbing Working Group](https://docs.google.com/document/d/1oE93V3SgOGWJ4O1zeD1UmpeToa0ZiiO6LqRAmZBPFWM/edit).
 
-Multus may be deployed as a Daemonset, and is provided in this guide along with Flannel. Flannel is deployed as a pod-to-pod network that is used as our "default network". Each network attachment is made in addition to this default network.
+Multus is one of the projects in the [Baremetal Container Experience kit](https://networkbuilders.intel.com/network-technologies/container-experience-kits)
+
+### Multi-Homed pod
+
+Here's an illustration of the network interfaces attached to a pod, as provisioned by Multus CNI. The diagram shows the pod with three interfaces: `eth0`, `net0` and `net1`. `eth0` connects kubernetes cluster network to connect with kubernetes server/services (e.g. kubernetes api-server, kubelet and so on). `net0` and `net1` are additional network attachments and connect to other networks by using [other CNI plugins](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) (e.g. vlan/vxlan/ptp).
+
+![multus-pod-image](doc/images/multus-pod-image.svg)
+
+## Quickstart Installation Guide
+
+Multus may be deployed as a Daemonset, and is provided in this guide along with Flannel. Flannel is deployed as a pod-to-pod network that is used as our "default network" (a network interface that every pod will be created with). Each network attachment is made in addition to this default network.
 
 Firstly, clone this GitHub repository. We'll apply files to `kubectl` from this repo.
 
@@ -48,561 +32,22 @@ We apply these files as such:
 $ cat ./images/{multus-daemonset.yml,flannel-daemonset.yml} | kubectl apply -f -
 ```
 
-Create a CNI configuration loaded as a CRD object, in this case a macvlan CNI configuration is defined. You may replace the `config` field with any valid CNI configuration where the CNI binary is available on the nodes.
+This will configure your systems to be ready to use Multus CNI, but, to get started with adding additional interfaces to your pods, refer to our complete [quick-start guide](doc/quickstart.md)
 
-```
-cat <<EOF | kubectl create -f -
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: macvlan-conf
-spec: 
-  config: '{
-      "cniVersion": "0.3.0",
-      "type": "macvlan",
-      "master": "eth0",
-      "mode": "bridge",
-      "ipam": {
-        "type": "host-local",
-        "subnet": "192.168.1.0/24",
-        "rangeStart": "192.168.1.200",
-        "rangeEnd": "192.168.1.216",
-        "routes": [
-          { "dst": "0.0.0.0/0" }
-        ],
-        "gateway": "192.168.1.1"
-      }
-    }'
-EOF
-```
+## Additional installation Options
 
-You may then create a pod which attached this additional interface, where the annotation correlates to the `name` in the `NetworkAttachmentDefinition` above.
+- Install via daemonset using the quick-start guide, above.
+- Download binaries from [release page](https://github.com/intel/multus-cni/releases)
+- By Docker image from [Docker Hub](https://hub.docker.com/r/nfvpe/multus/tags/)
+- Or, roll-you-own and build from source
+  - See [Development](doc/development.md)
 
-```
-cat <<EOF | kubectl create -f -
-apiVersion: v1
-kind: Pod
-metadata:
-  name: samplepod
-  annotations:
-    k8s.v1.cni.cncf.io/networks: macvlan-conf
-spec:
-  containers:
-  - name: samplepod
-    command: ["/bin/bash", "-c", "sleep 2000000000000"]
-    image: dougbtv/centos-network
-EOF
-```
+## Comprehensive Documentation
 
-You may now inspect the pod and see that there is an additional interface configured, like so:
+- [How to use](doc/how-to-use.md)
+- [Configuration](doc/configuration.md)
+- [Development](doc/development.md)
 
-```
-$ kubectl exec -it samplepod -- ip a
-```
-
-# Kubernetes Network Custom Resource Definition De-facto Standard - Reference implementation
-
-* This project is a reference implementation for Kubernetes Network Custom Resource Definition De-facto Standard. For more information refer [Network Plumbing Working Group Agenda](https://docs.google.com/document/d/1oE93V3SgOGWJ4O1zeD1UmpeToa0ZiiO6LqRAmZBPFWM/edit)
-* Kubernetes Network Custom Resource Definition De-facto Standard [documentation link](https://docs.google.com/document/d/1Ny03h6IDVy_e_vmElOqR7UdTPAG_RNydhVE1Kx54kFQ/edit)
-* Reference implementation support following modes
-   * CNI config JSON in network object
-   * Not using CNI config (“thick” plugin usecase)
-   * CNI configuration stored in on-disk file
-   > refer the section 3.2 Network Object Definition for more details in Kubernetes Network Custom Resource Definition De-facto Standard
-* Refer the reference implementation presentation and demo details - [link](https://docs.google.com/presentation/d/1dbCin6MnhK-BjjcVun5YiPTL99VA2uSiyWAtWAPNlIc/edit?usp=sharing)
-* Release version from v2.0 is not compatible with v1.1 and v1.2 network CRD 
-  * [MULTUS CNI plugin](#multus-cni-plugin)specifications.
-
-## Multi-Homed pod
-<p align="center">
-   <img src="doc/images/multus_cni_pod.png" width="1008" />
-</p>
-
-## Building from source
-
-**This plugin requires Go 1.8 (or later) to build.**
-
-```
-#./build
-```
-
-## Work flow
-<p align="center">
-   <img src="doc/images/workflow.png" width="1008" />
-</p>
-
-## Network configuration reference
-
-- name (string, required): the name of the network
-- type (string, required): &quot;multus&quot;
-- kubeconfig (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/intel/multus-cni/blob/master/doc/node-kubeconfig.yaml)
-- delegates (([]map,required): number of delegate details in the Multus
-
-## Usage with Kubernetes CRD based network objects
-
-Kubelet is responsible for establishing network interfaces for pods; it does this by invoking its configured CNI plugin. When Multus is invoked it retrieves network references from Pod annotation. Multus then uses these network references to get network configurations. Network configurations are defined as Kubernetes Custom Resource Object (CRD). These configurations describe which CNI plugins to invoke and what their configurations are. The order of plugin invocation is important as it identifies the primary plugin. This order is taken from network object references given in a Pod spec.
-
-<p align="center">
-   <img src="doc/images/multus_crd_usage_diagram.JPG" width="1008" />
-</p>
-
-### Creating &quot;Network&quot; resources in Kubernetes
-
-You may wish to create the `network-attachment-definition` manually if you haven't installed using the daemonset technique, which includes the CRD, and you can verify if it's loaded with `kubectl get crd` and look for the presence of `network-attachment-definition`.
-
-1. Create a Custom Resource Definition (CRD) `crdnetwork.yaml`; for the network object using the YAML from the examples directory.
-
-```
-$ kubectl create -f ./examples/crd.yml
-customresourcedefinition.apiextensions.k8s.io/network-attachment-definitions.k8s.cni.cncf.io created
-```
-
-3. Run kubectl get command to check the Network CRD creation
-
-```
-$ kubectl get crd
-NAME                      KIND
-network-attachment-definitions.k8s.cni.cncf.io   CustomResourceDefinition.v1beta1.apiextensions.k8s.io
-```
-
-### Creating CRD network resources in Kubernetes
-
-1. After creating CRD network object you can create network resources in Kubernetes. These network resources may contain additional underlying CNI plugin parameters given in JSON format. In the following example shown below the args field contains parameters that will be passed into Flannel plugin.
-
-2. Save the following YAML to flannel-network.yaml
-
-```
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: flannel-networkobj
-spec: 
-  config: '{
-    "cniVersion": "0.3.0",
-    "type": "flannel",
-    "delegate": {
-      "isDefaultGateway": true
-    }
-  }'
-```
-
-3. Create the custom resource definition
-
-```
-$ kubectl create -f ./flannel-network.yaml
-network "flannel-networkobj" created
-
-$ kubectl get net-attach-def
-NAME                         AGE
-flannel-networkobj           26s
-```
-
-4. Get the custom network object details
-
-```
-apiVersion: k8s.cni.cncf.io/v1
-kind: NetworkAttachmentDefinition
-metadata:
-  clusterName: ""
-  creationTimestamp: 2018-05-17T09:13:20Z
-  deletionGracePeriodSeconds: null
-  deletionTimestamp: null
-  initializers: null
-  name: flannel-networkobj
-  namespace: default
-  resourceVersion: "21176114"
-  selfLink: /apis/k8s.cni.cncf.io/v1/namespaces/default/networks/flannel-networkobj
-  uid: 8ac8f873-59b2-11e8-8308-a4bf01024e6f
-spec:
-  config: '{ "cniVersion": "0.3.0", "type": "flannel", "delegate": { "isDefaultGateway":
-    true } }'
-```
-
-5. Save the following YAML to sriov-network.yaml to creating sriov network object. ( Refer to [Intel - SR-IOV CNI](https://github.com/Intel-Corp/sriov-cni) or contact @kural in [Intel-Corp Slack](https://intel-corp.herokuapp.com/) for running the DPDK based workloads in Kubernetes)
-
-```
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: sriov-conf
-spec:
-  config: '{
-    "type": "sriov",
-    "if0": "enp12s0f1",
-    "ipam": {
-            "type": "host-local",
-            "subnet": "10.56.217.0/24",
-            "rangeStart": "10.56.217.171",
-            "rangeEnd": "10.56.217.181",
-            "routes": [
-                    { "dst": "0.0.0.0/0" }
-            ],
-            "gateway": "10.56.217.1"
-    }
-  }'
-```
-
-6. Likewise save the following YAML to sriov-vlanid-l2enable-network.yaml to create another sriov based network object:
-
-```
-apiVersion: "k8s.cni.cncf.io/v1"
-kind: NetworkAttachmentDefinition
-metadata:
-  name: sriov-vlanid-l2enable-conf
-spec:
-  config: '{
-    "type": "sriov",
-    "if0": "enp2s0",
-    "vlan": 210,
-    "l2enable": true
-  }'
-```
-
-7. Follow step 3 above to create &quot;sriov-vlanid-l2enable-conf&quot; and &quot;sriov-conf&quot; network objects
-
-8. View network objects using kubectl
-
-```
-# kubectl get net-attach-def
-NAME                         AGE
-flannel-networkobj           29m
-sriov-conf                   6m
-sriov-vlanid-l2enable-conf   2m
-
-```
-
-### Configuring Multus to use the kubeconfig
-
-1. Create a Mutlus CNI configuration file on each Kubernetes node. This file should be created in: /etc/cni/net.d/multus-cni.conf with the content shown below. Use only the absolute path to point to the kubeconfig file (as it may change depending upon your cluster env). We are assuming all CNI plugin binaries are default location (`/opt/cni/bin dir`)
-
-```
-{
-    "name": "node-cni-network",
-    "type": "multus",
-    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml"
-}
-```
-
-### Configuring Multus to use kubeconfig and a default network
-
-1. Many users want Kubernetes default networking feature along with network objects. Refer to issues [#14](https://github.com/intel/multus-cni/issues/14) &amp; [#17](https://github.com/intel/multus-cni/issues/17) for more information. In the following Multus configuration, Weave act as the default network in the absence of network field in the pod metadata annotation.
-
-```
-{
-    "name": "node-cni-network",
-    "type": "multus",
-    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
-    "delegates": [{
-        "type": "weave-net",
-        "hairpinMode": true
-    }]
-}
-```
-
-Configurations referenced in annotations are created in addition to the default network.
-
-### Configuring Pod to use the CRD network objects
-
-1. Save the following YAML to pod-multi-network.yaml. In this case flannel-conf network object acts as the primary network.
-```
-# cat pod-multi-network.yaml 
-apiVersion: v1
-kind: Pod
-metadata:
-  name: multus-multi-net-poc
-  annotations:
-    k8s.v1.cni.cncf.io/networks: '[
-            { "name": "flannel-conf" },
-            { "name": "sriov-conf" },
-            { "name": "sriov-vlanid-l2enable-conf",
-              "interfaceRequest": "north" }
-    ]'
-spec:  # specification of the pod's contents
-  containers:
-  - name: multus-multi-net-poc
-    image: "busybox"
-    command: ["top"]
-    stdin: true
-    tty: true
-```
-
-2. Create Multiple network based pod from the master node
-
-```
-# kubectl create -f ./pod-multi-network.yaml
-pod "multus-multi-net-poc" created
-```
-
-3. Get the details of the running pod from the master
-
-```
-# kubectl get pods
-NAME                   READY     STATUS    RESTARTS   AGE
-multus-multi-net-poc   1/1       Running   0          30s
-```
-
-### Verifying Pod network interfaces
-
-1. Run `ifconfig` command in Pod:
-
-```
-# kubectl exec -it multus-multi-net-poc -- ifconfig       
-eth0      Link encap:Ethernet  HWaddr C6:43:7C:09:B4:9C
-          inet addr:10.128.0.4  Bcast:0.0.0.0  Mask:255.255.255.0
-          UP BROADCAST RUNNING MULTICAST  MTU:1450  Metric:1
-          RX packets:8 errors:0 dropped:0 overruns:0 frame:0
-          TX packets:1 errors:0 dropped:0 overruns:0 carrier:0
-          collisions:0 txqueuelen:0
-          RX bytes:648 (648.0 B)  TX bytes:42 (42.0 B)
-
-lo        Link encap:Local Loopback  
-          inet addr:127.0.0.1  Mask:255.0.0.0
-          inet6 addr: ::1/128 Scope:Host
-          UP LOOPBACK RUNNING  MTU:65536  Metric:1
-          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
-          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
-          collisions:0 txqueuelen:1 
-          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)
-          
-net0      Link encap:Ethernet  HWaddr 06:21:91:2D:74:B9  
-          inet addr:192.168.42.3  Bcast:0.0.0.0  Mask:255.255.255.0
-          inet6 addr: fe80::421:91ff:fe2d:74b9/64 Scope:Link
-          UP BROADCAST RUNNING MULTICAST  MTU:1450  Metric:1
-          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
-          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
-          collisions:0 txqueuelen:0 
-          RX bytes:0 (0.0 B)  TX bytes:648 (648.0 B)
-
-net1      Link encap:Ethernet  HWaddr D2:94:98:82:00:00  
-          inet addr:10.56.217.171  Bcast:0.0.0.0  Mask:255.255.255.0
-          inet6 addr: fe80::d094:98ff:fe82:0/64 Scope:Link
-          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
-          RX packets:2 errors:0 dropped:0 overruns:0 frame:0
-          TX packets:8 errors:0 dropped:0 overruns:0 carrier:0
-          collisions:0 txqueuelen:1000 
-          RX bytes:120 (120.0 B)  TX bytes:648 (648.0 B)
-
-north     Link encap:Ethernet  HWaddr BE:F2:48:42:83:12  
-          inet6 addr: fe80::bcf2:48ff:fe42:8312/64 Scope:Link
-          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
-          RX packets:1420 errors:0 dropped:0 overruns:0 frame:0
-          TX packets:1276 errors:0 dropped:0 overruns:0 carrier:0
-          collisions:0 txqueuelen:1000 
-          RX bytes:95956 (93.7 KiB)  TX bytes:82200 (80.2 KiB)
-```
-
-| Interface name | Description |
-| --- | --- |
-| lo | loopback |
-| eth0 | weave network interface |
-| net0 | Flannel network tap interface |
-| net1 | VF0 of NIC 1 assigned to the container by [Intel - SR-IOV CNI](https://github.com/intel/sriov-cni) plugin |
-| north | VF0 of NIC 2 assigned with VLAN ID 210 to the container by SR-IOV CNI plugin |
-
-2. Check the vlan ID of the NIC 2 VFs
-
-```
-# ip link show enp2s0
-20: enp2s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
-    link/ether 24:8a:07:e8:7d:40 brd ff:ff:ff:ff:ff:ff
-    vf 0 MAC 00:00:00:00:00:00, vlan 210, spoof checking off, link-state auto
-    vf 1 MAC 00:00:00:00:00:00, vlan 4095, spoof checking off, link-state auto
-    vf 2 MAC 00:00:00:00:00:00, vlan 4095, spoof checking off, link-state auto
-    vf 3 MAC 00:00:00:00:00:00, vlan 4095, spoof checking off, link-state auto
-```
-
-## Using with Multus conf file
-
-Given the following network configuration:
-
-```
-# tee /etc/cni/net.d/multus-cni.conf <<-'EOF'
-{
-    "name": "multus-demo-network",
-    "type": "multus",
-    "delegates": [
-        {
-                "type": "sriov",
-                #part of sriov plugin conf
-                "if0": "enp12s0f0", 
-                "ipam": {
-                        "type": "host-local",
-                        "subnet": "10.56.217.0/24",
-                        "rangeStart": "10.56.217.131",
-                        "rangeEnd": "10.56.217.190",
-                        "routes": [
-                                { "dst": "0.0.0.0/0" }
-                        ],
-                        "gateway": "10.56.217.1"
-                }
-        },
-        {
-                "type": "ptp",
-                "ipam": {
-                        "type": "host-local",
-                        "subnet": "10.168.1.0/24",
-                        "rangeStart": "10.168.1.11",
-                        "rangeEnd": "10.168.1.20",
-                        "routes": [
-                                { "dst": "0.0.0.0/0" }
-                        ],
-                        "gateway": "10.168.1.1"
-                }
-        },
-        {
-                "type": "flannel",
-                "delegate": {
-                        "isDefaultGateway": true
-                }
-        }
-    ]
-}
-EOF
-```
-
-## Logging Options
-
-You may wish to enable some enhanced logging for Multus, especially during the process where you're configuring Multus and need to understand what is or isn't working with your particular configuration. 
-
-Multus will always log via `STDERR`, which is the standard method by which CNI plugins communicate errors, and these errors are logged by the Kubelet. This method is always enabled.
-
-### Writing to a Log File
-
-Optionally, you may have Multus log to a file on the filesystem. This file will be written locally on each node where Multus is executed. You may configure this via the `LogFile` option in the CNI configuration. By default this additional logging to a flat file is disabled.
-
-For example in your CNI configuration, you may set:
-
-```
-    "LogFile": "/var/log/multus.log",
-```
-
-### Logging Level
-
-The default logging level is set as `panic` -- this will log only the most critical errors, and is the least verbose logging level.
-
-The available logging level values, in decreasing order of verbosity are:
-
-* `debug`
-* `error`
-* `panic`
-
-You may configure the logging level by using the `LogLevel` option in your CNI configuration. For example:
-
-```
-    "LogLevel": "debug",
-```
-
-## Testing Multus CNI
-
-### Multiple flannel networks
-
-Github user [YYGCui](https://github.com/YYGCui) has used multiple flannel network to work with Multus CNI plugin. Please refer to this [closed issue](https://github.com/intel/multus-cni/issues/7) for ,multiple overlay network support with Multus CNI.
-
-Make sure that the multus, [sriov](https://github.com/Intel-Corp/sriov-cni), [flannel](https://github.com/containernetworking/cni/blob/master/Documentation/flannel.md), and [ptp](https://github.com/containernetworking/cni/blob/master/Documentation/ptp.md) binaries are in the /opt/cni/bin directories and follow the steps as mentioned in the [CNI](https://github.com/containernetworking/cni/#running-a-docker-container-with-network-namespace-set-up-by-cni-plugins)
-
-#### Configure Kubernetes with CNI
-
-Kubelet must be configured to run with the CNI network plugin. Edit `/etc/kubernetes/kubelet` file and add `--network-plugin=cni` flags in `KUBELET\_OPTS `as shown below:
-
-```
-KUBELET_OPTS="...
---network-plugin-dir=/etc/cni/net.d
---network-plugin=cni
-"
-```
-
-Refer to the Kubernetes User Guide and network plugin for more information.
-- [Single Node](https://kubernetes.io/docs/getting-started-guides/fedora/fedora_manual_config/)
-- [Multi Node](https://kubernetes.io/docs/getting-started-guides/fedora/flannel_multi_node_cluster/)
-- [Network plugin](https://kubernetes.io/docs/admin/network-plugins/)
-
-#### Launching workloads in Kubernetes
-
-With Multus CNI configured as described in sections above each workload launched via a Kubernetes Pod will have multiple network interfacesLaunch the workload using yaml file in the kubernetes master, with above configuration in the multus CNI, each pod should have multiple interfaces.
-
-Note: To verify whether Multus CNI plugin is working correctly, create a pod containing one `busybox` container and execute `ip link` command to check if interfaces management follows configuration.
-
-1. Create `multus-test.yaml` file containing below configuration. Created pod will consist of one `busybox` container running `top` command.
-
-```
-apiVersion: v1
-kind: Pod
-metadata:
-  name: multus-test
-spec:  # specification of the pod's contents
-  restartPolicy: Never
-  containers:
-  - name: test1
-    image: "busybox"
-    command: ["top"]
-    stdin: true
-    tty: true
-
-```
-
-2. Create pod using command:
-
-```
-# kubectl create -f multus-test.yaml
-pod "multus-test" created
-```
-
-3. Run &quot;ip link&quot; command inside the container:
-
-```
-# 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1
-    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
-3: eth0@if41: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue
-    link/ether 26:52:6b:d8:44:2d brd ff:ff:ff:ff:ff:ff
-20: net0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq qlen 1000
-    link/ether f6:fb:21:4f:1d:63 brd ff:ff:ff:ff:ff:ff
-21: net1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq qlen 1000
-    link/ether 76:13:b1:60:00:00 brd ff:ff:ff:ff:ff:ff 
-```
-
-| Interface name | Description |
-| --- | --- |
-| lo | loopback |
-| eth0@if41 | Flannel network tap interface |
-| net0 | VF assigned to the container by [SR-IOV CNI](https://github.com/Intel-Corp/sriov-cni) plugin |
-| net1 | ptp localhost interface |
-
-## Multus additional plugins
-
-- [DPDK-SRIOV CNI](https://github.com/Intel-Corp/sriov-cni)
-- [Vhostuser CNI](https://github.com/intel/vhost-user-net-plugin) - a Dataplane network plugin - Supports OVS-DPDK &amp; VPP
-- [Bond CNI](https://github.com/Intel-Corp/bond-cni) - For fail-over and high availability of networking
-
-## NFV based networking in Kubernetes
-
-- KubeCon workshop on [&quot;Enabling NFV features in Kubernetes&quot;](https://kccncna17.sched.com/event/Cvnw/enabling-nfv-features-in-kubernetes-hosted-by-kuralamudhan-ramakrishnan-ivan-coughlan-intel) presentation [slide deck](https://www.slideshare.net/KuralamudhanRamakris/enabling-nfv-features-in-kubernetes-83923352)
-- Feature brief
-  - [Multiple Network Interface Support in Kubernetes](https://builders.intel.com/docs/networkbuilders/multiple-network-interfaces-support-in-kubernetes-feature-brief.pdf)
-  - [Enhanced Platform Awareness in Kubernetes](https://builders.intel.com/docs/networkbuilders/enhanced-platform-awareness-feature-brief.pdf)
-- Application note
-  - [Multiple Network Interfaces in Kubernetes and Container Bare Metal](https://builders.intel.com/docs/networkbuilders/multiple-network-interfaces-in-kubernetes-application-note.pdf)
-  - [Enhanced Platform Awareness Features in Kubernetes](https://builders.intel.com/docs/networkbuilders/enhanced-platform-awareness-in-kubernetes-application-note.pdf)
-- White paper
-  - [Enabling New Features with Kubernetes for NFV](https://builders.intel.com/docs/networkbuilders/enabling_new_features_in_kubernetes_for_NFV.pdf)
-- Multus&#39;s related project github pages
-  - [Multus](https://github.com/Intel-Corp/multus-cni)
-  - [SRIOV - DPDK CNI](https://github.com/Intel-Corp/sriov-cni)
-  - [Vhostuser - VPP &amp; OVS - DPDK CNI](https://github.com/intel/vhost-user-net-plugin)
-  - [Bond CNI](https://github.com/Intel-Corp/bond-cni)
-  - [Node Feature Discovery](https://github.com/kubernetes-incubator/node-feature-discovery)
-  - [CPU Manager for Kubernetes](https://github.com/Intel-Corp/CPU-Manager-for-Kubernetes)
-
-
-## Need help
-
-- Read [Containers Experience Kits](https://networkbuilders.intel.com/network-technologies/container-experience-kits)
-- Try our container exp kit demo - KubeCon workshop on [Enabling NFV Features in Kubernetes](https://github.com/intel/container-experience-kits-demo-area/)
-- Join us on [#intel-sddsg-slack](https://intel-corp.herokuapp.com/) slack channel and ask question in [#general-discussion](https://intel-corp-team.slack.com/messages/C4C5RSEER)
-- You can also [email](mailto:kuralamudhan.ramakrishnan@intel.com) us
-- Feel free to [submit](https://github.com/Intel-Corp/multus-cni/issues/new) an issue
-
-Please fill in the Questions/feedback - [google-form](https://goo.gl/forms/upBWyGs8Wmq69IEi2)!
-
-## Contacts
-For any questions about Multus CNI, please reach out on github issue or feel free to contact the developer @kural in our [Intel-Corp Slack](https://intel-corp.herokuapp.com/)
+## Contact Us
 
+For any questions about Multus CNI, feel free to ask a question in #general in the [Intel-Corp Slack](https://intel-corp.herokuapp.com/), or open up a GitHub issue. 

build

@@ -14,4 +14,4 @@ export GOBIN=${PWD}/bin
 export GOPATH=${PWD}/gopath
 
 echo "Building plugins"
-go install "$@" ${REPO_PATH}/multus
+CGO_ENABLED=0 go install "$@" -tags no_openssl ${REPO_PATH}/multus

checkpoint/checkpoint.go

@@ -0,0 +1,110 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package checkpoint
+
+import (
+	"encoding/json"
+	"io/ioutil"
+
+	"github.com/intel/multus-cni/logging"
+	"github.com/intel/multus-cni/types"
+	v1 "k8s.io/api/core/v1"
+)
+
+const (
+	checkPointfile = "/var/lib/kubelet/device-plugins/kubelet_internal_checkpoint"
+)
+
+type PodDevicesEntry struct {
+	PodUID        string
+	ContainerName string
+	ResourceName  string
+	DeviceIDs     []string
+	AllocResp     []byte
+}
+
+type checkpointData struct {
+	PodDeviceEntries  []PodDevicesEntry
+	RegisteredDevices map[string][]string
+}
+
+type Data struct {
+	Data     checkpointData
+	Checksum uint64
+}
+
+type checkpoint struct {
+	fileName   string
+	podEntires []PodDevicesEntry
+}
+
+// GetCheckpoint returns an instance of Checkpoint
+func GetCheckpoint() (types.ResourceClient, error) {
+	logging.Debugf("GetCheckpoint(): invoked")
+	return getCheckpoint(checkPointfile)
+}
+
+func getCheckpoint(filePath string) (types.ResourceClient, error) {
+	cp := &checkpoint{fileName: filePath}
+	err := cp.getPodEntries()
+	if err != nil {
+		return nil, err
+	}
+	logging.Debugf("getCheckpoint(): created checkpoint instance with file: %s", filePath)
+	return cp, nil
+}
+
+// getPodEntries gets all Pod device allocation entries from checkpoint file
+func (cp *checkpoint) getPodEntries() error {
+
+	cpd := &Data{}
+	rawBytes, err := ioutil.ReadFile(cp.fileName)
+	if err != nil {
+		return logging.Errorf("getPodEntries(): error reading file %s\n%v\n", checkPointfile, err)
+	}
+
+	if err = json.Unmarshal(rawBytes, cpd); err != nil {
+		return logging.Errorf("getPodEntries(): error unmarshalling raw bytes %v", err)
+	}
+
+	cp.podEntires = cpd.Data.PodDeviceEntries
+	logging.Debugf("getPodEntries(): podEntires %+v", cp.podEntires)
+	return nil
+}
+
+// GetComputeDeviceMap returns an instance of a map of ResourceInfo
+func (cp *checkpoint) GetPodResourceMap(pod *v1.Pod) (map[string]*types.ResourceInfo, error) {
+	podID := string(pod.UID)
+	resourceMap := make(map[string]*types.ResourceInfo)
+
+	if podID == "" {
+		return nil, logging.Errorf("GetPodResourceMap(): invalid Pod cannot be empty")
+	}
+
+	for _, pod := range cp.podEntires {
+		if pod.PodUID == podID {
+			entry, ok := resourceMap[pod.ResourceName]
+			if ok {
+				// already exists; append to it
+				entry.DeviceIDs = append(entry.DeviceIDs, pod.DeviceIDs...)
+			} else {
+				// new entry
+				resourceMap[pod.ResourceName] = &types.ResourceInfo{DeviceIDs: pod.DeviceIDs}
+			}
+		}
+	}
+	return resourceMap, nil
+}

checkpoint/checkpoint_test.go

@@ -0,0 +1,131 @@
+package checkpoint
+
+import (
+	"os"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"io/ioutil"
+	"testing"
+
+	"github.com/intel/multus-cni/types"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8sTypes "k8s.io/apimachinery/pkg/types"
+)
+
+const (
+	fakeTempFile = "/tmp/kubelet_internal_checkpoint"
+)
+
+type fakeCheckpoint struct {
+	fileName string
+}
+
+func (fc *fakeCheckpoint) WriteToFile(inBytes []byte) error {
+	return ioutil.WriteFile(fc.fileName, inBytes, 0600)
+}
+
+func (fc *fakeCheckpoint) DeleteFile() error {
+	return os.Remove(fc.fileName)
+}
+
+func TestCheckpoint(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Checkpoint")
+}
+
+var _ = BeforeSuite(func() {
+	sampleData := `{
+		"Data": {
+			"PodDeviceEntries": [
+			{
+				"PodUID": "970a395d-bb3b-11e8-89df-408d5c537d23",
+				"ContainerName": "appcntr1",
+				"ResourceName": "intel.com/sriov_net_A",
+				"DeviceIDs": [
+				"0000:03:02.3",
+				"0000:03:02.0"
+				],
+				"AllocResp": "CikKC3NyaW92X25ldF9BEhogMDAwMDowMzowMi4zIDAwMDA6MDM6MDIuMA=="
+			}
+			],
+			"RegisteredDevices": {
+			"intel.com/sriov_net_A": [
+				"0000:03:02.1",
+				"0000:03:02.2",
+				"0000:03:02.3",
+				"0000:03:02.0"
+			],
+			"intel.com/sriov_net_B": [
+				"0000:03:06.3",
+				"0000:03:06.0",
+				"0000:03:06.1",
+				"0000:03:06.2"
+			]
+			}
+		},
+		"Checksum": 229855270
+		}`
+
+	fakeCheckpoint := &fakeCheckpoint{fileName: fakeTempFile}
+	err := fakeCheckpoint.WriteToFile([]byte(sampleData))
+	Expect(err).NotTo(HaveOccurred())
+})
+
+var _ = Describe("Kubelet checkpoint data read operations", func() {
+	Context("Using /tmp/kubelet_internal_checkpoint file", func() {
+		var (
+			cp            types.ResourceClient
+			err           error
+			resourceMap   map[string]*types.ResourceInfo
+			resourceInfo  *types.ResourceInfo
+			resourceAnnot = "intel.com/sriov_net_A"
+		)
+
+		It("should get a Checkpoint instance from file", func() {
+			cp, err = getCheckpoint(fakeTempFile)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should return a ResourceMap instance", func() {
+			podUID := k8sTypes.UID("970a395d-bb3b-11e8-89df-408d5c537d23")
+			fakePod := &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "fakePod",
+					Namespace: "podNamespace",
+					UID:       podUID,
+				},
+			}
+			rmap, err := cp.GetPodResourceMap(fakePod)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(rmap).NotTo(BeEmpty())
+			resourceMap = rmap
+		})
+
+		It("resourceMap should have value for \"intel.com/sriov_net_A\"", func() {
+			rInfo, ok := resourceMap[resourceAnnot]
+			Expect(ok).To(BeTrue())
+			resourceInfo = rInfo
+		})
+
+		It("should have 2 deviceIDs", func() {
+			Expect(len(resourceInfo.DeviceIDs)).To(BeEquivalentTo(2))
+		})
+
+		It("should have \"0000:03:02.3\" in deviceIDs[0]", func() {
+			Expect(resourceInfo.DeviceIDs[0]).To(BeEquivalentTo("0000:03:02.3"))
+		})
+
+		It("should have \"0000:03:02.0\" in deviceIDs[1]", func() {
+			Expect(resourceInfo.DeviceIDs[1]).To(BeEquivalentTo("0000:03:02.0"))
+		})
+	})
+})
+
+var _ = AfterSuite(func() {
+	fakeCheckpoint := &fakeCheckpoint{fileName: fakeTempFile}
+	err := fakeCheckpoint.DeleteFile()
+	Expect(err).NotTo(HaveOccurred())
+})

deployment/webhook/certs.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# Copyright (c) 2018 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# create temp dir to store intermediate files
+tmp=$(mktemp -d)
+
+# generate private key
+echo "Generating private RSA key..."
+openssl genrsa -out ${tmp}/webhook-key.pem 2048 >/dev/null 2>&1
+
+# generate CSR
+echo "Generating CSR configuration file..."
+cat <<EOF >> ${tmp}/webhook.conf
+[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+[req_distinguished_name]
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage = serverAuth
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = multus-webhook-service
+DNS.2 = multus-webhook-service.default
+DNS.3 = multus-webhook-service.default.svc
+EOF
+openssl req -new -key ${tmp}/webhook-key.pem -subj "/CN=multus-webhook-service.default.svc" -out ${tmp}/server.csr -config ${tmp}/webhook.conf
+
+# push CSR to Kubernetes API server
+echo "Sending CSR to Kubernetes..."
+csr_name="multus-webhook-service.default"
+kubectl delete csr ${csr_name} >/dev/null 2>&1
+cat <<EOF | kubectl create -f -
+apiVersion: certificates.k8s.io/v1beta1
+kind: CertificateSigningRequest
+metadata:
+  name: ${csr_name}
+spec:
+  request: $(cat ${tmp}/server.csr | base64 -w0)
+  groups:
+  - system:authenticated
+  usages:
+  - digital signature
+  - key encipherment
+  - server auth
+EOF
+
+# approve certificate
+echo "Approving CSR..."
+kubectl certificate approve ${csr_name}
+
+# wait for the cert to be issued
+echo -n "Waiting for the certificate to be issued..."
+cert=""
+for sec in $(seq 15); do
+  cert=$(kubectl get csr ${csr_name} -o jsonpath='{.status.certificate}')
+  if [[ $cert != "" ]]; then
+    echo -e "\nCertificate issued succesfully."
+    echo $cert | base64 --decode > ${tmp}/webhook-cert.pem
+    break
+  fi
+  echo -n "."; sleep 1
+done
+if [[ $cert == "" ]]; then
+  echo -e "\nError: certificate not issued. Verify that the API for signing certificates is enabled."
+  exit
+fi
+
+# create secret
+echo "Creating secret..."
+kubectl delete secret "multus-webhook-secret"
+kubectl create secret generic --from-file=key.pem=${tmp}/webhook-key.pem --from-file=cert.pem=${tmp}/webhook-cert.pem "multus-webhook-secret"
+
+# set cert in webhook configuration
+echo "Patching configuration file with certificate..."
+if [[ -f configuration-template.yaml ]]; then
+  sed "s/__CERT__/${cert}/" configuration-template.yaml > configuration.yaml
+  echo "File configuration.yaml patched."
+else
+  echo -e "Error: validating configuration template file 'configuration-template.yaml' is missing. Please update it with cert.pem value from the secret manually."
+fi

deployment/webhook/configuration-template.yaml

@@ -0,0 +1,38 @@
+# Copyright (c) 2018 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    app: multus-webhook
+  name: multus-webhook-config
+webhooks:
+- clientConfig:
+    caBundle: __CERT__
+    service:
+      name: multus-webhook-service
+      namespace: default
+      path: /validate
+  failurePolicy: Fail
+  name: multus-webhook.k8s.cni.cncf.io
+  rules:
+  - apiGroups:
+    - k8s.cni.cncf.io
+    apiVersions:
+    - v1
+    resources:
+    - network-attachment-definitions
+    operations:
+    - CREATE

deployment/webhook/deployment.yaml

@@ -0,0 +1,50 @@
+# Copyright (c) 2018 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: multus-webhook
+  name: multus-webhook-deployment
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: multus-webhook
+  template:
+    metadata:
+      labels:
+        app: multus-webhook
+    spec:
+      containers:
+      - name: multus-webhook
+        image: multus-webhook
+        command:
+        - /webhook/webhook
+        args:
+        - --bind-address=0.0.0.0
+        - --port=443
+        - --tls-private-key-file=/webhook/tls/key.pem
+        - --tls-cert-file=/webhook/tls/cert.pem
+        volumeMounts:
+        - mountPath: /webhook/tls
+          name: multus-webhook-secret
+          readOnly: True
+        imagePullPolicy: IfNotPresent
+      volumes:
+      - name: multus-webhook-secret
+        secret:
+          secretName: multus-webhook-secret

deployment/webhook/service.yaml

@@ -0,0 +1,27 @@
+# Copyright (c) 2018 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: multus-webhook-service
+  labels:
+    app: multus-webhook
+  namespace: default
+spec:
+  ports:
+  - port: 443
+    targetPort: 443
+  selector:
+    app: multus-webhook

doc/configuration.md

@@ -0,0 +1,251 @@
+## Multus-cni Configuration Reference
+
+Following is the example of multus config file, in `/etc/cni/net.d/`.
+(`"Note1"` and `"Note2"` are just comments, so you can remove them at your configuration)
+
+```
+{
+    "name": "node-cni-network",
+    "type": "multus",
+    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+    "confDir": "/etc/cni/multus/net.d",
+    "cniDir": "/var/lib/cni/multus",
+    "binDir": "/opt/cni/bin",
+    "logFile": "/var/log/multus.log",
+    "logLevel": "debug",
+    "capabilities": {
+        "portMappings": true
+    },    
+    "readinessindicatorfile": "",
+    "namespaceIsolation": false,
+"Note1":"NOTE: you can set clusterNetwork+defaultNetworks OR delegates!!",
+    "clusterNetwork": "defaultCRD",
+    "defaultNetworks": ["sidecarCRD", "flannel"],
+    "systemNamespaces": ["kube-system", "admin"],
+    "multusNamespace": "kube-system",
+"Note2":"NOTE: If you use clusterNetwork/defaultNetworks, delegates is ignored",
+    "delegates": [{
+        "type": "weave-net",
+        "hairpinMode": true
+    }, {
+        "type": "macvlan",
+        ... (snip)
+    }]
+}
+```
+
+* `name` (string, required): the name of the network
+* `type` (string, required): "multus"
+* `confDir` (string, optional): directory for CNI config file that multus reads. default `/etc/cni/multus/net.d`
+* `cniDir` (string, optional): Multus CNI data directory, default `/var/lib/cni/multus`
+* `binDir` (string, optional): directory for CNI plugins which multus calls. default `/opt/cni/bin`
+* `kubeconfig` (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/intel/multus-cni/blob/master/doc/node-kubeconfig.yaml). If you would like to use CRD (i.e. network attachment definition), this is required
+* `logFile` (string, optional): file path for log file. multus puts log in given file
+* `logLevel` (string, optional): logging level ("debug", "error" or "panic")
+* `namespaceIsolation` (boolean, optional): Enables a security feature where pods are only allowed to access `NetworkAttachmentDefinitions` in the namespace where the pod resides. Defaults to false.
+* `capabilities` ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings capability for now). See the [example](https://github.com/intel/multus-cni/blob/master/examples/multus-ptp-portmap.conf).
+* `readinessindicatorfile`: The path to a file whose existance denotes that the default network is ready
+
+User should chose following parameters combination (`clusterNetwork`+`defaultNetworks` or `delegates`):
+
+* `clusterNetwork` (string, required): default CNI network for pods, used in kubernetes cluster (Pod IP and so on): name of network-attachment-definition, CNI json file name (without extention, .conf/.conflist) or directory for CNI config file
+* `defaultNetworks` ([]string, required): default CNI network attachment: name of network-attachment-definition, CNI json file name (without extention, .conf/.conflist) or directory for CNI config file
+* `systemNamespaces` ([]string, optional): list of namespaces for Kubernetes system (namespaces listed here will not have `defaultNetworks` added)
+* `multusNamespace` (string, optional): namespace for `clusterNetwork`/`defaultNetworks`
+* `delegates` ([]map,required): number of delegate details in the Multus
+
+### Network selection flow of clusterNetwork/defaultNetworks
+
+Multus will find network for clusterNetwork/defaultNetworks as following sequences:
+
+1. CRD object for given network name, in 'kube-system' namespace
+1. CNI json config file in `confDir`. Given name should be without extention, like .conf/.conflist. (e.g. "test" for "test.conf")
+1. Directory for CNI json config file. Multus will find alphabetically first file for the network
+1. Multus failed to find network. Multus raise error message
+
+## Miscellaneous config
+
+### Default Network Readiness Indicator
+
+You may wish for your "default network" (that is, the CNI plugin & its configuration you specify as your default delegate) to become ready before you attach networks with Multus. This is disabled by default and not used unless you add the readiness check option(s) to your CNI configuration file.
+
+For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, hence, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists.
+
+In this manner, you may prevent pods from crash looping, and instead wait for that default network to be ready.
+
+Only one option is necessary to configure this functionality:
+
+* `readinessindicatorfile`: The path to a file whose existance denotes that the default network is ready.
+
+*NOTE*: If `readinessindicatorfile` is unset, or is an empty string, this functionality will be disabled, and is disabled by default.
+
+
+### Logging
+
+You may wish to enable some enhanced logging for Multus, especially during the process where you're configuring Multus and need to understand what is or isn't working with your particular configuration.
+
+Multus will always log via `STDERR`, which is the standard method by which CNI plugins communicate errors, and these errors are logged by the Kubelet. This method is always enabled.
+
+#### Writing to a Log File
+
+Optionally, you may have Multus log to a file on the filesystem. This file will be written locally on each node where Multus is executed. You may configure this via the `LogFile` option in the CNI configuration. By default this additional logging to a flat file is disabled.
+
+For example in your CNI configuration, you may set:
+
+```
+    "LogFile": "/var/log/multus.log",
+```
+
+#### Logging Level
+
+The default logging level is set as `panic` -- this will log only the most critical errors, and is the least verbose logging level.
+
+The available logging level values, in decreasing order of verbosity are:
+
+* `debug`
+* `error`
+* `panic`
+
+You may configure the logging level by using the `LogLevel` option in your CNI configuration. For example:
+
+```
+    "LogLevel": "debug",
+```
+
+### Namespace Isolation
+
+The functionality provided by the `namespaceIsolation` configuration option enables a mode where Multus only allows pods to access custom resources (the `NetworkAttachmentDefinitions`) within the namespace where that pod resides. In other words, the `NetworkAttachmentDefinitions` are isolated to usage within the namespace in which they're created. 
+
+For example, if a pod is created in the namespace called `development`, Multus will not allow networks to be attached when defined by custom resources created in a different namespace, say in the `default` network.
+
+Consider the situation where you have a system that has users of different privilege levels -- as an example, a platform which has two administrators: a Senior Administrator and a Junior Administrator. The Senior Administrator may have access to all namespaces, and some network configurations as used by Multus are considered to be privileged in that they allow access to some protected resources available on the network. However, the Junior Administrator has access to only a subset of namespaces, and therefore it should be assumed that the Junior Administrator cannot create pods in their limited subset of namespaces. The `namespaceIsolation` feature provides for this isolation, allowing pods created in given namespaces to only access custom resources in the same namespace as the pod.
+
+Namespace Isolation is disabled by default.
+
+#### Configuration example
+
+```
+  "namespaceIsolation": true,
+```
+
+#### Usage example
+
+Let's setup an example where we:
+
+* Create a custom resource in a namespace called `privileged`
+* Create a pod in a namespace called `development`, and have annotations that reference a custom resource in the `privileged` namespace. The creation of this pod should be disallowed by Multus (as we'll have the use of the custom resources limited only to those custom resources created within the same namespace as the pod).
+
+Given the above scenario with a Junior & Senior Administrator. You may assume that the Senior Administrator has access to all namespaces, whereas the Junior Administrator has access only to the `development` namespace.
+
+Firstly, we show that we have a number of namespaces available:
+
+```
+# List the available namespaces
+[user@kube-master ~]$ kubectl get namespaces
+NAME          STATUS   AGE
+default       Active   7h27m
+development   Active   3h
+kube-public   Active   7h27m
+kube-system   Active   7h27m
+privileged    Active   4s
+```
+
+We'll create a `NetworkAttachmentDefinition` in the `privileged` namespace.
+
+```
+# Show the network attachment definition we're creating.
+[user@kube-master ~]$ cat cr.yml
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf
+spec:
+  config: '{
+      "cniVersion": "0.3.0",
+      "type": "macvlan",
+      "master": "eth0",
+      "mode": "bridge",
+      "ipam": {
+        "type": "host-local",
+        "subnet": "192.168.1.0/24",
+        "rangeStart": "192.168.1.200",
+        "rangeEnd": "192.168.1.216",
+        "routes": [
+          { "dst": "0.0.0.0/0" }
+        ],
+        "gateway": "192.168.1.1"
+      }
+    }'
+
+# Create that network attachment definition in the privileged namespace
+[user@kube-master ~]$ kubectl create -f cr.yml -n privileged
+networkattachmentdefinition.k8s.cni.cncf.io/macvlan-conf created
+
+# List the available network attachment definitions in the privileged namespace.
+[user@kube-master ~]$ kubectl get networkattachmentdefinition.k8s.cni.cncf.io -n privileged
+NAME           AGE
+macvlan-conf   11s
+```
+
+Next, we'll create a pod with an annotation that references the privileged namespace. Pay particular attention to the annotation that reads `k8s.v1.cni.cncf.io/networks: privileged/macvlan-conf` -- where it contains a reference to a `namespace/configuration-name` formatted network attachment name. In this case referring to the `macvlan-conf` in the namespace called `privileged`.
+
+```
+# Show the yaml for a pod.
+[user@kube-master ~]$ cat example.pod.yml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: privileged/macvlan-conf
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "sleep 2000000000000"]
+    image: dougbtv/centos-network
+
+# Create that pod.
+[user@kube-master ~]$ kubectl create -f example.pod.yml -n development
+pod/samplepod created
+```
+
+You'll note that pod fails to spawn successfully. If you check the Multus logs, you'll see an entry such as:
+
+```
+2018-12-18T21:41:32Z [error] GetPodNetwork: namespace isolation violation: podnamespace: development / target namespace: privileged
+```
+
+This error expresses that the pod resides in the namespace named `development` but refers to a `NetworkAttachmentDefinition` outside of that namespace, in this case, the namespace named `privileged`.
+
+In a positive example, you'd instead create the `NetworkAttachmentDefinition` in the `development` namespace, and you'd have an annotation that either A. does not reference a namespace, or B. refers to the same annotation.
+
+A positive example may be:
+
+```
+# Create the same NetworkAttachmentDefinition as above, however in the development namespace
+[user@kube-master ~]$ kubectl create -f cr.yml -n development
+networkattachmentdefinition.k8s.cni.cncf.io/macvlan-conf created
+
+# Show the yaml for a sample pod which references macvlan-conf without a namspace/ format
+[user@kube-master ~]$ cat positive.example.pod
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "sleep 2000000000000"]
+    image: dougbtv/centos-network
+
+# Create that pod.
+[user@kube-master ~]$ kubectl create -f positive.example.pod -n development
+pod/samplepod created
+
+# We can see that this pod has been launched successfully.
+[user@kube-master ~]$ kubectl get pods -n development
+NAME        READY   STATUS    RESTARTS   AGE
+samplepod   1/1     Running   0          31s
+```

doc/development.md

@@ -0,0 +1,30 @@
+## Development Information
+
+## How to build the multus-cni?
+
+```
+git clone https://github.com/intel/multus-cni.git
+cd multus-cni
+./build
+```
+
+## How to run CI tests?
+
+Multus has go unit tests (based on ginkgo framework). Following commands drive CI tests manually in your environment:
+
+```
+sudo ./test.sh
+```
+
+## Logging Best Practices
+
+Followings are multus logging best practices:
+
+* Add `logging.Debugf()` at the begining of function
+* In case of error handling, use `logging.Errorf()` with given error info
+* `logging.Panicf()` only be used at very critical error (it should NOT used usually)
+
+
+## CI Introduction
+
+TBD

doc/how-to-use.md

@@ -0,0 +1,530 @@
+## How to use multus-cni?
+
+### Prerequisites
+
+* Kubelet configured to use CNI 
+* Kubernetes version with CRD support (generally )
+
+Your Kubelet(s) must be configured to run with the CNI network plugin. Please see [Kubernetes document for CNI](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/#cni) for more details.
+
+### Install multus
+
+Generally we recommend two options: Manually place a Multus binary in your `/opt/cni/bin`, or use our [quick-start method](quickstart.md) -- which creates a daemonset that has an opinionated way of how to install & configure Multus CNI (recommended).
+
+*Copy Multus Binary into place*
+
+You may acquire the Multus binary via compilation (see the [developer guide](development.md)) or download the a binary from the [GitHub releases](https://github.com/intel/multus-cni/releases) page. Copy multus binary into CNI binary directory, usually `/opt/cni/bin`. Perform this on all nodes in your cluster (master and nodes).
+
+    $ cp multus /opt/cni/bin
+
+*Via Daemonset method*
+
+As a [quickstart](quickstart.md), you may apply these YAML files (included in the clone of this repository). Run this command (typically you would run this on the master, or wherever you have access to the `kubectl` command to manage yoru cluster). 
+
+    $ cat ./images/{multus-daemonset.yml,flannel-daemonset.yml} | kubectl apply -f -
+
+If you need more comprehensive detail, continue along with this guide, otherwise, you may wish to either [follow the quickstart guide]() or skip to the ['Create network attachment definition'](https://github.com/s1061123/multus-cni/blob/dev/update-readme/doc/how-to-use.md#create-network-attachment-definition) section.
+
+### Set up conf file in /etc/cni/net.d/ (Installed automatically by Daemonset)
+
+**If you use daemonset to install multus, skip this section and go to "Create network attachment"**
+
+You put CNI config file in `/etc/cni/net.d`. Kubernetes CNI runtime uses the alphabetically first file in the directory. (`"NOTE1"`, `"NOTE2"` are just comments, you can remove them at your configuration)
+
+Execute following commands at all Kubernetes nodes (i.e. master and minions)
+
+```
+$ mkdir -p /etc/cni/net.d
+$ cat >/etc/cni/net.d/30-multus.conf <<EOF
+{
+  "name": "multus-cni-network",
+  "type": "multus",
+  "readinessindicatorfile": "/var/run/flannel/subnet.env",
+  "delegates": [
+    {
+      "NOTE1": "This is example, wrote your CNI config in delegates",
+      "NOTE2": "If you use flannel, you also need to run flannel daemonset before!",
+      "type": "flannel",
+      "name": "flannel.1",
+      "delegate": {
+        "isDefaultGateway": true
+      }
+    }
+  ],
+  "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
+}
+EOF
+```
+
+For the detail, please take a look into [Configuration Reference](configuration.md)
+
+**NOTE: You can use "clusterNetwork"/"defaultNetworks" instead of "delegates", see []() for the detail**
+
+As above config, you need to set `"kubeconfig"` in the config file for NetworkAttachmentDefinition(CRD).
+
+##### Which network will be used for "Pod IP"?
+
+In case of "delegates", the first delegates network will be used for "Pod IP". Otherwise, "clusterNetwork" will be used for "Pod IP".
+
+#### Create ServiceAccount, ClusterRole and its binding
+
+Create resources for multus to access CRD objects as following command:
+
+```
+# Execute following commands at Kubernetes master
+$ cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+EOF
+```
+
+#### Set up kubeconfig file
+
+Create kubeconfig at master node as following commands:
+
+```
+# Execute following command at Kubernetes master
+$ mkdir -p /etc/cni/multus.d
+$ SERVICEACCOUNT_CA=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data."ca.crt"')
+$ SERVICEACCOUNT_TOKEN=$(kubectl get secrets -n=kube-system -o json | jq -r '.items[]|select(.metadata.annotations."kubernetes.io/service-account.name"=="multus")| .data.token' | base64 -d )
+$ KUBERNETES_SERVICE_PROTO=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].name)
+$ KUBERNETES_SERVICE_HOST=$(kubectl get all -o json | jq -r .items[0].spec.clusterIP)
+$ KUBERNETES_SERVICE_PORT=$(kubectl get all -o json | jq -r .items[0].spec.ports[0].port)
+$ cat > /etc/cni/net.d/multus.d/multus.kubeconfig <<EOF
+# Kubeconfig file for Multus CNI plugin.
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}
+    certificate-authority-data: ${SERVICEACCOUNT_CA}
+users:
+- name: multus
+  user:
+    token: "${SERVICEACCOUNT_TOKEN}"
+contexts:
+- name: multus-context
+  context:
+    cluster: local
+    user: multus
+current-context: multus-context
+EOF
+```
+
+Copy `/etc/cni/net.d/multus.d/multus.kubeconfig` into other Kubernetes nodes
+**NOTE: Recommend to exec 'chmod 600 /etc/cni/net.d/multus.d/multus.kubeconfig' to keep secure**
+
+```
+$ scp /etc/cni/net.d/multus.d/multus.kubeconfig ...
+```
+
+### Setup CRDs (daemonset automatically does)
+
+**If you use daemonset to install multus, skip this section and go to "Create network attachment"**
+
+Create CRD definition in Kubernetes as following command at master node:
+
+```
+# Execute following command at Kubernetes master
+$ cat <<EOF | kubectl create -f -
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  version: v1
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          properties:
+            config:
+                 type: string
+EOF
+```
+
+### Create network attachment definition
+
+The 'NetworkAttachmentDefinition' is used to setup the network attachment, i.e. secondary interface for the pod, There are two ways to configure the 'NetworkAttachmentDefinition' as following:
+
+- NetworkAttachmentDefinition with json CNI config
+- NetworkAttachmentDefinition with CNI config file
+
+#### NetworkAttachmentDefinition with json CNI config:
+
+Following command creates NetworkAttachmentDefinition. CNI config is in `config:` field.
+
+```
+# Execute following command at Kubernetes master
+$ cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf-1
+spec:
+  config: '{
+            "cniVersion": "0.3.0",
+            "type": "macvlan",
+            "master": "eth1",
+            "mode": "bridge",
+            "ipam": {
+                "type": "host-local",
+                "ranges": [
+                    [ {
+                         "subnet": "10.10.0.0/16",
+                         "rangeStart": "10.10.1.20",
+                         "rangeEnd": "10.10.3.50",
+                         "gateway": "10.10.0.254"
+                    } ]
+                ]
+            }
+        }'
+EOF
+```
+
+#### NetworkAttachmentDefinition with CNI config file:
+
+If NetworkAttachmentDefinition has no spec, multus find a file in defaultConfDir ('/etc/cni/multus/net.d', with same name in the 'name' field of CNI config.
+
+```
+# Execute following command at Kubernetes master
+$ cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf-2
+EOF
+```
+
+```
+# Execute following commands at all Kubernetes nodes (i.e. master and minions)
+$ cat <<EOF > /etc/cni/multus/net.d/macvlan2.conf
+{
+  "cniVersion": "0.3.0",
+  "type": "macvlan",
+  "name": "macvlan-conf-2",
+  "master": "eth1",
+  "mode": "bridge",
+  "ipam": {
+      "type": "host-local",
+      "ranges": [
+          [ {
+               "subnet": "11.10.0.0/16",
+               "rangeStart": "11.10.1.20",
+               "rangeEnd": "11.10.3.50"
+          } ]
+      ]
+  }
+}
+```
+
+### Run pod with network annotation
+
+#### Lauch pod with text annotation
+
+```
+# Execute following command at Kubernetes master
+$ cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-01
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf-1, macvlan-conf-2
+spec:
+  containers:
+  - name: pod-case-01
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Lauch pod with text annotation for NetworkAttachmentDefinition in different namespace
+
+You can also specify NetworkAttachmentDefinition with its namespace as adding `/<namespace>`
+
+```
+# Execute following command at Kubernetes master
+$ cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf-3
+  namespace: testns1
+spec:
+  config: '{
+            "cniVersion": "0.3.0",
+            "type": "macvlan",
+            "master": "eth1",
+            "mode": "bridge",
+            "ipam": {
+                "type": "host-local",
+                "ranges": [
+                    [ {
+                         "subnet": "12.10.0.0/16",
+                         "rangeStart": "12.10.1.20",
+                         "rangeEnd": "12.10.3.50"
+                    } ]
+                ]
+            }
+        }'
+EOF
+$ cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-02
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf-3/testns1
+spec:
+  containers:
+  - name: pod-case-02
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Lauch pod with text annotation with interface name
+
+You can also specify interface name as adding `@<ifname>`.
+
+```
+# Execute following command at Kubernetes master
+$ cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-03
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf-1@macvlan1
+spec:
+  containers:
+  - name: pod-case-03
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Lauch pod with json annotation
+
+```
+# Execute following command at Kubernetes master
+$ cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-04
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name" : "macvlan-conf-1" },
+            { "name" : "macvlan-conf-2" }
+    ]'
+spec:
+  containers:
+  - name: pod-case-04
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Lauch pod with json annotation for NetworkAttachmentDefinition in different namespace
+
+You can also specify NetworkAttachmentDefinition with its namespace as adding `"namespace": "<namespace>"`.
+
+```
+# Execute following command at Kubernetes master
+$ cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-05
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name" : "macvlan-conf-1",
+              "namespace": "testns1" }
+    ]'
+spec:
+  containers:
+  - name: pod-case-05
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+#### Lauch pod with json annotation with interface
+
+You can also specify interface name as adding `"interfaceRequest": "<ifname>"`.
+
+```
+# Execute following command at Kubernetes master
+$ cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-case-06
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[
+            { "name" : "macvlan-conf-1",
+              "interfaceRequest": "macvlan1" },
+            { "name" : "macvlan-conf-2" }
+    ]'
+spec:
+  containers:
+  - name: pod-case-06
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
+EOF
+```
+
+### Verifying pod network
+
+Following the example of `ip -d address` output of above pod, "pod-case-06":
+
+```
+# Execute following command at Kubernetes master
+$ kubectl exec -it pod-case-06 -- ip -d address
+1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
+    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 promiscuity 0 numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
+    inet 127.0.0.1/8 scope host lo
+       valid_lft forever preferred_lft forever
+    inet6 ::1/128 scope host
+       valid_lft forever preferred_lft forever
+3: eth0@if11: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
+    link/ether 0a:58:0a:f4:02:06 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0
+    veth numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
+    inet 10.244.2.6/24 scope global eth0
+       valid_lft forever preferred_lft forever
+    inet6 fe80::ac66:45ff:fe7c:3a19/64 scope link
+       valid_lft forever preferred_lft forever
+4: macvlan1@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
+    link/ether 4e:6d:7a:4e:14:87 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0
+    macvlan mode bridge numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
+    inet 10.10.1.22/16 scope global macvlan1
+       valid_lft forever preferred_lft forever
+    inet6 fe80::4c6d:7aff:fe4e:1487/64 scope link
+       valid_lft forever preferred_lft forever
+5: net2@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
+    link/ether 6e:e3:71:7f:86:f7 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 0
+    macvlan mode bridge numtxqueues 1 numrxqueues 1 gso_max_size 65536 gso_max_segs 65535
+    inet 11.10.1.22/16 scope global net2
+       valid_lft forever preferred_lft forever
+    inet6 fe80::6ce3:71ff:fe7f:86f7/64 scope link
+       valid_lft forever preferred_lft forever
+```
+
+| Interface name | Description |
+| --- | --- |
+| lo | loopback |
+| eth0 | Default network interface (flannel) |
+| macvlan1 | macvlan interface (macvlan-conf-1) |
+| net2 | macvlan interface (macvlan-conf-2) |
+
+## Entrypoint Parameters
+
+Multus CNI, when installed using the daemonset-style installation uses an entrypoint script which copies the Multus binary into place, places CNI configurations. This entrypoint takes a variety of parameters for customization.
+
+Typically, you'd modified the daemonset YAML itself to specify these parameters.
+
+For example, the `command` and `args` parameters in the `containers` section of the DaemonSet may look something like:
+
+```
+  command: ["/entrypoint.sh"]
+  args:
+  - "--multus-conf-file=auto"
+  - "--namespace-isolation=true"
+  - "--multus-log-level=verbose"
+```
+
+Note that some of the defaults have directories inside the root directory named `/host/`, this is because it is deployed as a container and we have host file system locations mapped into this directory inside the container. If you use other directories, you may have to change the mounted volumes.
+
+### Entrypoint script parameters
+
+Each parameter is shown with the default as the value.
+
+    --cni-conf-dir=/host/etc/cni/net.d
+
+This is the configuration directory where Multus will write its configuration file.
+
+    --cni-bin-dir=/host/opt/cni/bin
+
+This the directory in which the Multus binary will be installed.
+
+    --namespace-isolation=false
+
+Setting this option to true enables the Namespace isolation feature, which insists that custom resources must be created in the same namespace as the pods, otherwise it will refuse to attach those definitions as additional interfaces.
+
+    --multus-bin-file=/usr/src/multus-cni/bin/multus
+
+This option lets you set which binary executable to copy from the container onto the host (into the directory specified by `--cni-bin-dir`), allowing one to copy an alternate version or build of Multus CNI.
+
+    --multus-conf-file=/usr/src/multus-cni/images/70-multus.conf
+
+The `--multus-conf-file` is one of two options; it can be set to a source file to be copied into the location specified by `--cni-conf-dir`. Or, to a value of `auto`, that is: `--multus-conf-file=auto`.
+
+The automatic configuration option is used to automatically generate Multus configurations given existing on-disk CNI configurations for your default network.
+
+This can be used if you have your CNI configuration stored in an alternate location, or, you have constraints on race conditions where you'd like to generate your default network configuration first, and then only have Multus write its configuration when it finds that configuration -- allowing only Multus to write the CNI configuration in the `--cni-conf-dir`, therefore notifying the Kubelet that the node is in a ready state.
+
+    --multus-kubeconfig-file-host=/etc/cni/net.d/multus.d/multus.kubeconfig
+
+Used only with `--multus-conf-file=auto`. Allows you to specify an alternate path to the Kubeconfig.
+
+    --multus-log-level=
+    --multus-log-file=
+
+Used only with `--multus-conf-file=auto`. See the documentation for logging for which values are permitted.
+
+Used only with `--multus-conf-file=auto`. Allows you to specify CNI spec version. Please set if you need to speicfy CNI spec version.
+
+    --cni-version=
+

doc/quickstart.md

@@ -0,0 +1,176 @@
+# Quickstart Guide
+
+This guide is intended as a way to get you off the ground, and using Multus CNI to create Kubernetes pods with multiple interfaces. If you're already using Multus and need more detail, see the [comprehensive usage guide](how-to-use.md). This document a quickstart and a getting started guide in one, intended for your first run-through of Multus CNI.
+
+We'll first install Multus CNI, and then we'll setup some configurations so that you can see how multiple interfaces are created for pods.
+
+## Key Concepts
+
+Two things we'll refer to a number of times through this document are:
+
+* "Default network" -- This is your pod-to-pod network. This is how pods communicate among one another in your cluster, how they have connectivity. Generally speaking, this is presented as the interface named `eth0`. This interface is always attached to your pods, so that they can have connectivity among themselves. We'll add interfaces in addition to this.
+* "CRDs" -- Custom Resource Definitions. Custom Resources are a way that the Kubernetes API is extended. We use these here to store some information that Multus can read. Primarily, we use these to store the configurations for each of the additional interfaces that are attached to your pods.
+
+## Installation
+
+Our recommended quickstart method to deploy Multus is to deploy using a Daemonset. This method is provided in this guide along with [Flannel](https://github.com/coreos/flannel). Flannel is deployed as a pod-to-pod network that is used as our "default network" -- this provides connectivity between pods in your cluster. Each additional network attachment (i.e. for multiple interfaces in pods) is made in addition to this default network. This guide generally assumes a new Kubernetes cluster that hasn't yet had any networking configured. If it's your first time using Multus, you might consider using a fresh cluster to learn with, and then later configure it to work with an existing cluster.
+
+Firstly, clone this GitHub repository. 
+
+```
+git clone https://github.com/intel/multus-cni.git && cd multus-cni
+```
+
+We'll apply files to `kubectl` from this repo. The files we're applying here specify a "Daemonset" (pods that run on each node in the cluster), this Daemonset handles installing the Multus CNI binary, dropping a default configuration on each node in the cluster -- and then also installs Flannel to use as a default network.
+
+```
+$ cat ./images/{multus-daemonset.yml,flannel-daemonset.yml} | kubectl apply -f -
+```
+
+### Validating your installation
+
+Generally, the first step in validating your installation is to look at the `STATUS` field of your nodes, you can check it out by looking at:
+
+```
+$ kubectl get nodes
+```
+
+This will show each of the nodes in your cluster, take a look at the `STATUS` field, and look for `Ready` to appear for each of your nodes. This readiness is determined by the presence of a CNI configuration file on each of the nodes, and when that file appears.
+
+You may also wish to start any pod in your cluster (without any further configuration), and validate that it works as you'd otherwise expect -- especially that it can communicate over the default network.
+
+## Creating additional interfaces
+
+The first thing we'll do is create configurations for each of the additional interfaces that we attach to pods. We'll do this by creating Custom Resources. Part of the quickstart installation creates a "CRD" -- a custom resource definition that is the home where we keep these custom resources -- we'll store our configurations for each interface in these.
+
+### CNI Configurations
+
+Each configuration we'll add is a CNI configuration. If you're not familiar with them, let's break them down quickly. Here's an example CNI configuration:
+
+```
+{
+  "cniVersion": "0.3.0",
+  "type": "loopback",
+  "additional": "information"
+}
+```
+
+CNI configurations are JSON, and we have a structure here that has a few things we're interested in:
+
+1. `cniVersion`: Tells each CNI plugin which version is being used and can give the plugin information if it's using a too late (or too early) version.
+2. `type`: This tells CNI which binary to call on disk. Each CNI plugin is a binary that's called. Typically, these binaries are stored in `/opt/cni/bin` on each node, and CNI executes this binary. In this case we've specified the `loopback` binary (which create a loopback-type network interface). If this is your first time installing Multus, you might want to verify that the plugins that are in the "type" field are actually on disk in the `/opt/cni/bin` directory.
+3. `additional`: This field is put here as an example, each CNI plugin can specify whatever configuration parameters they'd like in JSON. These are specific to the binary you're calling in the `type` field.
+
+For an even further example -- take a look at the [bridge CNI plugin README](https://github.com/containernetworking/plugins/tree/master/plugins/main/bridge) which shows additional
+
+If you'd like more information about CNI configuration, you can read [the entire CNI specification](https://github.com/containernetworking/cni/blob/master/SPEC.md). It might also be useful to look at the [CNI reference plugins](https://github.com/containernetworking/plugins) and see how they're configured.
+
+You do not need to reload or refresh the Kubelets when CNI configurations  change. These are read on each creation & deletion of pods. So if you change a configuration, it'll apply the next time a pod is created. Existing pods may need to be restarted if they need the new configuration.
+
+### Storing a configuration as a Custom Resource
+
+So, we want to create an additional interface. Let's create a macvlan interface for pods to use. We'll create a custom resource that defines the CNI configuration for interfaces.
+
+Note in the following command that there's a `kind: NetworkAttachmentDefinition`. This is our fancy name for our configuration -- it's a custom extension of Kubernetes that defines how we attach networks to our pods. 
+
+Secondarily, note the `config` field. You'll see that this is a CNI configuration just like we explained earlier.
+
+Lastly but *very* importantly, note under `metadata` the `name` field -- here's where we give this configuration a name, and it's how we tell pods to use this configuration. The name here is `macvlan-conf` -- as we're creating a configuration for macvlan.
+
+Here's the command to create this example configuration:
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: macvlan-conf
+spec:
+  config: '{
+      "cniVersion": "0.3.0",
+      "type": "macvlan",
+      "master": "eth0",
+      "mode": "bridge",
+      "ipam": {
+        "type": "host-local",
+        "subnet": "192.168.1.0/24",
+        "rangeStart": "192.168.1.200",
+        "rangeEnd": "192.168.1.216",
+        "routes": [
+          { "dst": "0.0.0.0/0" }
+        ],
+        "gateway": "192.168.1.1"
+      }
+    }'
+EOF
+```
+
+You can see which configurations you've created using `kubectl` here's how you can do that:
+
+```
+kubectl get network-attachment-definitions
+```
+
+You can get more detail by describing them:
+
+```
+kubectl describe network-attachment-definitions macvlan-conf
+```
+
+### Creating a pod that attaches an additional interface
+
+We're going to create a pod. This will look familiar as any pod you might have created before, but, we'll have a special `annotations` field -- in this case we'll have an annotation called `k8s.v1.cni.cncf.io/networks`. This field takes a comma delimited list of the names of your `NetworkAttachmentDefinition`s as we created above. Note in the comand below that we have the annotation of `k8s.v1.cni.cncf.io/networks: macvlan-conf` where `macvlan-conf` is the name we used above when we created our configuration.
+
+Let's go ahead and create a pod (that just sleeps for a really long time) with this command:
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "sleep 2000000000000"]
+    image: dougbtv/centos-network
+EOF
+```
+
+You may now inspect the pod and see what interfaces interfaces are attached, like so:
+
+```
+$ kubectl exec -it samplepod -- ip a
+```
+
+You should note that there's 3 interfaces:
+
+* `lo` a loopback interface
+* `eth0` our default network
+* `net1` the new interface we created with the macvlan configuration.
+
+### What if I want more interfaces?
+
+You can add more interfaces to a pod by creating more custom resources and then referring to them in pod's annotation. You can also reuse configurations, so for example, to attach two macvlan interfaces to a pod, you could create a pod like so:
+
+```
+cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Pod
+metadata:
+  name: samplepod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: macvlan-conf,macvlan-conf
+spec:
+  containers:
+  - name: samplepod
+    command: ["/bin/bash", "-c", "sleep 2000000000000"]
+    image: dougbtv/centos-network
+EOF
+```
+
+Note that the annotation now reads `k8s.v1.cni.cncf.io/networks: macvlan-conf,macvlan-conf`. Where we have the same configuration used twice, separated by a comma. 
+
+If you were to create another custom resource with the name `foo` you could use that such as: `k8s.v1.cni.cncf.io/networks: foo,macvlan-conf`, and use any number of attachments.

doc/webhook/webhook.md

@@ -0,0 +1,112 @@
+# Validating admission webhook
+
+## Building Docker image
+
+From the root directory of Multus execute:
+```
+cd webhook
+./build
+```
+
+## Deploying webhook application
+
+Change working directory. From the root directory of Multus execute:
+```
+cd deployment/webhook
+```
+
+Create key and certificate pair and patch configuration-template.yaml file with base64-encoded certificate file. Run:
+```
+./certs.sh
+```
+*Note: Verify that Kubernetes controller manager has --cluster-signing-cert-file and --cluster-signing-key-file parameters set to paths to your CA keypair,
+to make sure that Certificates API is enabled in order to generate certificate signed by cluster CA.
+Script generates private key and certificate signing request, which is then pushed to the Kubernetes API server.
+Then script approves that CSR and API server issues the certificate. Certificate is obtained from the API server and used to create a secret.
+Script also patches `configuration-template.yaml` file with base64-encoded certificate and creates `configuration.yaml` file containing
+Validating Webhook Configuration specification, which is deployed in one of the next steps.
+More details about TLS certificates management in a cluster available [here](https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/).*
+
+Create service:
+```
+kubectl create -f service.yaml
+```
+
+Run deployment:
+```
+kubectl create -f deployment.yaml
+```
+
+Create Validating Webhook Configuration:
+```
+kubectl create -f configuration.yaml
+```
+
+## Verifying installation
+
+Try to create invalid Network Attachment Definition resource:
+```
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: invalid-net-attach-def
+spec:
+  config: '{
+    "invalid": "config"
+  }'
+EOF
+```
+Webhook should deny the request:
+```
+Error from server: error when creating "STDIN": admission webhook "multus-webhook.k8s.cni.cncf.io" denied the request: Invalid network config spec
+```
+
+Now, try to create correctly defined one:
+```
+cat <<EOF | kubectl create -f -
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: correct-net-attach-def
+spec:
+  config: '{
+    "cniVersion": "0.3.0",
+    "name": "a-bridge-network",
+    "type": "bridge",
+    "bridge": "br0",
+    "isGateway": true,
+    "ipam": {
+      "type": "host-local",
+      "subnet": "192.168.5.0/24",
+      "dataDir": "/mnt/cluster-ipam"
+    }
+  }'
+EOF
+```
+Resource should be allowed and created:
+```
+networkattachmentdefinition.k8s.cni.cncf.io/correct-net-attach-def created
+```
+
+## Troubleshooting
+Webhook server prints a lot of debug messages that could help to find the root cause of an issue.
+To display logs run:
+```
+kubectl logs -l app=multus-webhook
+```
+Example output showing logs for handling requests generated in the "Verifying installation section":
+```
+# kubectl logs multus-webhook-pod
+2018-08-22T13:33:09Z [debug] Starting Multus webhook server
+2018-08-22T13:33:32Z [debug] Validating network config spec: { "invalid": "config" }
+2018-08-22T13:33:32Z [debug] Spec is not a valid network config: error parsing configuration list: no name. Trying to parse into config list
+2018-08-22T13:33:32Z [debug] Spec is not a valid network config list: error parsing configuration: missing 'type'
+2018-08-22T13:33:32Z [error] Invalid config: error parsing configuration: missing 'type'
+2018-08-22T13:33:32Z [debug] Sending response to the API server
+2018-08-22T13:35:29Z [debug] Validating network config spec: { "cniVersion": "0.3.0", "name": "a-bridge-network", "type": "bridge", "bridge": "br0", "isGateway": true, "ipam": { "type": "host-local", "subnet": "192.168.5.0/24", "dataDir": "/mnt/cluster-ipam" } }
+2018-08-22T13:35:29Z [debug] Spec is not a valid network config: error parsing configuration list: no 'plugins' key. Trying to parse into config list
+2018-08-22T13:35:29Z [debug] Network Attachment Defintion is valid. Admission Review request allowed
+2018-08-22T13:35:29Z [debug] Sending response to the API server
+```
+

examples/README.md

@@ -60,3 +60,35 @@ A sample `cni-configuration.conf` is provided, typically this file is placed in
 ## Other considerations
 
 Primarily in this setup one thing that one should consider are the aspects of the `macvlan-conf.yml`, which is likely specific to the configuration of the node on which this resides.
+
+## Passing down device information
+Some CNI plugins require specific device information which maybe pre-allocated by K8s device plugin. This could be indicated by providing `k8s.v1.cni.cncf.io/resourceName` annotaton in its network attachment definition CRD. The file [`examples/sriov-net.yaml`](./sriov-net.yaml) shows an example on how to define a Network attachment definition with specific device allocation information. Multus will get allocated device information and make them available for CNI plugin to work on.
+
+In this exmaple (shown below), it is expected that an [SRIOV Device Plugin](https://github.com/intel/sriov-network-device-plugin/) making a pool of SRIOV VFs available to the K8s with `intel.com/sriov` as their resourceName. Any device allocated from this resource pool will be passed down by Multus to the [sriov-cni](https://github.com/intel/sriov-cni/tree/dev/k8s-deviceid-model) plugin in `deviceID` field. This is up to the sriov-cni plugin to capture this information and work with this specific device information.
+
+```yaml
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-a
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
+spec:
+  config: '{
+  "type": "sriov",
+  "vlan": 1000,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "10.56.217.0/24",
+    "rangeStart": "10.56.217.171",
+    "rangeEnd": "10.56.217.181",
+    "routes": [{
+      "dst": "0.0.0.0/0"
+    }],
+    "gateway": "10.56.217.1"
+  }
+}'
+```
+The [net-resource-sample-pod.yaml](./net-resource-sample-pod.yaml) is an exmaple Pod manifest file that requesting a SRIOV device from a host which is then configured using the above network attachement definition.
+
+>For further information on how to configure SRIOV Device Plugin and SRIOV-CNI please refer to the links given above.

examples/clusterrole.yml

@@ -1,16 +1,19 @@
 ---
-apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
   name: multus-crd-overpowered
 rules:
-- apiGroups:
-  - '*'
-  resources:
-  - '*'
-  verbs:
-  - '*'
-- nonResourceURLs:
-  - '*'
-  verbs:
-  - '*'
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update

examples/multus-ptp-portmap.conf

@@ -0,0 +1,36 @@
+{
+    "name": "multus-cni-network",
+    "type": "multus"
+    "capabilities": {
+        "portMappings": true
+    },
+    "delegates": [
+        {
+            "cniVersion": "0.3.1",
+            "name": "ptp-tuning-conflist",
+            "plugins": [
+                {
+                    "dns": {
+                        "nameservers": [
+                            "172.16.1.1"
+                        ]
+                    },
+                    "ipMasq": true,
+                    "ipam": {
+                        "subnet": "172.16.0.0/24",
+                        "type": "host-local"
+                    },
+                    "mtu": 512,
+                    "type": "ptp"
+                },
+                {
+                    "capabilities": {
+                        "portMappings": true
+                    },
+                    "externalSetMarkChain": "KUBE-MARK-MASQ",
+                    "type": "portmap"
+                }
+            ]
+        }
+    ],
+}

examples/multus-with-flannel.yml

@@ -101,8 +101,7 @@ spec:
       nodeSelector:
         beta.kubernetes.io/arch: amd64
       tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
+      - operator: Exists
         effect: NoSchedule
       serviceAccountName: flannel
       initContainers:

examples/net-resource-sample-pod.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: testpod1
+  labels:
+    env: test
+  annotations:
+    k8s.v1.cni.cncf.io/networks: sriov-net-a
+spec:
+  containers:
+  - name: appcntr1
+    image: centos/tools
+    imagePullPolicy: IfNotPresent
+    command: [ "/bin/bash", "-c", "--" ]
+    args: [ "while true; do sleep 300000; done;" ]
+    resources:
+      requests:
+        intel.com/sriov: '1'
+      limits:
+        intel.com/sriov: '1'
+  restartPolicy: "Never"

examples/npwg-demo-1/12_pod_case2.yml

@@ -8,7 +8,7 @@ metadata:
             { "name": "macvlan-conf-2" },
             { "name": "vlan-conf-1-1",
               "namespace": "testns1",
-              "interfaceRequest": "vlan1-1" }
+              "interface": "vlan1-1" }
     ]'
 spec:
   containers:

examples/sriov-net.yaml

@@ -0,0 +1,21 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-a
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
+spec:
+  config: '{
+  "type": "sriov",
+  "vlan": 1000,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "10.56.217.0/24",
+    "rangeStart": "10.56.217.171",
+    "rangeEnd": "10.56.217.181",
+    "routes": [{
+      "dst": "0.0.0.0/0"
+    }],
+    "gateway": "10.56.217.1"
+  }
+}'

glide.lock

@@ -1,13 +1,11 @@
-hash: 0c4ea2a342364d2ff3b43242730cb3b1db3b7e8456f6cf43da3c51dbb67e18da
-updated: 2018-07-27T03:29:02.093332104+01:00
+hash: bdca35d4ed2194046b2d9825967a23fd10153eb1a31c9dfa1e2c21f856965ffe
+updated: 2019-08-30T16:53:47.105736373-04:00
 imports:
 - name: github.com/containernetworking/cni
   version: 07c1a6da47b7fbf8b357f4949ecce2113e598491
   subpackages:
   - libcni
   - pkg/invoke
-  - pkg/ip
-  - pkg/ipam
   - pkg/skel
   - pkg/types
   - pkg/types/020
@@ -16,6 +14,8 @@ imports:
 - name: github.com/containernetworking/plugins
   version: 2b8b1ac0af4568e928d96ccc5f47b075416eeabd
   subpackages:
+  - pkg/ip
+  - pkg/ipam
   - pkg/ns
   - pkg/testutils
 - name: github.com/ghodss/yaml
@@ -23,7 +23,9 @@ imports:
 - name: github.com/gogo/protobuf
   version: c0656edd0d9eab7c66d1eb0c568f9039345796f7
   subpackages:
+  - gogoproto
   - proto
+  - protoc-gen-gogo/descriptor
   - sortkeys
 - name: github.com/golang/glog
   version: 44145f04b68cf362d9c4df2182967c2275eaefed
@@ -53,6 +55,8 @@ imports:
   version: 6633656539c1639d9d78127b7d47c622b5d7b6dc
 - name: github.com/json-iterator/go
   version: f2b4162afba35581b6d4a50d3b8f34e33c144682
+- name: github.com/Microsoft/go-winio
+  version: 78439966b38d69bf38227fbf57ac8a6fee70f69a
 - name: github.com/modern-go/concurrent
   version: bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94
 - name: github.com/modern-go/reflect2
@@ -61,6 +65,7 @@ imports:
   version: 7f8ab55aaf3b86885aa55b762e803744d1674700
   subpackages:
   - config
+  - extensions/table
   - internal/codelocation
   - internal/containernode
   - internal/failer
@@ -91,15 +96,15 @@ imports:
 - name: github.com/peterbourgon/diskv
   version: 5f041e8faa004a95c88a202771f4cc3e991971e6
 - name: github.com/pkg/errors
-  version: 816c9085562cd7ee03e7f8188a1cfd942858cded
+  version: ba968bfe8b2f7e042a574c888954fccecfa385b4
 - name: github.com/spf13/pflag
   version: 583c0c0531f06d5278b7d917446061adc344b5cd
 - name: github.com/vishvananda/netlink
-  version: 6e453822d85ef5721799774b654d4d02fed62afb
+  version: b2de5d10e38ecce8607e6b438b6d174f389a004e
   subpackages:
   - nl
 - name: github.com/vishvananda/netns
-  version: 54f0e4339ce73702a0607f49922aaa1e749b418d
+  version: be1fbeda19366dea804f00efff2dd73a1642fdcc
 - name: golang.org/x/crypto
   version: 49796115aa4b964c318aad4f3084fdb41e9aa067
   subpackages:
@@ -111,9 +116,11 @@ imports:
   - http2
   - http2/hpack
   - idna
+  - internal/timeseries
   - lex/httplex
+  - trace
 - name: golang.org/x/sys
-  version: 95c6576299259db960f6c5b9b69ea52422860fce
+  version: 1c9583448a9c3aa0f9a6a5241bf73c0bd8aafded 
   subpackages:
   - unix
   - windows
@@ -128,6 +135,45 @@ imports:
   version: f51c12702a4d776e4c1fa9b0fabab841babae631
   subpackages:
   - rate
+- name: google.golang.org/genproto
+  version: 09f6ed296fc66555a25fe4ce95173148778dfa85
+  subpackages:
+  - googleapis/rpc/status
+- name: google.golang.org/grpc
+  version: 6eaf6f47437a6b4e2153a190160ef39a92c7eceb
+  subpackages:
+  - balancer
+  - balancer/base
+  - balancer/roundrobin
+  - binarylog/grpc_binarylog_v1
+  - codes
+  - connectivity
+  - credentials
+  - credentials/internal
+  - encoding
+  - encoding/proto
+  - grpclog
+  - internal
+  - internal/backoff
+  - internal/balancerload
+  - internal/binarylog
+  - internal/channelz
+  - internal/envconfig
+  - internal/grpcrand
+  - internal/grpcsync
+  - internal/syscall
+  - internal/transport
+  - keepalive
+  - metadata
+  - naming
+  - peer
+  - resolver
+  - resolver/dns
+  - resolver/passthrough
+  - serviceconfig
+  - stats
+  - status
+  - tap
 - name: gopkg.in/inf.v0
   version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4
 - name: gopkg.in/yaml.v2
@@ -135,6 +181,7 @@ imports:
 - name: k8s.io/api
   version: 2d6f90ab1293a1fb871cf149423ebb72aa7423aa
   subpackages:
+  - admission/v1beta1
   - admissionregistration/v1alpha1
   - admissionregistration/v1beta1
   - apps/v1
@@ -258,4 +305,12 @@ imports:
   - util/homedir
   - util/integer
   - util/retry
+- name: k8s.io/klog
+  version: 8139d8cb77af419532b33dfa7dd09fbc5f1d344f
+- name: k8s.io/kubernetes
+  version: ddf47ac13c1a9483ea035a79cd7c10005ff21a6d
+  subpackages:
+  - pkg/kubelet/apis/podresources
+  - pkg/kubelet/apis/podresources/v1alpha1
+  - pkg/kubelet/util
 testImports: []

glide.yaml

@@ -32,5 +32,16 @@ import:
   - kubernetes
   - tools/clientcmd
   - util/retry
+- package: k8s.io/kubernetes
+  version: v1.13.0
+  subpackages:
+  - pkg/kubelet/apis/podresources
+  - pkg/kubelet/apis/podresources/v1alpha1
+  - pkg/kubelet/util
 - package: github.com/vishvananda/netns
 - package: github.com/pkg/errors
+- package: google.golang.org/grpc
+  version: ^1.23.0
+- package: golang.org/x/sys
+  subpackages:
+  - unix

images/README.md

@@ -31,10 +31,12 @@ You can get get help with the `--help` flag.
 ```
 $ ./entrypoint.sh --help
 
-This is an entrypoint script for Multus CNI to overlay its
-binary and configuration into locations in a filesystem.
-The configuration & binary file will be copied to the 
-corresponding configuration directory.
+This is an entrypoint script for Multus CNI to overlay its binary and
+configuration into locations in a filesystem. The configuration & binary file
+will be copied to the corresponding configuration directory. When
+`--multus-conf-file=auto` is used, 00-multus.conf will be automatically
+generated from the CNI configuration file of the master plugin (the first file
+in lexicographical order in cni-conf-dir).
 
 ./entrypoint.sh
     -h --help
@@ -42,6 +44,7 @@ corresponding configuration directory.
     --cni-bin-dir=/host/opt/cni/bin
     --multus-conf-file=/usr/src/multus-cni/images/70-multus.conf
     --multus-bin-file=/usr/src/multus-cni/bin/multus
+    --multus-kubeconfig-file-host=/etc/cni/net.d/multus.d/multus.kubeconfig
 ```
 
 You must use an `=` to delimit the parameter name and the value. For example you may set a custom `cni-conf-dir` like so:
@@ -59,7 +62,7 @@ Note: You'll noticed that there's a `/host/...` directory from the root for the
 Example docker run command:
 
 ```
-$ docker run -it -v /opt/cni/bin/:/host/opt/cni/bin/ -v /etc/cni/net.d/:/host/etc/cni/net.d/ --entrypoint=/bin/bash dougbtv/multus 
+$ docker run -it -v /opt/cni/bin/:/host/opt/cni/bin/ -v /etc/cni/net.d/:/host/etc/cni/net.d/ --entrypoint=/bin/bash dougbtv/multus
 ```
 
-Originally inspired by and is a portmanteau of the [Flannel daemonset](https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml), the [Calico Daemonset](https://github.com/projectcalico/calico/blob/master/v2.0/getting-started/kubernetes/installation/hosted/k8s-backend-addon-manager/calico-daemonset.yaml), and the [Calico CNI install bash script](https://github.com/projectcalico/cni-plugin/blob/be4df4db2e47aa7378b1bdf6933724bac1f348d0/k8s-install/scripts/install-cni.sh#L104-L153).
+Originally inspired by and is a portmanteau of the [Flannel daemonset](https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml), the [Calico Daemonset](https://github.com/projectcalico/calico/blob/master/v2.0/getting-started/kubernetes/installation/hosted/k8s-backend-addon-manager/calico-daemonset.yaml), and the [Calico CNI install bash script](https://github.com/projectcalico/cni-plugin/blob/be4df4db2e47aa7378b1bdf6933724bac1f348d0/k8s-install/scripts/install-cni.sh#L104-L153).

images/entrypoint.sh

@@ -8,14 +8,20 @@ CNI_CONF_DIR="/host/etc/cni/net.d"
 CNI_BIN_DIR="/host/opt/cni/bin"
 MULTUS_CONF_FILE="/usr/src/multus-cni/images/70-multus.conf"
 MULTUS_BIN_FILE="/usr/src/multus-cni/bin/multus"
+MULTUS_KUBECONFIG_FILE_HOST="/etc/cni/net.d/multus.d/multus.kubeconfig"
+MULTUS_NAMESPACE_ISOLATION=false
+MULTUS_LOG_LEVEL=""
+MULTUS_LOG_FILE=""
 
 # Give help text for parameters.
 function usage()
 {
-    echo -e "This is an entrypoint script for Multus CNI to overlay its"
-    echo -e "binary and configuration into locations in a filesystem."
-    echo -e "The configuration & binary file will be copied to the "
-    echo -e "corresponding configuration directory."
+    echo -e "This is an entrypoint script for Multus CNI to overlay its binary and "
+    echo -e "configuration into locations in a filesystem. The configuration & binary file "
+    echo -e "will be copied to the corresponding configuration directory. When "
+    echo -e "'--multus-conf-file=auto' is used, 00-multus.conf will be automatically "
+    echo -e "generated from the CNI configuration file of the master plugin (the first file "
+    echo -e "in lexicographical order in cni-conf-dir)."
     echo -e ""
     echo -e "./entrypoint.sh"
     echo -e "\t-h --help"
@@ -23,6 +29,10 @@ function usage()
     echo -e "\t--cni-bin-dir=$CNI_BIN_DIR"
     echo -e "\t--multus-conf-file=$MULTUS_CONF_FILE"
     echo -e "\t--multus-bin-file=$MULTUS_BIN_FILE"
+    echo -e "\t--multus-kubeconfig-file-host=$MULTUS_KUBECONFIG_FILE_HOST"
+    echo -e "\t--namespace-isolation=$MULTUS_NAMESPACE_ISOLATION"
+    echo -e "\t--multus-log-level=$MULTUS_LOG_LEVEL (empty by default, used only with --multus-conf-file=auto)"
+    echo -e "\t--multus-log-file=$MULTUS_LOG_FILE (empty by default, used only with --multus-conf-file=auto)"
 }
 
 # Parse parameters given as arguments to this script.
@@ -34,6 +44,9 @@ while [ "$1" != "" ]; do
             usage
             exit
             ;;
+        --cni-version)
+            CNI_VERSION=$VALUE
+            ;;
         --cni-conf-dir)
             CNI_CONF_DIR=$VALUE
             ;;
@@ -46,10 +59,20 @@ while [ "$1" != "" ]; do
         --multus-bin-file)
             MULTUS_BIN_FILE=$VALUE
             ;;
+        --multus-kubeconfig-file-host)
+            MULTUS_KUBECONFIG_FILE_HOST=$VALUE
+            ;;
+        --namespace-isolation)
+            MULTUS_NAMESPACE_ISOLATION=$VALUE
+            ;;
+        --multus-log-level)
+            MULTUS_LOG_LEVEL=$VALUE
+            ;;
+        --multus-log-file)
+            MULTUS_LOG_FILE=$VALUE
+            ;;
         *)
-            echo "ERROR: unknown parameter \"$PARAM\""
-            usage
-            exit 1
+            echo "WARNING: unknown parameter \"$PARAM\""
             ;;
     esac
     shift
@@ -57,7 +80,11 @@ done
 
 
 # Create array of known locations
-declare -a arr=($CNI_CONF_DIR $CNI_BIN_DIR $MULTUS_CONF_FILE $MULTUS_BIN_FILE)
+declare -a arr=($CNI_CONF_DIR $CNI_BIN_DIR $MULTUS_BIN_FILE)
+if [ "$MULTUS_CONF_FILE" != "auto" ]; then
+  arr+=($MULTUS_CONF_FILE)
+fi
+
 
 # Loop through and verify each location each.
 for i in "${arr[@]}"
@@ -68,9 +95,12 @@ do
   fi
 done
 
-# Copy files into proper places.
-cp -f $MULTUS_CONF_FILE $CNI_CONF_DIR
-cp -f $MULTUS_BIN_FILE $CNI_BIN_DIR
+# Copy files into place and atomically move into final binary name
+cp -f $MULTUS_BIN_FILE $CNI_BIN_DIR/_multus
+mv -f $CNI_BIN_DIR/_multus $CNI_BIN_DIR/multus 
+if [ "$MULTUS_CONF_FILE" != "auto" ]; then
+  cp -f $MULTUS_CONF_FILE $CNI_CONF_DIR
+fi
 
 # Make a multus.d directory (for our kubeconfig)
 
@@ -134,6 +164,87 @@ fi
 
 # ---------------------- end Generate a "kube-config".
 
+# ------------------------------- Generate "00-multus.conf"
+
+if [ "$MULTUS_CONF_FILE" == "auto" ]; then
+  echo "Generating Multus configuration file ..."
+  found_master=false
+  tries=0
+  while [ $found_master == false ]; do
+    MASTER_PLUGIN="$(ls $CNI_CONF_DIR | grep -E '\.conf(list)?$' | grep -Ev '00-multus\.conf' | head -1)"
+    if [ "$MASTER_PLUGIN" == "" ]; then
+      if [ $tries -lt 600 ]; then
+        if ! (($tries % 5)); then
+          echo "Attemping to find master plugin configuration, attempt $tries"
+        fi
+        let "tries+=1"
+        sleep 1;
+      else
+        echo "Error: Multus could not be configured: no master plugin was found."
+        exit 1;
+      fi
+    else
+
+      found_master=true
+
+      ISOLATION_STRING=""
+      if [ "$MULTUS_NAMESPACE_ISOLATION" == true ]; then
+        ISOLATION_STRING="\"namespaceIsolation\": true,"
+      fi
+
+      LOG_LEVEL_STRING=""
+      if [ ! -z "${MULTUS_LOG_LEVEL// }" ]; then
+        case "$MULTUS_LOG_LEVEL" in
+          debug)
+              ;;
+          error)
+              ;;
+          panic)
+              ;;
+          verbose)
+              ;;
+          *)
+              echo "ERROR: Log levels should be one of: debug/verbose/error/panic, did not understand $MULTUS_LOG_LEVEL"
+              usage
+              exit 1     
+        esac
+        LOG_LEVEL_STRING="\"logLevel\": \"$MULTUS_LOG_LEVEL\","
+      fi
+
+      LOG_FILE_STRING=""
+      if [ ! -z "${MULTUS_LOG_FILE// }" ]; then
+        LOG_FILE_STRING="\"logFile\": \"$MULTUS_LOG_FILE\","
+      fi
+
+      CNI_VERSION_STRING=""
+      if [ ! -z "${CNI_VERSION// }" ]; then
+        CNI_VERSION_STRING="\"cniVersion\": \"$CNI_VERSION\","
+      fi
+
+      MASTER_PLUGIN_JSON="$(cat $CNI_CONF_DIR/$MASTER_PLUGIN)"
+      CONF=$(cat <<-EOF
+  			{
+          $CNI_VERSION_STRING
+  				"name": "multus-cni-network",
+  				"type": "multus",
+          $ISOLATION_STRING
+          $LOG_LEVEL_STRING
+          $LOG_FILE_STRING
+  				"kubeconfig": "$MULTUS_KUBECONFIG_FILE_HOST",
+  				"delegates": [
+  					$MASTER_PLUGIN_JSON
+  				]
+  			}
+EOF
+  		)
+      echo $CONF > $CNI_CONF_DIR/00-multus.conf
+      echo "Config file created @ $CNI_CONF_DIR/00-multus.conf"
+    fi
+  done
+fi
+
+# ---------------------- end Generate "00-multus.conf".
+
 echo "Entering sleep... (success)"
 
 # Sleep forever.

images/flannel-daemonset.yml

@@ -101,8 +101,7 @@ spec:
       nodeSelector:
         beta.kubernetes.io/arch: amd64
       tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
+      - operator: Exists
         effect: NoSchedule
       serviceAccountName: flannel
       # ------------------------------- Intentionally removed, Multus daemonset configures /etc/cni/net.d
@@ -181,8 +180,7 @@ spec:
       nodeSelector:
         beta.kubernetes.io/arch: arm64
       tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
+      - operator: Exists
         effect: NoSchedule
       serviceAccountName: flannel
       initContainers:
@@ -260,8 +258,7 @@ spec:
       nodeSelector:
         beta.kubernetes.io/arch: arm
       tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
+      - operator: Exists
         effect: NoSchedule
       serviceAccountName: flannel
       initContainers:
@@ -339,8 +336,7 @@ spec:
       nodeSelector:
         beta.kubernetes.io/arch: ppc64le
       tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
+      - operator: Exists
         effect: NoSchedule
       serviceAccountName: flannel
       initContainers:
@@ -418,8 +414,7 @@ spec:
       nodeSelector:
         beta.kubernetes.io/arch: s390x
       tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
+      - operator: Exists
         effect: NoSchedule
       serviceAccountName: flannel
       initContainers:
@@ -476,4 +471,4 @@ spec:
             path: /etc/cni/net.d
         - name: flannel-cfg
           configMap:
-            name: kube-flannel-cfg
+            name: kube-flannel-cfg

images/multus-daemonset.yml

@@ -26,16 +26,19 @@ apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
   name: multus
 rules:
-- apiGroups:
-  - '*'
-  resources:
-  - '*'
-  verbs:
-  - '*'
-- nonResourceURLs:
-  - '*'
-  verbs:
-  - '*'
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1beta1
@@ -111,8 +114,7 @@ spec:
       nodeSelector:
         beta.kubernetes.io/arch: amd64
       tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
+      - operator: Exists
         effect: NoSchedule
       serviceAccountName: multus
       containers:

k8sclient/k8sclient.go

@@ -31,10 +31,17 @@ import (
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/skel"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
+	"github.com/intel/multus-cni/kubeletclient"
 	"github.com/intel/multus-cni/logging"
 	"github.com/intel/multus-cni/types"
 )
 
+const (
+	resourceNameAnnot      = "k8s.v1.cni.cncf.io/resourceName"
+	defaultNetAnnot        = "v1.multus-cni.io/default-network"
+	networkAttachmentAnnot = "k8s.v1.cni.cncf.io/networks"
+)
+
 // NoK8sNetworkError indicates error, no network in kubernetes
 type NoK8sNetworkError struct {
 	message string
@@ -74,30 +81,45 @@ func setKubeClientInfo(c *clientInfo, client KubeClient, k8sArgs *types.K8sArgs)
 	c.Podname = string(k8sArgs.K8S_POD_NAME)
 }
 
-func SetNetworkStatus(k *clientInfo, netStatus []*types.NetworkStatus) error {
+func SetNetworkStatus(client KubeClient, k8sArgs *types.K8sArgs, netStatus []*types.NetworkStatus, conf *types.NetConf) error {
+	logging.Debugf("SetNetworkStatus: %v, %v, %v, %v", client, k8sArgs, netStatus, conf)
 
-	logging.Debugf("SetNetworkStatus: %v, %v", k, netStatus)
-	pod, err := k.Client.GetPod(k.Podnamespace, k.Podname)
+	client, err := GetK8sClient(conf.Kubeconfig, client)
 	if err != nil {
-		return logging.Errorf("SetNetworkStatus: failed to query the pod %v in out of cluster comm: %v", k.Podname, err)
+		return logging.Errorf("SetNetworkStatus: %v", err)
+	}
+	if client == nil {
+		if len(conf.Delegates) == 0 {
+			// No available kube client and no delegates, we can't do anything
+			return logging.Errorf("must have either Kubernetes config or delegates, refer to Multus documentation for usage instructions")
+		}
+		logging.Debugf("SetNetworkStatus: kube client info is not defined, skip network status setup")
+		return nil
 	}
 
-	var ns string
+	podName := string(k8sArgs.K8S_POD_NAME)
+	podNamespace := string(k8sArgs.K8S_POD_NAMESPACE)
+	pod, err := client.GetPod(podNamespace, podName)
+	if err != nil {
+		return logging.Errorf("SetNetworkStatus: failed to query the pod %v in out of cluster comm: %v", podName, err)
+	}
+
+	var networkStatuses string
 	if netStatus != nil {
 		var networkStatus []string
-		for _, nets := range netStatus {
-			data, err := json.MarshalIndent(nets, "", "    ")
+		for _, status := range netStatus {
+			data, err := json.MarshalIndent(status, "", "    ")
 			if err != nil {
 				return logging.Errorf("SetNetworkStatus: error with Marshal Indent: %v", err)
 			}
 			networkStatus = append(networkStatus, string(data))
 		}
 
-		ns = fmt.Sprintf("[%s]", strings.Join(networkStatus, ","))
+		networkStatuses = fmt.Sprintf("[%s]", strings.Join(networkStatus, ","))
 	}
-	_, err = setPodNetworkAnnotation(k.Client, k.Podnamespace, pod, ns)
+	_, err = setPodNetworkAnnotation(client, podNamespace, pod, networkStatuses)
 	if err != nil {
-		return logging.Errorf("SetNetworkStatus: failed to update the pod %v in out of cluster comm: %v", k.Podname, err)
+		return logging.Errorf("SetNetworkStatus: failed to update the pod %v in out of cluster comm: %v", podName, err)
 	}
 
 	return nil
@@ -131,18 +153,6 @@ func setPodNetworkAnnotation(client KubeClient, namespace string, pod *v1.Pod, n
 	return pod, nil
 }
 
-func getPodNetworkAnnotation(client KubeClient, k8sArgs *types.K8sArgs) (string, string, error) {
-	var err error
-
-	logging.Debugf("getPodNetworkAnnotation: %v, %v", client, k8sArgs)
-	pod, err := client.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-	if err != nil {
-		return "", "", logging.Errorf("getPodNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err)
-	}
-
-	return pod.Annotations["k8s.v1.cni.cncf.io/networks"], pod.ObjectMeta.Namespace, nil
-}
-
 func parsePodNetworkObjectName(podnetwork string) (string, string, string, error) {
 	var netNsName string
 	var netIfName string
@@ -250,7 +260,7 @@ func getCNIConfigFromFile(name string, confdir string) ([]byte, error) {
 				return nil, logging.Errorf("Error loading CNI conflist file %s: %v", confFile, err)
 			}
 
-			if confList.Name == name {
+			if confList.Name == name || name == "" {
 				return confList.Bytes, nil
 			}
 
@@ -260,7 +270,7 @@ func getCNIConfigFromFile(name string, confdir string) ([]byte, error) {
 				return nil, logging.Errorf("Error loading CNI config file %s: %v", confFile, err)
 			}
 
-			if conf.Network.Name == name {
+			if conf.Network.Name == name || name == "" {
 				// Ensure the config has a "type" so we know what plugin to run.
 				// Also catches the case where somebody put a conflist into a conf file.
 				if conf.Network.Type == "" {
@@ -326,30 +336,60 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit
 	return config, nil
 }
 
-func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string) (*types.DelegateNetConf, error) {
+func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string, pod *v1.Pod, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) {
+
 	logging.Debugf("getKubernetesDelegate: %v, %v, %s", client, net, confdir)
 	rawPath := fmt.Sprintf("/apis/k8s.cni.cncf.io/v1/namespaces/%s/network-attachment-definitions/%s", net.Namespace, net.Name)
 	netData, err := client.GetRawWithPath(rawPath)
 	if err != nil {
-		return nil, logging.Errorf("getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: %v", err)
+		return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: %v", err)
 	}
 
 	customResource := &types.NetworkAttachmentDefinition{}
 	if err := json.Unmarshal(netData, customResource); err != nil {
-		return nil, logging.Errorf("getKubernetesDelegate: failed to get the netplugin data: %v", err)
+		return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get the netplugin data: %v", err)
+	}
+
+	// Get resourceName annotation from NetworkAttachmentDefinition
+	deviceID := ""
+	resourceName, ok := customResource.Metadata.Annotations[resourceNameAnnot]
+	if ok && pod.Name != "" && pod.Namespace != "" {
+		// ResourceName annotation is found; try to get device info from resourceMap
+		logging.Debugf("getKubernetesDelegate: found resourceName annotation : %s", resourceName)
+
+		if resourceMap == nil {
+			ck, err := kubeletclient.GetResourceClient()
+			if err != nil {
+				return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get a ResourceClient instance: %v", err)
+			}
+			resourceMap, err = ck.GetPodResourceMap(pod)
+			if err != nil {
+				return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get resourceMap from ResourceClient: %v", err)
+			}
+			logging.Debugf("getKubernetesDelegate(): resourceMap instance: %+v", resourceMap)
+		}
+
+		entry, ok := resourceMap[resourceName]
+		if ok {
+			if idCount := len(entry.DeviceIDs); idCount > 0 && idCount > entry.Index {
+				deviceID = entry.DeviceIDs[entry.Index]
+				logging.Debugf("getKubernetesDelegate: podName: %s deviceID: %s", pod.Name, deviceID)
+				entry.Index++ // increment Index for next delegate
+			}
+		}
 	}
 
 	configBytes, err := cniConfigFromNetworkResource(customResource, confdir)
 	if err != nil {
-		return nil, err
+		return nil, resourceMap, err
 	}
 
-	delegate, err := types.LoadDelegateNetConf(configBytes, net.InterfaceRequest)
+	delegate, err := types.LoadDelegateNetConf(configBytes, net, deviceID)
 	if err != nil {
-		return nil, err
+		return nil, resourceMap, err
 	}
 
-	return delegate, nil
+	return delegate, resourceMap, nil
 }
 
 type KubeClient interface {
@@ -361,7 +401,7 @@ type KubeClient interface {
 func GetK8sArgs(args *skel.CmdArgs) (*types.K8sArgs, error) {
 	k8sArgs := &types.K8sArgs{}
 
-	logging.Debugf("GetK8sNetwork: %v", args)
+	logging.Debugf("GetK8sArgs: %v", args)
 	err := cnitypes.LoadArgs(args.Args, k8sArgs)
 	if err != nil {
 		return nil, err
@@ -372,11 +412,11 @@ func GetK8sArgs(args *skel.CmdArgs) (*types.K8sArgs, error) {
 
 // Attempts to load Kubernetes-defined delegates and add them to the Multus config.
 // Returns the number of Kubernetes-defined delegates added or an error.
-func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient KubeClient) (int, *clientInfo, error) {
+func TryLoadPodDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient KubeClient) (int, *clientInfo, error) {
 	var err error
 	clientInfo := &clientInfo{}
 
-	logging.Debugf("TryLoadK8sDelegates: %v, %v, %v", k8sArgs, conf, kubeClient)
+	logging.Debugf("TryLoadPodDelegates: %v, %v, %v", k8sArgs, conf, kubeClient)
 	kubeClient, err = GetK8sClient(conf.Kubeconfig, kubeClient)
 	if err != nil {
 		return 0, nil, err
@@ -391,19 +431,40 @@ func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient
 	}
 
 	setKubeClientInfo(clientInfo, kubeClient, k8sArgs)
-	delegates, err := GetK8sNetwork(kubeClient, k8sArgs, conf.ConfDir)
+	// Get the pod info. If cannot get it, we use cached delegates
+	pod, err := kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
 	if err != nil {
-		if _, ok := err.(*NoK8sNetworkError); ok {
-			return 0, clientInfo, nil
+		logging.Debugf("tryLoadK8sDelegates: Err in loading K8s cluster default network from pod annotation: %v, use cached delegates", err)
+		return 0, nil, nil
+	}
+
+	delegate, err := tryLoadK8sPodDefaultNetwork(kubeClient, pod, conf)
+	if err != nil {
+		return 0, nil, logging.Errorf("tryLoadK8sDelegates: Err in loading K8s cluster default network from pod annotation: %v", err)
+	}
+	if delegate != nil {
+		logging.Debugf("tryLoadK8sDelegates: Overwrite the cluster default network with %v from pod annotations", delegate)
+
+		conf.Delegates[0] = delegate
+	}
+
+	networks, err := GetPodNetwork(pod)
+	if networks != nil {
+		delegates, err := GetNetworkDelegates(kubeClient, pod, networks, conf.ConfDir, conf.NamespaceIsolation)
+
+		if err != nil {
+			if _, ok := err.(*NoK8sNetworkError); ok {
+				return 0, clientInfo, nil
+			}
+			return 0, nil, logging.Errorf("Multus: Err in getting k8s network from pod: %v", err)
 		}
-		return 0, nil, logging.Errorf("Multus: Err in getting k8s network from pod: %v", err)
-	}
 
-	if err = conf.AddDelegates(delegates); err != nil {
-		return 0, nil, err
+		if err = conf.AddDelegates(delegates); err != nil {
+			return 0, nil, err
+		}
+		return len(delegates), clientInfo, nil
 	}
-
-	return len(delegates), clientInfo, nil
+	return 0, clientInfo, nil
 }
 
 func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error) {
@@ -435,6 +496,10 @@ func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error)
 		return nil, nil
 	}
 
+	// Specify that we use gRPC
+	config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json"
+	config.ContentType = "application/vnd.kubernetes.protobuf"
+
 	// creates the clientset
 	client, err := kubernetes.NewForConfig(config)
 	if err != nil {
@@ -444,13 +509,11 @@ func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error)
 	return &defaultKubeClient{client: client}, nil
 }
 
-func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) ([]*types.DelegateNetConf, error) {
-	logging.Debugf("GetK8sNetwork: %v, %v, %v", k8sclient, k8sArgs, confdir)
+func GetPodNetwork(pod *v1.Pod) ([]*types.NetworkSelectionElement, error) {
+	logging.Debugf("GetPodNetwork: %v", pod)
 
-	netAnnot, defaultNamespace, err := getPodNetworkAnnotation(k8sclient, k8sArgs)
-	if err != nil {
-		return nil, err
-	}
+	netAnnot := pod.Annotations[networkAttachmentAnnot]
+	defaultNamespace := pod.ObjectMeta.Namespace
 
 	if len(netAnnot) == 0 {
 		return nil, &NoK8sNetworkError{"no kubernetes network found"}
@@ -460,16 +523,177 @@ func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string)
 	if err != nil {
 		return nil, err
 	}
+	return networks, nil
+}
+
+func GetNetworkDelegates(k8sclient KubeClient, pod *v1.Pod, networks []*types.NetworkSelectionElement, confdir string, confnamespaceIsolation bool) ([]*types.DelegateNetConf, error) {
+	logging.Debugf("GetNetworkDelegates: %v, %v, %v, %v, %v", k8sclient, pod, networks, confdir, confnamespaceIsolation)
+	// resourceMap holds Pod device allocation information; only initizized if CRD contains 'resourceName' annotation.
+	// This will only be initialized once and all delegate objects can reference this to look up device info.
+	var resourceMap map[string]*types.ResourceInfo
 
 	// Read all network objects referenced by 'networks'
 	var delegates []*types.DelegateNetConf
+	defaultNamespace := pod.ObjectMeta.Namespace
+
 	for _, net := range networks {
-		delegate, err := getKubernetesDelegate(k8sclient, net, confdir)
+
+		// The pods namespace (stored as defaultNamespace, does not equal the annotation's target namespace in net.Namespace)
+		// In the case that this is a mismatch when namespaceisolation is enabled, this should be an error.
+		if confnamespaceIsolation {
+			if defaultNamespace != net.Namespace {
+				return nil, logging.Errorf("GetPodNetwork: namespace isolation violation: podnamespace: %v / target namespace: %v", defaultNamespace, net.Namespace)
+			}
+		}
+
+		delegate, updatedResourceMap, err := getKubernetesDelegate(k8sclient, net, confdir, pod, resourceMap)
 		if err != nil {
-			return nil, logging.Errorf("GetK8sNetwork: failed getting the delegate: %v", err)
+			return nil, logging.Errorf("GetPodNetwork: failed getting the delegate: %v", err)
 		}
 		delegates = append(delegates, delegate)
+		resourceMap = updatedResourceMap
 	}
 
 	return delegates, nil
 }
+
+func getDefaultNetDelegateCRD(client KubeClient, net, confdir, namespace string) (*types.DelegateNetConf, error) {
+	logging.Debugf("getDefaultNetDelegateCRD: %v, %v, %s, %s", client, net, confdir, namespace)
+	rawPath := fmt.Sprintf("/apis/k8s.cni.cncf.io/v1/namespaces/%s/network-attachment-definitions/%s", namespace, net)
+	netData, err := client.GetRawWithPath(rawPath)
+	if err != nil {
+		return nil, logging.Errorf("getDefaultNetDelegateCRD: failed to get network resource, refer Multus README.md for the usage guide: %v", err)
+	}
+
+	customResource := &types.NetworkAttachmentDefinition{}
+	if err := json.Unmarshal(netData, customResource); err != nil {
+		return nil, logging.Errorf("getDefaultNetDelegateCRD: failed to get the netplugin data: %v", err)
+	}
+
+	configBytes, err := cniConfigFromNetworkResource(customResource, confdir)
+	if err != nil {
+		return nil, err
+	}
+
+	delegate, err := types.LoadDelegateNetConf(configBytes, nil, "")
+	if err != nil {
+		return nil, err
+	}
+
+	return delegate, nil
+}
+
+func getNetDelegate(client KubeClient, netname, confdir, namespace string) (*types.DelegateNetConf, error) {
+	logging.Debugf("getNetDelegate: %v, %v, %v, %s", client, netname, confdir, namespace)
+	// option1) search CRD object for the network
+	delegate, err := getDefaultNetDelegateCRD(client, netname, confdir, namespace)
+	if err == nil {
+		return delegate, nil
+	}
+
+	// option2) search CNI json config file
+	var configBytes []byte
+	configBytes, err = getCNIConfigFromFile(netname, confdir)
+	if err == nil {
+		delegate, err := types.LoadDelegateNetConf(configBytes, nil, "")
+		if err != nil {
+			return nil, err
+		}
+		return delegate, nil
+	}
+
+	// option3) search directry
+	fInfo, err := os.Stat(netname)
+	if err == nil {
+		if fInfo.IsDir() {
+			files, err := libcni.ConfFiles(netname, []string{".conf", ".conflist"})
+			if err != nil {
+				return nil, err
+			}
+			if len(files) > 0 {
+				var configBytes []byte
+				configBytes, err = getCNIConfigFromFile("", netname)
+				if err == nil {
+					delegate, err := types.LoadDelegateNetConf(configBytes, nil, "")
+					if err != nil {
+						return nil, err
+					}
+					return delegate, nil
+				}
+				return nil, err
+			}
+		}
+	}
+	return nil, logging.Errorf("getNetDelegate: cannot find network: %v", netname)
+}
+
+// GetDefaultNetwork parses 'defaultNetwork' config, gets network json and put it into netconf.Delegates.
+func GetDefaultNetworks(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient KubeClient) error {
+	logging.Debugf("GetDefaultNetworks: %v, %v, %v", k8sArgs, conf, kubeClient)
+	var delegates []*types.DelegateNetConf
+
+	kubeClient, err := GetK8sClient(conf.Kubeconfig, kubeClient)
+	if err != nil {
+		return err
+	}
+	if kubeClient == nil {
+		if len(conf.Delegates) == 0 {
+			// No available kube client and no delegates, we can't do anything
+			return logging.Errorf("must have either Kubernetes config or delegates, refer Multus README.md for the usage guide")
+		}
+		return nil
+	}
+
+	delegate, err := getNetDelegate(kubeClient, conf.ClusterNetwork, conf.ConfDir, conf.MultusNamespace)
+	if err != nil {
+		return err
+	}
+	delegate.MasterPlugin = true
+	delegates = append(delegates, delegate)
+
+	// Pod in kube-system namespace does not have default network for now.
+	if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAMESPACE), conf.SystemNamespaces) {
+		for _, netname := range conf.DefaultNetworks {
+			delegate, err := getNetDelegate(kubeClient, netname, conf.ConfDir, conf.MultusNamespace)
+			if err != nil {
+				return err
+			}
+			delegates = append(delegates, delegate)
+		}
+	}
+
+	if err = conf.AddDelegates(delegates); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// tryLoadK8sPodDefaultNetwork get pod default network from annotations
+func tryLoadK8sPodDefaultNetwork(kubeClient KubeClient, pod *v1.Pod, conf *types.NetConf) (*types.DelegateNetConf, error) {
+	var netAnnot string
+	logging.Debugf("tryLoadK8sPodDefaultNetwork: %v, %v, %v", kubeClient, pod, conf)
+
+	netAnnot, ok := pod.Annotations[defaultNetAnnot]
+	if !ok {
+		logging.Debugf("tryLoadK8sPodDefaultNetwork: Pod default network annotation is not defined")
+		return nil, nil
+	}
+
+	// The CRD object of default network should only be defined in multusNamespace
+	networks, err := parsePodNetworkAnnotation(netAnnot, conf.MultusNamespace)
+	if err != nil {
+		return nil, logging.Errorf("tryLoadK8sPodDefaultNetwork: failed to parse CRD object: %v", err)
+	}
+	if len(networks) > 1 {
+		return nil, logging.Errorf("tryLoadK8sPodDefaultNetwork: more than one default network is specified: %s", netAnnot)
+	}
+
+	delegate, _, err := getKubernetesDelegate(kubeClient, networks[0], conf.ConfDir, pod, nil)
+	if err != nil {
+		return nil, logging.Errorf("tryLoadK8sPodDefaultNetwork: failed getting the delegate: %v", err)
+	}
+	delegate.MasterPlugin = true
+
+	return delegate, nil
+}

k8sclient/k8sclient_test.go

@@ -25,6 +25,7 @@ import (
 	testutils "github.com/intel/multus-cni/testing"
 
 	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/intel/multus-cni/types"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -50,7 +51,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("retrieves delegates from kubernetes using simple format annotation", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2")
+		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
 		net1 := `{
 	"name": "net1",
 	"type": "mynet",
@@ -81,7 +82,10 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
+		pod, err := kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+		networks, err := GetPodNetwork(pod)
+		Expect(err).NotTo(HaveOccurred())
+		delegates, err := GetNetworkDelegates(kubeClient, pod, networks, tmpDir, false)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fKubeClient.PodCount).To(Equal(1))
 		Expect(fKubeClient.NetCount).To(Equal(2))
@@ -96,7 +100,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails when the network does not exist", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2")
+		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
 		net3 := `{
 	"name": "net3",
 	"type": "mynet3",
@@ -114,9 +118,12 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
+		pod, err := kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+		networks, err := GetPodNetwork(pod)
+		Expect(err).NotTo(HaveOccurred())
+		delegates, err := GetNetworkDelegates(kubeClient, pod, networks, tmpDir, false)
 		Expect(len(delegates)).To(Equal(0))
-		Expect(err).To(MatchError("GetK8sNetwork: failed getting the delegate: getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: resource not found"))
+		Expect(err).To(MatchError("GetPodNetwork: failed getting the delegate: getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: resource not found"))
 	})
 
 	It("retrieves delegates from kubernetes using JSON format annotation", func() {
@@ -131,7 +138,7 @@ var _ = Describe("k8sclient operations", func() {
   "name":"net3",
   "namespace":"other-ns"
 }
-]`)
+]`, "")
 		args := &skel.CmdArgs{
 			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
 		}
@@ -158,7 +165,10 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
+		pod, err := kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+		networks, err := GetPodNetwork(pod)
+		Expect(err).NotTo(HaveOccurred())
+		delegates, err := GetNetworkDelegates(kubeClient, pod, networks, tmpDir, false)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fKubeClient.PodCount).To(Equal(1))
 		Expect(fKubeClient.NetCount).To(Equal(3))
@@ -173,7 +183,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails when the JSON format annotation is invalid", func() {
-		fakePod := testutils.NewFakePod("testpod", "[adsfasdfasdfasf]")
+		fakePod := testutils.NewFakePod("testpod", "[adsfasdfasdfasf]", "")
 		args := &skel.CmdArgs{
 			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
 		}
@@ -185,13 +195,14 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
-		Expect(len(delegates)).To(Equal(0))
+		pod, err := kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+		networks, err := GetPodNetwork(pod)
+		Expect(len(networks)).To(Equal(0))
 		Expect(err).To(MatchError("parsePodNetworkAnnotation: failed to parse pod Network Attachment Selection Annotation JSON format: invalid character 'a' looking for beginning of value"))
 	})
 
 	It("retrieves delegates from kubernetes using on-disk config files", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2")
+		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
 		args := &skel.CmdArgs{
 			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
 		}
@@ -215,7 +226,10 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
+		pod, err := kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+		networks, err := GetPodNetwork(pod)
+		Expect(err).NotTo(HaveOccurred())
+		delegates, err := GetNetworkDelegates(kubeClient, pod, networks, tmpDir, false)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fKubeClient.PodCount).To(Equal(1))
 		Expect(fKubeClient.NetCount).To(Equal(2))
@@ -228,7 +242,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("injects network name into minimal thick plugin CNI config", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1")
+		fakePod := testutils.NewFakePod("testpod", "net1", "")
 		args := &skel.CmdArgs{
 			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
 		}
@@ -241,7 +255,10 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
+		pod, err := kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+		networks, err := GetPodNetwork(pod)
+		Expect(err).NotTo(HaveOccurred())
+		delegates, err := GetNetworkDelegates(kubeClient, pod, networks, tmpDir, false)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fKubeClient.PodCount).To(Equal(1))
 		Expect(fKubeClient.NetCount).To(Equal(1))
@@ -252,7 +269,7 @@ var _ = Describe("k8sclient operations", func() {
 	})
 
 	It("fails when on-disk config file is not valid", func() {
-		fakePod := testutils.NewFakePod("testpod", "net1,net2")
+		fakePod := testutils.NewFakePod("testpod", "net1,net2", "")
 		args := &skel.CmdArgs{
 			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
 		}
@@ -272,8 +289,326 @@ var _ = Describe("k8sclient operations", func() {
 		Expect(err).NotTo(HaveOccurred())
 		k8sArgs, err := GetK8sArgs(args)
 		Expect(err).NotTo(HaveOccurred())
-		delegates, err := GetK8sNetwork(kubeClient, k8sArgs, tmpDir)
+		pod, err := kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+		networks, err := GetPodNetwork(pod)
+		Expect(err).NotTo(HaveOccurred())
+		delegates, err := GetNetworkDelegates(kubeClient, pod, networks, tmpDir, false)
 		Expect(len(delegates)).To(Equal(0))
-		Expect(err).To(MatchError(fmt.Sprintf("GetK8sNetwork: failed getting the delegate: cniConfigFromNetworkResource: err in getCNIConfigFromFile: Error loading CNI config file %s: error parsing configuration: invalid character 'a' looking for beginning of value", net2Name)))
+		Expect(err).To(MatchError(fmt.Sprintf("GetPodNetwork: failed getting the delegate: cniConfigFromNetworkResource: err in getCNIConfigFromFile: Error loading CNI config file %s: error parsing configuration: invalid character 'a' looking for beginning of value", net2Name)))
+	})
+
+	It("retrieves cluster network from CRD", func() {
+		fakePod := testutils.NewFakePod("testpod", "", "")
+		conf := `{
+			"name":"node-cni-network",
+			"type":"multus",
+			"clusterNetwork": "myCRD1",
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
+		}`
+		netConf, err := types.LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+		}
+
+		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddNetConfig("kube-system", "myCRD1", "{\"type\": \"mynet\"}")
+		fKubeClient.AddPod(fakePod)
+		kubeClient, err := GetK8sClient("", fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		k8sArgs, err := GetK8sArgs(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = GetDefaultNetworks(k8sArgs, netConf, kubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(netConf.Delegates)).To(Equal(1))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("myCRD1"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
+	})
+
+	It("retrieves default networks from CRD", func() {
+		fakePod := testutils.NewFakePod("testpod", "", "")
+		conf := `{
+			"name":"node-cni-network",
+			"type":"multus",
+			"clusterNetwork": "myCRD1",
+			"defaultNetworks": ["myCRD2"],
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
+		}`
+		netConf, err := types.LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+		}
+
+		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddNetConfig("kube-system", "myCRD1", "{\"type\": \"mynet\"}")
+		fKubeClient.AddNetConfig("kube-system", "myCRD2", "{\"type\": \"mynet2\"}")
+		fKubeClient.AddPod(fakePod)
+		kubeClient, err := GetK8sClient("", fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		k8sArgs, err := GetK8sArgs(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = GetDefaultNetworks(k8sArgs, netConf, kubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(netConf.Delegates)).To(Equal(2))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("myCRD1"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
+		Expect(netConf.Delegates[1].Conf.Name).To(Equal("myCRD2"))
+		Expect(netConf.Delegates[1].Conf.Type).To(Equal("mynet2"))
+	})
+
+	It("ignore default networks from CRD in case of kube-system namespace", func() {
+		fakePod := testutils.NewFakePod("testpod", "", "")
+		// overwrite namespace
+		fakePod.ObjectMeta.Namespace = "kube-system"
+		conf := `{
+			"name":"node-cni-network",
+			"type":"multus",
+			"clusterNetwork": "myCRD1",
+			"defaultNetworks": ["myCRD2"],
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
+		}`
+		netConf, err := types.LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+		}
+
+		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddNetConfig("kube-system", "myCRD1", "{\"type\": \"mynet\"}")
+		fKubeClient.AddNetConfig("kube-system", "myCRD2", "{\"type\": \"mynet2\"}")
+		fKubeClient.AddPod(fakePod)
+		kubeClient, err := GetK8sClient("", fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		k8sArgs, err := GetK8sArgs(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = GetDefaultNetworks(k8sArgs, netConf, kubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(netConf.Delegates)).To(Equal(1))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("myCRD1"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
+	})
+
+	It("retrieves cluster network from file", func() {
+		fakePod := testutils.NewFakePod("testpod", "", "")
+		conf := `{
+			"name":"node-cni-network",
+			"type":"multus",
+			"clusterNetwork": "myFile1",
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
+		}`
+		netConf, err := types.LoadNetConf([]byte(conf))
+		netConf.ConfDir = tmpDir
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+		}
+
+		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
+		net1Name := filepath.Join(tmpDir, "10-net1.conf")
+		fKubeClient.AddNetFile(fakePod.ObjectMeta.Namespace, "net1", net1Name, `{
+	"name": "myFile1",
+	"type": "mynet",
+	"cniVersion": "0.2.0"
+}`)
+		kubeClient, err := GetK8sClient("", fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		k8sArgs, err := GetK8sArgs(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = GetDefaultNetworks(k8sArgs, netConf, kubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(netConf.Delegates)).To(Equal(1))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("myFile1"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
+	})
+
+	It("retrieves cluster network from path", func() {
+		fakePod := testutils.NewFakePod("testpod", "", "")
+		conf := fmt.Sprintf(`{
+			"name":"node-cni-network",
+			"type":"multus",
+			"clusterNetwork": "%s",
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
+		}`, tmpDir)
+		netConf, err := types.LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+		}
+
+		fKubeClient := testutils.NewFakeKubeClient()
+		net1Name := filepath.Join(tmpDir, "10-net1.conf")
+		fKubeClient.AddNetFile(fakePod.ObjectMeta.Namespace, "10-net1", net1Name, `{
+	"name": "net1",
+	"type": "mynet",
+	"cniVersion": "0.2.0"
+}`)
+		fKubeClient.AddPod(fakePod)
+		kubeClient, err := GetK8sClient("", fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		k8sArgs, err := GetK8sArgs(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = GetDefaultNetworks(k8sArgs, netConf, kubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(netConf.Delegates)).To(Equal(1))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet"))
+	})
+
+	It("Error in case of CRD not found", func() {
+		fakePod := testutils.NewFakePod("testpod", "", "")
+		conf := `{
+			"name":"node-cni-network",
+			"type":"multus",
+			"clusterNetwork": "myCRD1",
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
+		}`
+		netConf, err := types.LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+		}
+
+		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
+		kubeClient, err := GetK8sClient("", fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		k8sArgs, err := GetK8sArgs(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = GetDefaultNetworks(k8sArgs, netConf, kubeClient)
+		Expect(err).To(HaveOccurred())
+	})
+
+	It("overwrite cluster network when Pod annotation is set", func() {
+
+		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		conf := `{
+			"name":"node-cni-network",
+			"type":"multus",
+			"clusterNetwork": "net2",
+			"multusNamespace" : "kube-system",
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml"
+		}`
+		netConf, err := types.LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+		}
+
+		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddNetConfig("kube-system", "net1", "{\"type\": \"mynet1\"}")
+		fKubeClient.AddNetConfig("kube-system", "net2", "{\"type\": \"mynet2\"}")
+		fKubeClient.AddPod(fakePod)
+		kubeClient, err := GetK8sClient("", fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		k8sArgs, err := GetK8sArgs(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = GetDefaultNetworks(k8sArgs, netConf, kubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(len(netConf.Delegates)).To(Equal(1))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net2"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2"))
+
+		numK8sDelegates, _, err := TryLoadPodDelegates(k8sArgs, netConf, kubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(numK8sDelegates).To(Equal(0))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet1"))
+	})
+
+	It("overwrite multus config when Pod annotation is set", func() {
+
+		fakePod := testutils.NewFakePod("testpod", "", "net1")
+		conf := `{
+			"name":"node-cni-network",
+			"type":"multus",
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml",
+			"delegates": [{
+				"type": "mynet2",
+				"name": "net2"
+			}]
+		}`
+		netConf, err := types.LoadNetConf([]byte(conf))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net2"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet2"))
+		Expect(err).NotTo(HaveOccurred())
+
+		args := &skel.CmdArgs{
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+		}
+
+		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
+		fKubeClient.AddNetConfig("kube-system", "net1", "{\"type\": \"mynet1\"}")
+		kubeClient, err := GetK8sClient("", fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		k8sArgs, err := GetK8sArgs(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		numK8sDelegates, _, err := TryLoadPodDelegates(k8sArgs, netConf, kubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(numK8sDelegates).To(Equal(0))
+		Expect(netConf.Delegates[0].Conf.Name).To(Equal("net1"))
+		Expect(netConf.Delegates[0].Conf.Type).To(Equal("mynet1"))
+	})
+
+	It("Errors when namespace isolation is violated", func() {
+		fakePod := testutils.NewFakePod("testpod", "kube-system/net1", "")
+		conf := `{
+			"name":"node-cni-network",
+			"type":"multus",
+			"delegates": [{
+			"name": "weave1",
+				"cniVersion": "0.2.0",
+				"type": "weave-net"
+			}],
+			"kubeconfig":"/etc/kubernetes/node-kubeconfig.yaml",
+			"namespaceIsolation": true
+		}`
+
+		netConf, err := types.LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+
+		net1 := `{
+	"name": "net1",
+	"type": "mynet",
+	"cniVersion": "0.2.0"
+}`
+
+		args := &skel.CmdArgs{
+			Args: fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+		}
+
+		fKubeClient := testutils.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
+		fKubeClient.AddNetConfig("kube-system", "net1", net1)
+
+		kubeClient, err := GetK8sClient("", fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		k8sArgs, err := GetK8sArgs(args)
+		Expect(err).NotTo(HaveOccurred())
+
+		pod, err := kubeClient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+		networks, err := GetPodNetwork(pod)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = GetNetworkDelegates(kubeClient, pod, networks, tmpDir, netConf.NamespaceIsolation)
+		Expect(err).To(HaveOccurred())
+		Expect(err).To(MatchError("GetPodNetwork: namespace isolation violation: podnamespace: test / target namespace: kube-system"))
+
 	})
 })

kubeletclient/kubeletclient.go

@@ -0,0 +1,113 @@
+package kubeletclient
+
+import (
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/intel/multus-cni/checkpoint"
+	"github.com/intel/multus-cni/logging"
+	"github.com/intel/multus-cni/types"
+	"golang.org/x/net/context"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/kubernetes/pkg/kubelet/apis/podresources"
+	podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1"
+	"k8s.io/kubernetes/pkg/kubelet/util"
+)
+
+const (
+	defaultKubeletSocketFile   = "kubelet.sock"
+	defaultPodResourcesMaxSize = 1024 * 1024 * 16 // 16 Mb
+)
+
+var (
+	kubeletSocket           string
+	defaultPodResourcesPath = "/var/lib/kubelet/pod-resources"
+)
+
+// GetResourceClient returns an instance of ResourceClient interface initialized with Pod resource information
+func GetResourceClient() (types.ResourceClient, error) {
+	// If Kubelet resource API endpoint exist use that by default
+	// Or else fallback with checkpoint file
+	if hasKubeletAPIEndpoint() {
+		logging.Printf(logging.VerboseLevel, "GetResourceClient(): using Kubelet resource API endpoint")
+		return getKubeletClient()
+	} else {
+		logging.Printf(logging.VerboseLevel, "GetResourceClient(): using Kubelet device plugin checkpoint")
+		return checkpoint.GetCheckpoint()
+	}
+}
+
+func getKubeletClient() (types.ResourceClient, error) {
+	newClient := &kubeletClient{}
+	if kubeletSocket == "" {
+		kubeletSocket = util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket)
+	}
+
+	client, conn, err := podresources.GetClient(kubeletSocket, 10*time.Second, defaultPodResourcesMaxSize)
+	if err != nil {
+		return nil, logging.Errorf("GetResourceClient(): error getting grpc client: %v\n", err)
+	}
+	defer conn.Close()
+
+	if err := newClient.getPodResources(client); err != nil {
+		return nil, logging.Errorf("GetResourceClient(): error getting resource client: %v\n", err)
+	}
+
+	return newClient, nil
+}
+
+type kubeletClient struct {
+	resources []*podresourcesapi.PodResources
+}
+
+func (rc *kubeletClient) getPodResources(client podresourcesapi.PodResourcesListerClient) error {
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := client.List(ctx, &podresourcesapi.ListPodResourcesRequest{})
+	if err != nil {
+		return logging.Errorf("getPodResources(): %v.Get(_) = _, %v", client, err)
+	}
+
+	rc.resources = resp.PodResources
+	return nil
+}
+
+// GetPodResourceMap returns an instance of a map of Pod ResourceInfo given a (Pod name, namespace) tuple
+func (rc *kubeletClient) GetPodResourceMap(pod *v1.Pod) (map[string]*types.ResourceInfo, error) {
+	resourceMap := make(map[string]*types.ResourceInfo)
+
+	name := pod.Name
+	ns := pod.Namespace
+
+	if name == "" || ns == "" {
+		return nil, logging.Errorf("GetPodResourcesMap(): Pod name or namespace cannot be empty")
+	}
+
+	for _, pr := range rc.resources {
+		if pr.Name == name && pr.Namespace == ns {
+			for _, cnt := range pr.Containers {
+				for _, dev := range cnt.Devices {
+					if rInfo, ok := resourceMap[dev.ResourceName]; ok {
+						rInfo.DeviceIDs = append(rInfo.DeviceIDs, dev.DeviceIds...)
+					} else {
+						resourceMap[dev.ResourceName] = &types.ResourceInfo{DeviceIDs: dev.DeviceIds}
+					}
+				}
+			}
+		}
+	}
+	return resourceMap, nil
+}
+
+func hasKubeletAPIEndpoint() bool {
+	// Check for kubelet resource API socket file
+	kubeletAPISocket := filepath.Join(defaultPodResourcesPath, defaultKubeletSocketFile)
+	if _, err := os.Stat(kubeletAPISocket); err != nil {
+		logging.Verbosef("hasKubeletAPIEndpoint(): error looking up kubelet resource api socket file: %q", err)
+		return false
+	}
+	return true
+}

kubeletclient/kubeletclient_test.go

@@ -0,0 +1,209 @@
+package kubeletclient
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"google.golang.org/grpc"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8sTypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/kubelet/util"
+
+	mtypes "github.com/intel/multus-cni/types"
+	podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1"
+)
+
+var (
+	socketDir  string
+	socketName string
+	fakeServer *fakeResourceServer
+)
+
+type fakeResourceServer struct {
+	server *grpc.Server
+}
+
+func (m *fakeResourceServer) List(ctx context.Context, req *podresourcesapi.ListPodResourcesRequest) (*podresourcesapi.ListPodResourcesResponse, error) {
+	podName := "pod-name"
+	podNamespace := "pod-namespace"
+	containerName := "container-name"
+
+	devs := []*podresourcesapi.ContainerDevices{
+		{
+			ResourceName: "resource",
+			DeviceIds:    []string{"dev0", "dev1"},
+		},
+	}
+
+	resp := &podresourcesapi.ListPodResourcesResponse{
+		PodResources: []*podresourcesapi.PodResources{
+			{
+				Name:      podName,
+				Namespace: podNamespace,
+				Containers: []*podresourcesapi.ContainerResources{
+					{
+						Name:    containerName,
+						Devices: devs,
+					},
+				},
+			},
+		},
+	}
+	return resp, nil
+}
+
+func TestKubeletclient(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Kubeletclient Suite")
+}
+
+func setUp() error {
+	tempSocketDir, err := ioutil.TempDir("", "kubelet-resource-client")
+	if err != nil {
+		return err
+	}
+	defaultPodResourcesPath = filepath.Join(tempSocketDir, defaultPodResourcesPath)
+
+	if err := os.MkdirAll(defaultPodResourcesPath, os.ModeDir); err != nil {
+		return err
+	}
+
+	socketDir = defaultPodResourcesPath
+	socketName = filepath.Join(socketDir, "kubelet.sock")
+
+	fakeServer = &fakeResourceServer{server: grpc.NewServer()}
+	podresourcesapi.RegisterPodResourcesListerServer(fakeServer.server, fakeServer)
+	lis, err := util.CreateListener(socketName)
+	if err != nil {
+		return nil
+	}
+	go fakeServer.server.Serve(lis)
+	return nil
+}
+
+func tearDown(path string) error {
+	if fakeServer != nil {
+		fakeServer.server.Stop()
+	}
+	if err := os.RemoveAll(path); err != nil {
+		return err
+	}
+	return nil
+}
+
+var _ = BeforeSuite(func() {
+	err := setUp()
+	Expect(err).NotTo(HaveOccurred())
+})
+
+var _ = AfterSuite(func() {
+	err := tearDown(socketDir)
+	Expect(err).NotTo(HaveOccurred())
+})
+
+var _ = Describe("Kubelet resource endpoint data read operations", func() {
+
+	Context("GetResourceClient()", func() {
+		It("should return no error", func() {
+			kubeletSocket = socketName
+			_, err := GetResourceClient()
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+
+	Context("GetPodResourceMap() with valid pod name and namespace", func() {
+		It("should return no error", func() {
+			podUID := k8sTypes.UID("970a395d-bb3b-11e8-89df-408d5c537d23")
+			fakePod := &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod-name",
+					Namespace: "pod-namespace",
+					UID:       podUID,
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name: "container-name",
+						},
+					},
+				},
+			}
+			kubeletSocket = socketName
+			client, err := getKubeletClient()
+			Expect(err).NotTo(HaveOccurred())
+
+			outputRMap := map[string]*mtypes.ResourceInfo{
+				"resource": &mtypes.ResourceInfo{DeviceIDs: []string{"dev0", "dev1"}},
+			}
+			resourceMap, err := client.GetPodResourceMap(fakePod)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(resourceMap).ShouldNot(BeNil())
+			Expect(resourceMap).To(Equal(outputRMap))
+		})
+	})
+
+	Context("GetPodResourceMap() with empty podname", func() {
+		It("should return error", func() {
+			podUID := k8sTypes.UID("970a395d-bb3b-11e8-89df-408d5c537d23")
+			fakePod := &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "",
+					Namespace: "pod-namespace",
+					UID:       podUID,
+				},
+			}
+			kubeletSocket = socketName
+			client, err := getKubeletClient()
+			Expect(err).NotTo(HaveOccurred())
+			_, err = client.GetPodResourceMap(fakePod)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Context("GetPodResourceMap() with empty namespace", func() {
+		It("should return error", func() {
+			podUID := k8sTypes.UID("970a395d-bb3b-11e8-89df-408d5c537d23")
+			fakePod := &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod-name",
+					Namespace: "",
+					UID:       podUID,
+				},
+			}
+			kubeletSocket = socketName
+			client, err := getKubeletClient()
+			Expect(err).NotTo(HaveOccurred())
+			_, err = client.GetPodResourceMap(fakePod)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Context("GetPodResourceMap() with non-existent podname and namespace", func() {
+		It("should return no error", func() {
+			podUID := k8sTypes.UID("970a395d-bb3b-11e8-89df-408d5c537d23")
+			fakePod := &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "whateverpod",
+					Namespace: "whatevernamespace",
+					UID:       podUID,
+				},
+			}
+
+			kubeletSocket = socketName
+			client, err := getKubeletClient()
+			Expect(err).NotTo(HaveOccurred())
+
+			emptyRMap := map[string]*mtypes.ResourceInfo{}
+			resourceMap, err := client.GetPodResourceMap(fakePod)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(resourceMap).ShouldNot(BeNil())
+			Expect(resourceMap).To(Equal(emptyRMap))
+		})
+	})
+})

logging/logging.go

@@ -29,6 +29,7 @@ type Level uint32
 const (
 	PanicLevel Level = iota
 	ErrorLevel
+	VerboseLevel
 	DebugLevel
 	MaxLevel
 	UnknownLevel
@@ -44,6 +45,8 @@ func (l Level) String() string {
 	switch l {
 	case PanicLevel:
 		return "panic"
+	case VerboseLevel:
+		return "verbose"
 	case ErrorLevel:
 		return "error"
 	case DebugLevel:
@@ -76,6 +79,10 @@ func Debugf(format string, a ...interface{}) {
 	Printf(DebugLevel, format, a...)
 }
 
+func Verbosef(format string, a ...interface{}) {
+	Printf(VerboseLevel, format, a...)
+}
+
 func Errorf(format string, a ...interface{}) error {
 	Printf(ErrorLevel, format, a...)
 	return fmt.Errorf(format, a...)
@@ -88,10 +95,16 @@ func Panicf(format string, a ...interface{}) {
 	Printf(PanicLevel, "========= Stack trace output end ========")
 }
 
-func GetLoggingLevel(levelStr string) Level {
+func GetLoggingLevel() Level {
+	return loggingLevel
+}
+
+func getLoggingLevel(levelStr string) Level {
 	switch strings.ToLower(levelStr) {
 	case "debug":
 		return DebugLevel
+	case "verbose":
+		return VerboseLevel
 	case "error":
 		return ErrorLevel
 	case "panic":
@@ -102,7 +115,7 @@ func GetLoggingLevel(levelStr string) Level {
 }
 
 func SetLogLevel(levelStr string) {
-	level := GetLoggingLevel(levelStr)
+	level := getLoggingLevel(levelStr)
 	if level < MaxLevel {
 		loggingLevel = level
 	}

logging/logging_test.go

@@ -50,6 +50,8 @@ var _ = Describe("logging operations", func() {
 		Expect(loggingLevel).To(Equal(DebugLevel))
 		SetLogLevel("Error")
 		Expect(loggingLevel).To(Equal(ErrorLevel))
+		SetLogLevel("VERbose")
+		Expect(loggingLevel).To(Equal(VerboseLevel))
 		SetLogLevel("PANIC")
 		Expect(loggingLevel).To(Equal(PanicLevel))
 	})

multus/multus.go

@@ -22,8 +22,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
 	"path/filepath"
+	"strings"
+	"time"
 
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/pkg/invoke"
@@ -35,8 +38,16 @@ import (
 	"github.com/intel/multus-cni/logging"
 	"github.com/intel/multus-cni/types"
 	"github.com/vishvananda/netlink"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
+var defaultReadinessBackoff = wait.Backoff{
+	Steps:    4,
+	Duration: 250 * time.Millisecond,
+	Factor:   4.0,
+	Jitter:   0.1,
+}
+
 func saveScratchNetConf(containerID, dataDir string, netconf []byte) error {
 	logging.Debugf("saveScratchNetConf: %s, %s, %s", containerID, dataDir, string(netconf))
 	if err := os.MkdirAll(dataDir, 0700); err != nil {
@@ -53,12 +64,12 @@ func saveScratchNetConf(containerID, dataDir string, netconf []byte) error {
 	return err
 }
 
-func consumeScratchNetConf(containerID, dataDir string) ([]byte, error) {
+func consumeScratchNetConf(containerID, dataDir string) ([]byte, string, error) {
 	logging.Debugf("consumeScratchNetConf: %s, %s", containerID, dataDir)
 	path := filepath.Join(dataDir, containerID)
-	defer os.Remove(path)
 
-	return ioutil.ReadFile(path)
+	b, err := ioutil.ReadFile(path)
+	return b, path, err
 }
 
 func getIfname(delegate *types.DelegateNetConf, argif string, idx int) string {
@@ -84,12 +95,23 @@ func saveDelegates(containerID, dataDir string, delegates []*types.DelegateNetCo
 	}
 
 	if err = saveScratchNetConf(containerID, dataDir, delegatesBytes); err != nil {
-		return logging.Errorf("error in saving the  delegates : %v", err)
+		return logging.Errorf("error in saving the delegates : %v", err)
 	}
 
 	return err
 }
 
+func deleteDelegates(containerID, dataDir string) error {
+	logging.Debugf("deleteDelegates: %s, %s", containerID, dataDir)
+
+	path := filepath.Join(dataDir, containerID)
+	if err := os.Remove(path); err != nil {
+		return logging.Errorf("error in deleting the delegates : %v", err)
+	}
+
+	return nil
+}
+
 func validateIfName(nsname string, ifname string) error {
 	logging.Debugf("validateIfName: %s, %s", nsname, ifname)
 	podNs, err := ns.GetNS(nsname)
@@ -111,11 +133,12 @@ func validateIfName(nsname string, ifname string) error {
 	return err
 }
 
-func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string) (cnitypes.Result, error) {
+func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string, exec invoke.Exec) (cnitypes.Result, error) {
 	logging.Debugf("conflistAdd: %v, %s, %s", rt, string(rawnetconflist), binDir)
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go
-	binDirs := []string{binDir}
-	cniNet := libcni.CNIConfig{Path: binDirs}
+	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
+	binDirs = append(binDirs, binDir)
+	cniNet := libcni.NewCNIConfig(binDirs, exec)
 
 	confList, err := libcni.ConfListFromBytes(rawnetconflist)
 	if err != nil {
@@ -130,11 +153,12 @@ func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string) (
 	return result, nil
 }
 
-func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string) error {
+func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string, exec invoke.Exec) error {
 	logging.Debugf("conflistDel: %v, %s, %s", rt, string(rawnetconflist), binDir)
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go
-	binDirs := []string{binDir}
-	cniNet := libcni.CNIConfig{Path: binDirs}
+	binDirs := filepath.SplitList(os.Getenv("CNI_PATH"))
+	binDirs = append(binDirs, binDir)
+	cniNet := libcni.NewCNIConfig(binDirs, exec)
 
 	confList, err := libcni.ConfListFromBytes(rawnetconflist)
 	if err != nil {
@@ -149,7 +173,7 @@ func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string) e
 	return err
 }
 
-func delegateAdd(exec invoke.Exec, ifName string, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, binDir string) (cnitypes.Result, error) {
+func delegateAdd(exec invoke.Exec, ifName string, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, binDir string, cniArgs string) (cnitypes.Result, error) {
 	logging.Debugf("delegateAdd: %v, %s, %v, %v, %s", exec, ifName, delegate, rt, binDir)
 	if os.Setenv("CNI_IFNAME", ifName) != nil {
 		return nil, logging.Errorf("Multus: error in setting CNI_IFNAME")
@@ -159,18 +183,66 @@ func delegateAdd(exec invoke.Exec, ifName string, delegate *types.DelegateNetCon
 		return nil, logging.Errorf("cannot set %q ifname to %q: %v", delegate.Conf.Type, ifName, err)
 	}
 
-	if delegate.ConfListPlugin != false {
-		result, err := conflistAdd(rt, delegate.Bytes, binDir)
+	if delegate.MacRequest != "" || delegate.IPRequest != "" {
+		if cniArgs != "" {
+			cniArgs = fmt.Sprintf("%s;IgnoreUnknown=true", cniArgs)
+		} else {
+			cniArgs = "IgnoreUnknown=true"
+		}
+		if delegate.MacRequest != "" {
+			// validate Mac address
+			_, err := net.ParseMAC(delegate.MacRequest)
+			if err != nil {
+				return nil, logging.Errorf("failed to parse mac address %q", delegate.MacRequest)
+			}
+
+			cniArgs = fmt.Sprintf("%s;MAC=%s", cniArgs, delegate.MacRequest)
+			logging.Debugf("Set MAC address %q to %q", delegate.MacRequest, ifName)
+		}
+
+		if delegate.IPRequest != "" {
+			// validate IP address
+			if strings.Contains(delegate.IPRequest, "/") {
+				_, _, err := net.ParseCIDR(delegate.IPRequest)
+				if err != nil {
+					return nil, logging.Errorf("failed to parse CIDR %q", delegate.MacRequest)
+				}
+			} else if net.ParseIP(delegate.IPRequest) == nil {
+				return nil, logging.Errorf("failed to parse IP address %q", delegate.IPRequest)
+			}
+
+			cniArgs = fmt.Sprintf("%s;IP=%s", cniArgs, delegate.IPRequest)
+			logging.Debugf("Set IP address %q to %q", delegate.IPRequest, ifName)
+		}
+		if os.Setenv("CNI_ARGS", cniArgs) != nil {
+			return nil, logging.Errorf("cannot set %q mac to %q and ip to %q", delegate.Conf.Type, delegate.MacRequest, delegate.IPRequest)
+		}
+	}
+
+	var result cnitypes.Result
+	var err error
+	if delegate.ConfListPlugin {
+		result, err = conflistAdd(rt, delegate.Bytes, binDir, exec)
 		if err != nil {
 			return nil, logging.Errorf("Multus: error in invoke Conflist add - %q: %v", delegate.ConfList.Name, err)
 		}
-
-		return result, nil
+	} else {
+		result, err = invoke.DelegateAdd(delegate.Conf.Type, delegate.Bytes, exec)
+		if err != nil {
+			return nil, logging.Errorf("Multus: error in invoke Delegate add - %q: %v", delegate.Conf.Type, err)
+		}
 	}
 
-	result, err := invoke.DelegateAdd(delegate.Conf.Type, delegate.Bytes, exec)
-	if err != nil {
-		return nil, logging.Errorf("Multus: error in invoke Delegate add - %q: %v", delegate.Conf.Type, err)
+	if logging.GetLoggingLevel() >= logging.VerboseLevel {
+		data, _ := json.Marshal(result)
+		var confName string
+		if delegate.ConfListPlugin {
+			confName = delegate.ConfList.Name
+		} else {
+			confName = delegate.Conf.Name
+		}
+
+		logging.Verbosef("Add: %s:%s:%s:%s %s", rt.Args[1][1], rt.Args[2][1], confName, rt.IfName, string(data))
 	}
 
 	return result, nil
@@ -182,20 +254,29 @@ func delegateDel(exec invoke.Exec, ifName string, delegateConf *types.DelegateNe
 		return logging.Errorf("Multus: error in setting CNI_IFNAME")
 	}
 
-	if delegateConf.ConfListPlugin != false {
-		err := conflistDel(rt, delegateConf.Bytes, binDir)
+	if logging.GetLoggingLevel() >= logging.VerboseLevel {
+		var confName string
+		if delegateConf.ConfListPlugin {
+			confName = delegateConf.ConfList.Name
+		} else {
+			confName = delegateConf.Conf.Name
+		}
+		logging.Verbosef("Del: %s:%s:%s:%s %s", rt.Args[1][1], rt.Args[2][1], confName, rt.IfName, string(delegateConf.Bytes))
+	}
+
+	var err error
+	if delegateConf.ConfListPlugin {
+		err = conflistDel(rt, delegateConf.Bytes, binDir, exec)
 		if err != nil {
 			return logging.Errorf("Multus: error in invoke Conflist Del - %q: %v", delegateConf.ConfList.Name, err)
 		}
-
-		return err
+	} else {
+		if err = invoke.DelegateDel(delegateConf.Conf.Type, delegateConf.Bytes, exec); err != nil {
+			return logging.Errorf("Multus: error in invoke Delegate del - %q: %v", delegateConf.Conf.Type, err)
+		}
 	}
 
-	if err := invoke.DelegateDel(delegateConf.Conf.Type, delegateConf.Bytes, exec); err != nil {
-		return logging.Errorf("Multus: error in invoke Delegate del - %q: %v", delegateConf.Conf.Type, err)
-	}
-
-	return nil
+	return err
 }
 
 func delPlugins(exec invoke.Exec, argIfname string, delegates []*types.DelegateNetConf, lastIdx int, rt *libcni.RuntimeConf, binDir string) error {
@@ -204,20 +285,27 @@ func delPlugins(exec invoke.Exec, argIfname string, delegates []*types.DelegateN
 		return logging.Errorf("Multus: error in setting CNI_COMMAND to DEL")
 	}
 
+	var errorstrings []string
 	for idx := lastIdx; idx >= 0; idx-- {
 		ifName := getIfname(delegates[idx], argIfname, idx)
 		rt.IfName = ifName
+		// Attempt to delete all but do not error out, instead, collect all errors.
 		if err := delegateDel(exec, ifName, delegates[idx], rt, binDir); err != nil {
-			return err
+			errorstrings = append(errorstrings, err.Error())
 		}
 	}
 
+	// Check if we had any errors, and send them all back.
+	if len(errorstrings) > 0 {
+		return fmt.Errorf(strings.Join(errorstrings, " / "))
+	}
+
 	return nil
 }
 
 func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cnitypes.Result, error) {
-	logging.Debugf("cmdAdd: %v, %v, %v", args, exec, kubeClient)
 	n, err := types.LoadNetConf(args.StdinData)
+	logging.Debugf("cmdAdd: %v, %v, %v", args, exec, kubeClient)
 	if err != nil {
 		return nil, logging.Errorf("err in loading netconf: %v", err)
 	}
@@ -227,29 +315,51 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 		return nil, logging.Errorf("Multus: Err in getting k8s args: %v", err)
 	}
 
-	numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, n, kubeClient)
+	wait.ExponentialBackoff(defaultReadinessBackoff, func() (bool, error) {
+		_, err := os.Stat(n.ReadinessIndicatorFile)
+		switch {
+		case err == nil:
+			return true, nil
+		default:
+			return false, nil
+		}
+	})
+
+	if n.ClusterNetwork != "" {
+		err = k8s.GetDefaultNetworks(k8sArgs, n, kubeClient)
+		if err != nil {
+			return nil, logging.Errorf("Multus: Failed to get clusterNetwork/defaultNetworks: %v", err)
+		}
+		// First delegate is always the master plugin
+		n.Delegates[0].MasterPlugin = true
+	}
+
+	_, kc, err := k8s.TryLoadPodDelegates(k8sArgs, n, kubeClient)
 	if err != nil {
 		return nil, logging.Errorf("Multus: Err in loading K8s Delegates k8s args: %v", err)
 	}
 
-	if numK8sDelegates == 0 {
-		// cache the multus config if we have only Multus delegates
-		if err := saveDelegates(args.ContainerID, n.CNIDir, n.Delegates); err != nil {
-			return nil, logging.Errorf("Multus: Err in saving the delegates: %v", err)
-		}
+	// cache the multus config
+	if err := saveDelegates(args.ContainerID, n.CNIDir, n.Delegates); err != nil {
+		return nil, logging.Errorf("Multus: Err in saving the delegates: %v", err)
 	}
 
 	var result, tmpResult cnitypes.Result
 	var netStatus []*types.NetworkStatus
-	var rt *libcni.RuntimeConf
-	lastIdx := 0
+	cniArgs := os.Getenv("CNI_ARGS")
 	for idx, delegate := range n.Delegates {
-		lastIdx = idx
 		ifName := getIfname(delegate, args.IfName, idx)
-		rt, _ = types.LoadCNIRuntimeConf(args, k8sArgs, ifName)
-		tmpResult, err = delegateAdd(exec, ifName, delegate, rt, n.BinDir)
+		rt := types.CreateCNIRuntimeConf(args, k8sArgs, ifName, n.RuntimeConfig)
+		tmpResult, err = delegateAdd(exec, ifName, delegate, rt, n.BinDir, cniArgs)
 		if err != nil {
-			break
+			// If the add failed, tear down all networks we already added
+			netName := delegate.Conf.Name
+			if netName == "" {
+				netName = delegate.ConfList.Name
+			}
+			// Ignore errors; DEL must be idempotent anyway
+			_ = delPlugins(exec, args.IfName, n.Delegates, idx, rt, n.BinDir)
+			return nil, logging.Errorf("Multus: Err adding pod to network %q: %v", netName, err)
 		}
 
 		// Master plugin result is always used if present
@@ -259,10 +369,10 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 
 		//create the network status, only in case Multus as kubeconfig
 		if n.Kubeconfig != "" && kc != nil {
-			if kc.Podnamespace != "kube-system" {
+			if !types.CheckSystemNamespaces(kc.Podnamespace, n.SystemNamespaces) {
 				delegateNetStatus, err := types.LoadNetworkStatus(tmpResult, delegate.Conf.Name, delegate.MasterPlugin)
 				if err != nil {
-					return nil, logging.Errorf("Multus: Err in setting  networks status: %v", err)
+					return nil, logging.Errorf("Multus: Err in setting network status: %v", err)
 				}
 
 				netStatus = append(netStatus, delegateNetStatus)
@@ -270,16 +380,10 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 		}
 	}
 
-	if err != nil {
-		// Ignore errors; DEL must be idempotent anyway
-		_ = delPlugins(exec, args.IfName, n.Delegates, lastIdx, rt, n.BinDir)
-		return nil, logging.Errorf("Multus: Err in tearing down failed plugins: %v", err)
-	}
-
 	//set the network status annotation in apiserver, only in case Multus as kubeconfig
 	if n.Kubeconfig != "" && kc != nil {
-		if kc.Podnamespace != "kube-system" {
-			err = k8s.SetNetworkStatus(kc, netStatus)
+		if !types.CheckSystemNamespaces(kc.Podnamespace, n.SystemNamespaces) {
+			err = k8s.SetNetworkStatus(kubeClient, k8sArgs, netStatus, n)
 			if err != nil {
 				return nil, logging.Errorf("Multus: Err set the networks status: %v", err)
 			}
@@ -302,8 +406,8 @@ func cmdGet(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn
 }
 
 func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) error {
-	logging.Debugf("cmdDel: %v, %v, %v", args, exec, kubeClient)
 	in, err := types.LoadNetConf(args.StdinData)
+	logging.Debugf("cmdDel: %v, %v, %v", args, exec, kubeClient)
 	if err != nil {
 		return err
 	}
@@ -311,56 +415,91 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err
 	if args.Netns == "" {
 		return nil
 	}
+
+	netnsfound := true
 	netns, err := ns.GetNS(args.Netns)
 	if err != nil {
-		//  if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
+		// if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times
 		// so don't return an error if the device is already removed.
 		// https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444
 		_, ok := err.(ns.NSPathNotExistErr)
 		if ok {
-			return nil
+			netnsfound = false
+			logging.Debugf("cmdDel: WARNING netns may not exist, netns: %s, err: %s", netns, err)
+		} else {
+			return fmt.Errorf("failed to open netns %q: %v", netns, err)
 		}
-		return fmt.Errorf("failed to open netns %q: %v", netns, err)
 	}
-	defer netns.Close()
+
+	if netns != nil {
+		defer netns.Close()
+	}
 
 	k8sArgs, err := k8s.GetK8sArgs(args)
 	if err != nil {
 		return logging.Errorf("Multus: Err in getting k8s args: %v", err)
 	}
 
-	numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, in, kubeClient)
+	// Read the cache to get delegates json for the pod
+	netconfBytes, path, err := consumeScratchNetConf(args.ContainerID, in.CNIDir)
 	if err != nil {
-		return err
-	}
-
-	if numK8sDelegates == 0 {
-		// re-read the scratch multus config if we have only Multus delegates
-		netconfBytes, err := consumeScratchNetConf(args.ContainerID, in.CNIDir)
-		if err != nil {
-			if os.IsNotExist(err) {
-				// Per spec should ignore error if resources are missing / already removed
-				return nil
+		// Fetch delegates again if cache is not exist
+		if os.IsNotExist(err) {
+			if in.ClusterNetwork != "" {
+				err = k8s.GetDefaultNetworks(k8sArgs, in, kubeClient)
+				if err != nil {
+					return logging.Errorf("Multus: Failed to get clusterNetwork/defaultNetworks: %v", err)
+				}
+				// First delegate is always the master plugin
+				in.Delegates[0].MasterPlugin = true
 			}
-			return logging.Errorf("Multus: Err in  reading the delegates: %v", err)
-		}
 
+			// Get pod annotation and so on
+			_, _, err := k8s.TryLoadPodDelegates(k8sArgs, in, kubeClient)
+			if err != nil {
+				if len(in.Delegates) == 0 {
+					// No delegate available so send error
+					return logging.Errorf("Multus: failed to get delegates: %v", err)
+				}
+				// Get clusterNetwork before, so continue to delete
+				logging.Errorf("Multus: failed to get delegates: %v, but continue to delete clusterNetwork", err)
+			}
+		} else {
+			return logging.Errorf("Multus: Err in reading the delegates: %v", err)
+		}
+	} else {
+		defer os.Remove(path)
 		if err := json.Unmarshal(netconfBytes, &in.Delegates); err != nil {
 			return logging.Errorf("Multus: failed to load netconf: %v", err)
 		}
+		// First delegate is always the master plugin
+		in.Delegates[0].MasterPlugin = true
 	}
 
-	//unset the network status annotation in apiserver, only in case Multus as kubeconfig
-	if in.Kubeconfig != "" && kc != nil {
-		if kc.Podnamespace != "kube-system" {
-			err := k8s.SetNetworkStatus(kc, nil)
-			if err != nil {
-				return logging.Errorf("Multus: Err unset the networks status: %v", err)
-			}
+	// set CNIVersion in delegate CNI config if there is no CNIVersion and multus conf have CNIVersion.
+	for _, v := range in.Delegates {
+		if v.ConfListPlugin == true && v.ConfList.CNIVersion == "" && in.CNIVersion != "" {
+			v.ConfList.CNIVersion = in.CNIVersion
+			v.Bytes, err = json.Marshal(v.ConfList)
 		}
 	}
 
-	rt, _ := types.LoadCNIRuntimeConf(args, k8sArgs, "")
+	// unset the network status annotation in apiserver, only in case Multus as kubeconfig
+	if in.Kubeconfig != "" {
+		if netnsfound {
+			if !types.CheckSystemNamespaces(string(k8sArgs.K8S_POD_NAMESPACE), in.SystemNamespaces) {
+				err := k8s.SetNetworkStatus(kubeClient, k8sArgs, nil, in)
+				if err != nil {
+					// error happen but continue to delete
+					logging.Errorf("Multus: Err unset the networks status: %v", err)
+				}
+			}
+		} else {
+			logging.Debugf("WARNING: Unset SetNetworkStatus skipped due to netns not found.")
+		}
+	}
+
+	rt := types.CreateCNIRuntimeConf(args, k8sArgs, "", in.RuntimeConfig)
 	return delPlugins(exec, args.IfName, in.Delegates, len(in.Delegates)-1, rt, in.BinDir)
 }
 

multus/multus_test.go

@@ -123,8 +123,12 @@ func (f *fakeExec) ExecPlugin(pluginPath string, stdinData []byte, environ []str
 	if plugin.expectedIfname != "" {
 		Expect(os.Getenv("CNI_IFNAME")).To(Equal(plugin.expectedIfname))
 	}
+
 	if len(plugin.expectedEnv) > 0 {
-		matchArray(gatherCNIEnv(), plugin.expectedEnv)
+		cniEnv := gatherCNIEnv()
+		for _, expectedCniEnvVar := range plugin.expectedEnv {
+			Expect(cniEnv).Should(ContainElement(expectedCniEnvVar))
+		}
 	}
 
 	if plugin.err != nil {
@@ -173,6 +177,8 @@ var _ = Describe("multus operations", func() {
 			StdinData: []byte(`{
     "name": "node-cni-network",
     "type": "multus",
+    "defaultnetworkfile": "/tmp/foo.multus.conf",
+    "defaultnetworkwaitseconds": 3,
     "delegates": [{
         "name": "weave1",
         "cniVersion": "0.2.0",
@@ -185,6 +191,10 @@ var _ = Describe("multus operations", func() {
 }`),
 		}
 
+		// Touch the default network file.
+		configPath := "/tmp/foo.multus.conf"
+		os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755)
+
 		fExec := &fakeExec{}
 		expectedResult1 := &types020.Result{
 			CNIVersion: "0.2.0",
@@ -226,10 +236,152 @@ var _ = Describe("multus operations", func() {
 		err = cmdDel(args, fExec, nil)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+
+		// Cleanup default network file.
+		if _, errStat := os.Stat(configPath); errStat == nil {
+			errRemove := os.Remove(configPath)
+			Expect(errRemove).NotTo(HaveOccurred())
+		}
+
+	})
+
+	It("executes delegates and cleans up on failure", func() {
+		expectedConf1 := `{
+    "name": "weave1",
+    "cniVersion": "0.2.0",
+    "type": "weave-net"
+}`
+		expectedConf2 := `{
+    "name": "other1",
+    "cniVersion": "0.2.0",
+    "type": "other-plugin"
+}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(fmt.Sprintf(`{
+    "name": "node-cni-network",
+    "type": "multus",
+    "defaultnetworkfile": "/tmp/foo.multus.conf",
+    "defaultnetworkwaitseconds": 3,
+    "delegates": [%s,%s]
+}`, expectedConf1, expectedConf2)),
+		}
+
+		// Touch the default network file.
+		configPath := "/tmp/foo.multus.conf"
+		os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755)
+
+		fExec := &fakeExec{}
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil)
+
+		// This plugin invocation should fail
+		err := fmt.Errorf("expected plugin failure")
+		fExec.addPlugin(nil, "net1", expectedConf2, nil, err)
+
+		os.Setenv("CNI_COMMAND", "ADD")
+		os.Setenv("CNI_IFNAME", "eth0")
+		_, err = cmdAdd(args, fExec, nil)
+		Expect(fExec.addIndex).To(Equal(2))
+		Expect(fExec.delIndex).To(Equal(2))
+		Expect(err).To(MatchError("Multus: Err adding pod to network \"other1\": Multus: error in invoke Delegate add - \"other-plugin\": expected plugin failure"))
+
+		// Cleanup default network file.
+		if _, errStat := os.Stat(configPath); errStat == nil {
+			errRemove := os.Remove(configPath)
+			Expect(errRemove).NotTo(HaveOccurred())
+		}
+
+	})
+
+	It("executes delegates with interface name and MAC and IP addr", func() {
+		podNet := `[{"name":"net1",
+			 "interface": "test1",
+			 "ips":"1.2.3.4/24"},
+		{"name":"net2",
+		 "mac": "c2:11:22:33:44:66",
+		 "ips": "10.0.0.1"}
+]`
+		fakePod := testhelpers.NewFakePod("testpod", podNet, "")
+		net1 := `{
+	"name": "net1",
+	"type": "mynet",
+	"cniVersion": "0.2.0"
+}`
+		net2 := `{
+	"name": "net2",
+	"type": "mynet2",
+	"cniVersion": "0.2.0"
+}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+    "name": "node-cni-network",
+    "type": "multus",
+    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+    "delegates": [{
+        "name": "weave1",
+        "cniVersion": "0.2.0",
+        "type": "weave-net"
+    }]
+}`),
+		}
+
+		fExec := &fakeExec{}
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+    "name": "weave1",
+    "cniVersion": "0.2.0",
+    "type": "weave-net"
+}`
+		fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin([]string{"CNI_ARGS=IgnoreUnknown=true;IP=1.2.3.4/24"}, "test1", net1, &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.3/24"),
+			},
+		}, nil)
+		fExec.addPlugin([]string{"CNI_ARGS=IgnoreUnknown=true;MAC=c2:11:22:33:44:66;IP=10.0.0.1"}, "net2", net2, &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.4/24"),
+			},
+		}, nil)
+
+		fKubeClient := testhelpers.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
+		fKubeClient.AddNetConfig(fakePod.ObjectMeta.Namespace, "net1", net1)
+		fKubeClient.AddNetConfig(fakePod.ObjectMeta.Namespace, "net2", net2)
+
+		os.Setenv("CNI_COMMAND", "ADD")
+		os.Setenv("CNI_IFNAME", "eth0")
+		result, err := cmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		Expect(fKubeClient.PodCount).To(Equal(2))
+		Expect(fKubeClient.NetCount).To(Equal(2))
+		r := result.(*types020.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
 	})
 
 	It("executes delegates and kubernetes networks", func() {
-		fakePod := testhelpers.NewFakePod("testpod", "net1,net2")
+		fakePod := testhelpers.NewFakePod("testpod", "net1,net2", "")
 		net1 := `{
 	"name": "net1",
 	"type": "mynet",
@@ -306,4 +458,328 @@ var _ = Describe("multus operations", func() {
 		// plugin 1 is the masterplugin
 		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
 	})
+
+	It("executes kubernetes networks and delete it after pod removal", func() {
+		fakePod := testhelpers.NewFakePod("testpod", "net1", "")
+		net1 := `{
+	"name": "net1",
+	"type": "mynet",
+	"cniVersion": "0.2.0"
+}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+    "name": "node-cni-network",
+    "type": "multus",
+    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+    "delegates": [{
+        "name": "weave1",
+        "cniVersion": "0.2.0",
+        "type": "weave-net"
+    }]
+}`),
+		}
+
+		fExec := &fakeExec{}
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+    "name": "weave1",
+    "cniVersion": "0.2.0",
+    "type": "weave-net"
+}`
+		fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin(nil, "net1", net1, &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.3/24"),
+			},
+		}, nil)
+
+		fKubeClient := testhelpers.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
+		fKubeClient.AddNetConfig(fakePod.ObjectMeta.Namespace, "net1", net1)
+
+		os.Setenv("CNI_COMMAND", "ADD")
+		os.Setenv("CNI_IFNAME", "eth0")
+		result, err := cmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		Expect(fKubeClient.PodCount).To(Equal(2))
+		Expect(fKubeClient.NetCount).To(Equal(1))
+		r := result.(*types020.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		os.Setenv("CNI_COMMAND", "DEL")
+		os.Setenv("CNI_IFNAME", "eth0")
+		// set fKubeClient to nil to emulate no pod info
+		fKubeClient.DeletePod(fakePod)
+		err = cmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("ensure delegates get portmap runtime config", func() {
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			StdinData: []byte(`{
+    "name": "node-cni-network",
+    "type": "multus",
+    "delegates": [{
+        "cniVersion": "0.3.1",
+        "name": "mynet-confList",
+		"plugins": [
+			{
+				"type": "firstPlugin",
+                "capabilities": {"portMappings": true}
+            }
+		]
+	}],
+	"runtimeConfig": {
+        "portMappings": [
+            {"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
+		]
+    }
+}`),
+		}
+
+		fExec := &fakeExec{}
+		expectedConf1 := `{
+    "capabilities": {"portMappings": true},
+	"name": "mynet-confList",
+    "cniVersion": "0.3.1",
+    "type": "firstPlugin",
+    "runtimeConfig": {
+	    "portMappings": [
+            {"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
+		]
+    }
+}`
+		fExec.addPlugin(nil, "eth0", expectedConf1, nil, nil)
+		os.Setenv("CNI_COMMAND", "ADD")
+		os.Setenv("CNI_IFNAME", "eth0")
+		_, err := cmdAdd(args, fExec, nil)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("executes clusterNetwork delegate", func() {
+		fakePod := testhelpers.NewFakePod("testpod", "", "kube-system/net1")
+		net1 := `{
+	"name": "net1",
+	"type": "mynet",
+	"cniVersion": "0.2.0"
+}`
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(`{
+    "name": "node-cni-network",
+    "type": "multus",
+    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+    "defaultNetworks": [],
+    "clusterNetwork": "net1",
+    "delegates": []
+}`),
+		}
+
+		fExec := &fakeExec{}
+		fExec.addPlugin(nil, "eth0", net1, expectedResult1, nil)
+
+		fKubeClient := testhelpers.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
+		fKubeClient.AddNetConfig("kube-system", "net1", net1)
+
+		os.Setenv("CNI_COMMAND", "ADD")
+		os.Setenv("CNI_IFNAME", "eth0")
+		result, err := cmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		Expect(fKubeClient.PodCount).To(Equal(2))
+		Expect(fKubeClient.NetCount).To(Equal(2))
+		r := result.(*types020.Result)
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		os.Setenv("CNI_COMMAND", "DEL")
+		os.Setenv("CNI_IFNAME", "eth0")
+		err = cmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+	})
+
+	It("Verify the cache is created in dataDir", func() {
+		tmpCNIDir := tmpDir + "/cniData"
+		err := os.Mkdir(tmpCNIDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		fakePod := testhelpers.NewFakePod("testpod", "net1", "")
+		net1 := `{
+	"name": "net1",
+	"type": "mynet",
+	"cniVersion": "0.2.0"
+}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(fmt.Sprintf(`{
+    "name": "node-cni-network",
+    "type": "multus",
+    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+    "cniDir": "%s",
+    "delegates": [{
+        "name": "weave1",
+        "cniVersion": "0.2.0",
+        "type": "weave-net"
+    }]
+}`, tmpCNIDir)),
+		}
+
+		fExec := &fakeExec{}
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+    "name": "weave1",
+    "cniVersion": "0.2.0",
+    "type": "weave-net"
+}`
+		fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin(nil, "net1", net1, &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.3/24"),
+			},
+		}, nil)
+
+		fKubeClient := testhelpers.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
+		fKubeClient.AddNetConfig(fakePod.ObjectMeta.Namespace, "net1", net1)
+		os.Setenv("CNI_COMMAND", "ADD")
+		os.Setenv("CNI_IFNAME", "eth0")
+		result, err := cmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		Expect(fKubeClient.PodCount).To(Equal(2))
+		Expect(fKubeClient.NetCount).To(Equal(1))
+		r := result.(*types020.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		By("Verify cache file existence")
+		cacheFilePath := fmt.Sprintf("%s/%s", tmpCNIDir, "123456789")
+		_, err = os.Stat(cacheFilePath)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Delete and check net count is not incremented")
+		os.Setenv("CNI_COMMAND", "DEL")
+		os.Setenv("CNI_IFNAME", "eth0")
+		err = cmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+		Expect(fKubeClient.PodCount).To(Equal(3))
+		Expect(fKubeClient.NetCount).To(Equal(1))
+	})
+
+	It("Delete pod without cache", func() {
+		tmpCNIDir := tmpDir + "/cniData"
+		err := os.Mkdir(tmpCNIDir, 0777)
+		Expect(err).NotTo(HaveOccurred())
+
+		fakePod := testhelpers.NewFakePod("testpod", "net1", "")
+		net1 := `{
+	"name": "net1",
+	"type": "mynet",
+	"cniVersion": "0.2.0"
+}`
+		args := &skel.CmdArgs{
+			ContainerID: "123456789",
+			Netns:       testNS.Path(),
+			IfName:      "eth0",
+			Args:        fmt.Sprintf("K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s", fakePod.ObjectMeta.Name, fakePod.ObjectMeta.Namespace),
+			StdinData: []byte(fmt.Sprintf(`{
+    "name": "node-cni-network",
+    "type": "multus",
+    "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+    "cniDir": "%s",
+    "delegates": [{
+        "name": "weave1",
+        "cniVersion": "0.2.0",
+        "type": "weave-net"
+    }]
+}`, tmpCNIDir)),
+		}
+
+		fExec := &fakeExec{}
+		expectedResult1 := &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.2/24"),
+			},
+		}
+		expectedConf1 := `{
+    "name": "weave1",
+    "cniVersion": "0.2.0",
+    "type": "weave-net"
+}`
+		fExec.addPlugin(nil, "eth0", expectedConf1, expectedResult1, nil)
+		fExec.addPlugin(nil, "net1", net1, &types020.Result{
+			CNIVersion: "0.2.0",
+			IP4: &types020.IPConfig{
+				IP: *testhelpers.EnsureCIDR("1.1.1.3/24"),
+			},
+		}, nil)
+
+		fKubeClient := testhelpers.NewFakeKubeClient()
+		fKubeClient.AddPod(fakePod)
+		fKubeClient.AddNetConfig(fakePod.ObjectMeta.Namespace, "net1", net1)
+		os.Setenv("CNI_COMMAND", "ADD")
+		os.Setenv("CNI_IFNAME", "eth0")
+		result, err := cmdAdd(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.addIndex).To(Equal(len(fExec.plugins)))
+		Expect(fKubeClient.PodCount).To(Equal(2))
+		Expect(fKubeClient.NetCount).To(Equal(1))
+		r := result.(*types020.Result)
+		// plugin 1 is the masterplugin
+		Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue())
+
+		By("Verify cache file existence")
+		cacheFilePath := fmt.Sprintf("%s/%s", tmpCNIDir, "123456789")
+		_, err = os.Stat(cacheFilePath)
+		Expect(err).NotTo(HaveOccurred())
+
+		err = os.Remove(cacheFilePath)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Delete and check pod/net count is incremented")
+		os.Setenv("CNI_COMMAND", "DEL")
+		os.Setenv("CNI_IFNAME", "eth0")
+		err = cmdDel(args, fExec, fKubeClient)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(fExec.delIndex).To(Equal(len(fExec.plugins)))
+		Expect(fKubeClient.PodCount).To(Equal(4))
+		Expect(fKubeClient.NetCount).To(Equal(2))
+	})
 })

testing/testing.go

@@ -103,7 +103,12 @@ func (f *FakeKubeClient) AddPod(pod *v1.Pod) {
 	f.pods[key] = pod
 }
 
-func NewFakePod(name string, netAnnotation string) *v1.Pod {
+func (f *FakeKubeClient) DeletePod(pod *v1.Pod) {
+	key := fmt.Sprintf("%s/%s", pod.ObjectMeta.Namespace, pod.ObjectMeta.Name)
+	delete(f.pods, key)
+}
+
+func NewFakePod(name string, netAnnotation string, defaultNetAnnotation string) *v1.Pod {
 	pod := &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -115,13 +120,19 @@ func NewFakePod(name string, netAnnotation string) *v1.Pod {
 			},
 		},
 	}
+	annotations := make(map[string]string)
+
 	if netAnnotation != "" {
 		netAnnotation = strings.Replace(netAnnotation, "\n", "", -1)
 		netAnnotation = strings.Replace(netAnnotation, "\t", "", -1)
-		pod.ObjectMeta.Annotations = map[string]string{
-			"k8s.v1.cni.cncf.io/networks": netAnnotation,
-		}
+		annotations["k8s.v1.cni.cncf.io/networks"] = netAnnotation
 	}
+
+	if defaultNetAnnotation != "" {
+		annotations["v1.multus-cni.io/default-network"] = defaultNetAnnotation
+	}
+
+	pod.ObjectMeta.Annotations = annotations
 	return pod
 }
 

types/conf.go

@@ -27,9 +27,11 @@ import (
 )
 
 const (
-	defaultCNIDir  = "/var/lib/cni/multus"
-	defaultConfDir = "/etc/cni/multus/net.d"
-	defaultBinDir  = "/opt/cni/bin"
+	defaultCNIDir                 = "/var/lib/cni/multus"
+	defaultConfDir                = "/etc/cni/multus/net.d"
+	defaultBinDir                 = "/opt/cni/bin"
+	defaultReadinessIndicatorFile = ""
+	defaultMultusNamespace        = "kube-system"
 )
 
 func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error {
@@ -50,9 +52,11 @@ func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error
 }
 
 // Convert raw CNI JSON into a DelegateNetConf structure
-func LoadDelegateNetConf(bytes []byte, ifnameRequest string) (*DelegateNetConf, error) {
+func LoadDelegateNetConf(bytes []byte, net *NetworkSelectionElement, deviceID string) (*DelegateNetConf, error) {
+	var err error
+	logging.Debugf("LoadDelegateNetConf: %s, %v, %s", string(bytes), net, deviceID)
+
 	delegateConf := &DelegateNetConf{}
-	logging.Debugf("LoadDelegateNetConf: %s, %s", string(bytes), ifnameRequest)
 	if err := json.Unmarshal(bytes, &delegateConf.Conf); err != nil {
 		return nil, logging.Errorf("error in LoadDelegateNetConf - unmarshalling delegate config: %v", err)
 	}
@@ -62,10 +66,31 @@ func LoadDelegateNetConf(bytes []byte, ifnameRequest string) (*DelegateNetConf,
 		if err := LoadDelegateNetConfList(bytes, delegateConf); err != nil {
 			return nil, logging.Errorf("error in LoadDelegateNetConf: %v", err)
 		}
+		if deviceID != "" {
+			bytes, err = addDeviceIDInConfList(bytes, deviceID)
+			if err != nil {
+				return nil, logging.Errorf("LoadDelegateNetConf(): failed to add deviceID in NetConfList bytes: %v", err)
+			}
+		}
+	} else {
+		if deviceID != "" {
+			bytes, err = delegateAddDeviceID(bytes, deviceID)
+			if err != nil {
+				return nil, logging.Errorf("LoadDelegateNetConf(): failed to add deviceID in NetConf bytes: %v", err)
+			}
+		}
 	}
 
-	if ifnameRequest != "" {
-		delegateConf.IfnameRequest = ifnameRequest
+	if net != nil {
+		if net.InterfaceRequest != "" {
+			delegateConf.IfnameRequest = net.InterfaceRequest
+		}
+		if net.MacRequest != "" {
+			delegateConf.MacRequest = net.MacRequest
+		}
+		if net.IPRequest != "" {
+			delegateConf.IPRequest = net.IPRequest
+		}
 	}
 
 	delegateConf.Bytes = bytes
@@ -73,9 +98,9 @@ func LoadDelegateNetConf(bytes []byte, ifnameRequest string) (*DelegateNetConf,
 	return delegateConf, nil
 }
 
-func LoadCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string) (*libcni.RuntimeConf, error) {
+func CreateCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig) *libcni.RuntimeConf {
 
-	logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s", args, k8sArgs, ifName)
+	logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s, %v", args, k8sArgs, ifName, rc)
 	// In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go#buildCNIRuntimeConf
 	// Todo
 	// ingress, egress and bandwidth capability features as same as kubelet.
@@ -90,11 +115,17 @@ func LoadCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string) (*l
 			{"K8S_POD_INFRA_CONTAINER_ID", string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID)},
 		},
 	}
-	return rt, nil
+
+	if rc != nil {
+		rt.CapabilityArgs = map[string]interface{}{
+			"portMappings": rc.PortMaps,
+		}
+	}
+	return rt
 }
 
 func LoadNetworkStatus(r types.Result, netName string, defaultNet bool) (*NetworkStatus, error) {
-	logging.Debugf("LoadNetworkStatus: %v, %s, %s", r, netName, defaultNet)
+	logging.Debugf("LoadNetworkStatus: %v, %s, %t", r, netName, defaultNet)
 
 	// Convert whatever the IPAM result was into the current Result type
 	result, err := current.NewResultFromResult(r)
@@ -169,8 +200,8 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 	// the master plugin. Kubernetes CRD delegates are then appended to
 	// the existing delegate list and all delegates executed in-order.
 
-	if len(netconf.RawDelegates) == 0 {
-		return nil, logging.Errorf("at least one delegate must be specified")
+	if len(netconf.RawDelegates) == 0 && netconf.ClusterNetwork == "" {
+		return nil, logging.Errorf("at least one delegate/defaultNetwork must be specified")
 	}
 
 	if netconf.CNIDir == "" {
@@ -185,21 +216,40 @@ func LoadNetConf(bytes []byte) (*NetConf, error) {
 		netconf.BinDir = defaultBinDir
 	}
 
-	for idx, rawConf := range netconf.RawDelegates {
-		bytes, err := json.Marshal(rawConf)
-		if err != nil {
-			return nil, logging.Errorf("error marshalling delegate %d config: %v", idx, err)
-		}
-		delegateConf, err := LoadDelegateNetConf(bytes, "")
-		if err != nil {
-			return nil, logging.Errorf("failed to load delegate %d config: %v", idx, err)
-		}
-		netconf.Delegates = append(netconf.Delegates, delegateConf)
+	if netconf.ReadinessIndicatorFile == "" {
+		netconf.ReadinessIndicatorFile = defaultReadinessIndicatorFile
 	}
-	netconf.RawDelegates = nil
 
-	// First delegate is always the master plugin
-	netconf.Delegates[0].MasterPlugin = true
+	if len(netconf.SystemNamespaces) == 0 {
+		netconf.SystemNamespaces = []string{"kube-system"}
+	}
+
+	if netconf.MultusNamespace == "" {
+		netconf.MultusNamespace = defaultMultusNamespace
+	}
+
+	// get RawDelegates and put delegates field
+	if netconf.ClusterNetwork == "" {
+		// for Delegates
+		if len(netconf.RawDelegates) == 0 {
+			return nil, logging.Errorf("at least one delegate must be specified")
+		}
+		for idx, rawConf := range netconf.RawDelegates {
+			bytes, err := json.Marshal(rawConf)
+			if err != nil {
+				return nil, logging.Errorf("error marshalling delegate %d config: %v", idx, err)
+			}
+			delegateConf, err := LoadDelegateNetConf(bytes, nil, "")
+			if err != nil {
+				return nil, logging.Errorf("failed to load delegate %d config: %v", idx, err)
+			}
+			netconf.Delegates = append(netconf.Delegates, delegateConf)
+		}
+		netconf.RawDelegates = nil
+
+		// First delegate is always the master plugin
+		netconf.Delegates[0].MasterPlugin = true
+	}
 
 	return netconf, nil
 }
@@ -210,3 +260,67 @@ func (n *NetConf) AddDelegates(newDelegates []*DelegateNetConf) error {
 	n.Delegates = append(n.Delegates, newDelegates...)
 	return nil
 }
+
+// delegateAddDeviceID injects deviceID information in delegate bytes
+func delegateAddDeviceID(inBytes []byte, deviceID string) ([]byte, error) {
+	var rawConfig map[string]interface{}
+	var err error
+
+	err = json.Unmarshal(inBytes, &rawConfig)
+	if err != nil {
+		return nil, logging.Errorf("delegateAddDeviceID: failed to unmarshal inBytes: %v", err)
+	}
+	// Inject deviceID
+	rawConfig["deviceID"] = deviceID
+	configBytes, err := json.Marshal(rawConfig)
+	if err != nil {
+		return nil, logging.Errorf("delegateAddDeviceID: failed to re-marshal Spec.Config: %v", err)
+	}
+	logging.Debugf("delegateAddDeviceID(): updated configBytes %s", string(configBytes))
+	return configBytes, nil
+}
+
+// addDeviceIDInConfList injects deviceID information in delegate bytes
+func addDeviceIDInConfList(inBytes []byte, deviceID string) ([]byte, error) {
+	var rawConfig map[string]interface{}
+	var err error
+
+	err = json.Unmarshal(inBytes, &rawConfig)
+	if err != nil {
+		return nil, logging.Errorf("addDeviceIDInConfList(): failed to unmarshal inBytes: %v", err)
+	}
+
+	pList, ok := rawConfig["plugins"]
+	if !ok {
+		return nil, logging.Errorf("addDeviceIDInConfList(): unable to get plugin list")
+	}
+
+	pMap, ok := pList.([]interface{})
+	if !ok {
+		return nil, logging.Errorf("addDeviceIDInConfList(): unable to typecast plugin list")
+	}
+
+	firstPlugin, ok := pMap[0].(map[string]interface{})
+	if !ok {
+		return nil, logging.Errorf("addDeviceIDInConfList(): unable to typecast pMap")
+	}
+	// Inject deviceID
+	firstPlugin["deviceID"] = deviceID
+
+	configBytes, err := json.Marshal(rawConfig)
+	if err != nil {
+		return nil, logging.Errorf("addDeviceIDInConfList(): failed to re-marshal: %v", err)
+	}
+	logging.Debugf("addDeviceIDInConfList(): updated configBytes %s", string(configBytes))
+	return configBytes, nil
+}
+
+// CheckSystemNamespaces checks whether given namespace is in systemNamespaces or not.
+func CheckSystemNamespaces(namespace string, systemNamespaces []string) bool {
+	for _, nsname := range systemNamespaces {
+		if namespace == nsname {
+			return true
+		}
+	}
+	return false
+}

types/conf_test.go

@@ -35,13 +35,20 @@ var _ = Describe("config operations", func() {
     "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
     "delegates": [{
         "type": "weave-net"
-    }]
+    }],
+	"runtimeConfig": {
+      "portMappings": [
+        {"hostPort": 8080, "containerPort": 80, "protocol": "tcp"}
+      ]
+    }
+
 }`
 		netConf, err := LoadNetConf([]byte(conf))
 		Expect(err).NotTo(HaveOccurred())
 		Expect(len(netConf.Delegates)).To(Equal(1))
 		Expect(netConf.Delegates[0].Conf.Type).To(Equal("weave-net"))
 		Expect(netConf.Delegates[0].MasterPlugin).To(BeTrue())
+		Expect(len(netConf.RuntimeConfig.PortMaps)).To(Equal(1))
 	})
 
 	It("succeeds if only delegates are set", func() {
@@ -81,4 +88,47 @@ var _ = Describe("config operations", func() {
 		_, err := LoadNetConf([]byte(conf))
 		Expect(err).To(HaveOccurred())
 	})
+
+	It("has defaults set for network readiness", func() {
+		conf := `{
+    "name": "defaultnetwork",
+    "type": "multus",
+    "kubeconfig": "/etc/kubernetes/kubelet.conf",
+    "delegates": [{
+      "cniVersion": "0.3.0",
+      "name": "defaultnetwork",
+      "type": "flannel",
+      "isDefaultGateway": true
+    }]
+}`
+		netConf, err := LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(netConf.ReadinessIndicatorFile).To(Equal(""))
+	})
+
+	It("honors overrides for network readiness", func() {
+		conf := `{
+    "name": "defaultnetwork",
+    "type": "multus",
+    "readinessindicatorfile": "/etc/cni/net.d/foo",
+    "kubeconfig": "/etc/kubernetes/kubelet.conf",
+    "delegates": [{
+      "cniVersion": "0.3.0",
+      "name": "defaultnetwork",
+      "type": "flannel",
+      "isDefaultGateway": true
+    }]
+}`
+		netConf, err := LoadNetConf([]byte(conf))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(netConf.ReadinessIndicatorFile).To(Equal("/etc/cni/net.d/foo"))
+	})
+
+	It("check CheckSystemNamespaces() works fine", func() {
+		b1 := CheckSystemNamespaces("foobar", []string{"barfoo", "bafoo", "foobar"})
+		Expect(b1).To(Equal(true))
+		b2 := CheckSystemNamespaces("foobar1", []string{"barfoo", "bafoo", "foobar"})
+		Expect(b2).To(Equal(false))
+	})
+
 })

types/types.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/types"
 	"github.com/containernetworking/cni/pkg/types/current"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -36,12 +37,34 @@ type NetConf struct {
 	CNIDir  string `json:"cniDir"`
 	BinDir  string `json:"binDir"`
 	// RawDelegates is private to the NetConf class; use Delegates instead
-	RawDelegates []map[string]interface{} `json:"delegates"`
-	Delegates    []*DelegateNetConf       `json:"-"`
-	NetStatus    []*NetworkStatus         `json:"-"`
-	Kubeconfig   string                   `json:"kubeconfig"`
-	LogFile      string                   `json:"logFile"`
-	LogLevel     string                   `json:"logLevel"`
+	RawDelegates    []map[string]interface{} `json:"delegates"`
+	Delegates       []*DelegateNetConf       `json:"-"`
+	NetStatus       []*NetworkStatus         `json:"-"`
+	Kubeconfig      string                   `json:"kubeconfig"`
+	ClusterNetwork  string                   `json:"clusterNetwork"`
+	DefaultNetworks []string                 `json:"defaultNetworks"`
+	LogFile         string                   `json:"logFile"`
+	LogLevel        string                   `json:"logLevel"`
+	RuntimeConfig   *RuntimeConfig           `json:"runtimeConfig,omitempty"`
+	// Default network readiness options
+	ReadinessIndicatorFile string `json:"readinessindicatorfile"`
+	// Option to isolate the usage of CR's to the namespace in which a pod resides.
+	NamespaceIsolation bool `json:"namespaceIsolation"`
+	// Option to set system namespaces (to avoid to add defaultNetworks)
+	SystemNamespaces []string `json:"systemNamespaces"`
+	// Option to set the namespace that multus-cni uses (clusterNetwork/defaultNetworks)
+	MultusNamespace string `json:"multusNamespace"`
+}
+
+type RuntimeConfig struct {
+	PortMaps []PortMapEntry `json:"portMappings,omitempty"`
+}
+
+type PortMapEntry struct {
+	HostPort      int    `json:"hostPort"`
+	ContainerPort int    `json:"containerPort"`
+	Protocol      string `json:"protocol"`
+	HostIP        string `json:"hostIP,omitempty"`
 }
 
 type NetworkStatus struct {
@@ -57,6 +80,8 @@ type DelegateNetConf struct {
 	Conf          types.NetConf
 	ConfList      types.NetConfList
 	IfnameRequest string `json:"ifnameRequest,omitempty"`
+	MacRequest    string `json:"macRequest,omitempty"`
+	IPRequest     string `json:"ipRequest,omitempty"`
 	// MasterPlugin is only used internal housekeeping
 	MasterPlugin bool `json:"-"`
 	// Conflist plugin is only used internal housekeeping
@@ -102,13 +127,13 @@ type NetworkSelectionElement struct {
 	Namespace string `json:"namespace,omitempty"`
 	// IPRequest contains an optional requested IP address for this network
 	// attachment
-	IPRequest string `json:"ipRequest,omitempty"`
+	IPRequest string `json:"ips,omitempty"`
 	// MacRequest contains an optional requested MAC address for this
 	// network attachment
-	MacRequest string `json:"macRequest,omitempty"`
+	MacRequest string `json:"mac,omitempty"`
 	// InterfaceRequest contains an optional requested name for the
 	// network interface this attachment will create in the container
-	InterfaceRequest string `json:"interfaceRequest,omitempty"`
+	InterfaceRequest string `json:"interface,omitempty"`
 }
 
 // K8sArgs is the valid CNI_ARGS used for Kubernetes
@@ -119,3 +144,15 @@ type K8sArgs struct {
 	K8S_POD_NAMESPACE          types.UnmarshallableString
 	K8S_POD_INFRA_CONTAINER_ID types.UnmarshallableString
 }
+
+// ResourceInfo is struct to hold Pod device allocation information
+type ResourceInfo struct {
+	Index     int
+	DeviceIDs []string
+}
+
+// ResourceClient provides a kubelet Pod resource handle
+type ResourceClient interface {
+	// GetPodResourceMap returns an instance of a map of Pod ResourceInfo given a (Pod name, namespace) tuple
+	GetPodResourceMap(*v1.Pod) (map[string]*ResourceInfo, error)
+}

vendor/github.com/Microsoft/go-winio/.gitignore

@@ -0,0 +1 @@
+*.exe

vendor/github.com/Microsoft/go-winio/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Microsoft
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

vendor/github.com/Microsoft/go-winio/README.md

@@ -0,0 +1,22 @@
+# go-winio
+
+This repository contains utilities for efficiently performing Win32 IO operations in
+Go. Currently, this is focused on accessing named pipes and other file handles, and
+for using named pipes as a net transport.
+
+This code relies on IO completion ports to avoid blocking IO on system threads, allowing Go
+to reuse the thread to schedule another goroutine. This limits support to Windows Vista and
+newer operating systems. This is similar to the implementation of network sockets in Go's net
+package.
+
+Please see the LICENSE file for licensing information.
+
+This project has adopted the [Microsoft Open Source Code of
+Conduct](https://opensource.microsoft.com/codeofconduct/). For more information
+see the [Code of Conduct
+FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact
+[opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional
+questions or comments.
+
+Thanks to natefinch for the inspiration for this library. See https://github.com/natefinch/npipe
+for another named pipe implementation.

vendor/github.com/Microsoft/go-winio/archive/tar/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2012 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/Microsoft/go-winio/archive/tar/common.go

@@ -0,0 +1,344 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package tar implements access to tar archives.
+// It aims to cover most of the variations, including those produced
+// by GNU and BSD tars.
+//
+// References:
+//   http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5
+//   http://www.gnu.org/software/tar/manual/html_node/Standard.html
+//   http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html
+package tar
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"time"
+)
+
+const (
+	blockSize = 512
+
+	// Types
+	TypeReg           = '0'    // regular file
+	TypeRegA          = '\x00' // regular file
+	TypeLink          = '1'    // hard link
+	TypeSymlink       = '2'    // symbolic link
+	TypeChar          = '3'    // character device node
+	TypeBlock         = '4'    // block device node
+	TypeDir           = '5'    // directory
+	TypeFifo          = '6'    // fifo node
+	TypeCont          = '7'    // reserved
+	TypeXHeader       = 'x'    // extended header
+	TypeXGlobalHeader = 'g'    // global extended header
+	TypeGNULongName   = 'L'    // Next file has a long name
+	TypeGNULongLink   = 'K'    // Next file symlinks to a file w/ a long name
+	TypeGNUSparse     = 'S'    // sparse file
+)
+
+// A Header represents a single header in a tar archive.
+// Some fields may not be populated.
+type Header struct {
+	Name         string    // name of header file entry
+	Mode         int64     // permission and mode bits
+	Uid          int       // user id of owner
+	Gid          int       // group id of owner
+	Size         int64     // length in bytes
+	ModTime      time.Time // modified time
+	Typeflag     byte      // type of header entry
+	Linkname     string    // target name of link
+	Uname        string    // user name of owner
+	Gname        string    // group name of owner
+	Devmajor     int64     // major number of character or block device
+	Devminor     int64     // minor number of character or block device
+	AccessTime   time.Time // access time
+	ChangeTime   time.Time // status change time
+	CreationTime time.Time // creation time
+	Xattrs       map[string]string
+	Winheaders   map[string]string
+}
+
+// File name constants from the tar spec.
+const (
+	fileNameSize       = 100 // Maximum number of bytes in a standard tar name.
+	fileNamePrefixSize = 155 // Maximum number of ustar extension bytes.
+)
+
+// FileInfo returns an os.FileInfo for the Header.
+func (h *Header) FileInfo() os.FileInfo {
+	return headerFileInfo{h}
+}
+
+// headerFileInfo implements os.FileInfo.
+type headerFileInfo struct {
+	h *Header
+}
+
+func (fi headerFileInfo) Size() int64        { return fi.h.Size }
+func (fi headerFileInfo) IsDir() bool        { return fi.Mode().IsDir() }
+func (fi headerFileInfo) ModTime() time.Time { return fi.h.ModTime }
+func (fi headerFileInfo) Sys() interface{}   { return fi.h }
+
+// Name returns the base name of the file.
+func (fi headerFileInfo) Name() string {
+	if fi.IsDir() {
+		return path.Base(path.Clean(fi.h.Name))
+	}
+	return path.Base(fi.h.Name)
+}
+
+// Mode returns the permission and mode bits for the headerFileInfo.
+func (fi headerFileInfo) Mode() (mode os.FileMode) {
+	// Set file permission bits.
+	mode = os.FileMode(fi.h.Mode).Perm()
+
+	// Set setuid, setgid and sticky bits.
+	if fi.h.Mode&c_ISUID != 0 {
+		// setuid
+		mode |= os.ModeSetuid
+	}
+	if fi.h.Mode&c_ISGID != 0 {
+		// setgid
+		mode |= os.ModeSetgid
+	}
+	if fi.h.Mode&c_ISVTX != 0 {
+		// sticky
+		mode |= os.ModeSticky
+	}
+
+	// Set file mode bits.
+	// clear perm, setuid, setgid and sticky bits.
+	m := os.FileMode(fi.h.Mode) &^ 07777
+	if m == c_ISDIR {
+		// directory
+		mode |= os.ModeDir
+	}
+	if m == c_ISFIFO {
+		// named pipe (FIFO)
+		mode |= os.ModeNamedPipe
+	}
+	if m == c_ISLNK {
+		// symbolic link
+		mode |= os.ModeSymlink
+	}
+	if m == c_ISBLK {
+		// device file
+		mode |= os.ModeDevice
+	}
+	if m == c_ISCHR {
+		// Unix character device
+		mode |= os.ModeDevice
+		mode |= os.ModeCharDevice
+	}
+	if m == c_ISSOCK {
+		// Unix domain socket
+		mode |= os.ModeSocket
+	}
+
+	switch fi.h.Typeflag {
+	case TypeSymlink:
+		// symbolic link
+		mode |= os.ModeSymlink
+	case TypeChar:
+		// character device node
+		mode |= os.ModeDevice
+		mode |= os.ModeCharDevice
+	case TypeBlock:
+		// block device node
+		mode |= os.ModeDevice
+	case TypeDir:
+		// directory
+		mode |= os.ModeDir
+	case TypeFifo:
+		// fifo node
+		mode |= os.ModeNamedPipe
+	}
+
+	return mode
+}
+
+// sysStat, if non-nil, populates h from system-dependent fields of fi.
+var sysStat func(fi os.FileInfo, h *Header) error
+
+// Mode constants from the tar spec.
+const (
+	c_ISUID  = 04000   // Set uid
+	c_ISGID  = 02000   // Set gid
+	c_ISVTX  = 01000   // Save text (sticky bit)
+	c_ISDIR  = 040000  // Directory
+	c_ISFIFO = 010000  // FIFO
+	c_ISREG  = 0100000 // Regular file
+	c_ISLNK  = 0120000 // Symbolic link
+	c_ISBLK  = 060000  // Block special file
+	c_ISCHR  = 020000  // Character special file
+	c_ISSOCK = 0140000 // Socket
+)
+
+// Keywords for the PAX Extended Header
+const (
+	paxAtime        = "atime"
+	paxCharset      = "charset"
+	paxComment      = "comment"
+	paxCtime        = "ctime" // please note that ctime is not a valid pax header.
+	paxCreationTime = "LIBARCHIVE.creationtime"
+	paxGid          = "gid"
+	paxGname        = "gname"
+	paxLinkpath     = "linkpath"
+	paxMtime        = "mtime"
+	paxPath         = "path"
+	paxSize         = "size"
+	paxUid          = "uid"
+	paxUname        = "uname"
+	paxXattr        = "SCHILY.xattr."
+	paxWindows      = "MSWINDOWS."
+	paxNone         = ""
+)
+
+// FileInfoHeader creates a partially-populated Header from fi.
+// If fi describes a symlink, FileInfoHeader records link as the link target.
+// If fi describes a directory, a slash is appended to the name.
+// Because os.FileInfo's Name method returns only the base name of
+// the file it describes, it may be necessary to modify the Name field
+// of the returned header to provide the full path name of the file.
+func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) {
+	if fi == nil {
+		return nil, errors.New("tar: FileInfo is nil")
+	}
+	fm := fi.Mode()
+	h := &Header{
+		Name:    fi.Name(),
+		ModTime: fi.ModTime(),
+		Mode:    int64(fm.Perm()), // or'd with c_IS* constants later
+	}
+	switch {
+	case fm.IsRegular():
+		h.Mode |= c_ISREG
+		h.Typeflag = TypeReg
+		h.Size = fi.Size()
+	case fi.IsDir():
+		h.Typeflag = TypeDir
+		h.Mode |= c_ISDIR
+		h.Name += "/"
+	case fm&os.ModeSymlink != 0:
+		h.Typeflag = TypeSymlink
+		h.Mode |= c_ISLNK
+		h.Linkname = link
+	case fm&os.ModeDevice != 0:
+		if fm&os.ModeCharDevice != 0 {
+			h.Mode |= c_ISCHR
+			h.Typeflag = TypeChar
+		} else {
+			h.Mode |= c_ISBLK
+			h.Typeflag = TypeBlock
+		}
+	case fm&os.ModeNamedPipe != 0:
+		h.Typeflag = TypeFifo
+		h.Mode |= c_ISFIFO
+	case fm&os.ModeSocket != 0:
+		h.Mode |= c_ISSOCK
+	default:
+		return nil, fmt.Errorf("archive/tar: unknown file mode %v", fm)
+	}
+	if fm&os.ModeSetuid != 0 {
+		h.Mode |= c_ISUID
+	}
+	if fm&os.ModeSetgid != 0 {
+		h.Mode |= c_ISGID
+	}
+	if fm&os.ModeSticky != 0 {
+		h.Mode |= c_ISVTX
+	}
+	// If possible, populate additional fields from OS-specific
+	// FileInfo fields.
+	if sys, ok := fi.Sys().(*Header); ok {
+		// This FileInfo came from a Header (not the OS). Use the
+		// original Header to populate all remaining fields.
+		h.Uid = sys.Uid
+		h.Gid = sys.Gid
+		h.Uname = sys.Uname
+		h.Gname = sys.Gname
+		h.AccessTime = sys.AccessTime
+		h.ChangeTime = sys.ChangeTime
+		if sys.Xattrs != nil {
+			h.Xattrs = make(map[string]string)
+			for k, v := range sys.Xattrs {
+				h.Xattrs[k] = v
+			}
+		}
+		if sys.Typeflag == TypeLink {
+			// hard link
+			h.Typeflag = TypeLink
+			h.Size = 0
+			h.Linkname = sys.Linkname
+		}
+	}
+	if sysStat != nil {
+		return h, sysStat(fi, h)
+	}
+	return h, nil
+}
+
+var zeroBlock = make([]byte, blockSize)
+
+// POSIX specifies a sum of the unsigned byte values, but the Sun tar uses signed byte values.
+// We compute and return both.
+func checksum(header []byte) (unsigned int64, signed int64) {
+	for i := 0; i < len(header); i++ {
+		if i == 148 {
+			// The chksum field (header[148:156]) is special: it should be treated as space bytes.
+			unsigned += ' ' * 8
+			signed += ' ' * 8
+			i += 7
+			continue
+		}
+		unsigned += int64(header[i])
+		signed += int64(int8(header[i]))
+	}
+	return
+}
+
+type slicer []byte
+
+func (sp *slicer) next(n int) (b []byte) {
+	s := *sp
+	b, *sp = s[0:n], s[n:]
+	return
+}
+
+func isASCII(s string) bool {
+	for _, c := range s {
+		if c >= 0x80 {
+			return false
+		}
+	}
+	return true
+}
+
+func toASCII(s string) string {
+	if isASCII(s) {
+		return s
+	}
+	var buf bytes.Buffer
+	for _, c := range s {
+		if c < 0x80 {
+			buf.WriteByte(byte(c))
+		}
+	}
+	return buf.String()
+}
+
+// isHeaderOnlyType checks if the given type flag is of the type that has no
+// data section even if a size is specified.
+func isHeaderOnlyType(flag byte) bool {
+	switch flag {
+	case TypeLink, TypeSymlink, TypeChar, TypeBlock, TypeDir, TypeFifo:
+		return true
+	default:
+		return false
+	}
+}

vendor/github.com/Microsoft/go-winio/archive/tar/example_test.go

@@ -0,0 +1,80 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tar_test
+
+import (
+	"archive/tar"
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"os"
+)
+
+func Example() {
+	// Create a buffer to write our archive to.
+	buf := new(bytes.Buffer)
+
+	// Create a new tar archive.
+	tw := tar.NewWriter(buf)
+
+	// Add some files to the archive.
+	var files = []struct {
+		Name, Body string
+	}{
+		{"readme.txt", "This archive contains some text files."},
+		{"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"},
+		{"todo.txt", "Get animal handling license."},
+	}
+	for _, file := range files {
+		hdr := &tar.Header{
+			Name: file.Name,
+			Mode: 0600,
+			Size: int64(len(file.Body)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			log.Fatalln(err)
+		}
+		if _, err := tw.Write([]byte(file.Body)); err != nil {
+			log.Fatalln(err)
+		}
+	}
+	// Make sure to check the error on Close.
+	if err := tw.Close(); err != nil {
+		log.Fatalln(err)
+	}
+
+	// Open the tar archive for reading.
+	r := bytes.NewReader(buf.Bytes())
+	tr := tar.NewReader(r)
+
+	// Iterate through the files in the archive.
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			log.Fatalln(err)
+		}
+		fmt.Printf("Contents of %s:\n", hdr.Name)
+		if _, err := io.Copy(os.Stdout, tr); err != nil {
+			log.Fatalln(err)
+		}
+		fmt.Println()
+	}
+
+	// Output:
+	// Contents of readme.txt:
+	// This archive contains some text files.
+	// Contents of gopher.txt:
+	// Gopher names:
+	// George
+	// Geoffrey
+	// Gonzo
+	// Contents of todo.txt:
+	// Get animal handling license.
+}

vendor/github.com/Microsoft/go-winio/archive/tar/stat_atim.go

@@ -0,0 +1,20 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux dragonfly openbsd solaris
+
+package tar
+
+import (
+	"syscall"
+	"time"
+)
+
+func statAtime(st *syscall.Stat_t) time.Time {
+	return time.Unix(st.Atim.Unix())
+}
+
+func statCtime(st *syscall.Stat_t) time.Time {
+	return time.Unix(st.Ctim.Unix())
+}

vendor/github.com/Microsoft/go-winio/archive/tar/stat_atimespec.go

@@ -0,0 +1,20 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin freebsd netbsd
+
+package tar
+
+import (
+	"syscall"
+	"time"
+)
+
+func statAtime(st *syscall.Stat_t) time.Time {
+	return time.Unix(st.Atimespec.Unix())
+}
+
+func statCtime(st *syscall.Stat_t) time.Time {
+	return time.Unix(st.Ctimespec.Unix())
+}

vendor/github.com/Microsoft/go-winio/archive/tar/stat_unix.go

@@ -0,0 +1,32 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux darwin dragonfly freebsd openbsd netbsd solaris
+
+package tar
+
+import (
+	"os"
+	"syscall"
+)
+
+func init() {
+	sysStat = statUnix
+}
+
+func statUnix(fi os.FileInfo, h *Header) error {
+	sys, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return nil
+	}
+	h.Uid = int(sys.Uid)
+	h.Gid = int(sys.Gid)
+	// TODO(bradfitz): populate username & group.  os/user
+	// doesn't cache LookupId lookups, and lacks group
+	// lookup functions.
+	h.AccessTime = statAtime(sys)
+	h.ChangeTime = statCtime(sys)
+	// TODO(bradfitz): major/minor device numbers?
+	return nil
+}

vendor/github.com/Microsoft/go-winio/archive/tar/tar_test.go

@@ -0,0 +1,325 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tar
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestFileInfoHeader(t *testing.T) {
+	fi, err := os.Stat("testdata/small.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	h, err := FileInfoHeader(fi, "")
+	if err != nil {
+		t.Fatalf("FileInfoHeader: %v", err)
+	}
+	if g, e := h.Name, "small.txt"; g != e {
+		t.Errorf("Name = %q; want %q", g, e)
+	}
+	if g, e := h.Mode, int64(fi.Mode().Perm())|c_ISREG; g != e {
+		t.Errorf("Mode = %#o; want %#o", g, e)
+	}
+	if g, e := h.Size, int64(5); g != e {
+		t.Errorf("Size = %v; want %v", g, e)
+	}
+	if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) {
+		t.Errorf("ModTime = %v; want %v", g, e)
+	}
+	// FileInfoHeader should error when passing nil FileInfo
+	if _, err := FileInfoHeader(nil, ""); err == nil {
+		t.Fatalf("Expected error when passing nil to FileInfoHeader")
+	}
+}
+
+func TestFileInfoHeaderDir(t *testing.T) {
+	fi, err := os.Stat("testdata")
+	if err != nil {
+		t.Fatal(err)
+	}
+	h, err := FileInfoHeader(fi, "")
+	if err != nil {
+		t.Fatalf("FileInfoHeader: %v", err)
+	}
+	if g, e := h.Name, "testdata/"; g != e {
+		t.Errorf("Name = %q; want %q", g, e)
+	}
+	// Ignoring c_ISGID for golang.org/issue/4867
+	if g, e := h.Mode&^c_ISGID, int64(fi.Mode().Perm())|c_ISDIR; g != e {
+		t.Errorf("Mode = %#o; want %#o", g, e)
+	}
+	if g, e := h.Size, int64(0); g != e {
+		t.Errorf("Size = %v; want %v", g, e)
+	}
+	if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) {
+		t.Errorf("ModTime = %v; want %v", g, e)
+	}
+}
+
+func TestFileInfoHeaderSymlink(t *testing.T) {
+	h, err := FileInfoHeader(symlink{}, "some-target")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if g, e := h.Name, "some-symlink"; g != e {
+		t.Errorf("Name = %q; want %q", g, e)
+	}
+	if g, e := h.Linkname, "some-target"; g != e {
+		t.Errorf("Linkname = %q; want %q", g, e)
+	}
+}
+
+type symlink struct{}
+
+func (symlink) Name() string       { return "some-symlink" }
+func (symlink) Size() int64        { return 0 }
+func (symlink) Mode() os.FileMode  { return os.ModeSymlink }
+func (symlink) ModTime() time.Time { return time.Time{} }
+func (symlink) IsDir() bool        { return false }
+func (symlink) Sys() interface{}   { return nil }
+
+func TestRoundTrip(t *testing.T) {
+	data := []byte("some file contents")
+
+	var b bytes.Buffer
+	tw := NewWriter(&b)
+	hdr := &Header{
+		Name:    "file.txt",
+		Uid:     1 << 21, // too big for 8 octal digits
+		Size:    int64(len(data)),
+		ModTime: time.Now(),
+	}
+	// tar only supports second precision.
+	hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond)
+	if err := tw.WriteHeader(hdr); err != nil {
+		t.Fatalf("tw.WriteHeader: %v", err)
+	}
+	if _, err := tw.Write(data); err != nil {
+		t.Fatalf("tw.Write: %v", err)
+	}
+	if err := tw.Close(); err != nil {
+		t.Fatalf("tw.Close: %v", err)
+	}
+
+	// Read it back.
+	tr := NewReader(&b)
+	rHdr, err := tr.Next()
+	if err != nil {
+		t.Fatalf("tr.Next: %v", err)
+	}
+	if !reflect.DeepEqual(rHdr, hdr) {
+		t.Errorf("Header mismatch.\n got %+v\nwant %+v", rHdr, hdr)
+	}
+	rData, err := ioutil.ReadAll(tr)
+	if err != nil {
+		t.Fatalf("Read: %v", err)
+	}
+	if !bytes.Equal(rData, data) {
+		t.Errorf("Data mismatch.\n got %q\nwant %q", rData, data)
+	}
+}
+
+type headerRoundTripTest struct {
+	h  *Header
+	fm os.FileMode
+}
+
+func TestHeaderRoundTrip(t *testing.T) {
+	golden := []headerRoundTripTest{
+		// regular file.
+		{
+			h: &Header{
+				Name:     "test.txt",
+				Mode:     0644 | c_ISREG,
+				Size:     12,
+				ModTime:  time.Unix(1360600916, 0),
+				Typeflag: TypeReg,
+			},
+			fm: 0644,
+		},
+		// symbolic link.
+		{
+			h: &Header{
+				Name:     "link.txt",
+				Mode:     0777 | c_ISLNK,
+				Size:     0,
+				ModTime:  time.Unix(1360600852, 0),
+				Typeflag: TypeSymlink,
+			},
+			fm: 0777 | os.ModeSymlink,
+		},
+		// character device node.
+		{
+			h: &Header{
+				Name:     "dev/null",
+				Mode:     0666 | c_ISCHR,
+				Size:     0,
+				ModTime:  time.Unix(1360578951, 0),
+				Typeflag: TypeChar,
+			},
+			fm: 0666 | os.ModeDevice | os.ModeCharDevice,
+		},
+		// block device node.
+		{
+			h: &Header{
+				Name:     "dev/sda",
+				Mode:     0660 | c_ISBLK,
+				Size:     0,
+				ModTime:  time.Unix(1360578954, 0),
+				Typeflag: TypeBlock,
+			},
+			fm: 0660 | os.ModeDevice,
+		},
+		// directory.
+		{
+			h: &Header{
+				Name:     "dir/",
+				Mode:     0755 | c_ISDIR,
+				Size:     0,
+				ModTime:  time.Unix(1360601116, 0),
+				Typeflag: TypeDir,
+			},
+			fm: 0755 | os.ModeDir,
+		},
+		// fifo node.
+		{
+			h: &Header{
+				Name:     "dev/initctl",
+				Mode:     0600 | c_ISFIFO,
+				Size:     0,
+				ModTime:  time.Unix(1360578949, 0),
+				Typeflag: TypeFifo,
+			},
+			fm: 0600 | os.ModeNamedPipe,
+		},
+		// setuid.
+		{
+			h: &Header{
+				Name:     "bin/su",
+				Mode:     0755 | c_ISREG | c_ISUID,
+				Size:     23232,
+				ModTime:  time.Unix(1355405093, 0),
+				Typeflag: TypeReg,
+			},
+			fm: 0755 | os.ModeSetuid,
+		},
+		// setguid.
+		{
+			h: &Header{
+				Name:     "group.txt",
+				Mode:     0750 | c_ISREG | c_ISGID,
+				Size:     0,
+				ModTime:  time.Unix(1360602346, 0),
+				Typeflag: TypeReg,
+			},
+			fm: 0750 | os.ModeSetgid,
+		},
+		// sticky.
+		{
+			h: &Header{
+				Name:     "sticky.txt",
+				Mode:     0600 | c_ISREG | c_ISVTX,
+				Size:     7,
+				ModTime:  time.Unix(1360602540, 0),
+				Typeflag: TypeReg,
+			},
+			fm: 0600 | os.ModeSticky,
+		},
+		// hard link.
+		{
+			h: &Header{
+				Name:     "hard.txt",
+				Mode:     0644 | c_ISREG,
+				Size:     0,
+				Linkname: "file.txt",
+				ModTime:  time.Unix(1360600916, 0),
+				Typeflag: TypeLink,
+			},
+			fm: 0644,
+		},
+		// More information.
+		{
+			h: &Header{
+				Name:     "info.txt",
+				Mode:     0600 | c_ISREG,
+				Size:     0,
+				Uid:      1000,
+				Gid:      1000,
+				ModTime:  time.Unix(1360602540, 0),
+				Uname:    "slartibartfast",
+				Gname:    "users",
+				Typeflag: TypeReg,
+			},
+			fm: 0600,
+		},
+	}
+
+	for i, g := range golden {
+		fi := g.h.FileInfo()
+		h2, err := FileInfoHeader(fi, "")
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		if strings.Contains(fi.Name(), "/") {
+			t.Errorf("FileInfo of %q contains slash: %q", g.h.Name, fi.Name())
+		}
+		name := path.Base(g.h.Name)
+		if fi.IsDir() {
+			name += "/"
+		}
+		if got, want := h2.Name, name; got != want {
+			t.Errorf("i=%d: Name: got %v, want %v", i, got, want)
+		}
+		if got, want := h2.Size, g.h.Size; got != want {
+			t.Errorf("i=%d: Size: got %v, want %v", i, got, want)
+		}
+		if got, want := h2.Uid, g.h.Uid; got != want {
+			t.Errorf("i=%d: Uid: got %d, want %d", i, got, want)
+		}
+		if got, want := h2.Gid, g.h.Gid; got != want {
+			t.Errorf("i=%d: Gid: got %d, want %d", i, got, want)
+		}
+		if got, want := h2.Uname, g.h.Uname; got != want {
+			t.Errorf("i=%d: Uname: got %q, want %q", i, got, want)
+		}
+		if got, want := h2.Gname, g.h.Gname; got != want {
+			t.Errorf("i=%d: Gname: got %q, want %q", i, got, want)
+		}
+		if got, want := h2.Linkname, g.h.Linkname; got != want {
+			t.Errorf("i=%d: Linkname: got %v, want %v", i, got, want)
+		}
+		if got, want := h2.Typeflag, g.h.Typeflag; got != want {
+			t.Logf("%#v %#v", g.h, fi.Sys())
+			t.Errorf("i=%d: Typeflag: got %q, want %q", i, got, want)
+		}
+		if got, want := h2.Mode, g.h.Mode; got != want {
+			t.Errorf("i=%d: Mode: got %o, want %o", i, got, want)
+		}
+		if got, want := fi.Mode(), g.fm; got != want {
+			t.Errorf("i=%d: fi.Mode: got %o, want %o", i, got, want)
+		}
+		if got, want := h2.AccessTime, g.h.AccessTime; got != want {
+			t.Errorf("i=%d: AccessTime: got %v, want %v", i, got, want)
+		}
+		if got, want := h2.ChangeTime, g.h.ChangeTime; got != want {
+			t.Errorf("i=%d: ChangeTime: got %v, want %v", i, got, want)
+		}
+		if got, want := h2.ModTime, g.h.ModTime; got != want {
+			t.Errorf("i=%d: ModTime: got %v, want %v", i, got, want)
+		}
+		if sysh, ok := fi.Sys().(*Header); !ok || sysh != g.h {
+			t.Errorf("i=%d: Sys didn't return original *Header", i)
+		}
+	}
+}

vendor/github.com/Microsoft/go-winio/archive/tar/testdata/small.txt

@@ -0,0 +1 @@
+Kilts

vendor/github.com/Microsoft/go-winio/archive/tar/testdata/small2.txt

@@ -0,0 +1 @@
+Google.com

vendor/github.com/Microsoft/go-winio/archive/tar/writer.go

@@ -0,0 +1,444 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tar
+
+// TODO(dsymonds):
+// - catch more errors (no first header, etc.)
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var (
+	ErrWriteTooLong    = errors.New("archive/tar: write too long")
+	ErrFieldTooLong    = errors.New("archive/tar: header field too long")
+	ErrWriteAfterClose = errors.New("archive/tar: write after close")
+	errInvalidHeader   = errors.New("archive/tar: header field too long or contains invalid values")
+)
+
+// A Writer provides sequential writing of a tar archive in POSIX.1 format.
+// A tar archive consists of a sequence of files.
+// Call WriteHeader to begin a new file, and then call Write to supply that file's data,
+// writing at most hdr.Size bytes in total.
+type Writer struct {
+	w          io.Writer
+	err        error
+	nb         int64 // number of unwritten bytes for current file entry
+	pad        int64 // amount of padding to write after current file entry
+	closed     bool
+	usedBinary bool            // whether the binary numeric field extension was used
+	preferPax  bool            // use pax header instead of binary numeric header
+	hdrBuff    [blockSize]byte // buffer to use in writeHeader when writing a regular header
+	paxHdrBuff [blockSize]byte // buffer to use in writeHeader when writing a pax header
+}
+
+type formatter struct {
+	err error // Last error seen
+}
+
+// NewWriter creates a new Writer writing to w.
+func NewWriter(w io.Writer) *Writer { return &Writer{w: w, preferPax: true} }
+
+// Flush finishes writing the current file (optional).
+func (tw *Writer) Flush() error {
+	if tw.nb > 0 {
+		tw.err = fmt.Errorf("archive/tar: missed writing %d bytes", tw.nb)
+		return tw.err
+	}
+
+	n := tw.nb + tw.pad
+	for n > 0 && tw.err == nil {
+		nr := n
+		if nr > blockSize {
+			nr = blockSize
+		}
+		var nw int
+		nw, tw.err = tw.w.Write(zeroBlock[0:nr])
+		n -= int64(nw)
+	}
+	tw.nb = 0
+	tw.pad = 0
+	return tw.err
+}
+
+// Write s into b, terminating it with a NUL if there is room.
+func (f *formatter) formatString(b []byte, s string) {
+	if len(s) > len(b) {
+		f.err = ErrFieldTooLong
+		return
+	}
+	ascii := toASCII(s)
+	copy(b, ascii)
+	if len(ascii) < len(b) {
+		b[len(ascii)] = 0
+	}
+}
+
+// Encode x as an octal ASCII string and write it into b with leading zeros.
+func (f *formatter) formatOctal(b []byte, x int64) {
+	s := strconv.FormatInt(x, 8)
+	// leading zeros, but leave room for a NUL.
+	for len(s)+1 < len(b) {
+		s = "0" + s
+	}
+	f.formatString(b, s)
+}
+
+// fitsInBase256 reports whether x can be encoded into n bytes using base-256
+// encoding. Unlike octal encoding, base-256 encoding does not require that the
+// string ends with a NUL character. Thus, all n bytes are available for output.
+//
+// If operating in binary mode, this assumes strict GNU binary mode; which means
+// that the first byte can only be either 0x80 or 0xff. Thus, the first byte is
+// equivalent to the sign bit in two's complement form.
+func fitsInBase256(n int, x int64) bool {
+	var binBits = uint(n-1) * 8
+	return n >= 9 || (x >= -1<<binBits && x < 1<<binBits)
+}
+
+// Write x into b, as binary (GNUtar/star extension).
+func (f *formatter) formatNumeric(b []byte, x int64) {
+	if fitsInBase256(len(b), x) {
+		for i := len(b) - 1; i >= 0; i-- {
+			b[i] = byte(x)
+			x >>= 8
+		}
+		b[0] |= 0x80 // Highest bit indicates binary format
+		return
+	}
+
+	f.formatOctal(b, 0) // Last resort, just write zero
+	f.err = ErrFieldTooLong
+}
+
+var (
+	minTime = time.Unix(0, 0)
+	// There is room for 11 octal digits (33 bits) of mtime.
+	maxTime = minTime.Add((1<<33 - 1) * time.Second)
+)
+
+// WriteHeader writes hdr and prepares to accept the file's contents.
+// WriteHeader calls Flush if it is not the first header.
+// Calling after a Close will return ErrWriteAfterClose.
+func (tw *Writer) WriteHeader(hdr *Header) error {
+	return tw.writeHeader(hdr, true)
+}
+
+// WriteHeader writes hdr and prepares to accept the file's contents.
+// WriteHeader calls Flush if it is not the first header.
+// Calling after a Close will return ErrWriteAfterClose.
+// As this method is called internally by writePax header to allow it to
+// suppress writing the pax header.
+func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
+	if tw.closed {
+		return ErrWriteAfterClose
+	}
+	if tw.err == nil {
+		tw.Flush()
+	}
+	if tw.err != nil {
+		return tw.err
+	}
+
+	// a map to hold pax header records, if any are needed
+	paxHeaders := make(map[string]string)
+
+	// TODO(shanemhansen): we might want to use PAX headers for
+	// subsecond time resolution, but for now let's just capture
+	// too long fields or non ascii characters
+
+	var f formatter
+	var header []byte
+
+	// We need to select which scratch buffer to use carefully,
+	// since this method is called recursively to write PAX headers.
+	// If allowPax is true, this is the non-recursive call, and we will use hdrBuff.
+	// If allowPax is false, we are being called by writePAXHeader, and hdrBuff is
+	// already being used by the non-recursive call, so we must use paxHdrBuff.
+	header = tw.hdrBuff[:]
+	if !allowPax {
+		header = tw.paxHdrBuff[:]
+	}
+	copy(header, zeroBlock)
+	s := slicer(header)
+
+	// Wrappers around formatter that automatically sets paxHeaders if the
+	// argument extends beyond the capacity of the input byte slice.
+	var formatString = func(b []byte, s string, paxKeyword string) {
+		needsPaxHeader := paxKeyword != paxNone && len(s) > len(b) || !isASCII(s)
+		if needsPaxHeader {
+			paxHeaders[paxKeyword] = s
+			return
+		}
+		f.formatString(b, s)
+	}
+	var formatNumeric = func(b []byte, x int64, paxKeyword string) {
+		// Try octal first.
+		s := strconv.FormatInt(x, 8)
+		if len(s) < len(b) {
+			f.formatOctal(b, x)
+			return
+		}
+
+		// If it is too long for octal, and PAX is preferred, use a PAX header.
+		if paxKeyword != paxNone && tw.preferPax {
+			f.formatOctal(b, 0)
+			s := strconv.FormatInt(x, 10)
+			paxHeaders[paxKeyword] = s
+			return
+		}
+
+		tw.usedBinary = true
+		f.formatNumeric(b, x)
+	}
+	var formatTime = func(b []byte, t time.Time, paxKeyword string) {
+		var unixTime int64
+		if !t.Before(minTime) && !t.After(maxTime) {
+			unixTime = t.Unix()
+		}
+		formatNumeric(b, unixTime, paxNone)
+
+		// Write a PAX header if the time didn't fit precisely.
+		if paxKeyword != "" && tw.preferPax && allowPax && (t.Nanosecond() != 0 || !t.Before(minTime) || !t.After(maxTime)) {
+			paxHeaders[paxKeyword] = formatPAXTime(t)
+		}
+	}
+
+	// keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
+	pathHeaderBytes := s.next(fileNameSize)
+
+	formatString(pathHeaderBytes, hdr.Name, paxPath)
+
+	f.formatOctal(s.next(8), hdr.Mode)               // 100:108
+	formatNumeric(s.next(8), int64(hdr.Uid), paxUid) // 108:116
+	formatNumeric(s.next(8), int64(hdr.Gid), paxGid) // 116:124
+	formatNumeric(s.next(12), hdr.Size, paxSize)     // 124:136
+	formatTime(s.next(12), hdr.ModTime, paxMtime)    // 136:148
+	s.next(8)                                        // chksum (148:156)
+	s.next(1)[0] = hdr.Typeflag                      // 156:157
+
+	formatString(s.next(100), hdr.Linkname, paxLinkpath)
+
+	copy(s.next(8), []byte("ustar\x0000"))          // 257:265
+	formatString(s.next(32), hdr.Uname, paxUname)   // 265:297
+	formatString(s.next(32), hdr.Gname, paxGname)   // 297:329
+	formatNumeric(s.next(8), hdr.Devmajor, paxNone) // 329:337
+	formatNumeric(s.next(8), hdr.Devminor, paxNone) // 337:345
+
+	// keep a reference to the prefix to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
+	prefixHeaderBytes := s.next(155)
+	formatString(prefixHeaderBytes, "", paxNone) // 345:500  prefix
+
+	// Use the GNU magic instead of POSIX magic if we used any GNU extensions.
+	if tw.usedBinary {
+		copy(header[257:265], []byte("ustar  \x00"))
+	}
+
+	_, paxPathUsed := paxHeaders[paxPath]
+	// try to use a ustar header when only the name is too long
+	if !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed {
+		prefix, suffix, ok := splitUSTARPath(hdr.Name)
+		if ok {
+			// Since we can encode in USTAR format, disable PAX header.
+			delete(paxHeaders, paxPath)
+
+			// Update the path fields
+			formatString(pathHeaderBytes, suffix, paxNone)
+			formatString(prefixHeaderBytes, prefix, paxNone)
+		}
+	}
+
+	// The chksum field is terminated by a NUL and a space.
+	// This is different from the other octal fields.
+	chksum, _ := checksum(header)
+	f.formatOctal(header[148:155], chksum) // Never fails
+	header[155] = ' '
+
+	// Check if there were any formatting errors.
+	if f.err != nil {
+		tw.err = f.err
+		return tw.err
+	}
+
+	if allowPax {
+		if !hdr.AccessTime.IsZero() {
+			paxHeaders[paxAtime] = formatPAXTime(hdr.AccessTime)
+		}
+		if !hdr.ChangeTime.IsZero() {
+			paxHeaders[paxCtime] = formatPAXTime(hdr.ChangeTime)
+		}
+		if !hdr.CreationTime.IsZero() {
+			paxHeaders[paxCreationTime] = formatPAXTime(hdr.CreationTime)
+		}
+		for k, v := range hdr.Xattrs {
+			paxHeaders[paxXattr+k] = v
+		}
+		for k, v := range hdr.Winheaders {
+			paxHeaders[paxWindows+k] = v
+		}
+	}
+
+	if len(paxHeaders) > 0 {
+		if !allowPax {
+			return errInvalidHeader
+		}
+		if err := tw.writePAXHeader(hdr, paxHeaders); err != nil {
+			return err
+		}
+	}
+	tw.nb = int64(hdr.Size)
+	tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize
+
+	_, tw.err = tw.w.Write(header)
+	return tw.err
+}
+
+func formatPAXTime(t time.Time) string {
+	sec := t.Unix()
+	usec := t.Nanosecond()
+	s := strconv.FormatInt(sec, 10)
+	if usec != 0 {
+		s = fmt.Sprintf("%s.%09d", s, usec)
+	}
+	return s
+}
+
+// splitUSTARPath splits a path according to USTAR prefix and suffix rules.
+// If the path is not splittable, then it will return ("", "", false).
+func splitUSTARPath(name string) (prefix, suffix string, ok bool) {
+	length := len(name)
+	if length <= fileNameSize || !isASCII(name) {
+		return "", "", false
+	} else if length > fileNamePrefixSize+1 {
+		length = fileNamePrefixSize + 1
+	} else if name[length-1] == '/' {
+		length--
+	}
+
+	i := strings.LastIndex(name[:length], "/")
+	nlen := len(name) - i - 1 // nlen is length of suffix
+	plen := i                 // plen is length of prefix
+	if i <= 0 || nlen > fileNameSize || nlen == 0 || plen > fileNamePrefixSize {
+		return "", "", false
+	}
+	return name[:i], name[i+1:], true
+}
+
+// writePaxHeader writes an extended pax header to the
+// archive.
+func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error {
+	// Prepare extended header
+	ext := new(Header)
+	ext.Typeflag = TypeXHeader
+	// Setting ModTime is required for reader parsing to
+	// succeed, and seems harmless enough.
+	ext.ModTime = hdr.ModTime
+	// The spec asks that we namespace our pseudo files
+	// with the current pid.  However, this results in differing outputs
+	// for identical inputs.  As such, the constant 0 is now used instead.
+	// golang.org/issue/12358
+	dir, file := path.Split(hdr.Name)
+	fullName := path.Join(dir, "PaxHeaders.0", file)
+
+	ascii := toASCII(fullName)
+	if len(ascii) > 100 {
+		ascii = ascii[:100]
+	}
+	ext.Name = ascii
+	// Construct the body
+	var buf bytes.Buffer
+
+	// Keys are sorted before writing to body to allow deterministic output.
+	var keys []string
+	for k := range paxHeaders {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, k := range keys {
+		fmt.Fprint(&buf, formatPAXRecord(k, paxHeaders[k]))
+	}
+
+	ext.Size = int64(len(buf.Bytes()))
+	if err := tw.writeHeader(ext, false); err != nil {
+		return err
+	}
+	if _, err := tw.Write(buf.Bytes()); err != nil {
+		return err
+	}
+	if err := tw.Flush(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// formatPAXRecord formats a single PAX record, prefixing it with the
+// appropriate length.
+func formatPAXRecord(k, v string) string {
+	const padding = 3 // Extra padding for ' ', '=', and '\n'
+	size := len(k) + len(v) + padding
+	size += len(strconv.Itoa(size))
+	record := fmt.Sprintf("%d %s=%s\n", size, k, v)
+
+	// Final adjustment if adding size field increased the record size.
+	if len(record) != size {
+		size = len(record)
+		record = fmt.Sprintf("%d %s=%s\n", size, k, v)
+	}
+	return record
+}
+
+// Write writes to the current entry in the tar archive.
+// Write returns the error ErrWriteTooLong if more than
+// hdr.Size bytes are written after WriteHeader.
+func (tw *Writer) Write(b []byte) (n int, err error) {
+	if tw.closed {
+		err = ErrWriteAfterClose
+		return
+	}
+	overwrite := false
+	if int64(len(b)) > tw.nb {
+		b = b[0:tw.nb]
+		overwrite = true
+	}
+	n, err = tw.w.Write(b)
+	tw.nb -= int64(n)
+	if err == nil && overwrite {
+		err = ErrWriteTooLong
+		return
+	}
+	tw.err = err
+	return
+}
+
+// Close closes the tar archive, flushing any unwritten
+// data to the underlying writer.
+func (tw *Writer) Close() error {
+	if tw.err != nil || tw.closed {
+		return tw.err
+	}
+	tw.Flush()
+	tw.closed = true
+	if tw.err != nil {
+		return tw.err
+	}
+
+	// trailer: two zero blocks
+	for i := 0; i < 2; i++ {
+		_, tw.err = tw.w.Write(zeroBlock)
+		if tw.err != nil {
+			break
+		}
+	}
+	return tw.err
+}

vendor/github.com/Microsoft/go-winio/archive/tar/writer_test.go

@@ -0,0 +1,739 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package tar
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math"
+	"os"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+	"testing/iotest"
+	"time"
+)
+
+type writerTestEntry struct {
+	header   *Header
+	contents string
+}
+
+type writerTest struct {
+	file    string // filename of expected output
+	entries []*writerTestEntry
+}
+
+var writerTests = []*writerTest{
+	// The writer test file was produced with this command:
+	// tar (GNU tar) 1.26
+	//   ln -s small.txt link.txt
+	//   tar -b 1 --format=ustar -c -f writer.tar small.txt small2.txt link.txt
+	{
+		file: "testdata/writer.tar",
+		entries: []*writerTestEntry{
+			{
+				header: &Header{
+					Name:     "small.txt",
+					Mode:     0640,
+					Uid:      73025,
+					Gid:      5000,
+					Size:     5,
+					ModTime:  time.Unix(1246508266, 0),
+					Typeflag: '0',
+					Uname:    "dsymonds",
+					Gname:    "eng",
+				},
+				contents: "Kilts",
+			},
+			{
+				header: &Header{
+					Name:     "small2.txt",
+					Mode:     0640,
+					Uid:      73025,
+					Gid:      5000,
+					Size:     11,
+					ModTime:  time.Unix(1245217492, 0),
+					Typeflag: '0',
+					Uname:    "dsymonds",
+					Gname:    "eng",
+				},
+				contents: "Google.com\n",
+			},
+			{
+				header: &Header{
+					Name:     "link.txt",
+					Mode:     0777,
+					Uid:      1000,
+					Gid:      1000,
+					Size:     0,
+					ModTime:  time.Unix(1314603082, 0),
+					Typeflag: '2',
+					Linkname: "small.txt",
+					Uname:    "strings",
+					Gname:    "strings",
+				},
+				// no contents
+			},
+		},
+	},
+	// The truncated test file was produced using these commands:
+	//   dd if=/dev/zero bs=1048576 count=16384 > /tmp/16gig.txt
+	//   tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar
+	{
+		file: "testdata/writer-big.tar",
+		entries: []*writerTestEntry{
+			{
+				header: &Header{
+					Name:     "tmp/16gig.txt",
+					Mode:     0640,
+					Uid:      73025,
+					Gid:      5000,
+					Size:     16 << 30,
+					ModTime:  time.Unix(1254699560, 0),
+					Typeflag: '0',
+					Uname:    "dsymonds",
+					Gname:    "eng",
+				},
+				// fake contents
+				contents: strings.Repeat("\x00", 4<<10),
+			},
+		},
+	},
+	// The truncated test file was produced using these commands:
+	//   dd if=/dev/zero bs=1048576 count=16384 > (longname/)*15 /16gig.txt
+	//   tar -b 1 -c -f- (longname/)*15 /16gig.txt | dd bs=512 count=8 > writer-big-long.tar
+	{
+		file: "testdata/writer-big-long.tar",
+		entries: []*writerTestEntry{
+			{
+				header: &Header{
+					Name:     strings.Repeat("longname/", 15) + "16gig.txt",
+					Mode:     0644,
+					Uid:      1000,
+					Gid:      1000,
+					Size:     16 << 30,
+					ModTime:  time.Unix(1399583047, 0),
+					Typeflag: '0',
+					Uname:    "guillaume",
+					Gname:    "guillaume",
+				},
+				// fake contents
+				contents: strings.Repeat("\x00", 4<<10),
+			},
+		},
+	},
+	// This file was produced using gnu tar 1.17
+	// gnutar  -b 4 --format=ustar (longname/)*15 + file.txt
+	{
+		file: "testdata/ustar.tar",
+		entries: []*writerTestEntry{
+			{
+				header: &Header{
+					Name:     strings.Repeat("longname/", 15) + "file.txt",
+					Mode:     0644,
+					Uid:      0765,
+					Gid:      024,
+					Size:     06,
+					ModTime:  time.Unix(1360135598, 0),
+					Typeflag: '0',
+					Uname:    "shane",
+					Gname:    "staff",
+				},
+				contents: "hello\n",
+			},
+		},
+	},
+	// This file was produced using gnu tar 1.26
+	// echo "Slartibartfast" > file.txt
+	// ln file.txt hard.txt
+	// tar -b 1 --format=ustar -c -f hardlink.tar file.txt hard.txt
+	{
+		file: "testdata/hardlink.tar",
+		entries: []*writerTestEntry{
+			{
+				header: &Header{
+					Name:     "file.txt",
+					Mode:     0644,
+					Uid:      1000,
+					Gid:      100,
+					Size:     15,
+					ModTime:  time.Unix(1425484303, 0),
+					Typeflag: '0',
+					Uname:    "vbatts",
+					Gname:    "users",
+				},
+				contents: "Slartibartfast\n",
+			},
+			{
+				header: &Header{
+					Name:     "hard.txt",
+					Mode:     0644,
+					Uid:      1000,
+					Gid:      100,
+					Size:     0,
+					ModTime:  time.Unix(1425484303, 0),
+					Typeflag: '1',
+					Linkname: "file.txt",
+					Uname:    "vbatts",
+					Gname:    "users",
+				},
+				// no contents
+			},
+		},
+	},
+}
+
+// Render byte array in a two-character hexadecimal string, spaced for easy visual inspection.
+func bytestr(offset int, b []byte) string {
+	const rowLen = 32
+	s := fmt.Sprintf("%04x ", offset)
+	for _, ch := range b {
+		switch {
+		case '0' <= ch && ch <= '9', 'A' <= ch && ch <= 'Z', 'a' <= ch && ch <= 'z':
+			s += fmt.Sprintf("  %c", ch)
+		default:
+			s += fmt.Sprintf(" %02x", ch)
+		}
+	}
+	return s
+}
+
+// Render a pseudo-diff between two blocks of bytes.
+func bytediff(a []byte, b []byte) string {
+	const rowLen = 32
+	s := fmt.Sprintf("(%d bytes vs. %d bytes)\n", len(a), len(b))
+	for offset := 0; len(a)+len(b) > 0; offset += rowLen {
+		na, nb := rowLen, rowLen
+		if na > len(a) {
+			na = len(a)
+		}
+		if nb > len(b) {
+			nb = len(b)
+		}
+		sa := bytestr(offset, a[0:na])
+		sb := bytestr(offset, b[0:nb])
+		if sa != sb {
+			s += fmt.Sprintf("-%v\n+%v\n", sa, sb)
+		}
+		a = a[na:]
+		b = b[nb:]
+	}
+	return s
+}
+
+func TestWriter(t *testing.T) {
+testLoop:
+	for i, test := range writerTests {
+		expected, err := ioutil.ReadFile(test.file)
+		if err != nil {
+			t.Errorf("test %d: Unexpected error: %v", i, err)
+			continue
+		}
+
+		buf := new(bytes.Buffer)
+		tw := NewWriter(iotest.TruncateWriter(buf, 4<<10)) // only catch the first 4 KB
+		big := false
+		for j, entry := range test.entries {
+			big = big || entry.header.Size > 1<<10
+			if err := tw.WriteHeader(entry.header); err != nil {
+				t.Errorf("test %d, entry %d: Failed writing header: %v", i, j, err)
+				continue testLoop
+			}
+			if _, err := io.WriteString(tw, entry.contents); err != nil {
+				t.Errorf("test %d, entry %d: Failed writing contents: %v", i, j, err)
+				continue testLoop
+			}
+		}
+		// Only interested in Close failures for the small tests.
+		if err := tw.Close(); err != nil && !big {
+			t.Errorf("test %d: Failed closing archive: %v", i, err)
+			continue testLoop
+		}
+
+		actual := buf.Bytes()
+		if !bytes.Equal(expected, actual) {
+			t.Errorf("test %d: Incorrect result: (-=expected, +=actual)\n%v",
+				i, bytediff(expected, actual))
+		}
+		if testing.Short() { // The second test is expensive.
+			break
+		}
+	}
+}
+
+func TestPax(t *testing.T) {
+	// Create an archive with a large name
+	fileinfo, err := os.Stat("testdata/small.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	hdr, err := FileInfoHeader(fileinfo, "")
+	if err != nil {
+		t.Fatalf("os.Stat: %v", err)
+	}
+	// Force a PAX long name to be written
+	longName := strings.Repeat("ab", 100)
+	contents := strings.Repeat(" ", int(hdr.Size))
+	hdr.Name = longName
+	var buf bytes.Buffer
+	writer := NewWriter(&buf)
+	if err := writer.WriteHeader(hdr); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = writer.Write([]byte(contents)); err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatal(err)
+	}
+	// Simple test to make sure PAX extensions are in effect
+	if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.0")) {
+		t.Fatal("Expected at least one PAX header to be written.")
+	}
+	// Test that we can get a long name back out of the archive.
+	reader := NewReader(&buf)
+	hdr, err = reader.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if hdr.Name != longName {
+		t.Fatal("Couldn't recover long file name")
+	}
+}
+
+func TestPaxSymlink(t *testing.T) {
+	// Create an archive with a large linkname
+	fileinfo, err := os.Stat("testdata/small.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	hdr, err := FileInfoHeader(fileinfo, "")
+	hdr.Typeflag = TypeSymlink
+	if err != nil {
+		t.Fatalf("os.Stat:1 %v", err)
+	}
+	// Force a PAX long linkname to be written
+	longLinkname := strings.Repeat("1234567890/1234567890", 10)
+	hdr.Linkname = longLinkname
+
+	hdr.Size = 0
+	var buf bytes.Buffer
+	writer := NewWriter(&buf)
+	if err := writer.WriteHeader(hdr); err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatal(err)
+	}
+	// Simple test to make sure PAX extensions are in effect
+	if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.0")) {
+		t.Fatal("Expected at least one PAX header to be written.")
+	}
+	// Test that we can get a long name back out of the archive.
+	reader := NewReader(&buf)
+	hdr, err = reader.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if hdr.Linkname != longLinkname {
+		t.Fatal("Couldn't recover long link name")
+	}
+}
+
+func TestPaxNonAscii(t *testing.T) {
+	// Create an archive with non ascii. These should trigger a pax header
+	// because pax headers have a defined utf-8 encoding.
+	fileinfo, err := os.Stat("testdata/small.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hdr, err := FileInfoHeader(fileinfo, "")
+	if err != nil {
+		t.Fatalf("os.Stat:1 %v", err)
+	}
+
+	// some sample data
+	chineseFilename := "文件名"
+	chineseGroupname := "組"
+	chineseUsername := "用戶名"
+
+	hdr.Name = chineseFilename
+	hdr.Gname = chineseGroupname
+	hdr.Uname = chineseUsername
+
+	contents := strings.Repeat(" ", int(hdr.Size))
+
+	var buf bytes.Buffer
+	writer := NewWriter(&buf)
+	if err := writer.WriteHeader(hdr); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = writer.Write([]byte(contents)); err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatal(err)
+	}
+	// Simple test to make sure PAX extensions are in effect
+	if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.0")) {
+		t.Fatal("Expected at least one PAX header to be written.")
+	}
+	// Test that we can get a long name back out of the archive.
+	reader := NewReader(&buf)
+	hdr, err = reader.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if hdr.Name != chineseFilename {
+		t.Fatal("Couldn't recover unicode name")
+	}
+	if hdr.Gname != chineseGroupname {
+		t.Fatal("Couldn't recover unicode group")
+	}
+	if hdr.Uname != chineseUsername {
+		t.Fatal("Couldn't recover unicode user")
+	}
+}
+
+func TestPaxXattrs(t *testing.T) {
+	xattrs := map[string]string{
+		"user.key": "value",
+	}
+
+	// Create an archive with an xattr
+	fileinfo, err := os.Stat("testdata/small.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	hdr, err := FileInfoHeader(fileinfo, "")
+	if err != nil {
+		t.Fatalf("os.Stat: %v", err)
+	}
+	contents := "Kilts"
+	hdr.Xattrs = xattrs
+	var buf bytes.Buffer
+	writer := NewWriter(&buf)
+	if err := writer.WriteHeader(hdr); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = writer.Write([]byte(contents)); err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatal(err)
+	}
+	// Test that we can get the xattrs back out of the archive.
+	reader := NewReader(&buf)
+	hdr, err = reader.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(hdr.Xattrs, xattrs) {
+		t.Fatalf("xattrs did not survive round trip: got %+v, want %+v",
+			hdr.Xattrs, xattrs)
+	}
+}
+
+func TestPaxHeadersSorted(t *testing.T) {
+	fileinfo, err := os.Stat("testdata/small.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	hdr, err := FileInfoHeader(fileinfo, "")
+	if err != nil {
+		t.Fatalf("os.Stat: %v", err)
+	}
+	contents := strings.Repeat(" ", int(hdr.Size))
+
+	hdr.Xattrs = map[string]string{
+		"foo": "foo",
+		"bar": "bar",
+		"baz": "baz",
+		"qux": "qux",
+	}
+
+	var buf bytes.Buffer
+	writer := NewWriter(&buf)
+	if err := writer.WriteHeader(hdr); err != nil {
+		t.Fatal(err)
+	}
+	if _, err = writer.Write([]byte(contents)); err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatal(err)
+	}
+	// Simple test to make sure PAX extensions are in effect
+	if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.0")) {
+		t.Fatal("Expected at least one PAX header to be written.")
+	}
+
+	// xattr bar should always appear before others
+	indices := []int{
+		bytes.Index(buf.Bytes(), []byte("bar=bar")),
+		bytes.Index(buf.Bytes(), []byte("baz=baz")),
+		bytes.Index(buf.Bytes(), []byte("foo=foo")),
+		bytes.Index(buf.Bytes(), []byte("qux=qux")),
+	}
+	if !sort.IntsAreSorted(indices) {
+		t.Fatal("PAX headers are not sorted")
+	}
+}
+
+func TestUSTARLongName(t *testing.T) {
+	// Create an archive with a path that failed to split with USTAR extension in previous versions.
+	fileinfo, err := os.Stat("testdata/small.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	hdr, err := FileInfoHeader(fileinfo, "")
+	hdr.Typeflag = TypeDir
+	if err != nil {
+		t.Fatalf("os.Stat:1 %v", err)
+	}
+	// Force a PAX long name to be written. The name was taken from a practical example
+	// that fails and replaced ever char through numbers to anonymize the sample.
+	longName := "/0000_0000000/00000-000000000/0000_0000000/00000-0000000000000/0000_0000000/00000-0000000-00000000/0000_0000000/00000000/0000_0000000/000/0000_0000000/00000000v00/0000_0000000/000000/0000_0000000/0000000/0000_0000000/00000y-00/0000/0000/00000000/0x000000/"
+	hdr.Name = longName
+
+	hdr.Size = 0
+	var buf bytes.Buffer
+	writer := NewWriter(&buf)
+	if err := writer.WriteHeader(hdr); err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.Close(); err != nil {
+		t.Fatal(err)
+	}
+	// Test that we can get a long name back out of the archive.
+	reader := NewReader(&buf)
+	hdr, err = reader.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if hdr.Name != longName {
+		t.Fatal("Couldn't recover long name")
+	}
+}
+
+func TestValidTypeflagWithPAXHeader(t *testing.T) {
+	var buffer bytes.Buffer
+	tw := NewWriter(&buffer)
+
+	fileName := strings.Repeat("ab", 100)
+
+	hdr := &Header{
+		Name:     fileName,
+		Size:     4,
+		Typeflag: 0,
+	}
+	if err := tw.WriteHeader(hdr); err != nil {
+		t.Fatalf("Failed to write header: %s", err)
+	}
+	if _, err := tw.Write([]byte("fooo")); err != nil {
+		t.Fatalf("Failed to write the file's data: %s", err)
+	}
+	tw.Close()
+
+	tr := NewReader(&buffer)
+
+	for {
+		header, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatalf("Failed to read header: %s", err)
+		}
+		if header.Typeflag != 0 {
+			t.Fatalf("Typeflag should've been 0, found %d", header.Typeflag)
+		}
+	}
+}
+
+func TestWriteAfterClose(t *testing.T) {
+	var buffer bytes.Buffer
+	tw := NewWriter(&buffer)
+
+	hdr := &Header{
+		Name: "small.txt",
+		Size: 5,
+	}
+	if err := tw.WriteHeader(hdr); err != nil {
+		t.Fatalf("Failed to write header: %s", err)
+	}
+	tw.Close()
+	if _, err := tw.Write([]byte("Kilts")); err != ErrWriteAfterClose {
+		t.Fatalf("Write: got %v; want ErrWriteAfterClose", err)
+	}
+}
+
+func TestSplitUSTARPath(t *testing.T) {
+	var sr = strings.Repeat
+
+	var vectors = []struct {
+		input  string // Input path
+		prefix string // Expected output prefix
+		suffix string // Expected output suffix
+		ok     bool   // Split success?
+	}{
+		{"", "", "", false},
+		{"abc", "", "", false},
+		{"用戶名", "", "", false},
+		{sr("a", fileNameSize), "", "", false},
+		{sr("a", fileNameSize) + "/", "", "", false},
+		{sr("a", fileNameSize) + "/a", sr("a", fileNameSize), "a", true},
+		{sr("a", fileNamePrefixSize) + "/", "", "", false},
+		{sr("a", fileNamePrefixSize) + "/a", sr("a", fileNamePrefixSize), "a", true},
+		{sr("a", fileNameSize+1), "", "", false},
+		{sr("/", fileNameSize+1), sr("/", fileNameSize-1), "/", true},
+		{sr("a", fileNamePrefixSize) + "/" + sr("b", fileNameSize),
+			sr("a", fileNamePrefixSize), sr("b", fileNameSize), true},
+		{sr("a", fileNamePrefixSize) + "//" + sr("b", fileNameSize), "", "", false},
+		{sr("a/", fileNameSize), sr("a/", 77) + "a", sr("a/", 22), true},
+	}
+
+	for _, v := range vectors {
+		prefix, suffix, ok := splitUSTARPath(v.input)
+		if prefix != v.prefix || suffix != v.suffix || ok != v.ok {
+			t.Errorf("splitUSTARPath(%q):\ngot  (%q, %q, %v)\nwant (%q, %q, %v)",
+				v.input, prefix, suffix, ok, v.prefix, v.suffix, v.ok)
+		}
+	}
+}
+
+func TestFormatPAXRecord(t *testing.T) {
+	var medName = strings.Repeat("CD", 50)
+	var longName = strings.Repeat("AB", 100)
+
+	var vectors = []struct {
+		inputKey string
+		inputVal string
+		output   string
+	}{
+		{"k", "v", "6 k=v\n"},
+		{"path", "/etc/hosts", "19 path=/etc/hosts\n"},
+		{"path", longName, "210 path=" + longName + "\n"},
+		{"path", medName, "110 path=" + medName + "\n"},
+		{"foo", "ba", "9 foo=ba\n"},
+		{"foo", "bar", "11 foo=bar\n"},
+		{"foo", "b=\nar=\n==\x00", "18 foo=b=\nar=\n==\x00\n"},
+		{"foo", "hello9 foo=ba\nworld", "27 foo=hello9 foo=ba\nworld\n"},
+		{"☺☻☹", "日a本b語ç", "27 ☺☻☹=日a本b語ç\n"},
+		{"\x00hello", "\x00world", "17 \x00hello=\x00world\n"},
+	}
+
+	for _, v := range vectors {
+		output := formatPAXRecord(v.inputKey, v.inputVal)
+		if output != v.output {
+			t.Errorf("formatPAXRecord(%q, %q): got %q, want %q",
+				v.inputKey, v.inputVal, output, v.output)
+		}
+	}
+}
+
+func TestFitsInBase256(t *testing.T) {
+	var vectors = []struct {
+		input int64
+		width int
+		ok    bool
+	}{
+		{+1, 8, true},
+		{0, 8, true},
+		{-1, 8, true},
+		{1 << 56, 8, false},
+		{(1 << 56) - 1, 8, true},
+		{-1 << 56, 8, true},
+		{(-1 << 56) - 1, 8, false},
+		{121654, 8, true},
+		{-9849849, 8, true},
+		{math.MaxInt64, 9, true},
+		{0, 9, true},
+		{math.MinInt64, 9, true},
+		{math.MaxInt64, 12, true},
+		{0, 12, true},
+		{math.MinInt64, 12, true},
+	}
+
+	for _, v := range vectors {
+		ok := fitsInBase256(v.width, v.input)
+		if ok != v.ok {
+			t.Errorf("checkNumeric(%d, %d): got %v, want %v", v.input, v.width, ok, v.ok)
+		}
+	}
+}
+
+func TestFormatNumeric(t *testing.T) {
+	var vectors = []struct {
+		input  int64
+		output string
+		ok     bool
+	}{
+		// Test base-256 (binary) encoded values.
+		{-1, "\xff", true},
+		{-1, "\xff\xff", true},
+		{-1, "\xff\xff\xff", true},
+		{(1 << 0), "0", false},
+		{(1 << 8) - 1, "\x80\xff", true},
+		{(1 << 8), "0\x00", false},
+		{(1 << 16) - 1, "\x80\xff\xff", true},
+		{(1 << 16), "00\x00", false},
+		{-1 * (1 << 0), "\xff", true},
+		{-1*(1<<0) - 1, "0", false},
+		{-1 * (1 << 8), "\xff\x00", true},
+		{-1*(1<<8) - 1, "0\x00", false},
+		{-1 * (1 << 16), "\xff\x00\x00", true},
+		{-1*(1<<16) - 1, "00\x00", false},
+		{537795476381659745, "0000000\x00", false},
+		{537795476381659745, "\x80\x00\x00\x00\x07\x76\xa2\x22\xeb\x8a\x72\x61", true},
+		{-615126028225187231, "0000000\x00", false},
+		{-615126028225187231, "\xff\xff\xff\xff\xf7\x76\xa2\x22\xeb\x8a\x72\x61", true},
+		{math.MaxInt64, "0000000\x00", false},
+		{math.MaxInt64, "\x80\x00\x00\x00\x7f\xff\xff\xff\xff\xff\xff\xff", true},
+		{math.MinInt64, "0000000\x00", false},
+		{math.MinInt64, "\xff\xff\xff\xff\x80\x00\x00\x00\x00\x00\x00\x00", true},
+		{math.MaxInt64, "\x80\x7f\xff\xff\xff\xff\xff\xff\xff", true},
+		{math.MinInt64, "\xff\x80\x00\x00\x00\x00\x00\x00\x00", true},
+	}
+
+	for _, v := range vectors {
+		var f formatter
+		output := make([]byte, len(v.output))
+		f.formatNumeric(output, v.input)
+		ok := (f.err == nil)
+		if ok != v.ok {
+			if v.ok {
+				t.Errorf("formatNumeric(%d): got formatting failure, want success", v.input)
+			} else {
+				t.Errorf("formatNumeric(%d): got formatting success, want failure", v.input)
+			}
+		}
+		if string(output) != v.output {
+			t.Errorf("formatNumeric(%d): got %q, want %q", v.input, output, v.output)
+		}
+	}
+}
+
+func TestFormatPAXTime(t *testing.T) {
+	t1 := time.Date(2000, 1, 1, 11, 0, 0, 0, time.UTC)
+	t2 := time.Date(2000, 1, 1, 11, 0, 0, 100, time.UTC)
+	t3 := time.Date(1960, 1, 1, 11, 0, 0, 0, time.UTC)
+	t4 := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
+	verify := func(time time.Time, s string) {
+		p := formatPAXTime(time)
+		if p != s {
+			t.Errorf("for %v, expected %s, got %s", time, s, p)
+		}
+	}
+	verify(t1, "946724400")
+	verify(t2, "946724400.000000100")
+	verify(t3, "-315579600")
+	verify(t4, "0")
+}

vendor/github.com/Microsoft/go-winio/backup.go

@@ -0,0 +1,280 @@
+// +build windows
+
+package winio
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"runtime"
+	"syscall"
+	"unicode/utf16"
+)
+
+//sys backupRead(h syscall.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupRead
+//sys backupWrite(h syscall.Handle, b []byte, bytesWritten *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupWrite
+
+const (
+	BackupData = uint32(iota + 1)
+	BackupEaData
+	BackupSecurity
+	BackupAlternateData
+	BackupLink
+	BackupPropertyData
+	BackupObjectId
+	BackupReparseData
+	BackupSparseBlock
+	BackupTxfsData
+)
+
+const (
+	StreamSparseAttributes = uint32(8)
+)
+
+const (
+	WRITE_DAC              = 0x40000
+	WRITE_OWNER            = 0x80000
+	ACCESS_SYSTEM_SECURITY = 0x1000000
+)
+
+// BackupHeader represents a backup stream of a file.
+type BackupHeader struct {
+	Id         uint32 // The backup stream ID
+	Attributes uint32 // Stream attributes
+	Size       int64  // The size of the stream in bytes
+	Name       string // The name of the stream (for BackupAlternateData only).
+	Offset     int64  // The offset of the stream in the file (for BackupSparseBlock only).
+}
+
+type win32StreamId struct {
+	StreamId   uint32
+	Attributes uint32
+	Size       uint64
+	NameSize   uint32
+}
+
+// BackupStreamReader reads from a stream produced by the BackupRead Win32 API and produces a series
+// of BackupHeader values.
+type BackupStreamReader struct {
+	r         io.Reader
+	bytesLeft int64
+}
+
+// NewBackupStreamReader produces a BackupStreamReader from any io.Reader.
+func NewBackupStreamReader(r io.Reader) *BackupStreamReader {
+	return &BackupStreamReader{r, 0}
+}
+
+// Next returns the next backup stream and prepares for calls to Read(). It skips the remainder of the current stream if
+// it was not completely read.
+func (r *BackupStreamReader) Next() (*BackupHeader, error) {
+	if r.bytesLeft > 0 {
+		if s, ok := r.r.(io.Seeker); ok {
+			// Make sure Seek on io.SeekCurrent sometimes succeeds
+			// before trying the actual seek.
+			if _, err := s.Seek(0, io.SeekCurrent); err == nil {
+				if _, err = s.Seek(r.bytesLeft, io.SeekCurrent); err != nil {
+					return nil, err
+				}
+				r.bytesLeft = 0
+			}
+		}
+		if _, err := io.Copy(ioutil.Discard, r); err != nil {
+			return nil, err
+		}
+	}
+	var wsi win32StreamId
+	if err := binary.Read(r.r, binary.LittleEndian, &wsi); err != nil {
+		return nil, err
+	}
+	hdr := &BackupHeader{
+		Id:         wsi.StreamId,
+		Attributes: wsi.Attributes,
+		Size:       int64(wsi.Size),
+	}
+	if wsi.NameSize != 0 {
+		name := make([]uint16, int(wsi.NameSize/2))
+		if err := binary.Read(r.r, binary.LittleEndian, name); err != nil {
+			return nil, err
+		}
+		hdr.Name = syscall.UTF16ToString(name)
+	}
+	if wsi.StreamId == BackupSparseBlock {
+		if err := binary.Read(r.r, binary.LittleEndian, &hdr.Offset); err != nil {
+			return nil, err
+		}
+		hdr.Size -= 8
+	}
+	r.bytesLeft = hdr.Size
+	return hdr, nil
+}
+
+// Read reads from the current backup stream.
+func (r *BackupStreamReader) Read(b []byte) (int, error) {
+	if r.bytesLeft == 0 {
+		return 0, io.EOF
+	}
+	if int64(len(b)) > r.bytesLeft {
+		b = b[:r.bytesLeft]
+	}
+	n, err := r.r.Read(b)
+	r.bytesLeft -= int64(n)
+	if err == io.EOF {
+		err = io.ErrUnexpectedEOF
+	} else if r.bytesLeft == 0 && err == nil {
+		err = io.EOF
+	}
+	return n, err
+}
+
+// BackupStreamWriter writes a stream compatible with the BackupWrite Win32 API.
+type BackupStreamWriter struct {
+	w         io.Writer
+	bytesLeft int64
+}
+
+// NewBackupStreamWriter produces a BackupStreamWriter on top of an io.Writer.
+func NewBackupStreamWriter(w io.Writer) *BackupStreamWriter {
+	return &BackupStreamWriter{w, 0}
+}
+
+// WriteHeader writes the next backup stream header and prepares for calls to Write().
+func (w *BackupStreamWriter) WriteHeader(hdr *BackupHeader) error {
+	if w.bytesLeft != 0 {
+		return fmt.Errorf("missing %d bytes", w.bytesLeft)
+	}
+	name := utf16.Encode([]rune(hdr.Name))
+	wsi := win32StreamId{
+		StreamId:   hdr.Id,
+		Attributes: hdr.Attributes,
+		Size:       uint64(hdr.Size),
+		NameSize:   uint32(len(name) * 2),
+	}
+	if hdr.Id == BackupSparseBlock {
+		// Include space for the int64 block offset
+		wsi.Size += 8
+	}
+	if err := binary.Write(w.w, binary.LittleEndian, &wsi); err != nil {
+		return err
+	}
+	if len(name) != 0 {
+		if err := binary.Write(w.w, binary.LittleEndian, name); err != nil {
+			return err
+		}
+	}
+	if hdr.Id == BackupSparseBlock {
+		if err := binary.Write(w.w, binary.LittleEndian, hdr.Offset); err != nil {
+			return err
+		}
+	}
+	w.bytesLeft = hdr.Size
+	return nil
+}
+
+// Write writes to the current backup stream.
+func (w *BackupStreamWriter) Write(b []byte) (int, error) {
+	if w.bytesLeft < int64(len(b)) {
+		return 0, fmt.Errorf("too many bytes by %d", int64(len(b))-w.bytesLeft)
+	}
+	n, err := w.w.Write(b)
+	w.bytesLeft -= int64(n)
+	return n, err
+}
+
+// BackupFileReader provides an io.ReadCloser interface on top of the BackupRead Win32 API.
+type BackupFileReader struct {
+	f               *os.File
+	includeSecurity bool
+	ctx             uintptr
+}
+
+// NewBackupFileReader returns a new BackupFileReader from a file handle. If includeSecurity is true,
+// Read will attempt to read the security descriptor of the file.
+func NewBackupFileReader(f *os.File, includeSecurity bool) *BackupFileReader {
+	r := &BackupFileReader{f, includeSecurity, 0}
+	return r
+}
+
+// Read reads a backup stream from the file by calling the Win32 API BackupRead().
+func (r *BackupFileReader) Read(b []byte) (int, error) {
+	var bytesRead uint32
+	err := backupRead(syscall.Handle(r.f.Fd()), b, &bytesRead, false, r.includeSecurity, &r.ctx)
+	if err != nil {
+		return 0, &os.PathError{"BackupRead", r.f.Name(), err}
+	}
+	runtime.KeepAlive(r.f)
+	if bytesRead == 0 {
+		return 0, io.EOF
+	}
+	return int(bytesRead), nil
+}
+
+// Close frees Win32 resources associated with the BackupFileReader. It does not close
+// the underlying file.
+func (r *BackupFileReader) Close() error {
+	if r.ctx != 0 {
+		backupRead(syscall.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx)
+		runtime.KeepAlive(r.f)
+		r.ctx = 0
+	}
+	return nil
+}
+
+// BackupFileWriter provides an io.WriteCloser interface on top of the BackupWrite Win32 API.
+type BackupFileWriter struct {
+	f               *os.File
+	includeSecurity bool
+	ctx             uintptr
+}
+
+// NewBackupFileWriter returns a new BackupFileWriter from a file handle. If includeSecurity is true,
+// Write() will attempt to restore the security descriptor from the stream.
+func NewBackupFileWriter(f *os.File, includeSecurity bool) *BackupFileWriter {
+	w := &BackupFileWriter{f, includeSecurity, 0}
+	return w
+}
+
+// Write restores a portion of the file using the provided backup stream.
+func (w *BackupFileWriter) Write(b []byte) (int, error) {
+	var bytesWritten uint32
+	err := backupWrite(syscall.Handle(w.f.Fd()), b, &bytesWritten, false, w.includeSecurity, &w.ctx)
+	if err != nil {
+		return 0, &os.PathError{"BackupWrite", w.f.Name(), err}
+	}
+	runtime.KeepAlive(w.f)
+	if int(bytesWritten) != len(b) {
+		return int(bytesWritten), errors.New("not all bytes could be written")
+	}
+	return len(b), nil
+}
+
+// Close frees Win32 resources associated with the BackupFileWriter. It does not
+// close the underlying file.
+func (w *BackupFileWriter) Close() error {
+	if w.ctx != 0 {
+		backupWrite(syscall.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx)
+		runtime.KeepAlive(w.f)
+		w.ctx = 0
+	}
+	return nil
+}
+
+// OpenForBackup opens a file or directory, potentially skipping access checks if the backup
+// or restore privileges have been acquired.
+//
+// If the file opened was a directory, it cannot be used with Readdir().
+func OpenForBackup(path string, access uint32, share uint32, createmode uint32) (*os.File, error) {
+	winPath, err := syscall.UTF16FromString(path)
+	if err != nil {
+		return nil, err
+	}
+	h, err := syscall.CreateFile(&winPath[0], access, share, nil, createmode, syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OPEN_REPARSE_POINT, 0)
+	if err != nil {
+		err = &os.PathError{Op: "open", Path: path, Err: err}
+		return nil, err
+	}
+	return os.NewFile(uintptr(h), path), nil
+}

vendor/github.com/Microsoft/go-winio/backup_test.go

@@ -0,0 +1,255 @@
+package winio
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"syscall"
+	"testing"
+)
+
+var testFileName string
+
+func TestMain(m *testing.M) {
+	f, err := ioutil.TempFile("", "tmp")
+	if err != nil {
+		panic(err)
+	}
+	testFileName = f.Name()
+	f.Close()
+	defer os.Remove(testFileName)
+	os.Exit(m.Run())
+}
+
+func makeTestFile(makeADS bool) error {
+	os.Remove(testFileName)
+	f, err := os.Create(testFileName)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = f.Write([]byte("testing 1 2 3\n"))
+	if err != nil {
+		return err
+	}
+	if makeADS {
+		a, err := os.Create(testFileName + ":ads.txt")
+		if err != nil {
+			return err
+		}
+		defer a.Close()
+		_, err = a.Write([]byte("alternate data stream\n"))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func TestBackupRead(t *testing.T) {
+	err := makeTestFile(true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f, err := os.Open(testFileName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	r := NewBackupFileReader(f, false)
+	defer r.Close()
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(b) == 0 {
+		t.Fatal("no data")
+	}
+}
+
+func TestBackupStreamRead(t *testing.T) {
+	err := makeTestFile(true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f, err := os.Open(testFileName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	r := NewBackupFileReader(f, false)
+	defer r.Close()
+
+	br := NewBackupStreamReader(r)
+	gotData := false
+	gotAltData := false
+	for {
+		hdr, err := br.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		switch hdr.Id {
+		case BackupData:
+			if gotData {
+				t.Fatal("duplicate data")
+			}
+			if hdr.Name != "" {
+				t.Fatalf("unexpected name %s", hdr.Name)
+			}
+			b, err := ioutil.ReadAll(br)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if string(b) != "testing 1 2 3\n" {
+				t.Fatalf("incorrect data %v", b)
+			}
+			gotData = true
+		case BackupAlternateData:
+			if gotAltData {
+				t.Fatal("duplicate alt data")
+			}
+			if hdr.Name != ":ads.txt:$DATA" {
+				t.Fatalf("incorrect name %s", hdr.Name)
+			}
+			b, err := ioutil.ReadAll(br)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if string(b) != "alternate data stream\n" {
+				t.Fatalf("incorrect data %v", b)
+			}
+			gotAltData = true
+		default:
+			t.Fatalf("unknown stream ID %d", hdr.Id)
+		}
+	}
+	if !gotData || !gotAltData {
+		t.Fatal("missing stream")
+	}
+}
+
+func TestBackupStreamWrite(t *testing.T) {
+	f, err := os.Create(testFileName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	w := NewBackupFileWriter(f, false)
+	defer w.Close()
+
+	data := "testing 1 2 3\n"
+	altData := "alternate stream\n"
+
+	br := NewBackupStreamWriter(w)
+	err = br.WriteHeader(&BackupHeader{Id: BackupData, Size: int64(len(data))})
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := br.Write([]byte(data))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != len(data) {
+		t.Fatal("short write")
+	}
+
+	err = br.WriteHeader(&BackupHeader{Id: BackupAlternateData, Size: int64(len(altData)), Name: ":ads.txt:$DATA"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = br.Write([]byte(altData))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != len(altData) {
+		t.Fatal("short write")
+	}
+
+	f.Close()
+
+	b, err := ioutil.ReadFile(testFileName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(b) != data {
+		t.Fatalf("wrong data %v", b)
+	}
+
+	b, err = ioutil.ReadFile(testFileName + ":ads.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(b) != altData {
+		t.Fatalf("wrong data %v", b)
+	}
+}
+
+func makeSparseFile() error {
+	os.Remove(testFileName)
+	f, err := os.Create(testFileName)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	const (
+		FSCTL_SET_SPARSE    = 0x000900c4
+		FSCTL_SET_ZERO_DATA = 0x000980c8
+	)
+
+	err = syscall.DeviceIoControl(syscall.Handle(f.Fd()), FSCTL_SET_SPARSE, nil, 0, nil, 0, nil, nil)
+	if err != nil {
+		return err
+	}
+
+	_, err = f.Write([]byte("testing 1 2 3\n"))
+	if err != nil {
+		return err
+	}
+
+	_, err = f.Seek(1000000, 0)
+	if err != nil {
+		return err
+	}
+
+	_, err = f.Write([]byte("more data later\n"))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func TestBackupSparseFile(t *testing.T) {
+	err := makeSparseFile()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f, err := os.Open(testFileName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	r := NewBackupFileReader(f, false)
+	defer r.Close()
+
+	br := NewBackupStreamReader(r)
+	for {
+		hdr, err := br.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		t.Log(hdr)
+	}
+}

vendor/github.com/Microsoft/go-winio/backuptar/noop.go

@@ -0,0 +1,4 @@
+// +build !windows
+// This file only exists to allow go get on non-Windows platforms.
+
+package backuptar

vendor/github.com/Microsoft/go-winio/backuptar/tar.go

@@ -0,0 +1,439 @@
+// +build windows
+
+package backuptar
+
+import (
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/Microsoft/go-winio"
+	"github.com/Microsoft/go-winio/archive/tar" // until archive/tar supports pax extensions in its interface
+)
+
+const (
+	c_ISUID  = 04000   // Set uid
+	c_ISGID  = 02000   // Set gid
+	c_ISVTX  = 01000   // Save text (sticky bit)
+	c_ISDIR  = 040000  // Directory
+	c_ISFIFO = 010000  // FIFO
+	c_ISREG  = 0100000 // Regular file
+	c_ISLNK  = 0120000 // Symbolic link
+	c_ISBLK  = 060000  // Block special file
+	c_ISCHR  = 020000  // Character special file
+	c_ISSOCK = 0140000 // Socket
+)
+
+const (
+	hdrFileAttributes        = "fileattr"
+	hdrSecurityDescriptor    = "sd"
+	hdrRawSecurityDescriptor = "rawsd"
+	hdrMountPoint            = "mountpoint"
+	hdrEaPrefix              = "xattr."
+)
+
+func writeZeroes(w io.Writer, count int64) error {
+	buf := make([]byte, 8192)
+	c := len(buf)
+	for i := int64(0); i < count; i += int64(c) {
+		if int64(c) > count-i {
+			c = int(count - i)
+		}
+		_, err := w.Write(buf[:c])
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func copySparse(t *tar.Writer, br *winio.BackupStreamReader) error {
+	curOffset := int64(0)
+	for {
+		bhdr, err := br.Next()
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		if err != nil {
+			return err
+		}
+		if bhdr.Id != winio.BackupSparseBlock {
+			return fmt.Errorf("unexpected stream %d", bhdr.Id)
+		}
+
+		// archive/tar does not support writing sparse files
+		// so just write zeroes to catch up to the current offset.
+		err = writeZeroes(t, bhdr.Offset-curOffset)
+		if bhdr.Size == 0 {
+			break
+		}
+		n, err := io.Copy(t, br)
+		if err != nil {
+			return err
+		}
+		curOffset = bhdr.Offset + n
+	}
+	return nil
+}
+
+// BasicInfoHeader creates a tar header from basic file information.
+func BasicInfoHeader(name string, size int64, fileInfo *winio.FileBasicInfo) *tar.Header {
+	hdr := &tar.Header{
+		Name:         filepath.ToSlash(name),
+		Size:         size,
+		Typeflag:     tar.TypeReg,
+		ModTime:      time.Unix(0, fileInfo.LastWriteTime.Nanoseconds()),
+		ChangeTime:   time.Unix(0, fileInfo.ChangeTime.Nanoseconds()),
+		AccessTime:   time.Unix(0, fileInfo.LastAccessTime.Nanoseconds()),
+		CreationTime: time.Unix(0, fileInfo.CreationTime.Nanoseconds()),
+		Winheaders:   make(map[string]string),
+	}
+	hdr.Winheaders[hdrFileAttributes] = fmt.Sprintf("%d", fileInfo.FileAttributes)
+
+	if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
+		hdr.Mode |= c_ISDIR
+		hdr.Size = 0
+		hdr.Typeflag = tar.TypeDir
+	}
+	return hdr
+}
+
+// WriteTarFileFromBackupStream writes a file to a tar writer using data from a Win32 backup stream.
+//
+// This encodes Win32 metadata as tar pax vendor extensions starting with MSWINDOWS.
+//
+// The additional Win32 metadata is:
+//
+// MSWINDOWS.fileattr: The Win32 file attributes, as a decimal value
+//
+// MSWINDOWS.rawsd: The Win32 security descriptor, in raw binary format
+//
+// MSWINDOWS.mountpoint: If present, this is a mount point and not a symlink, even though the type is '2' (symlink)
+func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size int64, fileInfo *winio.FileBasicInfo) error {
+	name = filepath.ToSlash(name)
+	hdr := BasicInfoHeader(name, size, fileInfo)
+
+	// If r can be seeked, then this function is two-pass: pass 1 collects the
+	// tar header data, and pass 2 copies the data stream. If r cannot be
+	// seeked, then some header data (in particular EAs) will be silently lost.
+	var (
+		restartPos int64
+		err        error
+	)
+	sr, readTwice := r.(io.Seeker)
+	if readTwice {
+		if restartPos, err = sr.Seek(0, io.SeekCurrent); err != nil {
+			readTwice = false
+		}
+	}
+
+	br := winio.NewBackupStreamReader(r)
+	var dataHdr *winio.BackupHeader
+	for dataHdr == nil {
+		bhdr, err := br.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		switch bhdr.Id {
+		case winio.BackupData:
+			hdr.Mode |= c_ISREG
+			if !readTwice {
+				dataHdr = bhdr
+			}
+		case winio.BackupSecurity:
+			sd, err := ioutil.ReadAll(br)
+			if err != nil {
+				return err
+			}
+			hdr.Winheaders[hdrRawSecurityDescriptor] = base64.StdEncoding.EncodeToString(sd)
+
+		case winio.BackupReparseData:
+			hdr.Mode |= c_ISLNK
+			hdr.Typeflag = tar.TypeSymlink
+			reparseBuffer, err := ioutil.ReadAll(br)
+			rp, err := winio.DecodeReparsePoint(reparseBuffer)
+			if err != nil {
+				return err
+			}
+			if rp.IsMountPoint {
+				hdr.Winheaders[hdrMountPoint] = "1"
+			}
+			hdr.Linkname = rp.Target
+
+		case winio.BackupEaData:
+			eab, err := ioutil.ReadAll(br)
+			if err != nil {
+				return err
+			}
+			eas, err := winio.DecodeExtendedAttributes(eab)
+			if err != nil {
+				return err
+			}
+			for _, ea := range eas {
+				// Use base64 encoding for the binary value. Note that there
+				// is no way to encode the EA's flags, since their use doesn't
+				// make any sense for persisted EAs.
+				hdr.Winheaders[hdrEaPrefix+ea.Name] = base64.StdEncoding.EncodeToString(ea.Value)
+			}
+
+		case winio.BackupAlternateData, winio.BackupLink, winio.BackupPropertyData, winio.BackupObjectId, winio.BackupTxfsData:
+			// ignore these streams
+		default:
+			return fmt.Errorf("%s: unknown stream ID %d", name, bhdr.Id)
+		}
+	}
+
+	err = t.WriteHeader(hdr)
+	if err != nil {
+		return err
+	}
+
+	if readTwice {
+		// Get back to the data stream.
+		if _, err = sr.Seek(restartPos, io.SeekStart); err != nil {
+			return err
+		}
+		for dataHdr == nil {
+			bhdr, err := br.Next()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return err
+			}
+			if bhdr.Id == winio.BackupData {
+				dataHdr = bhdr
+			}
+		}
+	}
+
+	if dataHdr != nil {
+		// A data stream was found. Copy the data.
+		if (dataHdr.Attributes & winio.StreamSparseAttributes) == 0 {
+			if size != dataHdr.Size {
+				return fmt.Errorf("%s: mismatch between file size %d and header size %d", name, size, dataHdr.Size)
+			}
+			_, err = io.Copy(t, br)
+			if err != nil {
+				return err
+			}
+		} else {
+			err = copySparse(t, br)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// Look for streams after the data stream. The only ones we handle are alternate data streams.
+	// Other streams may have metadata that could be serialized, but the tar header has already
+	// been written. In practice, this means that we don't get EA or TXF metadata.
+	for {
+		bhdr, err := br.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		switch bhdr.Id {
+		case winio.BackupAlternateData:
+			altName := bhdr.Name
+			if strings.HasSuffix(altName, ":$DATA") {
+				altName = altName[:len(altName)-len(":$DATA")]
+			}
+			if (bhdr.Attributes & winio.StreamSparseAttributes) == 0 {
+				hdr = &tar.Header{
+					Name:       name + altName,
+					Mode:       hdr.Mode,
+					Typeflag:   tar.TypeReg,
+					Size:       bhdr.Size,
+					ModTime:    hdr.ModTime,
+					AccessTime: hdr.AccessTime,
+					ChangeTime: hdr.ChangeTime,
+				}
+				err = t.WriteHeader(hdr)
+				if err != nil {
+					return err
+				}
+				_, err = io.Copy(t, br)
+				if err != nil {
+					return err
+				}
+
+			} else {
+				// Unsupported for now, since the size of the alternate stream is not present
+				// in the backup stream until after the data has been read.
+				return errors.New("tar of sparse alternate data streams is unsupported")
+			}
+		case winio.BackupEaData, winio.BackupLink, winio.BackupPropertyData, winio.BackupObjectId, winio.BackupTxfsData:
+			// ignore these streams
+		default:
+			return fmt.Errorf("%s: unknown stream ID %d after data", name, bhdr.Id)
+		}
+	}
+	return nil
+}
+
+// FileInfoFromHeader retrieves basic Win32 file information from a tar header, using the additional metadata written by
+// WriteTarFileFromBackupStream.
+func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *winio.FileBasicInfo, err error) {
+	name = hdr.Name
+	if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA {
+		size = hdr.Size
+	}
+	fileInfo = &winio.FileBasicInfo{
+		LastAccessTime: syscall.NsecToFiletime(hdr.AccessTime.UnixNano()),
+		LastWriteTime:  syscall.NsecToFiletime(hdr.ModTime.UnixNano()),
+		ChangeTime:     syscall.NsecToFiletime(hdr.ChangeTime.UnixNano()),
+		CreationTime:   syscall.NsecToFiletime(hdr.CreationTime.UnixNano()),
+	}
+	if attrStr, ok := hdr.Winheaders[hdrFileAttributes]; ok {
+		attr, err := strconv.ParseUint(attrStr, 10, 32)
+		if err != nil {
+			return "", 0, nil, err
+		}
+		fileInfo.FileAttributes = uintptr(attr)
+	} else {
+		if hdr.Typeflag == tar.TypeDir {
+			fileInfo.FileAttributes |= syscall.FILE_ATTRIBUTE_DIRECTORY
+		}
+	}
+	return
+}
+
+// WriteBackupStreamFromTarFile writes a Win32 backup stream from the current tar file. Since this function may process multiple
+// tar file entries in order to collect all the alternate data streams for the file, it returns the next
+// tar file that was not processed, or io.EOF is there are no more.
+func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (*tar.Header, error) {
+	bw := winio.NewBackupStreamWriter(w)
+	var sd []byte
+	var err error
+	// Maintaining old SDDL-based behavior for backward compatibility.  All new tar headers written
+	// by this library will have raw binary for the security descriptor.
+	if sddl, ok := hdr.Winheaders[hdrSecurityDescriptor]; ok {
+		sd, err = winio.SddlToSecurityDescriptor(sddl)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if sdraw, ok := hdr.Winheaders[hdrRawSecurityDescriptor]; ok {
+		sd, err = base64.StdEncoding.DecodeString(sdraw)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if len(sd) != 0 {
+		bhdr := winio.BackupHeader{
+			Id:   winio.BackupSecurity,
+			Size: int64(len(sd)),
+		}
+		err := bw.WriteHeader(&bhdr)
+		if err != nil {
+			return nil, err
+		}
+		_, err = bw.Write(sd)
+		if err != nil {
+			return nil, err
+		}
+	}
+	var eas []winio.ExtendedAttribute
+	for k, v := range hdr.Winheaders {
+		if !strings.HasPrefix(k, hdrEaPrefix) {
+			continue
+		}
+		data, err := base64.StdEncoding.DecodeString(v)
+		if err != nil {
+			return nil, err
+		}
+		eas = append(eas, winio.ExtendedAttribute{
+			Name:  k[len(hdrEaPrefix):],
+			Value: data,
+		})
+	}
+	if len(eas) != 0 {
+		eadata, err := winio.EncodeExtendedAttributes(eas)
+		if err != nil {
+			return nil, err
+		}
+		bhdr := winio.BackupHeader{
+			Id:   winio.BackupEaData,
+			Size: int64(len(eadata)),
+		}
+		err = bw.WriteHeader(&bhdr)
+		if err != nil {
+			return nil, err
+		}
+		_, err = bw.Write(eadata)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if hdr.Typeflag == tar.TypeSymlink {
+		_, isMountPoint := hdr.Winheaders[hdrMountPoint]
+		rp := winio.ReparsePoint{
+			Target:       filepath.FromSlash(hdr.Linkname),
+			IsMountPoint: isMountPoint,
+		}
+		reparse := winio.EncodeReparsePoint(&rp)
+		bhdr := winio.BackupHeader{
+			Id:   winio.BackupReparseData,
+			Size: int64(len(reparse)),
+		}
+		err := bw.WriteHeader(&bhdr)
+		if err != nil {
+			return nil, err
+		}
+		_, err = bw.Write(reparse)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA {
+		bhdr := winio.BackupHeader{
+			Id:   winio.BackupData,
+			Size: hdr.Size,
+		}
+		err := bw.WriteHeader(&bhdr)
+		if err != nil {
+			return nil, err
+		}
+		_, err = io.Copy(bw, t)
+		if err != nil {
+			return nil, err
+		}
+	}
+	// Copy all the alternate data streams and return the next non-ADS header.
+	for {
+		ahdr, err := t.Next()
+		if err != nil {
+			return nil, err
+		}
+		if ahdr.Typeflag != tar.TypeReg || !strings.HasPrefix(ahdr.Name, hdr.Name+":") {
+			return ahdr, nil
+		}
+		bhdr := winio.BackupHeader{
+			Id:   winio.BackupAlternateData,
+			Size: ahdr.Size,
+			Name: ahdr.Name[len(hdr.Name):] + ":$DATA",
+		}
+		err = bw.WriteHeader(&bhdr)
+		if err != nil {
+			return nil, err
+		}
+		_, err = io.Copy(bw, t)
+		if err != nil {
+			return nil, err
+		}
+	}
+}

vendor/github.com/Microsoft/go-winio/backuptar/tar_test.go

@@ -0,0 +1,84 @@
+package backuptar
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+
+	"github.com/Microsoft/go-winio"
+	"github.com/Microsoft/go-winio/archive/tar"
+)
+
+func ensurePresent(t *testing.T, m map[string]string, keys ...string) {
+	for _, k := range keys {
+		if _, ok := m[k]; !ok {
+			t.Error(k, "not present in tar header")
+		}
+	}
+}
+
+func TestRoundTrip(t *testing.T) {
+	f, err := ioutil.TempFile("", "tst")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	defer os.Remove(f.Name())
+
+	if _, err = f.Write([]byte("testing 1 2 3\n")); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = f.Seek(0, 0); err != nil {
+		t.Fatal(err)
+	}
+
+	fi, err := f.Stat()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bi, err := winio.GetFileBasicInfo(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	br := winio.NewBackupFileReader(f, true)
+	defer br.Close()
+
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+
+	err = WriteTarFileFromBackupStream(tw, br, f.Name(), fi.Size(), bi)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tr := tar.NewReader(&buf)
+	hdr, err := tr.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	name, size, bi2, err := FileInfoFromHeader(hdr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if name != filepath.ToSlash(f.Name()) {
+		t.Errorf("got name %s, expected %s", name, filepath.ToSlash(f.Name()))
+	}
+
+	if size != fi.Size() {
+		t.Errorf("got size %d, expected %d", size, fi.Size())
+	}
+
+	if !reflect.DeepEqual(*bi, *bi2) {
+		t.Errorf("got %#v, expected %#v", *bi, *bi2)
+	}
+
+	ensurePresent(t, hdr.Winheaders, "fileattr", "rawsd")
+}

vendor/github.com/Microsoft/go-winio/ea.go

@@ -0,0 +1,137 @@
+package winio
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+)
+
+type fileFullEaInformation struct {
+	NextEntryOffset uint32
+	Flags           uint8
+	NameLength      uint8
+	ValueLength     uint16
+}
+
+var (
+	fileFullEaInformationSize = binary.Size(&fileFullEaInformation{})
+
+	errInvalidEaBuffer = errors.New("invalid extended attribute buffer")
+	errEaNameTooLarge  = errors.New("extended attribute name too large")
+	errEaValueTooLarge = errors.New("extended attribute value too large")
+)
+
+// ExtendedAttribute represents a single Windows EA.
+type ExtendedAttribute struct {
+	Name  string
+	Value []byte
+	Flags uint8
+}
+
+func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
+	var info fileFullEaInformation
+	err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info)
+	if err != nil {
+		err = errInvalidEaBuffer
+		return
+	}
+
+	nameOffset := fileFullEaInformationSize
+	nameLen := int(info.NameLength)
+	valueOffset := nameOffset + int(info.NameLength) + 1
+	valueLen := int(info.ValueLength)
+	nextOffset := int(info.NextEntryOffset)
+	if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) {
+		err = errInvalidEaBuffer
+		return
+	}
+
+	ea.Name = string(b[nameOffset : nameOffset+nameLen])
+	ea.Value = b[valueOffset : valueOffset+valueLen]
+	ea.Flags = info.Flags
+	if info.NextEntryOffset != 0 {
+		nb = b[info.NextEntryOffset:]
+	}
+	return
+}
+
+// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION
+// buffer retrieved from BackupRead, ZwQueryEaFile, etc.
+func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) {
+	for len(b) != 0 {
+		ea, nb, err := parseEa(b)
+		if err != nil {
+			return nil, err
+		}
+
+		eas = append(eas, ea)
+		b = nb
+	}
+	return
+}
+
+func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error {
+	if int(uint8(len(ea.Name))) != len(ea.Name) {
+		return errEaNameTooLarge
+	}
+	if int(uint16(len(ea.Value))) != len(ea.Value) {
+		return errEaValueTooLarge
+	}
+	entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value))
+	withPadding := (entrySize + 3) &^ 3
+	nextOffset := uint32(0)
+	if !last {
+		nextOffset = withPadding
+	}
+	info := fileFullEaInformation{
+		NextEntryOffset: nextOffset,
+		Flags:           ea.Flags,
+		NameLength:      uint8(len(ea.Name)),
+		ValueLength:     uint16(len(ea.Value)),
+	}
+
+	err := binary.Write(buf, binary.LittleEndian, &info)
+	if err != nil {
+		return err
+	}
+
+	_, err = buf.Write([]byte(ea.Name))
+	if err != nil {
+		return err
+	}
+
+	err = buf.WriteByte(0)
+	if err != nil {
+		return err
+	}
+
+	_, err = buf.Write(ea.Value)
+	if err != nil {
+		return err
+	}
+
+	_, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize])
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION
+// buffer for use with BackupWrite, ZwSetEaFile, etc.
+func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) {
+	var buf bytes.Buffer
+	for i := range eas {
+		last := false
+		if i == len(eas)-1 {
+			last = true
+		}
+
+		err := writeEa(&buf, &eas[i], last)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return buf.Bytes(), nil
+}

vendor/github.com/Microsoft/go-winio/ea_test.go

@@ -0,0 +1,89 @@
+package winio
+
+import (
+	"io/ioutil"
+	"os"
+	"reflect"
+	"syscall"
+	"testing"
+	"unsafe"
+)
+
+var (
+	testEas = []ExtendedAttribute{
+		{Name: "foo", Value: []byte("bar")},
+		{Name: "fizz", Value: []byte("buzz")},
+	}
+
+	testEasEncoded   = []byte{16, 0, 0, 0, 0, 3, 3, 0, 102, 111, 111, 0, 98, 97, 114, 0, 0, 0, 0, 0, 0, 4, 4, 0, 102, 105, 122, 122, 0, 98, 117, 122, 122, 0, 0, 0}
+	testEasNotPadded = testEasEncoded[0 : len(testEasEncoded)-3]
+	testEasTruncated = testEasEncoded[0:20]
+)
+
+func Test_RoundTripEas(t *testing.T) {
+	b, err := EncodeExtendedAttributes(testEas)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(testEasEncoded, b) {
+		t.Fatalf("encoded mismatch %v %v", testEasEncoded, b)
+	}
+	eas, err := DecodeExtendedAttributes(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(testEas, eas) {
+		t.Fatalf("mismatch %+v %+v", testEas, eas)
+	}
+}
+
+func Test_EasDontNeedPaddingAtEnd(t *testing.T) {
+	eas, err := DecodeExtendedAttributes(testEasNotPadded)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(testEas, eas) {
+		t.Fatalf("mismatch %+v %+v", testEas, eas)
+	}
+}
+
+func Test_TruncatedEasFailCorrectly(t *testing.T) {
+	_, err := DecodeExtendedAttributes(testEasTruncated)
+	if err == nil {
+		t.Fatal("expected error")
+	}
+}
+
+func Test_NilEasEncodeAndDecodeAsNil(t *testing.T) {
+	b, err := EncodeExtendedAttributes(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(b) != 0 {
+		t.Fatal("expected empty")
+	}
+	eas, err := DecodeExtendedAttributes(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(eas) != 0 {
+		t.Fatal("expected empty")
+	}
+}
+
+// Test_SetFileEa makes sure that the test buffer is actually parsable by NtSetEaFile.
+func Test_SetFileEa(t *testing.T) {
+	f, err := ioutil.TempFile("", "winio")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(f.Name())
+	defer f.Close()
+	ntdll := syscall.MustLoadDLL("ntdll.dll")
+	ntSetEaFile := ntdll.MustFindProc("NtSetEaFile")
+	var iosb [2]uintptr
+	r, _, _ := ntSetEaFile.Call(f.Fd(), uintptr(unsafe.Pointer(&iosb[0])), uintptr(unsafe.Pointer(&testEasEncoded[0])), uintptr(len(testEasEncoded)))
+	if r != 0 {
+		t.Fatalf("NtSetEaFile failed with %08x", r)
+	}
+}

vendor/github.com/Microsoft/go-winio/file.go

@@ -0,0 +1,310 @@
+// +build windows
+
+package winio
+
+import (
+	"errors"
+	"io"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"time"
+)
+
+//sys cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) = CancelIoEx
+//sys createIoCompletionPort(file syscall.Handle, port syscall.Handle, key uintptr, threadCount uint32) (newport syscall.Handle, err error) = CreateIoCompletionPort
+//sys getQueuedCompletionStatus(port syscall.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) = GetQueuedCompletionStatus
+//sys setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err error) = SetFileCompletionNotificationModes
+//sys timeBeginPeriod(period uint32) (n int32) = winmm.timeBeginPeriod
+
+type atomicBool int32
+
+func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 }
+func (b *atomicBool) setFalse()   { atomic.StoreInt32((*int32)(b), 0) }
+func (b *atomicBool) setTrue()    { atomic.StoreInt32((*int32)(b), 1) }
+func (b *atomicBool) swap(new bool) bool {
+	var newInt int32
+	if new {
+		newInt = 1
+	}
+	return atomic.SwapInt32((*int32)(b), newInt) == 1
+}
+
+const (
+	cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS = 1
+	cFILE_SKIP_SET_EVENT_ON_HANDLE        = 2
+)
+
+var (
+	ErrFileClosed = errors.New("file has already been closed")
+	ErrTimeout    = &timeoutError{}
+)
+
+type timeoutError struct{}
+
+func (e *timeoutError) Error() string   { return "i/o timeout" }
+func (e *timeoutError) Timeout() bool   { return true }
+func (e *timeoutError) Temporary() bool { return true }
+
+type timeoutChan chan struct{}
+
+var ioInitOnce sync.Once
+var ioCompletionPort syscall.Handle
+
+// ioResult contains the result of an asynchronous IO operation
+type ioResult struct {
+	bytes uint32
+	err   error
+}
+
+// ioOperation represents an outstanding asynchronous Win32 IO
+type ioOperation struct {
+	o  syscall.Overlapped
+	ch chan ioResult
+}
+
+func initIo() {
+	h, err := createIoCompletionPort(syscall.InvalidHandle, 0, 0, 0xffffffff)
+	if err != nil {
+		panic(err)
+	}
+	ioCompletionPort = h
+	go ioCompletionProcessor(h)
+}
+
+// win32File implements Reader, Writer, and Closer on a Win32 handle without blocking in a syscall.
+// It takes ownership of this handle and will close it if it is garbage collected.
+type win32File struct {
+	handle        syscall.Handle
+	wg            sync.WaitGroup
+	wgLock        sync.RWMutex
+	closing       atomicBool
+	readDeadline  deadlineHandler
+	writeDeadline deadlineHandler
+}
+
+type deadlineHandler struct {
+	setLock     sync.Mutex
+	channel     timeoutChan
+	channelLock sync.RWMutex
+	timer       *time.Timer
+	timedout    atomicBool
+}
+
+// makeWin32File makes a new win32File from an existing file handle
+func makeWin32File(h syscall.Handle) (*win32File, error) {
+	f := &win32File{handle: h}
+	ioInitOnce.Do(initIo)
+	_, err := createIoCompletionPort(h, ioCompletionPort, 0, 0xffffffff)
+	if err != nil {
+		return nil, err
+	}
+	err = setFileCompletionNotificationModes(h, cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS|cFILE_SKIP_SET_EVENT_ON_HANDLE)
+	if err != nil {
+		return nil, err
+	}
+	f.readDeadline.channel = make(timeoutChan)
+	f.writeDeadline.channel = make(timeoutChan)
+	return f, nil
+}
+
+func MakeOpenFile(h syscall.Handle) (io.ReadWriteCloser, error) {
+	return makeWin32File(h)
+}
+
+// closeHandle closes the resources associated with a Win32 handle
+func (f *win32File) closeHandle() {
+	f.wgLock.Lock()
+	// Atomically set that we are closing, releasing the resources only once.
+	if !f.closing.swap(true) {
+		f.wgLock.Unlock()
+		// cancel all IO and wait for it to complete
+		cancelIoEx(f.handle, nil)
+		f.wg.Wait()
+		// at this point, no new IO can start
+		syscall.Close(f.handle)
+		f.handle = 0
+	} else {
+		f.wgLock.Unlock()
+	}
+}
+
+// Close closes a win32File.
+func (f *win32File) Close() error {
+	f.closeHandle()
+	return nil
+}
+
+// prepareIo prepares for a new IO operation.
+// The caller must call f.wg.Done() when the IO is finished, prior to Close() returning.
+func (f *win32File) prepareIo() (*ioOperation, error) {
+	f.wgLock.RLock()
+	if f.closing.isSet() {
+		f.wgLock.RUnlock()
+		return nil, ErrFileClosed
+	}
+	f.wg.Add(1)
+	f.wgLock.RUnlock()
+	c := &ioOperation{}
+	c.ch = make(chan ioResult)
+	return c, nil
+}
+
+// ioCompletionProcessor processes completed async IOs forever
+func ioCompletionProcessor(h syscall.Handle) {
+	// Set the timer resolution to 1. This fixes a performance regression in golang 1.6.
+	timeBeginPeriod(1)
+	for {
+		var bytes uint32
+		var key uintptr
+		var op *ioOperation
+		err := getQueuedCompletionStatus(h, &bytes, &key, &op, syscall.INFINITE)
+		if op == nil {
+			panic(err)
+		}
+		op.ch <- ioResult{bytes, err}
+	}
+}
+
+// asyncIo processes the return value from ReadFile or WriteFile, blocking until
+// the operation has actually completed.
+func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) {
+	if err != syscall.ERROR_IO_PENDING {
+		return int(bytes), err
+	}
+
+	if f.closing.isSet() {
+		cancelIoEx(f.handle, &c.o)
+	}
+
+	var timeout timeoutChan
+	if d != nil {
+		d.channelLock.Lock()
+		timeout = d.channel
+		d.channelLock.Unlock()
+	}
+
+	var r ioResult
+	select {
+	case r = <-c.ch:
+		err = r.err
+		if err == syscall.ERROR_OPERATION_ABORTED {
+			if f.closing.isSet() {
+				err = ErrFileClosed
+			}
+		}
+	case <-timeout:
+		cancelIoEx(f.handle, &c.o)
+		r = <-c.ch
+		err = r.err
+		if err == syscall.ERROR_OPERATION_ABORTED {
+			err = ErrTimeout
+		}
+	}
+
+	// runtime.KeepAlive is needed, as c is passed via native
+	// code to ioCompletionProcessor, c must remain alive
+	// until the channel read is complete.
+	runtime.KeepAlive(c)
+	return int(r.bytes), err
+}
+
+// Read reads from a file handle.
+func (f *win32File) Read(b []byte) (int, error) {
+	c, err := f.prepareIo()
+	if err != nil {
+		return 0, err
+	}
+	defer f.wg.Done()
+
+	if f.readDeadline.timedout.isSet() {
+		return 0, ErrTimeout
+	}
+
+	var bytes uint32
+	err = syscall.ReadFile(f.handle, b, &bytes, &c.o)
+	n, err := f.asyncIo(c, &f.readDeadline, bytes, err)
+	runtime.KeepAlive(b)
+
+	// Handle EOF conditions.
+	if err == nil && n == 0 && len(b) != 0 {
+		return 0, io.EOF
+	} else if err == syscall.ERROR_BROKEN_PIPE {
+		return 0, io.EOF
+	} else {
+		return n, err
+	}
+}
+
+// Write writes to a file handle.
+func (f *win32File) Write(b []byte) (int, error) {
+	c, err := f.prepareIo()
+	if err != nil {
+		return 0, err
+	}
+	defer f.wg.Done()
+
+	if f.writeDeadline.timedout.isSet() {
+		return 0, ErrTimeout
+	}
+
+	var bytes uint32
+	err = syscall.WriteFile(f.handle, b, &bytes, &c.o)
+	n, err := f.asyncIo(c, &f.writeDeadline, bytes, err)
+	runtime.KeepAlive(b)
+	return n, err
+}
+
+func (f *win32File) SetReadDeadline(deadline time.Time) error {
+	return f.readDeadline.set(deadline)
+}
+
+func (f *win32File) SetWriteDeadline(deadline time.Time) error {
+	return f.writeDeadline.set(deadline)
+}
+
+func (f *win32File) Flush() error {
+	return syscall.FlushFileBuffers(f.handle)
+}
+
+func (d *deadlineHandler) set(deadline time.Time) error {
+	d.setLock.Lock()
+	defer d.setLock.Unlock()
+
+	if d.timer != nil {
+		if !d.timer.Stop() {
+			<-d.channel
+		}
+		d.timer = nil
+	}
+	d.timedout.setFalse()
+
+	select {
+	case <-d.channel:
+		d.channelLock.Lock()
+		d.channel = make(chan struct{})
+		d.channelLock.Unlock()
+	default:
+	}
+
+	if deadline.IsZero() {
+		return nil
+	}
+
+	timeoutIO := func() {
+		d.timedout.setTrue()
+		close(d.channel)
+	}
+
+	now := time.Now()
+	duration := deadline.Sub(now)
+	if deadline.After(now) {
+		// Deadline is in the future, set a timer to wait
+		d.timer = time.AfterFunc(duration, timeoutIO)
+	} else {
+		// Deadline is in the past. Cancel all pending IO now.
+		timeoutIO()
+	}
+	return nil
+}

vendor/github.com/Microsoft/go-winio/fileinfo.go

@@ -0,0 +1,60 @@
+// +build windows
+
+package winio
+
+import (
+	"os"
+	"runtime"
+	"syscall"
+	"unsafe"
+)
+
+//sys getFileInformationByHandleEx(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = GetFileInformationByHandleEx
+//sys setFileInformationByHandle(h syscall.Handle, class uint32, buffer *byte, size uint32) (err error) = SetFileInformationByHandle
+
+const (
+	fileBasicInfo = 0
+	fileIDInfo    = 0x12
+)
+
+// FileBasicInfo contains file access time and file attributes information.
+type FileBasicInfo struct {
+	CreationTime, LastAccessTime, LastWriteTime, ChangeTime syscall.Filetime
+	FileAttributes                                          uintptr // includes padding
+}
+
+// GetFileBasicInfo retrieves times and attributes for a file.
+func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
+	bi := &FileBasicInfo{}
+	if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
+		return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
+	}
+	runtime.KeepAlive(f)
+	return bi, nil
+}
+
+// SetFileBasicInfo sets times and attributes for a file.
+func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error {
+	if err := setFileInformationByHandle(syscall.Handle(f.Fd()), fileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
+		return &os.PathError{Op: "SetFileInformationByHandle", Path: f.Name(), Err: err}
+	}
+	runtime.KeepAlive(f)
+	return nil
+}
+
+// FileIDInfo contains the volume serial number and file ID for a file. This pair should be
+// unique on a system.
+type FileIDInfo struct {
+	VolumeSerialNumber uint64
+	FileID             [16]byte
+}
+
+// GetFileID retrieves the unique (volume, file ID) pair for a file.
+func GetFileID(f *os.File) (*FileIDInfo, error) {
+	fileID := &FileIDInfo{}
+	if err := getFileInformationByHandleEx(syscall.Handle(f.Fd()), fileIDInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
+		return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
+	}
+	runtime.KeepAlive(f)
+	return fileID, nil
+}

vendor/github.com/Microsoft/go-winio/pipe.go

@@ -0,0 +1,404 @@
+// +build windows
+
+package winio
+
+import (
+	"errors"
+	"io"
+	"net"
+	"os"
+	"syscall"
+	"time"
+	"unsafe"
+)
+
+//sys connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) = ConnectNamedPipe
+//sys createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error)  [failretval==syscall.InvalidHandle] = CreateNamedPipeW
+//sys createFile(name string, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) [failretval==syscall.InvalidHandle] = CreateFileW
+//sys waitNamedPipe(name string, timeout uint32) (err error) = WaitNamedPipeW
+//sys getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) = GetNamedPipeInfo
+//sys getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW
+//sys localAlloc(uFlags uint32, length uint32) (ptr uintptr) = LocalAlloc
+
+const (
+	cERROR_PIPE_BUSY      = syscall.Errno(231)
+	cERROR_PIPE_CONNECTED = syscall.Errno(535)
+	cERROR_SEM_TIMEOUT    = syscall.Errno(121)
+
+	cPIPE_ACCESS_DUPLEX            = 0x3
+	cFILE_FLAG_FIRST_PIPE_INSTANCE = 0x80000
+	cSECURITY_SQOS_PRESENT         = 0x100000
+	cSECURITY_ANONYMOUS            = 0
+
+	cPIPE_REJECT_REMOTE_CLIENTS = 0x8
+
+	cPIPE_UNLIMITED_INSTANCES = 255
+
+	cNMPWAIT_USE_DEFAULT_WAIT = 0
+	cNMPWAIT_NOWAIT           = 1
+
+	cPIPE_TYPE_MESSAGE = 4
+
+	cPIPE_READMODE_MESSAGE = 2
+)
+
+var (
+	// ErrPipeListenerClosed is returned for pipe operations on listeners that have been closed.
+	// This error should match net.errClosing since docker takes a dependency on its text.
+	ErrPipeListenerClosed = errors.New("use of closed network connection")
+
+	errPipeWriteClosed = errors.New("pipe has been closed for write")
+)
+
+type win32Pipe struct {
+	*win32File
+	path string
+}
+
+type win32MessageBytePipe struct {
+	win32Pipe
+	writeClosed bool
+	readEOF     bool
+}
+
+type pipeAddress string
+
+func (f *win32Pipe) LocalAddr() net.Addr {
+	return pipeAddress(f.path)
+}
+
+func (f *win32Pipe) RemoteAddr() net.Addr {
+	return pipeAddress(f.path)
+}
+
+func (f *win32Pipe) SetDeadline(t time.Time) error {
+	f.SetReadDeadline(t)
+	f.SetWriteDeadline(t)
+	return nil
+}
+
+// CloseWrite closes the write side of a message pipe in byte mode.
+func (f *win32MessageBytePipe) CloseWrite() error {
+	if f.writeClosed {
+		return errPipeWriteClosed
+	}
+	err := f.win32File.Flush()
+	if err != nil {
+		return err
+	}
+	_, err = f.win32File.Write(nil)
+	if err != nil {
+		return err
+	}
+	f.writeClosed = true
+	return nil
+}
+
+// Write writes bytes to a message pipe in byte mode. Zero-byte writes are ignored, since
+// they are used to implement CloseWrite().
+func (f *win32MessageBytePipe) Write(b []byte) (int, error) {
+	if f.writeClosed {
+		return 0, errPipeWriteClosed
+	}
+	if len(b) == 0 {
+		return 0, nil
+	}
+	return f.win32File.Write(b)
+}
+
+// Read reads bytes from a message pipe in byte mode. A read of a zero-byte message on a message
+// mode pipe will return io.EOF, as will all subsequent reads.
+func (f *win32MessageBytePipe) Read(b []byte) (int, error) {
+	if f.readEOF {
+		return 0, io.EOF
+	}
+	n, err := f.win32File.Read(b)
+	if err == io.EOF {
+		// If this was the result of a zero-byte read, then
+		// it is possible that the read was due to a zero-size
+		// message. Since we are simulating CloseWrite with a
+		// zero-byte message, ensure that all future Read() calls
+		// also return EOF.
+		f.readEOF = true
+	}
+	return n, err
+}
+
+func (s pipeAddress) Network() string {
+	return "pipe"
+}
+
+func (s pipeAddress) String() string {
+	return string(s)
+}
+
+// DialPipe connects to a named pipe by path, timing out if the connection
+// takes longer than the specified duration. If timeout is nil, then the timeout
+// is the default timeout established by the pipe server.
+func DialPipe(path string, timeout *time.Duration) (net.Conn, error) {
+	var absTimeout time.Time
+	if timeout != nil {
+		absTimeout = time.Now().Add(*timeout)
+	}
+	var err error
+	var h syscall.Handle
+	for {
+		h, err = createFile(path, syscall.GENERIC_READ|syscall.GENERIC_WRITE, 0, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_OVERLAPPED|cSECURITY_SQOS_PRESENT|cSECURITY_ANONYMOUS, 0)
+		if err != cERROR_PIPE_BUSY {
+			break
+		}
+		now := time.Now()
+		var ms uint32
+		if absTimeout.IsZero() {
+			ms = cNMPWAIT_USE_DEFAULT_WAIT
+		} else if now.After(absTimeout) {
+			ms = cNMPWAIT_NOWAIT
+		} else {
+			ms = uint32(absTimeout.Sub(now).Nanoseconds() / 1000 / 1000)
+		}
+		err = waitNamedPipe(path, ms)
+		if err != nil {
+			if err == cERROR_SEM_TIMEOUT {
+				return nil, ErrTimeout
+			}
+			break
+		}
+	}
+	if err != nil {
+		return nil, &os.PathError{Op: "open", Path: path, Err: err}
+	}
+
+	var flags uint32
+	err = getNamedPipeInfo(h, &flags, nil, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var state uint32
+	err = getNamedPipeHandleState(h, &state, nil, nil, nil, nil, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	if state&cPIPE_READMODE_MESSAGE != 0 {
+		return nil, &os.PathError{Op: "open", Path: path, Err: errors.New("message readmode pipes not supported")}
+	}
+
+	f, err := makeWin32File(h)
+	if err != nil {
+		syscall.Close(h)
+		return nil, err
+	}
+
+	// If the pipe is in message mode, return a message byte pipe, which
+	// supports CloseWrite().
+	if flags&cPIPE_TYPE_MESSAGE != 0 {
+		return &win32MessageBytePipe{
+			win32Pipe: win32Pipe{win32File: f, path: path},
+		}, nil
+	}
+	return &win32Pipe{win32File: f, path: path}, nil
+}
+
+type acceptResponse struct {
+	f   *win32File
+	err error
+}
+
+type win32PipeListener struct {
+	firstHandle        syscall.Handle
+	path               string
+	securityDescriptor []byte
+	config             PipeConfig
+	acceptCh           chan (chan acceptResponse)
+	closeCh            chan int
+	doneCh             chan int
+}
+
+func makeServerPipeHandle(path string, securityDescriptor []byte, c *PipeConfig, first bool) (syscall.Handle, error) {
+	var flags uint32 = cPIPE_ACCESS_DUPLEX | syscall.FILE_FLAG_OVERLAPPED
+	if first {
+		flags |= cFILE_FLAG_FIRST_PIPE_INSTANCE
+	}
+
+	var mode uint32 = cPIPE_REJECT_REMOTE_CLIENTS
+	if c.MessageMode {
+		mode |= cPIPE_TYPE_MESSAGE
+	}
+
+	sa := &syscall.SecurityAttributes{}
+	sa.Length = uint32(unsafe.Sizeof(*sa))
+	if securityDescriptor != nil {
+		len := uint32(len(securityDescriptor))
+		sa.SecurityDescriptor = localAlloc(0, len)
+		defer localFree(sa.SecurityDescriptor)
+		copy((*[0xffff]byte)(unsafe.Pointer(sa.SecurityDescriptor))[:], securityDescriptor)
+	}
+	h, err := createNamedPipe(path, flags, mode, cPIPE_UNLIMITED_INSTANCES, uint32(c.OutputBufferSize), uint32(c.InputBufferSize), 0, sa)
+	if err != nil {
+		return 0, &os.PathError{Op: "open", Path: path, Err: err}
+	}
+	return h, nil
+}
+
+func (l *win32PipeListener) makeServerPipe() (*win32File, error) {
+	h, err := makeServerPipeHandle(l.path, l.securityDescriptor, &l.config, false)
+	if err != nil {
+		return nil, err
+	}
+	f, err := makeWin32File(h)
+	if err != nil {
+		syscall.Close(h)
+		return nil, err
+	}
+	return f, nil
+}
+
+func (l *win32PipeListener) listenerRoutine() {
+	closed := false
+	for !closed {
+		select {
+		case <-l.closeCh:
+			closed = true
+		case responseCh := <-l.acceptCh:
+			p, err := l.makeServerPipe()
+			if err == nil {
+				// Wait for the client to connect.
+				ch := make(chan error)
+				go func(p *win32File) {
+					ch <- connectPipe(p)
+				}(p)
+				select {
+				case err = <-ch:
+					if err != nil {
+						p.Close()
+						p = nil
+					}
+				case <-l.closeCh:
+					// Abort the connect request by closing the handle.
+					p.Close()
+					p = nil
+					err = <-ch
+					if err == nil || err == ErrFileClosed {
+						err = ErrPipeListenerClosed
+					}
+					closed = true
+				}
+			}
+			responseCh <- acceptResponse{p, err}
+		}
+	}
+	syscall.Close(l.firstHandle)
+	l.firstHandle = 0
+	// Notify Close() and Accept() callers that the handle has been closed.
+	close(l.doneCh)
+}
+
+// PipeConfig contain configuration for the pipe listener.
+type PipeConfig struct {
+	// SecurityDescriptor contains a Windows security descriptor in SDDL format.
+	SecurityDescriptor string
+
+	// MessageMode determines whether the pipe is in byte or message mode. In either
+	// case the pipe is read in byte mode by default. The only practical difference in
+	// this implementation is that CloseWrite() is only supported for message mode pipes;
+	// CloseWrite() is implemented as a zero-byte write, but zero-byte writes are only
+	// transferred to the reader (and returned as io.EOF in this implementation)
+	// when the pipe is in message mode.
+	MessageMode bool
+
+	// InputBufferSize specifies the size the input buffer, in bytes.
+	InputBufferSize int32
+
+	// OutputBufferSize specifies the size the input buffer, in bytes.
+	OutputBufferSize int32
+}
+
+// ListenPipe creates a listener on a Windows named pipe path, e.g. \\.\pipe\mypipe.
+// The pipe must not already exist.
+func ListenPipe(path string, c *PipeConfig) (net.Listener, error) {
+	var (
+		sd  []byte
+		err error
+	)
+	if c == nil {
+		c = &PipeConfig{}
+	}
+	if c.SecurityDescriptor != "" {
+		sd, err = SddlToSecurityDescriptor(c.SecurityDescriptor)
+		if err != nil {
+			return nil, err
+		}
+	}
+	h, err := makeServerPipeHandle(path, sd, c, true)
+	if err != nil {
+		return nil, err
+	}
+	// Immediately open and then close a client handle so that the named pipe is
+	// created but not currently accepting connections.
+	h2, err := createFile(path, 0, 0, nil, syscall.OPEN_EXISTING, cSECURITY_SQOS_PRESENT|cSECURITY_ANONYMOUS, 0)
+	if err != nil {
+		syscall.Close(h)
+		return nil, err
+	}
+	syscall.Close(h2)
+	l := &win32PipeListener{
+		firstHandle:        h,
+		path:               path,
+		securityDescriptor: sd,
+		config:             *c,
+		acceptCh:           make(chan (chan acceptResponse)),
+		closeCh:            make(chan int),
+		doneCh:             make(chan int),
+	}
+	go l.listenerRoutine()
+	return l, nil
+}
+
+func connectPipe(p *win32File) error {
+	c, err := p.prepareIo()
+	if err != nil {
+		return err
+	}
+	defer p.wg.Done()
+
+	err = connectNamedPipe(p.handle, &c.o)
+	_, err = p.asyncIo(c, nil, 0, err)
+	if err != nil && err != cERROR_PIPE_CONNECTED {
+		return err
+	}
+	return nil
+}
+
+func (l *win32PipeListener) Accept() (net.Conn, error) {
+	ch := make(chan acceptResponse)
+	select {
+	case l.acceptCh <- ch:
+		response := <-ch
+		err := response.err
+		if err != nil {
+			return nil, err
+		}
+		if l.config.MessageMode {
+			return &win32MessageBytePipe{
+				win32Pipe: win32Pipe{win32File: response.f, path: l.path},
+			}, nil
+		}
+		return &win32Pipe{win32File: response.f, path: l.path}, nil
+	case <-l.doneCh:
+		return nil, ErrPipeListenerClosed
+	}
+}
+
+func (l *win32PipeListener) Close() error {
+	select {
+	case l.closeCh <- 1:
+		<-l.doneCh
+	case <-l.doneCh:
+	}
+	return nil
+}
+
+func (l *win32PipeListener) Addr() net.Addr {
+	return pipeAddress(l.path)
+}

vendor/github.com/Microsoft/go-winio/pipe_test.go

@@ -0,0 +1,424 @@
+package winio
+
+import (
+	"bufio"
+	"io"
+	"net"
+	"os"
+	"syscall"
+	"testing"
+	"time"
+)
+
+var testPipeName = `\\.\pipe\winiotestpipe`
+
+var aLongTimeAgo = time.Unix(1, 0)
+
+func TestDialUnknownFailsImmediately(t *testing.T) {
+	_, err := DialPipe(testPipeName, nil)
+	if err.(*os.PathError).Err != syscall.ENOENT {
+		t.Fatalf("expected ENOENT got %v", err)
+	}
+}
+
+func TestDialListenerTimesOut(t *testing.T) {
+	l, err := ListenPipe(testPipeName, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+	var d = time.Duration(10 * time.Millisecond)
+	_, err = DialPipe(testPipeName, &d)
+	if err != ErrTimeout {
+		t.Fatalf("expected ErrTimeout, got %v", err)
+	}
+}
+
+func TestDialAccessDeniedWithRestrictedSD(t *testing.T) {
+	c := PipeConfig{
+		SecurityDescriptor: "D:P(A;;0x1200FF;;;WD)",
+	}
+	l, err := ListenPipe(testPipeName, &c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+	_, err = DialPipe(testPipeName, nil)
+	if err.(*os.PathError).Err != syscall.ERROR_ACCESS_DENIED {
+		t.Fatalf("expected ERROR_ACCESS_DENIED, got %v", err)
+	}
+}
+
+func getConnection(cfg *PipeConfig) (client net.Conn, server net.Conn, err error) {
+	l, err := ListenPipe(testPipeName, cfg)
+	if err != nil {
+		return
+	}
+	defer l.Close()
+
+	type response struct {
+		c   net.Conn
+		err error
+	}
+	ch := make(chan response)
+	go func() {
+		c, err := l.Accept()
+		ch <- response{c, err}
+	}()
+
+	c, err := DialPipe(testPipeName, nil)
+	if err != nil {
+		return
+	}
+
+	r := <-ch
+	if err = r.err; err != nil {
+		c.Close()
+		return
+	}
+
+	client = c
+	server = r.c
+	return
+}
+
+func TestReadTimeout(t *testing.T) {
+	c, s, err := getConnection(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	defer s.Close()
+
+	c.SetReadDeadline(time.Now().Add(10 * time.Millisecond))
+
+	buf := make([]byte, 10)
+	_, err = c.Read(buf)
+	if err != ErrTimeout {
+		t.Fatalf("expected ErrTimeout, got %v", err)
+	}
+}
+
+func server(l net.Listener, ch chan int) {
+	c, err := l.Accept()
+	if err != nil {
+		panic(err)
+	}
+	rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
+	s, err := rw.ReadString('\n')
+	if err != nil {
+		panic(err)
+	}
+	_, err = rw.WriteString("got " + s)
+	if err != nil {
+		panic(err)
+	}
+	err = rw.Flush()
+	if err != nil {
+		panic(err)
+	}
+	c.Close()
+	ch <- 1
+}
+
+func TestFullListenDialReadWrite(t *testing.T) {
+	l, err := ListenPipe(testPipeName, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+
+	ch := make(chan int)
+	go server(l, ch)
+
+	c, err := DialPipe(testPipeName, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+
+	rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
+	_, err = rw.WriteString("hello world\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = rw.Flush()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, err := rw.ReadString('\n')
+	if err != nil {
+		t.Fatal(err)
+	}
+	ms := "got hello world\n"
+	if s != ms {
+		t.Errorf("expected '%s', got '%s'", ms, s)
+	}
+
+	<-ch
+}
+
+func TestCloseAbortsListen(t *testing.T) {
+	l, err := ListenPipe(testPipeName, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ch := make(chan error)
+	go func() {
+		_, err := l.Accept()
+		ch <- err
+	}()
+
+	time.Sleep(30 * time.Millisecond)
+	l.Close()
+
+	err = <-ch
+	if err != ErrPipeListenerClosed {
+		t.Fatalf("expected ErrPipeListenerClosed, got %v", err)
+	}
+}
+
+func ensureEOFOnClose(t *testing.T, r io.Reader, w io.Closer) {
+	b := make([]byte, 10)
+	w.Close()
+	n, err := r.Read(b)
+	if n > 0 {
+		t.Errorf("unexpected byte count %d", n)
+	}
+	if err != io.EOF {
+		t.Errorf("expected EOF: %v", err)
+	}
+}
+
+func TestCloseClientEOFServer(t *testing.T) {
+	c, s, err := getConnection(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	defer s.Close()
+	ensureEOFOnClose(t, c, s)
+}
+
+func TestCloseServerEOFClient(t *testing.T) {
+	c, s, err := getConnection(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	defer s.Close()
+	ensureEOFOnClose(t, s, c)
+}
+
+func TestCloseWriteEOF(t *testing.T) {
+	cfg := &PipeConfig{
+		MessageMode: true,
+	}
+	c, s, err := getConnection(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	defer s.Close()
+
+	type closeWriter interface {
+		CloseWrite() error
+	}
+
+	err = c.(closeWriter).CloseWrite()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b := make([]byte, 10)
+	_, err = s.Read(b)
+	if err != io.EOF {
+		t.Fatal(err)
+	}
+}
+
+func TestAcceptAfterCloseFails(t *testing.T) {
+	l, err := ListenPipe(testPipeName, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	l.Close()
+	_, err = l.Accept()
+	if err != ErrPipeListenerClosed {
+		t.Fatalf("expected ErrPipeListenerClosed, got %v", err)
+	}
+}
+
+func TestDialTimesOutByDefault(t *testing.T) {
+	l, err := ListenPipe(testPipeName, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+	_, err = DialPipe(testPipeName, nil)
+	if err != ErrTimeout {
+		t.Fatalf("expected ErrTimeout, got %v", err)
+	}
+}
+
+func TestTimeoutPendingRead(t *testing.T) {
+	l, err := ListenPipe(testPipeName, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+
+	serverDone := make(chan struct{})
+
+	go func() {
+		s, err := l.Accept()
+		if err != nil {
+			t.Fatal(err)
+		}
+		time.Sleep(1 * time.Second)
+		s.Close()
+		close(serverDone)
+	}()
+
+	client, err := DialPipe(testPipeName, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+
+	clientErr := make(chan error)
+	go func() {
+		buf := make([]byte, 10)
+		_, err = client.Read(buf)
+		clientErr <- err
+	}()
+
+	time.Sleep(100 * time.Millisecond) // make *sure* the pipe is reading before we set the deadline
+	client.SetReadDeadline(aLongTimeAgo)
+
+	select {
+	case err = <-clientErr:
+		if err != ErrTimeout {
+			t.Fatalf("expected ErrTimeout, got %v", err)
+		}
+	case <-time.After(100 * time.Millisecond):
+		t.Fatalf("timed out while waiting for read to cancel")
+		<-clientErr
+	}
+	<-serverDone
+}
+
+func TestTimeoutPendingWrite(t *testing.T) {
+	l, err := ListenPipe(testPipeName, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+
+	serverDone := make(chan struct{})
+
+	go func() {
+		s, err := l.Accept()
+		if err != nil {
+			t.Fatal(err)
+		}
+		time.Sleep(1 * time.Second)
+		s.Close()
+		close(serverDone)
+	}()
+
+	client, err := DialPipe(testPipeName, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+
+	clientErr := make(chan error)
+	go func() {
+		_, err = client.Write([]byte("this should timeout"))
+		clientErr <- err
+	}()
+
+	time.Sleep(100 * time.Millisecond) // make *sure* the pipe is writing before we set the deadline
+	client.SetWriteDeadline(aLongTimeAgo)
+
+	select {
+	case err = <-clientErr:
+		if err != ErrTimeout {
+			t.Fatalf("expected ErrTimeout, got %v", err)
+		}
+	case <-time.After(100 * time.Millisecond):
+		t.Fatalf("timed out while waiting for write to cancel")
+		<-clientErr
+	}
+	<-serverDone
+}
+
+type CloseWriter interface {
+	CloseWrite() error
+}
+
+func TestEchoWithMessaging(t *testing.T) {
+	c := PipeConfig{
+		MessageMode:      true,  // Use message mode so that CloseWrite() is supported
+		InputBufferSize:  65536, // Use 64KB buffers to improve performance
+		OutputBufferSize: 65536,
+	}
+	l, err := ListenPipe(testPipeName, &c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+
+	listenerDone := make(chan bool)
+	clientDone := make(chan bool)
+	go func() {
+		// server echo
+		conn, e := l.Accept()
+		if e != nil {
+			t.Fatal(e)
+		}
+		defer conn.Close()
+
+		time.Sleep(500 * time.Millisecond) // make *sure* we don't begin to read before eof signal is sent
+		io.Copy(conn, conn)
+		conn.(CloseWriter).CloseWrite()
+		close(listenerDone)
+	}()
+	timeout := 1 * time.Second
+	client, err := DialPipe(testPipeName, &timeout)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+
+	go func() {
+		// client read back
+		bytes := make([]byte, 2)
+		n, e := client.Read(bytes)
+		if e != nil {
+			t.Fatal(e)
+		}
+		if n != 2 {
+			t.Fatalf("expected 2 bytes, got %v", n)
+		}
+		close(clientDone)
+	}()
+
+	payload := make([]byte, 2)
+	payload[0] = 0
+	payload[1] = 1
+
+	n, err := client.Write(payload)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 2 {
+		t.Fatalf("expected 2 bytes, got %v", n)
+	}
+	client.(CloseWriter).CloseWrite()
+	<-listenerDone
+	<-clientDone
+}

vendor/github.com/Microsoft/go-winio/privilege.go

@@ -0,0 +1,202 @@
+// +build windows
+
+package winio
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"runtime"
+	"sync"
+	"syscall"
+	"unicode/utf16"
+
+	"golang.org/x/sys/windows"
+)
+
+//sys adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) [true] = advapi32.AdjustTokenPrivileges
+//sys impersonateSelf(level uint32) (err error) = advapi32.ImpersonateSelf
+//sys revertToSelf() (err error) = advapi32.RevertToSelf
+//sys openThreadToken(thread syscall.Handle, accessMask uint32, openAsSelf bool, token *windows.Token) (err error) = advapi32.OpenThreadToken
+//sys getCurrentThread() (h syscall.Handle) = GetCurrentThread
+//sys lookupPrivilegeValue(systemName string, name string, luid *uint64) (err error) = advapi32.LookupPrivilegeValueW
+//sys lookupPrivilegeName(systemName string, luid *uint64, buffer *uint16, size *uint32) (err error) = advapi32.LookupPrivilegeNameW
+//sys lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) = advapi32.LookupPrivilegeDisplayNameW
+
+const (
+	SE_PRIVILEGE_ENABLED = 2
+
+	ERROR_NOT_ALL_ASSIGNED syscall.Errno = 1300
+
+	SeBackupPrivilege  = "SeBackupPrivilege"
+	SeRestorePrivilege = "SeRestorePrivilege"
+)
+
+const (
+	securityAnonymous = iota
+	securityIdentification
+	securityImpersonation
+	securityDelegation
+)
+
+var (
+	privNames     = make(map[string]uint64)
+	privNameMutex sync.Mutex
+)
+
+// PrivilegeError represents an error enabling privileges.
+type PrivilegeError struct {
+	privileges []uint64
+}
+
+func (e *PrivilegeError) Error() string {
+	s := ""
+	if len(e.privileges) > 1 {
+		s = "Could not enable privileges "
+	} else {
+		s = "Could not enable privilege "
+	}
+	for i, p := range e.privileges {
+		if i != 0 {
+			s += ", "
+		}
+		s += `"`
+		s += getPrivilegeName(p)
+		s += `"`
+	}
+	return s
+}
+
+// RunWithPrivilege enables a single privilege for a function call.
+func RunWithPrivilege(name string, fn func() error) error {
+	return RunWithPrivileges([]string{name}, fn)
+}
+
+// RunWithPrivileges enables privileges for a function call.
+func RunWithPrivileges(names []string, fn func() error) error {
+	privileges, err := mapPrivileges(names)
+	if err != nil {
+		return err
+	}
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+	token, err := newThreadToken()
+	if err != nil {
+		return err
+	}
+	defer releaseThreadToken(token)
+	err = adjustPrivileges(token, privileges, SE_PRIVILEGE_ENABLED)
+	if err != nil {
+		return err
+	}
+	return fn()
+}
+
+func mapPrivileges(names []string) ([]uint64, error) {
+	var privileges []uint64
+	privNameMutex.Lock()
+	defer privNameMutex.Unlock()
+	for _, name := range names {
+		p, ok := privNames[name]
+		if !ok {
+			err := lookupPrivilegeValue("", name, &p)
+			if err != nil {
+				return nil, err
+			}
+			privNames[name] = p
+		}
+		privileges = append(privileges, p)
+	}
+	return privileges, nil
+}
+
+// EnableProcessPrivileges enables privileges globally for the process.
+func EnableProcessPrivileges(names []string) error {
+	return enableDisableProcessPrivilege(names, SE_PRIVILEGE_ENABLED)
+}
+
+// DisableProcessPrivileges disables privileges globally for the process.
+func DisableProcessPrivileges(names []string) error {
+	return enableDisableProcessPrivilege(names, 0)
+}
+
+func enableDisableProcessPrivilege(names []string, action uint32) error {
+	privileges, err := mapPrivileges(names)
+	if err != nil {
+		return err
+	}
+
+	p, _ := windows.GetCurrentProcess()
+	var token windows.Token
+	err = windows.OpenProcessToken(p, windows.TOKEN_ADJUST_PRIVILEGES|windows.TOKEN_QUERY, &token)
+	if err != nil {
+		return err
+	}
+
+	defer token.Close()
+	return adjustPrivileges(token, privileges, action)
+}
+
+func adjustPrivileges(token windows.Token, privileges []uint64, action uint32) error {
+	var b bytes.Buffer
+	binary.Write(&b, binary.LittleEndian, uint32(len(privileges)))
+	for _, p := range privileges {
+		binary.Write(&b, binary.LittleEndian, p)
+		binary.Write(&b, binary.LittleEndian, action)
+	}
+	prevState := make([]byte, b.Len())
+	reqSize := uint32(0)
+	success, err := adjustTokenPrivileges(token, false, &b.Bytes()[0], uint32(len(prevState)), &prevState[0], &reqSize)
+	if !success {
+		return err
+	}
+	if err == ERROR_NOT_ALL_ASSIGNED {
+		return &PrivilegeError{privileges}
+	}
+	return nil
+}
+
+func getPrivilegeName(luid uint64) string {
+	var nameBuffer [256]uint16
+	bufSize := uint32(len(nameBuffer))
+	err := lookupPrivilegeName("", &luid, &nameBuffer[0], &bufSize)
+	if err != nil {
+		return fmt.Sprintf("<unknown privilege %d>", luid)
+	}
+
+	var displayNameBuffer [256]uint16
+	displayBufSize := uint32(len(displayNameBuffer))
+	var langID uint32
+	err = lookupPrivilegeDisplayName("", &nameBuffer[0], &displayNameBuffer[0], &displayBufSize, &langID)
+	if err != nil {
+		return fmt.Sprintf("<unknown privilege %s>", string(utf16.Decode(nameBuffer[:bufSize])))
+	}
+
+	return string(utf16.Decode(displayNameBuffer[:displayBufSize]))
+}
+
+func newThreadToken() (windows.Token, error) {
+	err := impersonateSelf(securityImpersonation)
+	if err != nil {
+		return 0, err
+	}
+
+	var token windows.Token
+	err = openThreadToken(getCurrentThread(), syscall.TOKEN_ADJUST_PRIVILEGES|syscall.TOKEN_QUERY, false, &token)
+	if err != nil {
+		rerr := revertToSelf()
+		if rerr != nil {
+			panic(rerr)
+		}
+		return 0, err
+	}
+	return token, nil
+}
+
+func releaseThreadToken(h windows.Token) {
+	err := revertToSelf()
+	if err != nil {
+		panic(err)
+	}
+	h.Close()
+}

vendor/github.com/Microsoft/go-winio/privileges_test.go

@@ -0,0 +1,17 @@
+package winio
+
+import "testing"
+
+func TestRunWithUnavailablePrivilege(t *testing.T) {
+	err := RunWithPrivilege("SeCreateTokenPrivilege", func() error { return nil })
+	if _, ok := err.(*PrivilegeError); err == nil || !ok {
+		t.Fatal("expected PrivilegeError")
+	}
+}
+
+func TestRunWithPrivileges(t *testing.T) {
+	err := RunWithPrivilege("SeShutdownPrivilege", func() error { return nil })
+	if err != nil {
+		t.Fatal(err)
+	}
+}

vendor/github.com/Microsoft/go-winio/reparse.go

@@ -0,0 +1,128 @@
+package winio
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"strings"
+	"unicode/utf16"
+	"unsafe"
+)
+
+const (
+	reparseTagMountPoint = 0xA0000003
+	reparseTagSymlink    = 0xA000000C
+)
+
+type reparseDataBuffer struct {
+	ReparseTag           uint32
+	ReparseDataLength    uint16
+	Reserved             uint16
+	SubstituteNameOffset uint16
+	SubstituteNameLength uint16
+	PrintNameOffset      uint16
+	PrintNameLength      uint16
+}
+
+// ReparsePoint describes a Win32 symlink or mount point.
+type ReparsePoint struct {
+	Target       string
+	IsMountPoint bool
+}
+
+// UnsupportedReparsePointError is returned when trying to decode a non-symlink or
+// mount point reparse point.
+type UnsupportedReparsePointError struct {
+	Tag uint32
+}
+
+func (e *UnsupportedReparsePointError) Error() string {
+	return fmt.Sprintf("unsupported reparse point %x", e.Tag)
+}
+
+// DecodeReparsePoint decodes a Win32 REPARSE_DATA_BUFFER structure containing either a symlink
+// or a mount point.
+func DecodeReparsePoint(b []byte) (*ReparsePoint, error) {
+	tag := binary.LittleEndian.Uint32(b[0:4])
+	return DecodeReparsePointData(tag, b[8:])
+}
+
+func DecodeReparsePointData(tag uint32, b []byte) (*ReparsePoint, error) {
+	isMountPoint := false
+	switch tag {
+	case reparseTagMountPoint:
+		isMountPoint = true
+	case reparseTagSymlink:
+	default:
+		return nil, &UnsupportedReparsePointError{tag}
+	}
+	nameOffset := 8 + binary.LittleEndian.Uint16(b[4:6])
+	if !isMountPoint {
+		nameOffset += 4
+	}
+	nameLength := binary.LittleEndian.Uint16(b[6:8])
+	name := make([]uint16, nameLength/2)
+	err := binary.Read(bytes.NewReader(b[nameOffset:nameOffset+nameLength]), binary.LittleEndian, &name)
+	if err != nil {
+		return nil, err
+	}
+	return &ReparsePoint{string(utf16.Decode(name)), isMountPoint}, nil
+}
+
+func isDriveLetter(c byte) bool {
+	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+}
+
+// EncodeReparsePoint encodes a Win32 REPARSE_DATA_BUFFER structure describing a symlink or
+// mount point.
+func EncodeReparsePoint(rp *ReparsePoint) []byte {
+	// Generate an NT path and determine if this is a relative path.
+	var ntTarget string
+	relative := false
+	if strings.HasPrefix(rp.Target, `\\?\`) {
+		ntTarget = `\??\` + rp.Target[4:]
+	} else if strings.HasPrefix(rp.Target, `\\`) {
+		ntTarget = `\??\UNC\` + rp.Target[2:]
+	} else if len(rp.Target) >= 2 && isDriveLetter(rp.Target[0]) && rp.Target[1] == ':' {
+		ntTarget = `\??\` + rp.Target
+	} else {
+		ntTarget = rp.Target
+		relative = true
+	}
+
+	// The paths must be NUL-terminated even though they are counted strings.
+	target16 := utf16.Encode([]rune(rp.Target + "\x00"))
+	ntTarget16 := utf16.Encode([]rune(ntTarget + "\x00"))
+
+	size := int(unsafe.Sizeof(reparseDataBuffer{})) - 8
+	size += len(ntTarget16)*2 + len(target16)*2
+
+	tag := uint32(reparseTagMountPoint)
+	if !rp.IsMountPoint {
+		tag = reparseTagSymlink
+		size += 4 // Add room for symlink flags
+	}
+
+	data := reparseDataBuffer{
+		ReparseTag:           tag,
+		ReparseDataLength:    uint16(size),
+		SubstituteNameOffset: 0,
+		SubstituteNameLength: uint16((len(ntTarget16) - 1) * 2),
+		PrintNameOffset:      uint16(len(ntTarget16) * 2),
+		PrintNameLength:      uint16((len(target16) - 1) * 2),
+	}
+
+	var b bytes.Buffer
+	binary.Write(&b, binary.LittleEndian, &data)
+	if !rp.IsMountPoint {
+		flags := uint32(0)
+		if relative {
+			flags |= 1
+		}
+		binary.Write(&b, binary.LittleEndian, flags)
+	}
+
+	binary.Write(&b, binary.LittleEndian, ntTarget16)
+	binary.Write(&b, binary.LittleEndian, target16)
+	return b.Bytes()
+}

vendor/github.com/Microsoft/go-winio/sd.go

@@ -0,0 +1,98 @@
+// +build windows
+
+package winio
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+//sys lookupAccountName(systemName *uint16, accountName string, sid *byte, sidSize *uint32, refDomain *uint16, refDomainSize *uint32, sidNameUse *uint32) (err error) = advapi32.LookupAccountNameW
+//sys convertSidToStringSid(sid *byte, str **uint16) (err error) = advapi32.ConvertSidToStringSidW
+//sys convertStringSecurityDescriptorToSecurityDescriptor(str string, revision uint32, sd *uintptr, size *uint32) (err error) = advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW
+//sys convertSecurityDescriptorToStringSecurityDescriptor(sd *byte, revision uint32, secInfo uint32, sddl **uint16, sddlSize *uint32) (err error) = advapi32.ConvertSecurityDescriptorToStringSecurityDescriptorW
+//sys localFree(mem uintptr) = LocalFree
+//sys getSecurityDescriptorLength(sd uintptr) (len uint32) = advapi32.GetSecurityDescriptorLength
+
+const (
+	cERROR_NONE_MAPPED = syscall.Errno(1332)
+)
+
+type AccountLookupError struct {
+	Name string
+	Err  error
+}
+
+func (e *AccountLookupError) Error() string {
+	if e.Name == "" {
+		return "lookup account: empty account name specified"
+	}
+	var s string
+	switch e.Err {
+	case cERROR_NONE_MAPPED:
+		s = "not found"
+	default:
+		s = e.Err.Error()
+	}
+	return "lookup account " + e.Name + ": " + s
+}
+
+type SddlConversionError struct {
+	Sddl string
+	Err  error
+}
+
+func (e *SddlConversionError) Error() string {
+	return "convert " + e.Sddl + ": " + e.Err.Error()
+}
+
+// LookupSidByName looks up the SID of an account by name
+func LookupSidByName(name string) (sid string, err error) {
+	if name == "" {
+		return "", &AccountLookupError{name, cERROR_NONE_MAPPED}
+	}
+
+	var sidSize, sidNameUse, refDomainSize uint32
+	err = lookupAccountName(nil, name, nil, &sidSize, nil, &refDomainSize, &sidNameUse)
+	if err != nil && err != syscall.ERROR_INSUFFICIENT_BUFFER {
+		return "", &AccountLookupError{name, err}
+	}
+	sidBuffer := make([]byte, sidSize)
+	refDomainBuffer := make([]uint16, refDomainSize)
+	err = lookupAccountName(nil, name, &sidBuffer[0], &sidSize, &refDomainBuffer[0], &refDomainSize, &sidNameUse)
+	if err != nil {
+		return "", &AccountLookupError{name, err}
+	}
+	var strBuffer *uint16
+	err = convertSidToStringSid(&sidBuffer[0], &strBuffer)
+	if err != nil {
+		return "", &AccountLookupError{name, err}
+	}
+	sid = syscall.UTF16ToString((*[0xffff]uint16)(unsafe.Pointer(strBuffer))[:])
+	localFree(uintptr(unsafe.Pointer(strBuffer)))
+	return sid, nil
+}
+
+func SddlToSecurityDescriptor(sddl string) ([]byte, error) {
+	var sdBuffer uintptr
+	err := convertStringSecurityDescriptorToSecurityDescriptor(sddl, 1, &sdBuffer, nil)
+	if err != nil {
+		return nil, &SddlConversionError{sddl, err}
+	}
+	defer localFree(sdBuffer)
+	sd := make([]byte, getSecurityDescriptorLength(sdBuffer))
+	copy(sd, (*[0xffff]byte)(unsafe.Pointer(sdBuffer))[:len(sd)])
+	return sd, nil
+}
+
+func SecurityDescriptorToSddl(sd []byte) (string, error) {
+	var sddl *uint16
+	// The returned string length seems to including an aribtrary number of terminating NULs.
+	// Don't use it.
+	err := convertSecurityDescriptorToStringSecurityDescriptor(&sd[0], 1, 0xff, &sddl, nil)
+	if err != nil {
+		return "", err
+	}
+	defer localFree(uintptr(unsafe.Pointer(sddl)))
+	return syscall.UTF16ToString((*[0xffff]uint16)(unsafe.Pointer(sddl))[:]), nil
+}

vendor/github.com/Microsoft/go-winio/sd_test.go

@@ -0,0 +1,26 @@
+package winio
+
+import "testing"
+
+func TestLookupInvalidSid(t *testing.T) {
+	_, err := LookupSidByName(".\\weoifjdsklfj")
+	aerr, ok := err.(*AccountLookupError)
+	if !ok || aerr.Err != cERROR_NONE_MAPPED {
+		t.Fatalf("expected AccountLookupError with ERROR_NONE_MAPPED, got %s", err)
+	}
+}
+
+func TestLookupValidSid(t *testing.T) {
+	sid, err := LookupSidByName("Everyone")
+	if err != nil || sid != "S-1-1-0" {
+		t.Fatal("expected S-1-1-0, got %s, %s", sid, err)
+	}
+}
+
+func TestLookupEmptyNameFails(t *testing.T) {
+	_, err := LookupSidByName(".\\weoifjdsklfj")
+	aerr, ok := err.(*AccountLookupError)
+	if !ok || aerr.Err != cERROR_NONE_MAPPED {
+		t.Fatalf("expected AccountLookupError with ERROR_NONE_MAPPED, got %s", err)
+	}
+}