Add containerd windows support on GCE for test.

2025-07-22 03:11:40 +00:00 · 2019-11-27 11:44:41 -08:00 · 2019-11-27 11:44:41 -08:00 · e66f4ca537
commit e66f4ca537
parent beaf3a2f04
7 changed files with 292 additions and 28 deletions
--- a/build/dependencies.yaml
+++ b/build/dependencies.yaml
@ -83,3 +83,5 @@ dependencies:
      match: CRI_TOOLS_VERSION =
    - path: cluster/gce/gci/configure.sh
      match: DEFAULT_CRICTL_VERSION=
+    - path: cluster/gce/windows/k8s-node-setup.psm1
+      match: CRICTL_VERSION =
--- a/cluster/gce/config-default.sh
+++ b/cluster/gce/config-default.sh
@ -98,9 +98,7 @@ CONTAINER_RUNTIME_NAME=${KUBE_CONTAINER_RUNTIME_NAME:-}
 LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-}
 if [[ "${CONTAINER_RUNTIME}" == "containerd" ]]; then
  CONTAINER_RUNTIME_NAME=${KUBE_CONTAINER_RUNTIME_NAME:-containerd}
-  CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-unix:///run/containerd/containerd.sock}
  LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-ctr -n=k8s.io images import}
-  KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS} --runtime-cgroups=/system.slice/containerd.service"
 fi

 # MASTER_EXTRA_METADATA is the extra instance metadata on master instance separated by commas.
--- a/cluster/gce/config-test.sh
+++ b/cluster/gce/config-test.sh
@ -104,9 +104,7 @@ LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-}
 GCI_DOCKER_VERSION=${KUBE_GCI_DOCKER_VERSION:-}
 if [[ "${CONTAINER_RUNTIME}" == "containerd" ]]; then
  CONTAINER_RUNTIME_NAME=${KUBE_CONTAINER_RUNTIME_NAME:-containerd}
-  CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-unix:///run/containerd/containerd.sock}
  LOAD_IMAGE_COMMAND=${KUBE_LOAD_IMAGE_COMMAND:-ctr -n=k8s.io images import}
-  KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS:-} --runtime-cgroups=/system.slice/containerd.service"
 fi

 # MASTER_EXTRA_METADATA is the extra instance metadata on master instance separated by commas.
--- a/cluster/gce/util.sh
+++ b/cluster/gce/util.sh
@ -796,7 +796,12 @@ function construct-linux-kubelet-flags {
  fi
  if [[ "${CONTAINER_RUNTIME:-}" != "docker" ]]; then
    flags+=" --container-runtime=remote"
+    if [[ "${CONTAINER_RUNTIME}" == "containerd" ]]; then
+      CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-unix:///run/containerd/containerd.sock}
+      flags+=" --runtime-cgroups=/system.slice/containerd.service"
+    fi
  fi
+
  if [[ -n "${CONTAINER_RUNTIME_ENDPOINT:-}" ]]; then
    flags+=" --container-runtime-endpoint=${CONTAINER_RUNTIME_ENDPOINT}"
  fi
@ -886,6 +891,14 @@ function construct-windows-kubelet-flags {
  # Force disable KubeletPodResources feature on Windows until #78628 is fixed.
  flags+=" --feature-gates=KubeletPodResources=false"

+  if [[ "${CONTAINER_RUNTIME:-}" != "docker" ]]; then
+    flags+=" --container-runtime=remote"
+    if [[ "${CONTAINER_RUNTIME}" == "containerd" ]]; then
+      CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-npipe:////./pipe/containerd-containerd}
+      flags+=" --container-runtime-endpoint=${CONTAINER_RUNTIME_ENDPOINT}"
+    fi
+  fi
+
  KUBELET_ARGS="${flags}"
 }

--- a/cluster/gce/windows/common.psm1
+++ b/cluster/gce/windows/common.psm1
@ -106,30 +106,33 @@ function Get-InstanceMetadataAttribute {
  return Get-InstanceMetadata "attributes/$Key" $Default
 }

-function Validate-SHA1 {
+function Validate-SHA {
  param(
    [parameter(Mandatory=$true)] [string]$Hash,
-    [parameter(Mandatory=$true)] [string]$Path
+    [parameter(Mandatory=$true)] [string]$Path,
+    [parameter(Mandatory=$true)] [string]$Algorithm
  )
-  $actual = Get-FileHash -Path $Path -Algorithm SHA1
+  $actual = Get-FileHash -Path $Path -Algorithm $Algorithm
  # Note: Powershell string comparisons are case-insensitive by default, and this
  # is important here because Linux shell scripts produce lowercase hashes but
  # Powershell Get-FileHash produces uppercase hashes. This must be case-insensitive
  # to work.
  if ($actual.Hash -ne $Hash) {
-    Log-Output "$Path corrupted, sha1 $actual doesn't match expected $Hash"
-    Throw ("$Path corrupted, sha1 $actual doesn't match expected $Hash")
+    Log-Output "$Path corrupted, $Algorithm $actual doesn't match expected $Hash"
+    Throw ("$Path corrupted, $Algorithm $actual doesn't match expected $Hash")
  }
 }

 # Attempts to download the file from URLs, trying each URL until it succeeds.
 # It will loop through the URLs list forever until it has a success. If
 # successful, it will write the file to OutFile. You can optionally provide a
-# SHA1 Hash argument, in which case it will attempt to validate the downloaded
-# file against the hash.
+# Hash argument with an optional Algorithm, in which case it will attempt to
+# validate the downloaded file against the hash. SHA1 will be used if Algorithm
+# is not provided.
 function MustDownload-File {
  param (
    [parameter(Mandatory=$false)] [string]$Hash,
+    [parameter(Mandatory=$false)] [string]$Algorithm = 'SHA1',
    [parameter(Mandatory=$true)] [string]$OutFile,
    [parameter(Mandatory=$true)] [System.Collections.Generic.List[String]]$URLs,
    [parameter(Mandatory=$false)] [System.Collections.IDictionary]$Headers = @{}
@ -156,13 +159,13 @@ function MustDownload-File {
      # Attempt to validate the hash
      if ($Hash) {
        Try {
-            Validate-SHA1 -Hash $Hash -Path $OutFile
+            Validate-SHA -Hash $Hash -Path $OutFile -Algorithm $Algorithm
        } Catch {
            $message = $_.Exception.ToString()
            Log-Output "Hash validation of $url failed. Will retry. Error: $message"
            continue
        }
-        Log-Output "Downloaded $url (SHA1 = $Hash)"
+        Log-Output "Downloaded $url ($Algorithm = $Hash)"
        return
      }
      Log-Output "Downloaded $url"
--- a/cluster/gce/windows/configure.ps1
+++ b/cluster/gce/windows/configure.ps1
@ -131,8 +131,8 @@ try {
  Create-Directories
  Download-HelperScripts

-  Create-DockerRegistryKey
-  Configure-Dockerd
+  DownloadAndInstall-Crictl
+  Setup-ContainerRuntime
  DownloadAndInstall-KubernetesBinaries
  Create-NodePki
  Create-KubeletKubeconfig
@ -156,6 +156,7 @@ try {
  Verify-WorkerServices

  $config = New-FileRotationConfig
+  # TODO(random-liu): Generate containerd log into the log directory.
  Schedule-LogRotation -Pattern '.*\.log$' -Path ${env:LOGS_DIR} -RepetitionInterval $(New-Timespan -Hour 1) -Config $config

  Pull-InfraContainer
--- a/cluster/gce/windows/k8s-node-setup.psm1
+++ b/cluster/gce/windows/k8s-node-setup.psm1
@ -209,7 +209,7 @@ function Fetch-KubeEnv {
 function Set_MachineEnvironmentVar {
  param (
    [parameter(Mandatory=$true)] [string]$Key,
-    [parameter(Mandatory=$true)] [string]$Value
+    [parameter(Mandatory=$true)] [AllowEmptyString()] [string]$Value
  )
  [Environment]::SetEnvironmentVariable($Key, $Value, "Machine")
 }
@ -218,7 +218,7 @@ function Set_MachineEnvironmentVar {
 function Set_CurrentShellEnvironmentVar {
  param (
    [parameter(Mandatory=$true)] [string]$Key,
-    [parameter(Mandatory=$true)] [string]$Value
+    [parameter(Mandatory=$true)] [AllowEmptyString()] [string]$Value
  )
  $expression = '$env:' + $Key + ' = "' + $Value + '"'
  Invoke-Expression ${expression}
@ -247,6 +247,9 @@ function Set-EnvironmentVars {
    "KUBELET_CERT_PATH" = ${kube_env}['PKI_DIR'] + '\kubelet.crt'
    "KUBELET_KEY_PATH" = ${kube_env}['PKI_DIR'] + '\kubelet.key'

+    "CONTAINER_RUNTIME" = ${kube_env}['CONTAINER_RUNTIME']
+    "CONTAINER_RUNTIME_ENDPOINT" = ${kube_env}['CONTAINER_RUNTIME_ENDPOINT']
+
    # TODO(pjh): these are only in flags, can be removed from env once flags are
    # moved to util.sh:
    "LOGS_DIR" = ${kube_env}['LOGS_DIR']
@ -344,6 +347,7 @@ function DownloadAndInstall-KubernetesBinaries {

  # Change the directory to the parent directory of ${env:K8S_DIR} and untar.
  # This (over-)writes ${dest_dir}/kubernetes/node/bin/*.exe files.
+  # TODO(pjh): clean this up, files not guaranteed to end up in NODE_DIR
  $dest_dir = (Get-Item ${env:K8S_DIR}).Parent.Fullname
  tar xzf ${tmp_dir}\${filename} -C ${dest_dir}

@ -836,6 +840,16 @@ Unblock-File $modulePath
 Import-Module -Name $modulePath'.replace('K8S_DIR', ${env:K8S_DIR})
 }

+# Setup cni network. This function supports both Docker
+# and containerd.
+function Configure-CniNetworking {
+  if (${env:CONTAINER_RUNTIME} -eq "containerd") {
+    Configure_Containerd_CniNetworking
+  } else {
+    Configure_Dockerd_CniNetworking
+  }
+}
+
 # Downloads the Windows CNI binaries and writes a CNI config file under
 # $env:CNI_CONFIG_DIR.
 #
@ -850,7 +864,7 @@ Import-Module -Name $modulePath'.replace('K8S_DIR', ${env:K8S_DIR})
 #   DNS_DOMAIN
 #   CLUSTER_IP_RANGE
 #   SERVICE_CLUSTER_IP_RANGE
-function Configure-CniNetworking {
+function Configure_Dockerd_CniNetworking {
  $CNI_RELEASE_VERSION = 'v0.8.2-gke.0'
  if ((ShouldWrite-File ${env:CNI_DIR}\win-bridge.exe) -or
      (ShouldWrite-File ${env:CNI_DIR}\host-local.exe)) {
@ -1115,28 +1129,74 @@ function Verify-WorkerServices {
  Log_Todo "run more verification commands."
 }

+function DownloadAndInstall-Crictl {
+  $CRICTL_VERSION = "v1.16.1"
+  $CRICTL_SHA256 = "7d092dcb3b1af2edf75477d5d049a70e8c0d1ac8242b1dff2de7e6aa084e3615"
+
+  # Assume that presence of crictl.exe indicates that the crictl binaries
+  # were already previously downloaded to this node.
+  if (-not (ShouldWrite-File ${env:NODE_DIR}\crictl.exe)) {
+    return
+  }
+  $tmp_dir = 'C:\crictl_tmp'
+  New-Item $tmp_dir -ItemType 'directory' -Force | Out-Null
+
+  $url = ('https://github.com/kubernetes-sigs/cri-tools/releases/' +
+      'download/' + $CRICTL_VERSION + '/crictl-' +
+      $CRICTL_VERSION + '-windows-amd64.tar.gz')
+  MustDownload-File `
+      -URLs $url `
+      -OutFile $tmp_dir\crictl.tgz `
+      -Hash $CRICTL_SHA256 `
+      -Algorithm SHA256
+
+  Push-Location $tmp_dir
+  # tar can only extract in the current directory.
+  tar -xvf $tmp_dir\crictl.tgz
+  Move-Item -Force crictl.exe ${env:NODE_DIR}\
+  if (${env:CONTAINER_RUNTIME_ENDPOINT}) {
+    crictl.exe config runtime-endpoint ${env:CONTAINER_RUNTIME_ENDPOINT}
+  }
+  Pop-Location
+  Remove-Item -Force -Recurse $tmp_dir
+}
+
 # Pulls the infra/pause container image onto the node so that it will be
 # immediately available when the kubelet tries to run pods.
 # TODO(pjh): downloading the container container image may take a few minutes;
 # figure out how to run this in the background while perform the rest of the
 # node startup steps!
+# Pull-InfraContainer must be called AFTER Verify-WorkerServices.
 function Pull-InfraContainer {
  $name, $label = $INFRA_CONTAINER -split ':',2
-  if (-not ("$(& docker image list)" -match "$name.*$label")) {
-    & docker pull $INFRA_CONTAINER
+  if (-not ("$(& crictl images)" -match "$name.*$label")) {
+    & crictl pull $INFRA_CONTAINER
    if (!$?) {
-      throw "Error running 'docker pull $INFRA_CONTAINER'"
+      throw "Error running 'crictl pull $INFRA_CONTAINER'"
    }
  }
-  $inspect = "$(& docker inspect $INFRA_CONTAINER | Out-String)"
+  $inspect = "$(& crictl inspecti $INFRA_CONTAINER | Out-String)"
  Log-Output "Infra/pause container:`n$inspect"
 }

+# Setup the container runtime on the node. It supports both
+# Docker and containerd.
+function Setup-ContainerRuntime {
+  if (${env:CONTAINER_RUNTIME} -eq "containerd") {
+    Install_Containerd
+    Start_Containerd
+  } else {
+    Create_DockerRegistryKey
+    Configure_Dockerd
+  }
+}
+
 # Add a registry key for docker in EventLog so that log messages are mapped
 # correctly. This is a workaround since the key is missing in the base image.
 # https://github.com/MicrosoftDocs/Virtualization-Documentation/pull/503
 # TODO: Fix this in the base image.
-function Create-DockerRegistryKey {
+# TODO(random-liu): Figure out whether we need this for containerd.
+function Create_DockerRegistryKey {
  $tmp_dir = 'C:\tmp_docker_reg'
  New-Item -Force -ItemType 'directory' ${tmp_dir} | Out-Null
  $reg_file = 'docker.reg'
@ -1153,7 +1213,7 @@ function Create-DockerRegistryKey {
 }

 # Configure Docker daemon and restart the service.
-function Configure-Dockerd {
+function Configure_Dockerd {
  Set-Content "C:\ProgramData\docker\config\daemon.json" @'
 {
  "log-driver": "json-file",
@ -1167,6 +1227,180 @@ function Configure-Dockerd {
 Restart-Service Docker
 }

+# Writes a CNI config file under $env:CNI_CONFIG_DIR for containerd.
+#
+# Prerequisites:
+#   $env:POD_CIDR is set (by Set-PodCidr).
+#   The "management" interface exists (Configure-HostNetworkingService).
+#   The HNS network for pod networking has been configured
+#     (Configure-HostNetworkingService).
+#   Containerd is installed (Install_Containerd).
+#
+# Required ${kube_env} keys:
+#   DNS_SERVER_IP
+#   DNS_DOMAIN
+#   CLUSTER_IP_RANGE
+#   SERVICE_CLUSTER_IP_RANGE
+function Configure_Containerd_CniNetworking {
+  $l2bridge_conf = "${env:CNI_CONFIG_DIR}\l2bridge.conf"
+  if (-not (ShouldWrite-File ${l2bridge_conf})) {
+    return
+  }
+
+  $mgmt_ip = (Get_MgmtNetAdapter |
+              Get-NetIPAddress -AddressFamily IPv4).IPAddress
+  $mgmt_subnet = Get_MgmtSubnet
+  Log-Output ("using mgmt IP ${mgmt_ip} and mgmt subnet ${mgmt_subnet} for " +
+              "CNI config")
+
+  $pod_gateway = Get_Endpoint_Gateway_From_CIDR(${env:POD_CIDR})
+
+  # Explanation of the CNI config values:
+  #   CLUSTER_CIDR: the cluster CIDR from which pod CIDRs are allocated.
+  #   POD_CIDR: the pod CIDR assigned to this node.
+  #   POD_GATEWAY: the gateway IP.
+  #   MGMT_SUBNET: the subnet on which the Windows pods + kubelet will
+  #     communicate with the rest of the cluster without NAT (i.e. the subnet
+  #     that VM internal IPs are allocated from).
+  #   MGMT_IP: the IP address assigned to the node's primary network interface
+  #     (i.e. the internal IP of the GCE VM).
+  #   SERVICE_CIDR: the CIDR used for kubernetes services.
+  #   DNS_SERVER_IP: the cluster's DNS server IP address.
+  #   DNS_DOMAIN: the cluster's DNS domain, e.g. "cluster.local".
+  New-Item -Force -ItemType file ${l2bridge_conf} | Out-Null
+  Set-Content ${l2bridge_conf} `
+'{
+  "cniVersion":  "0.2.0",
+  "name":  "l2bridge",
+  "type":  "sdnbridge",
+  "master": "Ethernet",
+  "capabilities":  {
+    "portMappings":  true,
+    "dns": true
+  },
+  "ipam":  {
+    "subnet": "POD_CIDR",
+    "routes": [
+      {
+        "GW": "POD_GATEWAY"
+      }
+    ]
+  },
+  "dns":  {
+    "Nameservers":  [
+      "DNS_SERVER_IP"
+    ],
+    "Search": [
+      "DNS_DOMAIN"
+    ]
+  },
+  "AdditionalArgs": [
+    {
+      "Name":  "EndpointPolicy",
+      "Value":  {
+        "Type":  "OutBoundNAT",
+	"Settings": {
+          "Exceptions":  [
+            "CLUSTER_CIDR",
+            "SERVICE_CIDR",
+            "MGMT_SUBNET"
+          ]
+        }
+      }
+    },
+    {
+      "Name":  "EndpointPolicy",
+      "Value":  {
+        "Type":  "SDNRoute",
+	"Settings": {
+          "DestinationPrefix":  "SERVICE_CIDR",
+          "NeedEncap":  true
+	}
+      }
+    },
+    {
+      "Name":  "EndpointPolicy",
+      "Value":  {
+        "Type":  "SDNRoute",
+	"Settings": {
+          "DestinationPrefix":  "MGMT_IP/32",
+          "NeedEncap":  true
+	}
+      }
+    }
+  ]
+}'.replace('POD_CIDR', ${env:POD_CIDR}).`
+  replace('POD_GATEWAY', ${pod_gateway}).`
+  replace('DNS_SERVER_IP', ${kube_env}['DNS_SERVER_IP']).`
+  replace('DNS_DOMAIN', ${kube_env}['DNS_DOMAIN']).`
+  replace('MGMT_IP', ${mgmt_ip}).`
+  replace('CLUSTER_CIDR', ${kube_env}['CLUSTER_IP_RANGE']).`
+  replace('SERVICE_CIDR', ${kube_env}['SERVICE_CLUSTER_IP_RANGE']).`
+  replace('MGMT_SUBNET', ${mgmt_subnet})
+
+  Log-Output "CNI config:`n$(Get-Content -Raw ${l2bridge_conf})"
+}
+
+# Download and install containerd and CNI binaries.
+function Install_Containerd {
+  # Assume that presence of containerd.exe indicates that all containerd binaries
+  # were already previously downloaded to this node.
+  if (-not (ShouldWrite-File ${env:NODE_DIR}\containerd.exe)) {
+    return
+  }
+
+  # https://storage.googleapis.com/cri-containerd-staging/cri-containerd-9f79be1b.windows-amd64.tar.gz
+  # TODO(random-liu): Change this to official release path after testing.
+  $CONTAINERD_GCS_BUCKET = "cri-containerd-staging/windows"
+
+  $tmp_dir = 'C:\containerd_tmp'
+  New-Item $tmp_dir -ItemType 'directory' -Force | Out-Null
+
+  $version_url = "https://storage.googleapis.com/$CONTAINERD_GCS_BUCKET/latest"
+  MustDownload-File -URLs $version_url -OutFile $tmp_dir\version
+  $version = $(Get-Content $tmp_dir\version)
+
+  $tar_url = "https://storage.googleapis.com/$CONTAINERD_GCS_BUCKET/cri-containerd-cni-$version.windows-amd64.tar.gz"
+  $sha_url = $tar_url + ".sha256"
+  MustDownload-File -URLs $sha_url -OutFile $tmp_dir\sha256
+  $sha = $(Get-Content $tmp_dir\sha256)
+
+  MustDownload-File `
+      -URLs $tar_url `
+      -OutFile $tmp_dir\containerd.tar.gz `
+      -Hash $sha `
+      -Algorithm SHA256
+
+  Push-Location $tmp_dir
+  # tar can only extract in the current directory.
+  tar -xvf $tmp_dir\containerd.tar.gz
+  Move-Item -Force cni\*.exe ${env:CNI_DIR}\
+  Move-Item -Force *.exe ${env:NODE_DIR}\
+  Pop-Location
+  Remove-Item -Force -Recurse $tmp_dir
+
+  # Generate containerd config
+  $config_dir = 'C:\Program Files\containerd'
+  New-Item $config_dir -ItemType 'directory' -Force | Out-Null
+  Set-Content "$config_dir\config.toml" @"
+[plugins.cri]
+  sandbox_image = 'INFRA_CONTAINER_IMAGE'
+[plugins.cri.cni]
+  bin_dir = 'CNI_BIN_DIR'
+  conf_dir = 'CNI_CONF_DIR'
+"@.replace('INFRA_CONTAINER_IMAGE', $INFRA_CONTAINER).`
+    replace('CNI_BIN_DIR', ${env:CNI_DIR}).`
+    replace('CNI_CONF_DIR', ${env:CNI_CONFIG_DIR})
+}
+
+# Register and start containerd service.
+function Start_Containerd {
+  Log-Output "Creating containerd service"
+  containerd.exe --register-service
+  Log-Output "Starting containerd service"
+  Start-Service containerd
+}
+
 # TODO(pjh): move the Stackdriver logging agent code below into a separate
 # module; it was put here temporarily to avoid disrupting the file layout in
 # the K8s release machinery.
@ -1261,6 +1495,11 @@ function Install-LoggingAgent {
      -ArgumentList "install","fluent-plugin-record-reformer" `
      -Wait

+  # Install the multi-format-parser plugin.
+  Start-Process "$STACKDRIVER_ROOT\LoggingAgent\Main\bin\fluent-gem" `
+      -ArgumentList "install","fluent-plugin-multi-format-parser" `
+      -Wait
+
  Remove-Item -Force -Recurse $tmp_dir
 }

@ -1329,7 +1568,8 @@ $FLUENTD_CONFIG = @'

 # Json Log Example:
 # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
-# TODO: Support CRI log format, which requires the multi_format plugin.
+# CRI Log Example:
+# 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here
 <source>
  @type tail
  path /var/log/containers/*.log
@ -1337,10 +1577,19 @@ $FLUENTD_CONFIG = @'
  # Tags at this point are in the format of:
  # reform.var.log.containers.<POD_NAME>_<NAMESPACE_NAME>_<CONTAINER_NAME>-<CONTAINER_ID>.log
  tag reform.*
-  format json
-  time_key time
-  time_format %Y-%m-%dT%H:%M:%S.%NZ
  read_from_head true
+  <parse>
+    @type multi_format
+    <pattern>
+      format json
+      time_key time
+      time_format %Y-%m-%dT%H:%M:%S.%NZ
+    </pattern>
+    <pattern>
+      format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
+      time_format %Y-%m-%dT%H:%M:%S.%N%:z
+    </pattern>
+  </parse>
 </source>

 # Example: