From 11245be0fed247e15669dec877a33e7957ab3253 Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Mon, 31 Oct 2016 16:04:35 +0800 Subject: [PATCH 1/3] CRI: Add devices to ContainerConfig --- pkg/kubelet/api/v1alpha1/runtime/api.proto | 37 +++++++++++++++------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/pkg/kubelet/api/v1alpha1/runtime/api.proto b/pkg/kubelet/api/v1alpha1/runtime/api.proto index 14d32388acf..aebe62a3331 100644 --- a/pkg/kubelet/api/v1alpha1/runtime/api.proto +++ b/pkg/kubelet/api/v1alpha1/runtime/api.proto @@ -428,6 +428,19 @@ message ContainerMetadata { optional uint32 attempt = 2; } +// Device specifies a host device to mount into a container. +message Device { + // The path of the device within the container. + optional string container_path = 1; + // The path of the device on the host. + optional string host_path = 2; + // Cgroups permissions of the device, candidates are one or more of + // * r - allows container to read from the specified device. + // * w - allows container to write to the specified device. + // * m - allows container to create device files that do not yet exist. + optional string permissions = 3; +} + // ContainerConfig holds all the required and optional fields for creating a // container. message ContainerConfig { @@ -444,25 +457,27 @@ message ContainerConfig { repeated string args = 4; // Current working directory of the command. optional string working_dir = 5; - // List of environment variable to set in the container + // List of environment variable to set in the container. repeated KeyValue envs = 6; - // Mounts specifies mounts for the container + // Mounts specifies mounts for the container. repeated Mount mounts = 7; + // Devices specifies devices for the container. + repeated Device devices = 8; // Labels are key value pairs that may be used to scope and select individual resources. // Label keys are of the form: // label-key ::= prefixed-name | name // prefixed-name ::= prefix '/' name // prefix ::= DNS_SUBDOMAIN // name ::= DNS_LABEL - map labels = 8; + map labels = 9; // Annotations is an unstructured key value map that may be set by external // tools to store and retrieve arbitrary metadata. - map annotations = 9; + map annotations = 10; // If set, run container in privileged mode. // Processes in privileged containers are essentially equivalent to root on the host. - optional bool privileged = 10; + optional bool privileged = 11; // If set, the root filesystem of the container is read-only. - optional bool readonly_rootfs = 11; + optional bool readonly_rootfs = 12; // Path relative to PodSandboxConfig.LogDirectory for container to store // the log (STDOUT and STDERR) on the host. // E.g., @@ -473,19 +488,19 @@ message ContainerConfig { // container logs are under active discussion in // https://issues.k8s.io/24677. There *may* be future change of direction // for logging as the discussion carries on. - optional string log_path = 12; + optional string log_path = 13; // The hash of container config // Variables for interactive containers, these have very specialized // use-cases (e.g. debugging). // TODO: Determine if we need to continue supporting these fields that are // part of Kubernetes's Container Spec. - optional bool stdin = 13; - optional bool stdin_once = 14; - optional bool tty = 15; + optional bool stdin = 14; + optional bool stdin_once = 15; + optional bool tty = 16; // Linux contains configuration specific to Linux containers. - optional LinuxContainerConfig linux = 16; + optional LinuxContainerConfig linux = 17; } message CreateContainerRequest { From c53fee77259272b6d7064610ee39560feefdedea Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Mon, 31 Oct 2016 16:04:48 +0800 Subject: [PATCH 2/3] Run hack/update-generated-runtime.sh --- pkg/kubelet/api/v1alpha1/runtime/api.pb.go | 546 +++++++++++---------- 1 file changed, 300 insertions(+), 246 deletions(-) diff --git a/pkg/kubelet/api/v1alpha1/runtime/api.pb.go b/pkg/kubelet/api/v1alpha1/runtime/api.pb.go index e7dc8e79243..9cd9e1c7580 100644 --- a/pkg/kubelet/api/v1alpha1/runtime/api.pb.go +++ b/pkg/kubelet/api/v1alpha1/runtime/api.pb.go @@ -58,6 +58,7 @@ It has these top-level messages: LinuxContainerConfig LinuxUser ContainerMetadata + Device ContainerConfig CreateContainerRequest CreateContainerResponse @@ -1373,6 +1374,46 @@ func (m *ContainerMetadata) GetAttempt() uint32 { return 0 } +// Device specifies a host device to mount into a container. +type Device struct { + // The path of the device within the container. + ContainerPath *string `protobuf:"bytes,1,opt,name=container_path,json=containerPath" json:"container_path,omitempty"` + // The path of the device on the host. + HostPath *string `protobuf:"bytes,2,opt,name=host_path,json=hostPath" json:"host_path,omitempty"` + // Cgroups permissions of the device, candidates are one or more of + // * r - allows container to read from the specified device. + // * w - allows container to write to the specified device. + // * m - allows container to create device files that do not yet exist. + Permissions *string `protobuf:"bytes,3,opt,name=permissions" json:"permissions,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Device) Reset() { *m = Device{} } +func (m *Device) String() string { return proto.CompactTextString(m) } +func (*Device) ProtoMessage() {} +func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{33} } + +func (m *Device) GetContainerPath() string { + if m != nil && m.ContainerPath != nil { + return *m.ContainerPath + } + return "" +} + +func (m *Device) GetHostPath() string { + if m != nil && m.HostPath != nil { + return *m.HostPath + } + return "" +} + +func (m *Device) GetPermissions() string { + if m != nil && m.Permissions != nil { + return *m.Permissions + } + return "" +} + // ContainerConfig holds all the required and optional fields for creating a // container. type ContainerConfig struct { @@ -1389,25 +1430,27 @@ type ContainerConfig struct { Args []string `protobuf:"bytes,4,rep,name=args" json:"args,omitempty"` // Current working directory of the command. WorkingDir *string `protobuf:"bytes,5,opt,name=working_dir,json=workingDir" json:"working_dir,omitempty"` - // List of environment variable to set in the container + // List of environment variable to set in the container. Envs []*KeyValue `protobuf:"bytes,6,rep,name=envs" json:"envs,omitempty"` - // Mounts specifies mounts for the container + // Mounts specifies mounts for the container. Mounts []*Mount `protobuf:"bytes,7,rep,name=mounts" json:"mounts,omitempty"` + // Devices specifies devices for the container. + Devices []*Device `protobuf:"bytes,8,rep,name=devices" json:"devices,omitempty"` // Labels are key value pairs that may be used to scope and select individual resources. // Label keys are of the form: // label-key ::= prefixed-name | name // prefixed-name ::= prefix '/' name // prefix ::= DNS_SUBDOMAIN // name ::= DNS_LABEL - Labels map[string]string `protobuf:"bytes,8,rep,name=labels" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Labels map[string]string `protobuf:"bytes,9,rep,name=labels" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // Annotations is an unstructured key value map that may be set by external // tools to store and retrieve arbitrary metadata. - Annotations map[string]string `protobuf:"bytes,9,rep,name=annotations" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Annotations map[string]string `protobuf:"bytes,10,rep,name=annotations" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // If set, run container in privileged mode. // Processes in privileged containers are essentially equivalent to root on the host. - Privileged *bool `protobuf:"varint,10,opt,name=privileged" json:"privileged,omitempty"` + Privileged *bool `protobuf:"varint,11,opt,name=privileged" json:"privileged,omitempty"` // If set, the root filesystem of the container is read-only. - ReadonlyRootfs *bool `protobuf:"varint,11,opt,name=readonly_rootfs,json=readonlyRootfs" json:"readonly_rootfs,omitempty"` + ReadonlyRootfs *bool `protobuf:"varint,12,opt,name=readonly_rootfs,json=readonlyRootfs" json:"readonly_rootfs,omitempty"` // Path relative to PodSandboxConfig.LogDirectory for container to store // the log (STDOUT and STDERR) on the host. // E.g., @@ -1418,23 +1461,23 @@ type ContainerConfig struct { // container logs are under active discussion in // https://issues.k8s.io/24677. There *may* be future change of direction // for logging as the discussion carries on. - LogPath *string `protobuf:"bytes,12,opt,name=log_path,json=logPath" json:"log_path,omitempty"` + LogPath *string `protobuf:"bytes,13,opt,name=log_path,json=logPath" json:"log_path,omitempty"` // Variables for interactive containers, these have very specialized // use-cases (e.g. debugging). // TODO: Determine if we need to continue supporting these fields that are // part of Kubernetes's Container Spec. - Stdin *bool `protobuf:"varint,13,opt,name=stdin" json:"stdin,omitempty"` - StdinOnce *bool `protobuf:"varint,14,opt,name=stdin_once,json=stdinOnce" json:"stdin_once,omitempty"` - Tty *bool `protobuf:"varint,15,opt,name=tty" json:"tty,omitempty"` + Stdin *bool `protobuf:"varint,14,opt,name=stdin" json:"stdin,omitempty"` + StdinOnce *bool `protobuf:"varint,15,opt,name=stdin_once,json=stdinOnce" json:"stdin_once,omitempty"` + Tty *bool `protobuf:"varint,16,opt,name=tty" json:"tty,omitempty"` // Linux contains configuration specific to Linux containers. - Linux *LinuxContainerConfig `protobuf:"bytes,16,opt,name=linux" json:"linux,omitempty"` + Linux *LinuxContainerConfig `protobuf:"bytes,17,opt,name=linux" json:"linux,omitempty"` XXX_unrecognized []byte `json:"-"` } func (m *ContainerConfig) Reset() { *m = ContainerConfig{} } func (m *ContainerConfig) String() string { return proto.CompactTextString(m) } func (*ContainerConfig) ProtoMessage() {} -func (*ContainerConfig) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{33} } +func (*ContainerConfig) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{34} } func (m *ContainerConfig) GetMetadata() *ContainerMetadata { if m != nil { @@ -1485,6 +1528,13 @@ func (m *ContainerConfig) GetMounts() []*Mount { return nil } +func (m *ContainerConfig) GetDevices() []*Device { + if m != nil { + return m.Devices + } + return nil +} + func (m *ContainerConfig) GetLabels() map[string]string { if m != nil { return m.Labels @@ -1564,7 +1614,7 @@ type CreateContainerRequest struct { func (m *CreateContainerRequest) Reset() { *m = CreateContainerRequest{} } func (m *CreateContainerRequest) String() string { return proto.CompactTextString(m) } func (*CreateContainerRequest) ProtoMessage() {} -func (*CreateContainerRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{34} } +func (*CreateContainerRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{35} } func (m *CreateContainerRequest) GetPodSandboxId() string { if m != nil && m.PodSandboxId != nil { @@ -1596,7 +1646,7 @@ type CreateContainerResponse struct { func (m *CreateContainerResponse) Reset() { *m = CreateContainerResponse{} } func (m *CreateContainerResponse) String() string { return proto.CompactTextString(m) } func (*CreateContainerResponse) ProtoMessage() {} -func (*CreateContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{35} } +func (*CreateContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{36} } func (m *CreateContainerResponse) GetContainerId() string { if m != nil && m.ContainerId != nil { @@ -1614,7 +1664,7 @@ type StartContainerRequest struct { func (m *StartContainerRequest) Reset() { *m = StartContainerRequest{} } func (m *StartContainerRequest) String() string { return proto.CompactTextString(m) } func (*StartContainerRequest) ProtoMessage() {} -func (*StartContainerRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{36} } +func (*StartContainerRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{37} } func (m *StartContainerRequest) GetContainerId() string { if m != nil && m.ContainerId != nil { @@ -1630,7 +1680,7 @@ type StartContainerResponse struct { func (m *StartContainerResponse) Reset() { *m = StartContainerResponse{} } func (m *StartContainerResponse) String() string { return proto.CompactTextString(m) } func (*StartContainerResponse) ProtoMessage() {} -func (*StartContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{37} } +func (*StartContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{38} } type StopContainerRequest struct { // The id of the container @@ -1643,7 +1693,7 @@ type StopContainerRequest struct { func (m *StopContainerRequest) Reset() { *m = StopContainerRequest{} } func (m *StopContainerRequest) String() string { return proto.CompactTextString(m) } func (*StopContainerRequest) ProtoMessage() {} -func (*StopContainerRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{38} } +func (*StopContainerRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{39} } func (m *StopContainerRequest) GetContainerId() string { if m != nil && m.ContainerId != nil { @@ -1666,7 +1716,7 @@ type StopContainerResponse struct { func (m *StopContainerResponse) Reset() { *m = StopContainerResponse{} } func (m *StopContainerResponse) String() string { return proto.CompactTextString(m) } func (*StopContainerResponse) ProtoMessage() {} -func (*StopContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{39} } +func (*StopContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{40} } type RemoveContainerRequest struct { // The id of the container @@ -1677,7 +1727,7 @@ type RemoveContainerRequest struct { func (m *RemoveContainerRequest) Reset() { *m = RemoveContainerRequest{} } func (m *RemoveContainerRequest) String() string { return proto.CompactTextString(m) } func (*RemoveContainerRequest) ProtoMessage() {} -func (*RemoveContainerRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{40} } +func (*RemoveContainerRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{41} } func (m *RemoveContainerRequest) GetContainerId() string { if m != nil && m.ContainerId != nil { @@ -1693,7 +1743,7 @@ type RemoveContainerResponse struct { func (m *RemoveContainerResponse) Reset() { *m = RemoveContainerResponse{} } func (m *RemoveContainerResponse) String() string { return proto.CompactTextString(m) } func (*RemoveContainerResponse) ProtoMessage() {} -func (*RemoveContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{41} } +func (*RemoveContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{42} } // ContainerFilter is used to filter containers. // All those fields are combined with 'AND' @@ -1714,7 +1764,7 @@ type ContainerFilter struct { func (m *ContainerFilter) Reset() { *m = ContainerFilter{} } func (m *ContainerFilter) String() string { return proto.CompactTextString(m) } func (*ContainerFilter) ProtoMessage() {} -func (*ContainerFilter) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{42} } +func (*ContainerFilter) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{43} } func (m *ContainerFilter) GetId() string { if m != nil && m.Id != nil { @@ -1752,7 +1802,7 @@ type ListContainersRequest struct { func (m *ListContainersRequest) Reset() { *m = ListContainersRequest{} } func (m *ListContainersRequest) String() string { return proto.CompactTextString(m) } func (*ListContainersRequest) ProtoMessage() {} -func (*ListContainersRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{43} } +func (*ListContainersRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{44} } func (m *ListContainersRequest) GetFilter() *ContainerFilter { if m != nil { @@ -1791,7 +1841,7 @@ type Container struct { func (m *Container) Reset() { *m = Container{} } func (m *Container) String() string { return proto.CompactTextString(m) } func (*Container) ProtoMessage() {} -func (*Container) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{44} } +func (*Container) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{45} } func (m *Container) GetId() string { if m != nil && m.Id != nil { @@ -1865,7 +1915,7 @@ type ListContainersResponse struct { func (m *ListContainersResponse) Reset() { *m = ListContainersResponse{} } func (m *ListContainersResponse) String() string { return proto.CompactTextString(m) } func (*ListContainersResponse) ProtoMessage() {} -func (*ListContainersResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{45} } +func (*ListContainersResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{46} } func (m *ListContainersResponse) GetContainers() []*Container { if m != nil { @@ -1883,7 +1933,7 @@ type ContainerStatusRequest struct { func (m *ContainerStatusRequest) Reset() { *m = ContainerStatusRequest{} } func (m *ContainerStatusRequest) String() string { return proto.CompactTextString(m) } func (*ContainerStatusRequest) ProtoMessage() {} -func (*ContainerStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{46} } +func (*ContainerStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{47} } func (m *ContainerStatusRequest) GetContainerId() string { if m != nil && m.ContainerId != nil { @@ -1930,7 +1980,7 @@ type ContainerStatus struct { func (m *ContainerStatus) Reset() { *m = ContainerStatus{} } func (m *ContainerStatus) String() string { return proto.CompactTextString(m) } func (*ContainerStatus) ProtoMessage() {} -func (*ContainerStatus) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{47} } +func (*ContainerStatus) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{48} } func (m *ContainerStatus) GetId() string { if m != nil && m.Id != nil { @@ -2039,7 +2089,7 @@ type ContainerStatusResponse struct { func (m *ContainerStatusResponse) Reset() { *m = ContainerStatusResponse{} } func (m *ContainerStatusResponse) String() string { return proto.CompactTextString(m) } func (*ContainerStatusResponse) ProtoMessage() {} -func (*ContainerStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{48} } +func (*ContainerStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{49} } func (m *ContainerStatusResponse) GetStatus() *ContainerStatus { if m != nil { @@ -2061,7 +2111,7 @@ type ExecSyncRequest struct { func (m *ExecSyncRequest) Reset() { *m = ExecSyncRequest{} } func (m *ExecSyncRequest) String() string { return proto.CompactTextString(m) } func (*ExecSyncRequest) ProtoMessage() {} -func (*ExecSyncRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{49} } +func (*ExecSyncRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{50} } func (m *ExecSyncRequest) GetContainerId() string { if m != nil && m.ContainerId != nil { @@ -2097,7 +2147,7 @@ type ExecSyncResponse struct { func (m *ExecSyncResponse) Reset() { *m = ExecSyncResponse{} } func (m *ExecSyncResponse) String() string { return proto.CompactTextString(m) } func (*ExecSyncResponse) ProtoMessage() {} -func (*ExecSyncResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{50} } +func (*ExecSyncResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{51} } func (m *ExecSyncResponse) GetStdout() []byte { if m != nil { @@ -2135,7 +2185,7 @@ type ExecRequest struct { func (m *ExecRequest) Reset() { *m = ExecRequest{} } func (m *ExecRequest) String() string { return proto.CompactTextString(m) } func (*ExecRequest) ProtoMessage() {} -func (*ExecRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{51} } +func (*ExecRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{52} } func (m *ExecRequest) GetContainerId() string { if m != nil && m.ContainerId != nil { @@ -2174,7 +2224,7 @@ type ExecResponse struct { func (m *ExecResponse) Reset() { *m = ExecResponse{} } func (m *ExecResponse) String() string { return proto.CompactTextString(m) } func (*ExecResponse) ProtoMessage() {} -func (*ExecResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{52} } +func (*ExecResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{53} } func (m *ExecResponse) GetUrl() string { if m != nil && m.Url != nil { @@ -2194,7 +2244,7 @@ type AttachRequest struct { func (m *AttachRequest) Reset() { *m = AttachRequest{} } func (m *AttachRequest) String() string { return proto.CompactTextString(m) } func (*AttachRequest) ProtoMessage() {} -func (*AttachRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{53} } +func (*AttachRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{54} } func (m *AttachRequest) GetContainerId() string { if m != nil && m.ContainerId != nil { @@ -2219,7 +2269,7 @@ type AttachResponse struct { func (m *AttachResponse) Reset() { *m = AttachResponse{} } func (m *AttachResponse) String() string { return proto.CompactTextString(m) } func (*AttachResponse) ProtoMessage() {} -func (*AttachResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{54} } +func (*AttachResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{55} } func (m *AttachResponse) GetUrl() string { if m != nil && m.Url != nil { @@ -2239,7 +2289,7 @@ type PortForwardRequest struct { func (m *PortForwardRequest) Reset() { *m = PortForwardRequest{} } func (m *PortForwardRequest) String() string { return proto.CompactTextString(m) } func (*PortForwardRequest) ProtoMessage() {} -func (*PortForwardRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{55} } +func (*PortForwardRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{56} } func (m *PortForwardRequest) GetPodSandboxId() string { if m != nil && m.PodSandboxId != nil { @@ -2264,7 +2314,7 @@ type PortForwardResponse struct { func (m *PortForwardResponse) Reset() { *m = PortForwardResponse{} } func (m *PortForwardResponse) String() string { return proto.CompactTextString(m) } func (*PortForwardResponse) ProtoMessage() {} -func (*PortForwardResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{56} } +func (*PortForwardResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{57} } func (m *PortForwardResponse) GetUrl() string { if m != nil && m.Url != nil { @@ -2282,7 +2332,7 @@ type ImageFilter struct { func (m *ImageFilter) Reset() { *m = ImageFilter{} } func (m *ImageFilter) String() string { return proto.CompactTextString(m) } func (*ImageFilter) ProtoMessage() {} -func (*ImageFilter) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{57} } +func (*ImageFilter) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{58} } func (m *ImageFilter) GetImage() *ImageSpec { if m != nil { @@ -2300,7 +2350,7 @@ type ListImagesRequest struct { func (m *ListImagesRequest) Reset() { *m = ListImagesRequest{} } func (m *ListImagesRequest) String() string { return proto.CompactTextString(m) } func (*ListImagesRequest) ProtoMessage() {} -func (*ListImagesRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{58} } +func (*ListImagesRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{59} } func (m *ListImagesRequest) GetFilter() *ImageFilter { if m != nil { @@ -2325,7 +2375,7 @@ type Image struct { func (m *Image) Reset() { *m = Image{} } func (m *Image) String() string { return proto.CompactTextString(m) } func (*Image) ProtoMessage() {} -func (*Image) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{59} } +func (*Image) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{60} } func (m *Image) GetId() string { if m != nil && m.Id != nil { @@ -2364,7 +2414,7 @@ type ListImagesResponse struct { func (m *ListImagesResponse) Reset() { *m = ListImagesResponse{} } func (m *ListImagesResponse) String() string { return proto.CompactTextString(m) } func (*ListImagesResponse) ProtoMessage() {} -func (*ListImagesResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{60} } +func (*ListImagesResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{61} } func (m *ListImagesResponse) GetImages() []*Image { if m != nil { @@ -2382,7 +2432,7 @@ type ImageStatusRequest struct { func (m *ImageStatusRequest) Reset() { *m = ImageStatusRequest{} } func (m *ImageStatusRequest) String() string { return proto.CompactTextString(m) } func (*ImageStatusRequest) ProtoMessage() {} -func (*ImageStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{61} } +func (*ImageStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{62} } func (m *ImageStatusRequest) GetImage() *ImageSpec { if m != nil { @@ -2400,7 +2450,7 @@ type ImageStatusResponse struct { func (m *ImageStatusResponse) Reset() { *m = ImageStatusResponse{} } func (m *ImageStatusResponse) String() string { return proto.CompactTextString(m) } func (*ImageStatusResponse) ProtoMessage() {} -func (*ImageStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{62} } +func (*ImageStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{63} } func (m *ImageStatusResponse) GetImage() *Image { if m != nil { @@ -2426,7 +2476,7 @@ type AuthConfig struct { func (m *AuthConfig) Reset() { *m = AuthConfig{} } func (m *AuthConfig) String() string { return proto.CompactTextString(m) } func (*AuthConfig) ProtoMessage() {} -func (*AuthConfig) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{63} } +func (*AuthConfig) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{64} } func (m *AuthConfig) GetUsername() string { if m != nil && m.Username != nil { @@ -2483,7 +2533,7 @@ type PullImageRequest struct { func (m *PullImageRequest) Reset() { *m = PullImageRequest{} } func (m *PullImageRequest) String() string { return proto.CompactTextString(m) } func (*PullImageRequest) ProtoMessage() {} -func (*PullImageRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{64} } +func (*PullImageRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{65} } func (m *PullImageRequest) GetImage() *ImageSpec { if m != nil { @@ -2513,7 +2563,7 @@ type PullImageResponse struct { func (m *PullImageResponse) Reset() { *m = PullImageResponse{} } func (m *PullImageResponse) String() string { return proto.CompactTextString(m) } func (*PullImageResponse) ProtoMessage() {} -func (*PullImageResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{65} } +func (*PullImageResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{66} } type RemoveImageRequest struct { // The spec of the image @@ -2524,7 +2574,7 @@ type RemoveImageRequest struct { func (m *RemoveImageRequest) Reset() { *m = RemoveImageRequest{} } func (m *RemoveImageRequest) String() string { return proto.CompactTextString(m) } func (*RemoveImageRequest) ProtoMessage() {} -func (*RemoveImageRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{66} } +func (*RemoveImageRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{67} } func (m *RemoveImageRequest) GetImage() *ImageSpec { if m != nil { @@ -2540,7 +2590,7 @@ type RemoveImageResponse struct { func (m *RemoveImageResponse) Reset() { *m = RemoveImageResponse{} } func (m *RemoveImageResponse) String() string { return proto.CompactTextString(m) } func (*RemoveImageResponse) ProtoMessage() {} -func (*RemoveImageResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{67} } +func (*RemoveImageResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{68} } type NetworkConfig struct { // The CIDR to use for pod IP addresses @@ -2551,7 +2601,7 @@ type NetworkConfig struct { func (m *NetworkConfig) Reset() { *m = NetworkConfig{} } func (m *NetworkConfig) String() string { return proto.CompactTextString(m) } func (*NetworkConfig) ProtoMessage() {} -func (*NetworkConfig) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{68} } +func (*NetworkConfig) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{69} } func (m *NetworkConfig) GetPodCidr() string { if m != nil && m.PodCidr != nil { @@ -2568,7 +2618,7 @@ type RuntimeConfig struct { func (m *RuntimeConfig) Reset() { *m = RuntimeConfig{} } func (m *RuntimeConfig) String() string { return proto.CompactTextString(m) } func (*RuntimeConfig) ProtoMessage() {} -func (*RuntimeConfig) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{69} } +func (*RuntimeConfig) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{70} } func (m *RuntimeConfig) GetNetworkConfig() *NetworkConfig { if m != nil { @@ -2585,7 +2635,7 @@ type UpdateRuntimeConfigRequest struct { func (m *UpdateRuntimeConfigRequest) Reset() { *m = UpdateRuntimeConfigRequest{} } func (m *UpdateRuntimeConfigRequest) String() string { return proto.CompactTextString(m) } func (*UpdateRuntimeConfigRequest) ProtoMessage() {} -func (*UpdateRuntimeConfigRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{70} } +func (*UpdateRuntimeConfigRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{71} } func (m *UpdateRuntimeConfigRequest) GetRuntimeConfig() *RuntimeConfig { if m != nil { @@ -2601,7 +2651,7 @@ type UpdateRuntimeConfigResponse struct { func (m *UpdateRuntimeConfigResponse) Reset() { *m = UpdateRuntimeConfigResponse{} } func (m *UpdateRuntimeConfigResponse) String() string { return proto.CompactTextString(m) } func (*UpdateRuntimeConfigResponse) ProtoMessage() {} -func (*UpdateRuntimeConfigResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{71} } +func (*UpdateRuntimeConfigResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{72} } func init() { proto.RegisterType((*VersionRequest)(nil), "runtime.VersionRequest") @@ -2637,6 +2687,7 @@ func init() { proto.RegisterType((*LinuxContainerConfig)(nil), "runtime.LinuxContainerConfig") proto.RegisterType((*LinuxUser)(nil), "runtime.LinuxUser") proto.RegisterType((*ContainerMetadata)(nil), "runtime.ContainerMetadata") + proto.RegisterType((*Device)(nil), "runtime.Device") proto.RegisterType((*ContainerConfig)(nil), "runtime.ContainerConfig") proto.RegisterType((*CreateContainerRequest)(nil), "runtime.CreateContainerRequest") proto.RegisterType((*CreateContainerResponse)(nil), "runtime.CreateContainerResponse") @@ -3503,201 +3554,204 @@ var _ImageService_serviceDesc = grpc.ServiceDesc{ } var fileDescriptorApi = []byte{ - // 3133 bytes of a gzipped FileDescriptorProto + // 3179 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x3a, 0x4d, 0x73, 0x1b, 0xc7, - 0xb1, 0x04, 0xc0, 0x0f, 0xa0, 0x41, 0x80, 0xe0, 0x90, 0x22, 0x21, 0xd0, 0x92, 0xa8, 0xb5, 0x65, - 0x4b, 0xb2, 0xad, 0xb2, 0xf0, 0xde, 0xb3, 0x9e, 0x65, 0x4b, 0x36, 0x4c, 0xd2, 0x0e, 0x2d, 0x09, - 0x62, 0x06, 0x92, 0x63, 0x97, 0x0f, 0x9b, 0x15, 0x76, 0x04, 0xae, 0x04, 0xec, 0xae, 0x67, 0x07, - 0xb4, 0x98, 0x6b, 0x2e, 0x39, 0xe4, 0x90, 0x6b, 0x6e, 0x39, 0xa4, 0xca, 0x87, 0xdc, 0x52, 0x95, - 0xaa, 0xfc, 0x87, 0x54, 0x7e, 0x48, 0x0e, 0x39, 0x27, 0xc7, 0xd4, 0x7c, 0xec, 0xec, 0xec, 0x07, - 0x68, 0x52, 0xae, 0x8a, 0x75, 0xdb, 0xe9, 0xe9, 0xee, 0xe9, 0xe9, 0xee, 0xe9, 0xe9, 0xee, 0x59, - 0xa8, 0x39, 0xa1, 0x77, 0x23, 0xa4, 0x01, 0x0b, 0xd0, 0x12, 0x9d, 0xfa, 0xcc, 0x9b, 0x10, 0xeb, - 0x3a, 0x34, 0xbf, 0x24, 0x34, 0xf2, 0x02, 0x1f, 0x93, 0x6f, 0xa7, 0x24, 0x62, 0xa8, 0x0d, 0x4b, - 0x47, 0x12, 0xd2, 0x2e, 0x6d, 0x97, 0xae, 0xd6, 0x70, 0x3c, 0xb4, 0xbe, 0x2f, 0xc1, 0x8a, 0x46, - 0x8e, 0xc2, 0xc0, 0x8f, 0xc8, 0x6c, 0x6c, 0x74, 0x19, 0x96, 0xd5, 0x22, 0xb6, 0xef, 0x4c, 0x48, - 0xbb, 0x2c, 0xa6, 0xeb, 0x0a, 0xd6, 0x77, 0x26, 0x04, 0xbd, 0x05, 0x2b, 0x31, 0x4a, 0xcc, 0xa4, - 0x22, 0xb0, 0x9a, 0x0a, 0xac, 0x56, 0x43, 0x37, 0x60, 0x2d, 0x46, 0x74, 0x42, 0x4f, 0x23, 0xcf, - 0x0b, 0xe4, 0x55, 0x35, 0xd5, 0x0b, 0x3d, 0x85, 0x6f, 0x7d, 0x03, 0xb5, 0xdd, 0xfe, 0x60, 0x27, - 0xf0, 0x9f, 0x7a, 0x23, 0x2e, 0x62, 0x44, 0x28, 0xa7, 0x69, 0x97, 0xb6, 0x2b, 0x5c, 0x44, 0x35, - 0x44, 0x1d, 0xa8, 0x46, 0xc4, 0xa1, 0xc3, 0x43, 0x12, 0xb5, 0xcb, 0x62, 0x4a, 0x8f, 0x39, 0x55, - 0x10, 0x32, 0x2f, 0xf0, 0xa3, 0x76, 0x45, 0x52, 0xa9, 0xa1, 0xf5, 0xfb, 0x12, 0xd4, 0x0f, 0x02, - 0xca, 0x1e, 0x38, 0x61, 0xe8, 0xf9, 0x23, 0xf4, 0x2e, 0x54, 0x85, 0x52, 0x87, 0xc1, 0x58, 0xe8, - 0xa0, 0xd9, 0x5d, 0xbd, 0xa1, 0x44, 0xba, 0x71, 0xa0, 0x26, 0xb0, 0x46, 0x41, 0x57, 0xa0, 0x39, - 0x0c, 0x7c, 0xe6, 0x78, 0x3e, 0xa1, 0x76, 0x18, 0x50, 0x26, 0x34, 0xb3, 0x80, 0x1b, 0x1a, 0xca, - 0x99, 0xa3, 0x2d, 0xa8, 0x1d, 0x06, 0x11, 0x93, 0x18, 0x15, 0x81, 0x51, 0xe5, 0x00, 0x31, 0xb9, - 0x09, 0x4b, 0x62, 0xd2, 0x0b, 0x95, 0x0e, 0x16, 0xf9, 0x70, 0x3f, 0xb4, 0x7e, 0x57, 0x82, 0x85, - 0x07, 0xc1, 0xd4, 0x67, 0x99, 0x65, 0x1c, 0x76, 0xa8, 0xec, 0x63, 0x2c, 0xe3, 0xb0, 0xc3, 0x64, - 0x19, 0x8e, 0x21, 0x4d, 0x24, 0x97, 0xe1, 0x93, 0x1d, 0xa8, 0x52, 0xe2, 0xb8, 0x81, 0x3f, 0x3e, - 0x16, 0x22, 0x54, 0xb1, 0x1e, 0x73, 0xdb, 0x45, 0x64, 0xec, 0xf9, 0xd3, 0x17, 0x36, 0x25, 0x63, - 0xe7, 0x09, 0x19, 0x0b, 0x51, 0xaa, 0xb8, 0xa9, 0xc0, 0x58, 0x42, 0xad, 0x67, 0xb0, 0xc2, 0x8d, - 0x1d, 0x85, 0xce, 0x90, 0x3c, 0x14, 0x2a, 0xe4, 0xae, 0x21, 0x16, 0xf5, 0x09, 0xfb, 0x2e, 0xa0, - 0xcf, 0x85, 0x64, 0x55, 0x5c, 0xe7, 0xb0, 0xbe, 0x04, 0xa1, 0xf3, 0x50, 0x95, 0x72, 0x79, 0xae, - 0x10, 0xab, 0x8a, 0xc5, 0x8e, 0x0f, 0x3c, 0x57, 0x4f, 0x79, 0xe1, 0x50, 0x49, 0xb5, 0x24, 0x77, - 0x3f, 0xb4, 0x7e, 0x5d, 0x82, 0x73, 0xf7, 0xf9, 0xe2, 0x07, 0x81, 0x3b, 0x70, 0x7c, 0xf7, 0x49, - 0xf0, 0x42, 0x39, 0xc1, 0xeb, 0xd0, 0x18, 0x8e, 0x68, 0x30, 0x0d, 0xed, 0xd0, 0xa1, 0xc4, 0x67, - 0x4a, 0x1b, 0xcb, 0x12, 0x78, 0x20, 0x60, 0x68, 0x0f, 0x56, 0xfd, 0x58, 0x54, 0x3b, 0xb6, 0x3e, - 0x5f, 0xbd, 0xde, 0x6d, 0x6b, 0x93, 0x66, 0x36, 0x83, 0x5b, 0x7e, 0x1a, 0x10, 0x59, 0x14, 0x50, - 0xb2, 0xfe, 0x03, 0xc2, 0x1c, 0xd7, 0x61, 0x0e, 0x42, 0x30, 0x2f, 0xce, 0x81, 0x5c, 0x58, 0x7c, - 0xa3, 0x16, 0x54, 0xa6, 0x6a, 0x83, 0x35, 0xcc, 0x3f, 0xd1, 0x6b, 0x50, 0xd3, 0xfc, 0xd4, 0x61, - 0x48, 0x00, 0xdc, 0x29, 0x1d, 0xc6, 0xc8, 0x24, 0x64, 0x42, 0xd9, 0x0d, 0x1c, 0x0f, 0xad, 0xbf, - 0xce, 0x43, 0x2b, 0xb7, 0xe9, 0x5b, 0x50, 0x9d, 0xa8, 0xe5, 0xc5, 0xb2, 0xf5, 0xee, 0x56, 0xe2, - 0x99, 0x39, 0x09, 0xb1, 0x46, 0xe6, 0x86, 0xe7, 0x2a, 0x35, 0xce, 0xad, 0x1e, 0x73, 0x4d, 0x8e, - 0x83, 0x91, 0xed, 0x7a, 0x94, 0x0c, 0x59, 0x40, 0x8f, 0x95, 0x94, 0xcb, 0xe3, 0x60, 0xb4, 0x1b, - 0xc3, 0xd0, 0x4d, 0x00, 0xd7, 0x8f, 0xec, 0xa1, 0x90, 0x43, 0xc8, 0x5a, 0xef, 0x22, 0xbd, 0xb6, - 0x3e, 0x9b, 0xb8, 0xe6, 0xfa, 0x91, 0x12, 0xf6, 0x03, 0x68, 0x70, 0x5f, 0xb7, 0x27, 0xf2, 0x58, - 0x45, 0xed, 0x85, 0xed, 0xca, 0xd5, 0x7a, 0x77, 0xdd, 0x90, 0x58, 0x9f, 0x39, 0xbc, 0x1c, 0x26, - 0x83, 0x08, 0xdd, 0x81, 0x45, 0xe1, 0x6b, 0x51, 0x7b, 0x51, 0xd0, 0x5c, 0x29, 0xd8, 0xa5, 0x5c, - 0xe5, 0xc6, 0x7d, 0x81, 0xb7, 0xe7, 0x33, 0x7a, 0x8c, 0x15, 0x11, 0xba, 0x0f, 0x75, 0xc7, 0xf7, - 0x03, 0xe6, 0x48, 0x83, 0x2f, 0x09, 0x1e, 0xd7, 0x67, 0xf3, 0xe8, 0x25, 0xc8, 0x92, 0x91, 0x49, - 0x8e, 0xfe, 0x17, 0x16, 0x84, 0xff, 0xb7, 0xab, 0x62, 0xd7, 0x17, 0x35, 0x9f, 0x42, 0xc7, 0xc4, - 0x12, 0xb9, 0xf3, 0x01, 0xd4, 0x0d, 0xd1, 0xb8, 0x63, 0x3c, 0x27, 0xc7, 0xca, 0x57, 0xf8, 0x27, - 0x5a, 0x87, 0x85, 0x23, 0x67, 0x3c, 0x8d, 0xed, 0x21, 0x07, 0xb7, 0xcb, 0xff, 0x5f, 0xea, 0xdc, - 0x85, 0x56, 0x56, 0xa2, 0xb3, 0xd0, 0x5b, 0xfb, 0xb0, 0x8e, 0xa7, 0x7e, 0x22, 0x58, 0x7c, 0x11, - 0xdc, 0x84, 0x45, 0x65, 0x3f, 0xe9, 0x3b, 0xe7, 0x67, 0x6a, 0x04, 0x2b, 0x44, 0xeb, 0x0e, 0x9c, - 0xcb, 0xb0, 0x52, 0xd7, 0xc4, 0x1b, 0xd0, 0x0c, 0x03, 0xd7, 0x8e, 0x24, 0xd8, 0xf6, 0xdc, 0xf8, - 0xfc, 0x85, 0x1a, 0x77, 0xdf, 0xe5, 0xe4, 0x03, 0x16, 0x84, 0x79, 0x51, 0x4e, 0x47, 0xde, 0x86, - 0x8d, 0x2c, 0xb9, 0x5c, 0xde, 0xfa, 0x18, 0x36, 0x31, 0x99, 0x04, 0x47, 0xe4, 0x65, 0x59, 0x77, - 0xa0, 0x9d, 0x67, 0x90, 0x30, 0x4f, 0xa0, 0x03, 0xe6, 0xb0, 0x69, 0x74, 0x36, 0xe6, 0xd7, 0x4c, - 0x06, 0x2a, 0x00, 0x4a, 0x3e, 0xa8, 0x09, 0x65, 0x2f, 0x54, 0x44, 0x65, 0x2f, 0xb4, 0xbe, 0x86, - 0x5a, 0xdf, 0x8c, 0x06, 0x66, 0x04, 0xad, 0xe1, 0x78, 0x88, 0xba, 0xc9, 0xe5, 0xf5, 0x43, 0xe1, - 0x4b, 0x5f, 0x6b, 0xf7, 0x72, 0xa1, 0x53, 0xc9, 0xd0, 0x05, 0xd0, 0x11, 0x28, 0x52, 0xbe, 0x80, - 0xf2, 0xfc, 0xb0, 0x81, 0x65, 0xfd, 0x31, 0x15, 0x8e, 0x8c, 0xcd, 0xb8, 0x7a, 0x33, 0x6e, 0x2a, - 0x3c, 0x95, 0xcf, 0x12, 0x9e, 0x6e, 0xc0, 0x42, 0xc4, 0x1c, 0x26, 0x03, 0x64, 0xd3, 0xd8, 0x5c, - 0x7a, 0x49, 0x82, 0x25, 0x1a, 0xba, 0x00, 0x30, 0xa4, 0xc4, 0x61, 0xc4, 0xb5, 0x1d, 0x19, 0x39, - 0x2b, 0xb8, 0xa6, 0x20, 0x3d, 0x86, 0x6e, 0x27, 0x7a, 0x5c, 0x10, 0x62, 0x6c, 0x17, 0x30, 0x4c, - 0xd9, 0x25, 0xd1, 0xb4, 0x3e, 0xed, 0x8b, 0x27, 0x9f, 0x76, 0x45, 0x27, 0x91, 0x8d, 0x80, 0xb5, - 0x34, 0x33, 0x60, 0x49, 0x8a, 0xd3, 0x04, 0xac, 0xea, 0xcc, 0x80, 0xa5, 0x78, 0x9c, 0x18, 0xb0, - 0x7e, 0xca, 0xd0, 0xf3, 0x00, 0xda, 0xf9, 0xa3, 0xa3, 0x42, 0xc6, 0x4d, 0x58, 0x8c, 0x04, 0xe4, - 0x84, 0xf0, 0xa3, 0x48, 0x14, 0xa2, 0xf5, 0x8f, 0x92, 0xe9, 0x75, 0x9f, 0x79, 0x63, 0x46, 0x68, - 0xce, 0xeb, 0xb4, 0xf3, 0x94, 0x4f, 0xe7, 0x3c, 0x03, 0x68, 0x0a, 0xb5, 0xdb, 0x11, 0x19, 0x8b, - 0xdb, 0x4d, 0xe4, 0x83, 0xf5, 0xee, 0x3b, 0x05, 0x84, 0x72, 0x49, 0x69, 0xb3, 0x81, 0x42, 0x97, - 0x1a, 0x6f, 0x8c, 0x4d, 0x58, 0xe7, 0x13, 0x40, 0x79, 0xa4, 0x33, 0xa9, 0xee, 0x0b, 0x7e, 0x5c, - 0x79, 0x3a, 0x58, 0x10, 0xb6, 0x9f, 0x0a, 0x31, 0x4e, 0xd0, 0x9b, 0x94, 0x13, 0x2b, 0x44, 0xeb, - 0x0f, 0x15, 0x80, 0x64, 0xf2, 0x95, 0x3d, 0xa7, 0xb7, 0xf4, 0xa9, 0x91, 0xa9, 0xc1, 0xa5, 0x02, - 0x7e, 0x85, 0xe7, 0xe5, 0xb3, 0xf4, 0x79, 0x91, 0x49, 0xc2, 0x1b, 0x45, 0xd4, 0xaf, 0xec, 0x49, - 0xd9, 0x81, 0x8d, 0xac, 0xb9, 0xd5, 0x39, 0xb9, 0x06, 0x0b, 0x1e, 0x23, 0x13, 0x59, 0xdc, 0xd4, - 0xbb, 0x6b, 0x05, 0xdb, 0xc2, 0x12, 0xc3, 0xba, 0x0c, 0xb5, 0xfd, 0x89, 0x33, 0x22, 0x83, 0x90, - 0x0c, 0xf9, 0x5a, 0x1e, 0x1f, 0xa8, 0xf5, 0xe5, 0xc0, 0xea, 0x42, 0xf5, 0x1e, 0x39, 0xfe, 0x92, - 0xaf, 0x7b, 0x5a, 0xf9, 0xac, 0xbf, 0x95, 0x60, 0x53, 0x84, 0xbb, 0x9d, 0xb8, 0xb4, 0xc0, 0x24, - 0x0a, 0xa6, 0x74, 0x48, 0x22, 0x61, 0xd2, 0x70, 0x6a, 0x87, 0x84, 0x7a, 0x81, 0xf4, 0x29, 0x6e, - 0xd2, 0x70, 0x7a, 0x20, 0x00, 0xbc, 0xfc, 0xe0, 0xd3, 0xdf, 0x4e, 0x03, 0xe5, 0x5b, 0x15, 0x5c, - 0x1d, 0x86, 0xd3, 0x9f, 0xf3, 0x71, 0x4c, 0x1b, 0x1d, 0x3a, 0x94, 0x44, 0xc2, 0x87, 0x24, 0xed, - 0x40, 0x00, 0xd0, 0x4d, 0x38, 0x37, 0x21, 0x93, 0x80, 0x1e, 0xdb, 0x63, 0x6f, 0xe2, 0x31, 0xdb, - 0xf3, 0xed, 0x27, 0xc7, 0x8c, 0x44, 0xca, 0x71, 0x90, 0x9c, 0xbc, 0xcf, 0xe7, 0xf6, 0xfd, 0x4f, - 0xf9, 0x0c, 0xb2, 0xa0, 0x11, 0x04, 0x13, 0x3b, 0x1a, 0x06, 0x94, 0xd8, 0x8e, 0xfb, 0x4c, 0xc4, - 0xfb, 0x0a, 0xae, 0x07, 0xc1, 0x64, 0xc0, 0x61, 0x3d, 0xf7, 0x99, 0xe5, 0x40, 0x63, 0xb0, 0x27, - 0xb6, 0xa3, 0xaa, 0x15, 0x04, 0xf3, 0xd3, 0x48, 0x1d, 0xa7, 0x1a, 0x16, 0xdf, 0x1c, 0x46, 0x83, - 0x71, 0xac, 0x07, 0xf1, 0xcd, 0x61, 0xec, 0x38, 0x8c, 0xb3, 0x76, 0xf1, 0xcd, 0x15, 0x36, 0x26, - 0x47, 0xaa, 0x36, 0xaa, 0x61, 0x39, 0xb0, 0x5c, 0x80, 0x1d, 0x27, 0x74, 0x9e, 0x78, 0x63, 0x8f, - 0x1d, 0xa3, 0x6b, 0xd0, 0x72, 0x5c, 0xd7, 0x1e, 0xc6, 0x10, 0x8f, 0xc4, 0x85, 0xea, 0x8a, 0xe3, - 0xba, 0x3b, 0x06, 0x18, 0xbd, 0x0d, 0xab, 0x2e, 0x0d, 0xc2, 0x34, 0xae, 0xac, 0x5c, 0x5b, 0x7c, - 0xc2, 0x44, 0xb6, 0xfe, 0x5d, 0x82, 0xf5, 0xb4, 0x59, 0x54, 0xa6, 0x7d, 0x17, 0x6a, 0x34, 0x36, - 0x90, 0x0a, 0x12, 0xdb, 0xe9, 0x7b, 0x2b, 0x6f, 0x48, 0x9c, 0x90, 0xa0, 0x5b, 0xb0, 0x9c, 0x11, - 0xa0, 0x94, 0x72, 0xbc, 0x64, 0x6f, 0x38, 0x85, 0x88, 0x3e, 0x4e, 0x6a, 0xc6, 0xa4, 0xb6, 0xe6, - 0xb4, 0x1b, 0x9a, 0x36, 0xa5, 0x7a, 0x5d, 0x4b, 0xaa, 0xca, 0x0a, 0xbd, 0xa9, 0x4c, 0x91, 0x2d, - 0x28, 0x04, 0xcd, 0xe3, 0x88, 0x50, 0x69, 0x1e, 0xeb, 0x2b, 0xa8, 0x69, 0x50, 0x5c, 0x64, 0x49, - 0xdf, 0x13, 0x45, 0x56, 0x0b, 0x2a, 0x23, 0x55, 0x76, 0x55, 0x30, 0xff, 0xe4, 0xd5, 0xac, 0xe3, - 0xba, 0x1e, 0x5f, 0xc5, 0x19, 0xdb, 0x23, 0xcf, 0x95, 0x55, 0x7f, 0x05, 0x37, 0x13, 0xf0, 0xe7, - 0x9e, 0x1b, 0x59, 0x3d, 0x58, 0xd5, 0xca, 0x39, 0xb1, 0xb4, 0x33, 0x4a, 0xb5, 0x72, 0xba, 0x54, - 0xfb, 0xd7, 0x02, 0xac, 0x64, 0x4d, 0xf2, 0x7e, 0xae, 0x52, 0xeb, 0x24, 0xea, 0xcc, 0xae, 0x67, - 0x44, 0xd8, 0xab, 0xf1, 0x21, 0x2e, 0x67, 0x34, 0xa2, 0xcf, 0xb9, 0x3a, 0xd8, 0x5c, 0x9e, 0x61, - 0x30, 0x99, 0x38, 0xbe, 0x1b, 0xf7, 0x33, 0xd4, 0x90, 0x4b, 0xef, 0xd0, 0x11, 0x3f, 0x36, 0x1c, - 0x2c, 0xbe, 0xd1, 0x25, 0xa8, 0xf3, 0xf4, 0xc6, 0xf3, 0x45, 0xa1, 0x27, 0x8e, 0x49, 0x0d, 0x83, - 0x02, 0xed, 0x7a, 0x14, 0x5d, 0x81, 0x79, 0xe2, 0x1f, 0xc5, 0xb1, 0x34, 0x69, 0x78, 0xc4, 0xc1, - 0x03, 0x8b, 0x69, 0xf4, 0x26, 0x2c, 0x4e, 0x82, 0xa9, 0xcf, 0xe2, 0x44, 0xa7, 0xa9, 0x11, 0x45, - 0x97, 0x02, 0xab, 0x59, 0xf4, 0x91, 0x0e, 0xed, 0xd5, 0x4c, 0x70, 0xce, 0x68, 0xaa, 0x30, 0xbe, - 0xdf, 0x4b, 0xc7, 0xf7, 0x9a, 0x60, 0x71, 0x6d, 0x26, 0x8b, 0x93, 0xeb, 0xb7, 0x8b, 0x00, 0x21, - 0xf5, 0x8e, 0xbc, 0x31, 0x19, 0x11, 0xb7, 0x0d, 0xa2, 0xc1, 0x60, 0x40, 0x44, 0xd3, 0x4a, 0x35, - 0x41, 0x6c, 0x1a, 0x04, 0xec, 0x69, 0xd4, 0xae, 0xcb, 0xc6, 0x47, 0x0c, 0xc6, 0x02, 0x8a, 0xce, - 0x43, 0x95, 0x17, 0xca, 0xa2, 0xb3, 0xb2, 0x2c, 0xf3, 0xf3, 0x71, 0x30, 0x12, 0x8d, 0x95, 0x75, - 0x7e, 0x31, 0xba, 0x9e, 0xdf, 0x6e, 0x08, 0x4a, 0x39, 0xe0, 0xf1, 0x4e, 0x7c, 0xd8, 0x81, 0x3f, - 0x24, 0xed, 0xa6, 0x98, 0xaa, 0x09, 0xc8, 0x43, 0x7f, 0x28, 0xc2, 0x31, 0x63, 0xc7, 0xed, 0x15, - 0x01, 0xe7, 0x9f, 0xe8, 0x7f, 0xe2, 0xe4, 0xb3, 0x25, 0xac, 0x7f, 0x61, 0xc6, 0x21, 0x7e, 0x65, - 0x2a, 0xcd, 0x3f, 0x97, 0x60, 0x63, 0x47, 0x5c, 0xe7, 0x46, 0x80, 0x39, 0x43, 0xa5, 0x84, 0xde, - 0xd3, 0x25, 0x69, 0xb6, 0xac, 0xc9, 0x6e, 0x56, 0xe1, 0xa1, 0x4f, 0xa0, 0x19, 0xf3, 0x54, 0x94, - 0x95, 0x1f, 0x2a, 0x66, 0x1b, 0x91, 0x39, 0xb4, 0x3e, 0x82, 0xcd, 0x9c, 0xcc, 0xea, 0xea, 0xbd, - 0x0c, 0xcb, 0x49, 0x8f, 0x4d, 0x8b, 0x5c, 0xd7, 0xb0, 0x7d, 0xd7, 0xba, 0xcd, 0x4b, 0x5a, 0x87, - 0xb2, 0xdc, 0x86, 0x4f, 0x41, 0x2b, 0xea, 0xd9, 0x34, 0xad, 0x2a, 0x39, 0x07, 0xb0, 0xce, 0x2b, - 0xdd, 0x97, 0x60, 0xca, 0xe3, 0x00, 0xdf, 0x76, 0x30, 0x65, 0x2a, 0xfe, 0xc5, 0x43, 0x6b, 0x53, - 0x56, 0xdf, 0xf9, 0xd5, 0x3e, 0x84, 0x0d, 0x59, 0xfc, 0xbe, 0xcc, 0x26, 0xce, 0xc7, 0xa5, 0x77, - 0x9e, 0xef, 0x6f, 0xcb, 0x46, 0x20, 0x9c, 0x91, 0xad, 0xbf, 0x9b, 0xce, 0xd6, 0x37, 0xf3, 0x06, - 0x4f, 0x65, 0x90, 0x79, 0x37, 0xaa, 0x14, 0xb8, 0x11, 0xce, 0xa5, 0xf4, 0xf3, 0x22, 0x64, 0xbc, - 0x9d, 0xe7, 0xfe, 0x5f, 0xcc, 0xe8, 0xf7, 0x65, 0x46, 0xaf, 0x97, 0xd6, 0x5d, 0x84, 0xf7, 0x32, - 0x19, 0x7d, 0x7b, 0x96, 0x98, 0x3a, 0xa1, 0xff, 0xcd, 0x3c, 0xd4, 0xf4, 0x5c, 0x4e, 0xa7, 0x79, - 0x25, 0x95, 0x0b, 0x94, 0x64, 0x5e, 0x49, 0x95, 0x97, 0xb9, 0x92, 0xe6, 0x7f, 0xe8, 0x4a, 0xda, - 0x82, 0x9a, 0xf8, 0xb0, 0x29, 0x79, 0xaa, 0xae, 0x98, 0xaa, 0x00, 0x60, 0xf2, 0x34, 0x31, 0xfc, - 0xe2, 0xa9, 0x0c, 0x9f, 0x2e, 0x1d, 0x96, 0xb2, 0xa5, 0xc3, 0xfb, 0x99, 0xfb, 0xe5, 0x62, 0x9e, - 0x5d, 0xe1, 0xcd, 0xb2, 0x57, 0x74, 0xb3, 0xbc, 0x5e, 0x40, 0xfc, 0xca, 0x16, 0x0e, 0xf7, 0x65, - 0xe1, 0x60, 0x7a, 0x95, 0x8a, 0x5e, 0x5d, 0x00, 0x7d, 0x50, 0xe3, 0xea, 0x01, 0xe5, 0xb7, 0x86, - 0x0d, 0x2c, 0x1e, 0x0a, 0x52, 0xfa, 0x4f, 0x5a, 0x5d, 0xa7, 0x08, 0x05, 0x7f, 0x32, 0x13, 0x9f, - 0x19, 0x3d, 0xa1, 0xf7, 0x73, 0xb5, 0xe6, 0xe9, 0xbc, 0xee, 0xdd, 0x74, 0xa9, 0x79, 0x36, 0x77, - 0xc9, 0x55, 0x9a, 0xe2, 0x26, 0x76, 0xa8, 0x9a, 0x96, 0x45, 0x42, 0x4d, 0x41, 0x7a, 0x8c, 0x67, - 0x47, 0x4f, 0x3d, 0xdf, 0x8b, 0x0e, 0xe5, 0xfc, 0xa2, 0x98, 0x87, 0x18, 0xd4, 0x13, 0x8f, 0x37, - 0xe4, 0x85, 0xc7, 0xec, 0x61, 0xe0, 0x12, 0xe1, 0x8c, 0x0b, 0xb8, 0xca, 0x01, 0x3b, 0x81, 0x4b, - 0x92, 0x03, 0x52, 0x3d, 0xd3, 0x01, 0xa9, 0x65, 0x0e, 0xc8, 0x06, 0x2c, 0x52, 0xe2, 0x44, 0x81, - 0x2f, 0x72, 0x94, 0x1a, 0x56, 0x23, 0x1e, 0xe0, 0x27, 0x24, 0x8a, 0xf8, 0x02, 0x75, 0x99, 0x75, - 0xa8, 0xa1, 0x91, 0x64, 0x2d, 0xcf, 0x4a, 0xb2, 0x4e, 0x68, 0x3a, 0x65, 0x92, 0xac, 0xc6, 0xac, - 0x24, 0xeb, 0x34, 0x3d, 0x27, 0x23, 0x2f, 0x6c, 0x9e, 0x94, 0x17, 0xfe, 0x94, 0x07, 0xe7, 0x1e, - 0x6c, 0xe6, 0x5c, 0x5d, 0x9d, 0x9c, 0xf7, 0x32, 0xad, 0xa9, 0xf6, 0x2c, 0x2d, 0xe8, 0xce, 0xd4, - 0x2f, 0x61, 0x65, 0xef, 0x05, 0x19, 0x0e, 0x8e, 0xfd, 0xe1, 0x19, 0xee, 0xea, 0x16, 0x54, 0x86, - 0x13, 0x57, 0x15, 0x78, 0xfc, 0xd3, 0xbc, 0xbd, 0x2b, 0xe9, 0xdb, 0xdb, 0x86, 0x56, 0xb2, 0x82, - 0x92, 0x73, 0x83, 0xcb, 0xe9, 0x72, 0x64, 0xce, 0x7c, 0x19, 0xab, 0x91, 0x82, 0x13, 0x4a, 0xc5, - 0xae, 0x25, 0x9c, 0x50, 0x9a, 0x76, 0xdb, 0x4a, 0xda, 0x6d, 0xad, 0x67, 0x50, 0xe7, 0x0b, 0xfc, - 0x28, 0xf1, 0x55, 0x0a, 0x5b, 0x49, 0x52, 0x58, 0x9d, 0x09, 0xcf, 0x1b, 0x99, 0xb0, 0xb5, 0x0d, - 0xcb, 0x72, 0x2d, 0xb5, 0x11, 0x5e, 0xc2, 0xd1, 0x71, 0x6c, 0xb7, 0x29, 0x1d, 0x5b, 0x3f, 0x83, - 0x46, 0x8f, 0x31, 0x67, 0x78, 0x78, 0x06, 0x79, 0xf4, 0x5a, 0x65, 0x73, 0x2d, 0x0b, 0x9a, 0x31, - 0xa7, 0x99, 0xab, 0xf5, 0x01, 0x1d, 0x04, 0x94, 0x7d, 0x16, 0xd0, 0xef, 0x1c, 0xea, 0x9e, 0x2d, - 0x67, 0x45, 0x30, 0xaf, 0x5e, 0x79, 0x2b, 0x57, 0x17, 0xb0, 0xf8, 0xb6, 0xde, 0x82, 0xb5, 0x14, - 0xbf, 0x99, 0x0b, 0xdf, 0x82, 0xba, 0x88, 0x0a, 0x2a, 0x3b, 0xba, 0x6a, 0xf6, 0x6c, 0x4e, 0x0a, - 0x1d, 0xbc, 0x4e, 0xe5, 0x61, 0x5f, 0xc0, 0x75, 0x8c, 0x7e, 0x27, 0x93, 0x48, 0xac, 0xa7, 0xe9, - 0x33, 0x49, 0xc4, 0x73, 0x58, 0x10, 0xe0, 0x5c, 0x8c, 0xde, 0x82, 0x1a, 0x25, 0x61, 0x60, 0x33, - 0x67, 0xa4, 0xdf, 0xcd, 0x39, 0xe0, 0x91, 0x33, 0x8a, 0xc4, 0xb3, 0x3f, 0x9f, 0x74, 0xbd, 0x11, - 0x89, 0x58, 0xfc, 0x78, 0x5e, 0xe7, 0xb0, 0x5d, 0x09, 0xe2, 0x1a, 0x89, 0xbc, 0x5f, 0xc9, 0x04, - 0x61, 0x1e, 0x8b, 0x6f, 0xeb, 0x23, 0x40, 0xa6, 0xbc, 0x4a, 0x21, 0x6f, 0xc2, 0xa2, 0xd8, 0x4e, - 0x7c, 0x3d, 0x35, 0xd3, 0x02, 0x63, 0x35, 0x6b, 0xdd, 0x05, 0x24, 0x35, 0x90, 0xba, 0x92, 0x4e, - 0xaf, 0xad, 0x0f, 0x61, 0x2d, 0x45, 0xaf, 0x5f, 0xad, 0x52, 0x0c, 0xb2, 0xab, 0x2b, 0xe2, 0xbf, - 0x97, 0x00, 0x7a, 0x53, 0x76, 0xa8, 0x4a, 0xf9, 0x0e, 0x54, 0xa7, 0x11, 0xa1, 0x46, 0x43, 0x40, - 0x8f, 0xf9, 0x5c, 0xe8, 0x44, 0xd1, 0x77, 0x01, 0x8d, 0x73, 0x2e, 0x3d, 0x16, 0x65, 0xf8, 0x94, - 0x1d, 0xc6, 0xed, 0x23, 0xfe, 0x8d, 0xae, 0x40, 0x53, 0xfe, 0xab, 0x60, 0x3b, 0xae, 0x4b, 0x49, - 0x14, 0xa9, 0x3e, 0x52, 0x43, 0x42, 0x7b, 0x12, 0xc8, 0xd1, 0x3c, 0x97, 0xf8, 0xcc, 0x63, 0xc7, - 0x36, 0x0b, 0x9e, 0x13, 0x5f, 0x65, 0x53, 0x8d, 0x18, 0xfa, 0x88, 0x03, 0x39, 0x1a, 0x25, 0x23, - 0x2f, 0x62, 0x34, 0x46, 0x5b, 0x94, 0x68, 0x31, 0x54, 0xa0, 0x59, 0xdf, 0x97, 0xa0, 0x75, 0x30, - 0x1d, 0x8f, 0xe5, 0x26, 0xcf, 0xaa, 0x4b, 0xf4, 0x96, 0xda, 0x47, 0xb6, 0x2b, 0x94, 0xa8, 0x48, - 0x6d, 0xee, 0xc7, 0x97, 0x66, 0x6b, 0xb0, 0x6a, 0x08, 0xaa, 0xaa, 0x8a, 0xbb, 0x80, 0x64, 0xc1, - 0xf1, 0x72, 0xf2, 0x5b, 0xe7, 0x60, 0x2d, 0x45, 0xaf, 0xd8, 0x5e, 0x87, 0x86, 0x7a, 0x02, 0x52, - 0x76, 0x3e, 0x0f, 0x55, 0x7e, 0xfa, 0x87, 0x9e, 0x1b, 0xb7, 0x06, 0x97, 0xc2, 0xc0, 0xdd, 0xf1, - 0x5c, 0x6a, 0xf5, 0xa1, 0x81, 0x25, 0x7b, 0x85, 0x7b, 0x07, 0x9a, 0xea, 0xc1, 0xc8, 0x4e, 0x3d, - 0xa9, 0x26, 0x7d, 0xaf, 0x14, 0x6f, 0xdc, 0xf0, 0xcd, 0xa1, 0xf5, 0x0d, 0x74, 0x1e, 0x87, 0x2e, - 0x4f, 0x5f, 0x4c, 0xae, 0xf1, 0xd6, 0xee, 0x40, 0xfc, 0xbb, 0xcc, 0x2c, 0xe6, 0x69, 0xb2, 0x06, - 0x35, 0x87, 0xd6, 0x05, 0xd8, 0x2a, 0x64, 0x2e, 0xf7, 0x7d, 0xfd, 0x35, 0xa8, 0xc6, 0x3f, 0xb1, - 0xa0, 0x25, 0xa8, 0x3c, 0xda, 0x39, 0x68, 0xcd, 0xf1, 0x8f, 0xc7, 0xbb, 0x07, 0xad, 0xd2, 0xf5, - 0xdb, 0xb0, 0x92, 0xe9, 0xe5, 0xa3, 0x55, 0x68, 0x0c, 0x7a, 0xfd, 0xdd, 0x4f, 0x1f, 0x7e, 0x65, - 0xe3, 0xbd, 0xde, 0xee, 0xd7, 0xad, 0x39, 0xb4, 0x0e, 0xad, 0x18, 0xd4, 0x7f, 0xf8, 0x48, 0x42, - 0x4b, 0xd7, 0x9f, 0x43, 0x33, 0x9d, 0x9c, 0xa1, 0x73, 0xb0, 0xba, 0xf3, 0xb0, 0xff, 0xa8, 0xb7, - 0xdf, 0xdf, 0xc3, 0xf6, 0x0e, 0xde, 0xeb, 0x3d, 0xda, 0xdb, 0x6d, 0xcd, 0xa5, 0xc1, 0xf8, 0x71, - 0xbf, 0xbf, 0xdf, 0xff, 0xbc, 0x55, 0xe2, 0x5c, 0x13, 0xf0, 0xde, 0x57, 0xfb, 0x1c, 0xb9, 0x9c, - 0x46, 0x7e, 0xdc, 0xbf, 0xd7, 0x7f, 0xf8, 0x8b, 0x7e, 0xab, 0xd2, 0xfd, 0x27, 0x40, 0x53, 0x6d, - 0x70, 0x40, 0xe8, 0x91, 0x37, 0x24, 0xe8, 0x2e, 0x2c, 0xc5, 0xff, 0x17, 0x25, 0xe9, 0x62, 0xfa, - 0x67, 0xa8, 0x4e, 0x3b, 0x3f, 0xa1, 0xfc, 0x61, 0x0e, 0x1d, 0x08, 0x2b, 0x1b, 0xef, 0x26, 0x17, - 0x4c, 0x85, 0xe7, 0x1e, 0x66, 0x3a, 0x17, 0x67, 0x4d, 0x6b, 0x8e, 0x03, 0x68, 0xa6, 0x1f, 0xb0, - 0x51, 0x42, 0x53, 0xf8, 0x30, 0xde, 0xb9, 0x34, 0x73, 0x5e, 0x33, 0xfd, 0x1a, 0x5a, 0xd9, 0xa7, - 0x6b, 0x94, 0xb4, 0x7b, 0x67, 0x3c, 0x8b, 0x77, 0x2e, 0x9f, 0x80, 0x61, 0xb2, 0xce, 0x3d, 0xf2, - 0x6e, 0xcf, 0x7e, 0xa6, 0xcb, 0xb1, 0x9e, 0xf5, 0xf6, 0x27, 0x55, 0x91, 0x7e, 0xef, 0x40, 0xe6, - 0xd3, 0x6a, 0xc1, 0xbb, 0x97, 0xa1, 0x8a, 0xe2, 0x87, 0x12, 0x6b, 0x0e, 0x7d, 0x09, 0x2b, 0x99, - 0x56, 0x0e, 0x4a, 0xa8, 0x8a, 0x1b, 0x53, 0x9d, 0xed, 0xd9, 0x08, 0x69, 0xbb, 0x99, 0x8d, 0x9a, - 0x94, 0xdd, 0x0a, 0xba, 0x3f, 0x29, 0xbb, 0x15, 0x76, 0x78, 0x84, 0x7b, 0xa5, 0xda, 0x31, 0x86, - 0x7b, 0x15, 0xf5, 0x7e, 0x3a, 0x17, 0x67, 0x4d, 0x9b, 0xdb, 0xcf, 0xb4, 0x62, 0x8c, 0xed, 0x17, - 0x77, 0x78, 0x3a, 0xdb, 0xb3, 0x11, 0xb2, 0xb6, 0x4a, 0x4a, 0xcc, 0x8c, 0xad, 0x72, 0x1d, 0x8d, - 0x8c, 0xad, 0xf2, 0xb5, 0xa9, 0xb2, 0x55, 0xa6, 0x56, 0xbc, 0x34, 0x33, 0xcd, 0xce, 0xdb, 0xaa, - 0x38, 0x73, 0xb7, 0xe6, 0x50, 0x0f, 0xaa, 0x71, 0x9e, 0x8c, 0x92, 0xd3, 0x9d, 0x49, 0xce, 0x3b, - 0xe7, 0x0b, 0x66, 0x34, 0x8b, 0xff, 0x83, 0x79, 0x0e, 0x45, 0xeb, 0x29, 0xa4, 0x98, 0xf4, 0x5c, - 0x06, 0xaa, 0xc9, 0x3e, 0x84, 0x45, 0x99, 0x68, 0xa2, 0x24, 0x32, 0xa7, 0x72, 0xd8, 0xce, 0x66, - 0x0e, 0xae, 0x89, 0xbf, 0x90, 0xff, 0x1c, 0xaa, 0x8c, 0x11, 0x6d, 0xa5, 0xfe, 0x8a, 0x4a, 0xe7, - 0xa5, 0x9d, 0xd7, 0x8a, 0x27, 0x35, 0xaf, 0x27, 0xb0, 0x56, 0x10, 0xf1, 0x51, 0xd2, 0xd6, 0x98, - 0x7d, 0xd9, 0x74, 0xde, 0x38, 0x19, 0x29, 0x5e, 0xa3, 0xfb, 0x97, 0x32, 0x2c, 0xcb, 0xab, 0x55, - 0x45, 0xdb, 0xcf, 0x01, 0x92, 0x04, 0x0f, 0x75, 0x52, 0x0e, 0x90, 0xca, 0x52, 0x3b, 0x5b, 0x85, - 0x73, 0xa6, 0x26, 0x8c, 0x5c, 0xcd, 0xd0, 0x44, 0x3e, 0x03, 0x34, 0x34, 0x51, 0x90, 0xde, 0x59, - 0x73, 0x68, 0x17, 0x6a, 0x3a, 0x81, 0x40, 0x46, 0xde, 0x91, 0xc9, 0x7e, 0x3a, 0x9d, 0xa2, 0x29, - 0x53, 0x22, 0x23, 0x63, 0x30, 0x24, 0xca, 0xe7, 0x21, 0x86, 0x44, 0x45, 0x49, 0xc6, 0xdc, 0x7f, - 0x02, 0x00, 0x00, 0xff, 0xff, 0x45, 0x43, 0x5d, 0x1f, 0xa4, 0x2b, 0x00, 0x00, + 0xb1, 0x04, 0x40, 0x82, 0x40, 0x83, 0x00, 0xc1, 0x21, 0x45, 0x42, 0xa0, 0x25, 0x51, 0x6b, 0xcb, + 0x96, 0x64, 0x5b, 0x65, 0xf1, 0xbd, 0x67, 0x3d, 0xcb, 0x96, 0x6c, 0x98, 0xa4, 0xfd, 0x68, 0x49, + 0x10, 0xdf, 0x40, 0x72, 0xec, 0xf2, 0x61, 0xb3, 0xc2, 0x8e, 0xc0, 0x95, 0x80, 0xdd, 0xf5, 0xec, + 0x80, 0x16, 0x73, 0xcd, 0x25, 0x87, 0x1c, 0x72, 0xcd, 0x2d, 0x95, 0x4a, 0x95, 0x0f, 0xb9, 0xa5, + 0x2a, 0x55, 0xf9, 0x0f, 0xa9, 0xfc, 0x90, 0x1c, 0x72, 0xcf, 0x31, 0x35, 0x1f, 0x3b, 0x3b, 0xfb, + 0x01, 0x9a, 0x94, 0x53, 0xb1, 0x6e, 0x3b, 0x3d, 0xdd, 0x3d, 0x3d, 0xdd, 0x3d, 0x3d, 0xdd, 0x3d, + 0x0b, 0x75, 0x27, 0xf4, 0x6e, 0x84, 0x34, 0x60, 0x01, 0x5a, 0xa4, 0x53, 0x9f, 0x79, 0x13, 0x62, + 0x5d, 0x87, 0xd6, 0x97, 0x84, 0x46, 0x5e, 0xe0, 0x63, 0xf2, 0xed, 0x94, 0x44, 0x0c, 0x75, 0x60, + 0xf1, 0x48, 0x42, 0x3a, 0xa5, 0xad, 0xd2, 0xd5, 0x3a, 0x8e, 0x87, 0xd6, 0xf7, 0x25, 0x58, 0xd6, + 0xc8, 0x51, 0x18, 0xf8, 0x11, 0x99, 0x8d, 0x8d, 0x2e, 0xc3, 0x92, 0x5a, 0xc4, 0xf6, 0x9d, 0x09, + 0xe9, 0x94, 0xc5, 0x74, 0x43, 0xc1, 0xfa, 0xce, 0x84, 0xa0, 0xb7, 0x60, 0x39, 0x46, 0x89, 0x99, + 0x54, 0x04, 0x56, 0x4b, 0x81, 0xd5, 0x6a, 0xe8, 0x06, 0xac, 0xc6, 0x88, 0x4e, 0xe8, 0x69, 0xe4, + 0x79, 0x81, 0xbc, 0xa2, 0xa6, 0x7a, 0xa1, 0xa7, 0xf0, 0xad, 0x6f, 0xa0, 0xbe, 0xdb, 0x1f, 0xec, + 0x04, 0xfe, 0x53, 0x6f, 0xc4, 0x45, 0x8c, 0x08, 0xe5, 0x34, 0x9d, 0xd2, 0x56, 0x85, 0x8b, 0xa8, + 0x86, 0xa8, 0x0b, 0xb5, 0x88, 0x38, 0x74, 0x78, 0x48, 0xa2, 0x4e, 0x59, 0x4c, 0xe9, 0x31, 0xa7, + 0x0a, 0x42, 0xe6, 0x05, 0x7e, 0xd4, 0xa9, 0x48, 0x2a, 0x35, 0xb4, 0x7e, 0x5b, 0x82, 0xc6, 0x41, + 0x40, 0xd9, 0x03, 0x27, 0x0c, 0x3d, 0x7f, 0x84, 0xde, 0x85, 0x9a, 0x50, 0xea, 0x30, 0x18, 0x0b, + 0x1d, 0xb4, 0xb6, 0x57, 0x6e, 0x28, 0x91, 0x6e, 0x1c, 0xa8, 0x09, 0xac, 0x51, 0xd0, 0x15, 0x68, + 0x0d, 0x03, 0x9f, 0x39, 0x9e, 0x4f, 0xa8, 0x1d, 0x06, 0x94, 0x09, 0xcd, 0x2c, 0xe0, 0xa6, 0x86, + 0x72, 0xe6, 0x68, 0x13, 0xea, 0x87, 0x41, 0xc4, 0x24, 0x46, 0x45, 0x60, 0xd4, 0x38, 0x40, 0x4c, + 0x6e, 0xc0, 0xa2, 0x98, 0xf4, 0x42, 0xa5, 0x83, 0x2a, 0x1f, 0xee, 0x87, 0xd6, 0x6f, 0x4a, 0xb0, + 0xf0, 0x20, 0x98, 0xfa, 0x2c, 0xb3, 0x8c, 0xc3, 0x0e, 0x95, 0x7d, 0x8c, 0x65, 0x1c, 0x76, 0x98, + 0x2c, 0xc3, 0x31, 0xa4, 0x89, 0xe4, 0x32, 0x7c, 0xb2, 0x0b, 0x35, 0x4a, 0x1c, 0x37, 0xf0, 0xc7, + 0xc7, 0x42, 0x84, 0x1a, 0xd6, 0x63, 0x6e, 0xbb, 0x88, 0x8c, 0x3d, 0x7f, 0xfa, 0xc2, 0xa6, 0x64, + 0xec, 0x3c, 0x21, 0x63, 0x21, 0x4a, 0x0d, 0xb7, 0x14, 0x18, 0x4b, 0xa8, 0xf5, 0x0c, 0x96, 0xb9, + 0xb1, 0xa3, 0xd0, 0x19, 0x92, 0x87, 0x42, 0x85, 0xdc, 0x35, 0xc4, 0xa2, 0x3e, 0x61, 0xdf, 0x05, + 0xf4, 0xb9, 0x90, 0xac, 0x86, 0x1b, 0x1c, 0xd6, 0x97, 0x20, 0x74, 0x1e, 0x6a, 0x52, 0x2e, 0xcf, + 0x15, 0x62, 0xd5, 0xb0, 0xd8, 0xf1, 0x81, 0xe7, 0xea, 0x29, 0x2f, 0x1c, 0x2a, 0xa9, 0x16, 0xe5, + 0xee, 0x87, 0xd6, 0x2f, 0x4b, 0x70, 0xee, 0x3e, 0x5f, 0xfc, 0x20, 0x70, 0x07, 0x8e, 0xef, 0x3e, + 0x09, 0x5e, 0x28, 0x27, 0x78, 0x1d, 0x9a, 0xc3, 0x11, 0x0d, 0xa6, 0xa1, 0x1d, 0x3a, 0x94, 0xf8, + 0x4c, 0x69, 0x63, 0x49, 0x02, 0x0f, 0x04, 0x0c, 0xed, 0xc1, 0x8a, 0x1f, 0x8b, 0x6a, 0xc7, 0xd6, + 0xe7, 0xab, 0x37, 0xb6, 0x3b, 0xda, 0xa4, 0x99, 0xcd, 0xe0, 0xb6, 0x9f, 0x06, 0x44, 0x16, 0x05, + 0x94, 0xac, 0xff, 0x80, 0x30, 0xc7, 0x75, 0x98, 0x83, 0x10, 0xcc, 0x8b, 0x73, 0x20, 0x17, 0x16, + 0xdf, 0xa8, 0x0d, 0x95, 0xa9, 0xda, 0x60, 0x1d, 0xf3, 0x4f, 0xf4, 0x1a, 0xd4, 0x35, 0x3f, 0x75, + 0x18, 0x12, 0x00, 0x77, 0x4a, 0x87, 0x31, 0x32, 0x09, 0x99, 0x50, 0x76, 0x13, 0xc7, 0x43, 0xeb, + 0x2f, 0xf3, 0xd0, 0xce, 0x6d, 0xfa, 0x16, 0xd4, 0x26, 0x6a, 0x79, 0xb1, 0x6c, 0x63, 0x7b, 0x33, + 0xf1, 0xcc, 0x9c, 0x84, 0x58, 0x23, 0x73, 0xc3, 0x73, 0x95, 0x1a, 0xe7, 0x56, 0x8f, 0xb9, 0x26, + 0xc7, 0xc1, 0xc8, 0x76, 0x3d, 0x4a, 0x86, 0x2c, 0xa0, 0xc7, 0x4a, 0xca, 0xa5, 0x71, 0x30, 0xda, + 0x8d, 0x61, 0xe8, 0x26, 0x80, 0xeb, 0x47, 0xf6, 0x50, 0xc8, 0x21, 0x64, 0x6d, 0x6c, 0x23, 0xbd, + 0xb6, 0x3e, 0x9b, 0xb8, 0xee, 0xfa, 0x91, 0x12, 0xf6, 0x03, 0x68, 0x72, 0x5f, 0xb7, 0x27, 0xf2, + 0x58, 0x45, 0x9d, 0x85, 0xad, 0xca, 0xd5, 0xc6, 0xf6, 0x9a, 0x21, 0xb1, 0x3e, 0x73, 0x78, 0x29, + 0x4c, 0x06, 0x11, 0xba, 0x03, 0x55, 0xe1, 0x6b, 0x51, 0xa7, 0x2a, 0x68, 0xae, 0x14, 0xec, 0x52, + 0xae, 0x72, 0xe3, 0xbe, 0xc0, 0xdb, 0xf3, 0x19, 0x3d, 0xc6, 0x8a, 0x08, 0xdd, 0x87, 0x86, 0xe3, + 0xfb, 0x01, 0x73, 0xa4, 0xc1, 0x17, 0x05, 0x8f, 0xeb, 0xb3, 0x79, 0xf4, 0x12, 0x64, 0xc9, 0xc8, + 0x24, 0x47, 0xff, 0x0d, 0x0b, 0xc2, 0xff, 0x3b, 0x35, 0xb1, 0xeb, 0x8b, 0x9a, 0x4f, 0xa1, 0x63, + 0x62, 0x89, 0xdc, 0xfd, 0x00, 0x1a, 0x86, 0x68, 0xdc, 0x31, 0x9e, 0x93, 0x63, 0xe5, 0x2b, 0xfc, + 0x13, 0xad, 0xc1, 0xc2, 0x91, 0x33, 0x9e, 0xc6, 0xf6, 0x90, 0x83, 0xdb, 0xe5, 0xff, 0x2d, 0x75, + 0xef, 0x42, 0x3b, 0x2b, 0xd1, 0x59, 0xe8, 0xad, 0x7d, 0x58, 0xc3, 0x53, 0x3f, 0x11, 0x2c, 0xbe, + 0x08, 0x6e, 0x42, 0x55, 0xd9, 0x4f, 0xfa, 0xce, 0xf9, 0x99, 0x1a, 0xc1, 0x0a, 0xd1, 0xba, 0x03, + 0xe7, 0x32, 0xac, 0xd4, 0x35, 0xf1, 0x06, 0xb4, 0xc2, 0xc0, 0xb5, 0x23, 0x09, 0xb6, 0x3d, 0x37, + 0x3e, 0x7f, 0xa1, 0xc6, 0xdd, 0x77, 0x39, 0xf9, 0x80, 0x05, 0x61, 0x5e, 0x94, 0xd3, 0x91, 0x77, + 0x60, 0x3d, 0x4b, 0x2e, 0x97, 0xb7, 0x3e, 0x86, 0x0d, 0x4c, 0x26, 0xc1, 0x11, 0x79, 0x59, 0xd6, + 0x5d, 0xe8, 0xe4, 0x19, 0x24, 0xcc, 0x13, 0xe8, 0x80, 0x39, 0x6c, 0x1a, 0x9d, 0x8d, 0xf9, 0x35, + 0x93, 0x81, 0x0a, 0x80, 0x92, 0x0f, 0x6a, 0x41, 0xd9, 0x0b, 0x15, 0x51, 0xd9, 0x0b, 0xad, 0xaf, + 0xa1, 0xde, 0x37, 0xa3, 0x81, 0x19, 0x41, 0xeb, 0x38, 0x1e, 0xa2, 0xed, 0xe4, 0xf2, 0xfa, 0xa1, + 0xf0, 0xa5, 0xaf, 0xb5, 0x7b, 0xb9, 0xd0, 0xa9, 0x64, 0xd8, 0x06, 0xd0, 0x11, 0x28, 0x52, 0xbe, + 0x80, 0xf2, 0xfc, 0xb0, 0x81, 0x65, 0xfd, 0x21, 0x15, 0x8e, 0x8c, 0xcd, 0xb8, 0x7a, 0x33, 0x6e, + 0x2a, 0x3c, 0x95, 0xcf, 0x12, 0x9e, 0x6e, 0xc0, 0x42, 0xc4, 0x1c, 0x26, 0x03, 0x64, 0xcb, 0xd8, + 0x5c, 0x7a, 0x49, 0x82, 0x25, 0x1a, 0xba, 0x00, 0x30, 0xa4, 0xc4, 0x61, 0xc4, 0xb5, 0x1d, 0x19, + 0x39, 0x2b, 0xb8, 0xae, 0x20, 0x3d, 0x86, 0x6e, 0x27, 0x7a, 0x5c, 0x10, 0x62, 0x6c, 0x15, 0x30, + 0x4c, 0xd9, 0x25, 0xd1, 0xb4, 0x3e, 0xed, 0xd5, 0x93, 0x4f, 0xbb, 0xa2, 0x93, 0xc8, 0x46, 0xc0, + 0x5a, 0x9c, 0x19, 0xb0, 0x24, 0xc5, 0x69, 0x02, 0x56, 0x6d, 0x66, 0xc0, 0x52, 0x3c, 0x4e, 0x0c, + 0x58, 0x3f, 0x65, 0xe8, 0x79, 0x00, 0x9d, 0xfc, 0xd1, 0x51, 0x21, 0xe3, 0x26, 0x54, 0x23, 0x01, + 0x39, 0x21, 0xfc, 0x28, 0x12, 0x85, 0x68, 0xfd, 0xbd, 0x64, 0x7a, 0xdd, 0x67, 0xde, 0x98, 0x11, + 0x9a, 0xf3, 0x3a, 0xed, 0x3c, 0xe5, 0xd3, 0x39, 0xcf, 0x00, 0x5a, 0x42, 0xed, 0x76, 0x44, 0xc6, + 0xe2, 0x76, 0x13, 0xf9, 0x60, 0x63, 0xfb, 0x9d, 0x02, 0x42, 0xb9, 0xa4, 0xb4, 0xd9, 0x40, 0xa1, + 0x4b, 0x8d, 0x37, 0xc7, 0x26, 0xac, 0xfb, 0x09, 0xa0, 0x3c, 0xd2, 0x99, 0x54, 0xf7, 0x05, 0x3f, + 0xae, 0x3c, 0x1d, 0x2c, 0x08, 0xdb, 0x4f, 0x85, 0x18, 0x27, 0xe8, 0x4d, 0xca, 0x89, 0x15, 0xa2, + 0xf5, 0xbb, 0x0a, 0x40, 0x32, 0xf9, 0xca, 0x9e, 0xd3, 0x5b, 0xfa, 0xd4, 0xc8, 0xd4, 0xe0, 0x52, + 0x01, 0xbf, 0xc2, 0xf3, 0xf2, 0x59, 0xfa, 0xbc, 0xc8, 0x24, 0xe1, 0x8d, 0x22, 0xea, 0x57, 0xf6, + 0xa4, 0xec, 0xc0, 0x7a, 0xd6, 0xdc, 0xea, 0x9c, 0x5c, 0x83, 0x05, 0x8f, 0x91, 0x89, 0x2c, 0x6e, + 0x1a, 0xdb, 0xab, 0x05, 0xdb, 0xc2, 0x12, 0xc3, 0xba, 0x0c, 0xf5, 0xfd, 0x89, 0x33, 0x22, 0x83, + 0x90, 0x0c, 0xf9, 0x5a, 0x1e, 0x1f, 0xa8, 0xf5, 0xe5, 0xc0, 0xda, 0x86, 0xda, 0x3d, 0x72, 0xfc, + 0x25, 0x5f, 0xf7, 0xb4, 0xf2, 0x59, 0x7f, 0x2d, 0xc1, 0x86, 0x08, 0x77, 0x3b, 0x71, 0x69, 0x81, + 0x49, 0x14, 0x4c, 0xe9, 0x90, 0x44, 0xc2, 0xa4, 0xe1, 0xd4, 0x0e, 0x09, 0xf5, 0x02, 0xe9, 0x53, + 0xdc, 0xa4, 0xe1, 0xf4, 0x40, 0x00, 0x78, 0xf9, 0xc1, 0xa7, 0xbf, 0x9d, 0x06, 0xca, 0xb7, 0x2a, + 0xb8, 0x36, 0x0c, 0xa7, 0xff, 0xcf, 0xc7, 0x31, 0x6d, 0x74, 0xe8, 0x50, 0x12, 0x09, 0x1f, 0x92, + 0xb4, 0x03, 0x01, 0x40, 0x37, 0xe1, 0xdc, 0x84, 0x4c, 0x02, 0x7a, 0x6c, 0x8f, 0xbd, 0x89, 0xc7, + 0x6c, 0xcf, 0xb7, 0x9f, 0x1c, 0x33, 0x12, 0x29, 0xc7, 0x41, 0x72, 0xf2, 0x3e, 0x9f, 0xdb, 0xf7, + 0x3f, 0xe5, 0x33, 0xc8, 0x82, 0x66, 0x10, 0x4c, 0xec, 0x68, 0x18, 0x50, 0x62, 0x3b, 0xee, 0x33, + 0x11, 0xef, 0x2b, 0xb8, 0x11, 0x04, 0x93, 0x01, 0x87, 0xf5, 0xdc, 0x67, 0x96, 0x03, 0xcd, 0xc1, + 0x9e, 0xd8, 0x8e, 0xaa, 0x56, 0x10, 0xcc, 0x4f, 0x23, 0x75, 0x9c, 0xea, 0x58, 0x7c, 0x73, 0x18, + 0x0d, 0xc6, 0xb1, 0x1e, 0xc4, 0x37, 0x87, 0xb1, 0xe3, 0x30, 0xce, 0xda, 0xc5, 0x37, 0x57, 0xd8, + 0x98, 0x1c, 0xa9, 0xda, 0xa8, 0x8e, 0xe5, 0xc0, 0x72, 0x01, 0x76, 0x9c, 0xd0, 0x79, 0xe2, 0x8d, + 0x3d, 0x76, 0x8c, 0xae, 0x41, 0xdb, 0x71, 0x5d, 0x7b, 0x18, 0x43, 0x3c, 0x12, 0x17, 0xaa, 0xcb, + 0x8e, 0xeb, 0xee, 0x18, 0x60, 0xf4, 0x36, 0xac, 0xb8, 0x34, 0x08, 0xd3, 0xb8, 0xb2, 0x72, 0x6d, + 0xf3, 0x09, 0x13, 0xd9, 0xfa, 0x67, 0x09, 0xd6, 0xd2, 0x66, 0x51, 0x99, 0xf6, 0x5d, 0xa8, 0xd3, + 0xd8, 0x40, 0x2a, 0x48, 0x6c, 0xa5, 0xef, 0xad, 0xbc, 0x21, 0x71, 0x42, 0x82, 0x6e, 0xc1, 0x52, + 0x46, 0x80, 0x52, 0xca, 0xf1, 0x92, 0xbd, 0xe1, 0x14, 0x22, 0xfa, 0x38, 0xa9, 0x19, 0x93, 0xda, + 0x9a, 0xd3, 0xae, 0x6b, 0xda, 0x94, 0xea, 0x75, 0x2d, 0xa9, 0x2a, 0x2b, 0xf4, 0xa6, 0x32, 0x45, + 0xb6, 0xa0, 0x10, 0x34, 0x8f, 0x23, 0x42, 0xa5, 0x79, 0xac, 0xaf, 0xa0, 0xae, 0x41, 0x71, 0x91, + 0x25, 0x7d, 0x4f, 0x14, 0x59, 0x6d, 0xa8, 0x8c, 0x54, 0xd9, 0x55, 0xc1, 0xfc, 0x93, 0x57, 0xb3, + 0x8e, 0xeb, 0x7a, 0x7c, 0x15, 0x67, 0x6c, 0x8f, 0x3c, 0x57, 0x56, 0xfd, 0x15, 0xdc, 0x4a, 0xc0, + 0x9f, 0x7b, 0x6e, 0x64, 0xf5, 0x60, 0x45, 0x2b, 0xe7, 0xc4, 0xd2, 0xce, 0x28, 0xd5, 0xca, 0xe9, + 0x52, 0xcd, 0x87, 0xea, 0x2e, 0x39, 0xf2, 0x86, 0xe4, 0xdf, 0x52, 0xa3, 0x6f, 0x41, 0x23, 0x24, + 0x74, 0xe2, 0x45, 0x91, 0xd6, 0x67, 0x1d, 0x9b, 0x20, 0xeb, 0xf7, 0x55, 0x58, 0xce, 0xba, 0xc0, + 0xfb, 0xb9, 0xca, 0xb0, 0x9b, 0x98, 0x2f, 0xbb, 0x3f, 0x23, 0xa2, 0x5f, 0x8d, 0x83, 0x46, 0x39, + 0x63, 0x01, 0x1d, 0x57, 0x54, 0x20, 0xe1, 0xfb, 0x1f, 0x06, 0x93, 0x89, 0xe3, 0xbb, 0x71, 0xff, + 0x44, 0x0d, 0xb9, 0xb6, 0x1c, 0x3a, 0xe2, 0xc7, 0x94, 0x83, 0xc5, 0x37, 0xba, 0x04, 0x0d, 0x9e, + 0x4e, 0x79, 0xbe, 0x28, 0x2c, 0xc5, 0xb1, 0xac, 0x63, 0x50, 0xa0, 0x5d, 0x8f, 0xa2, 0x2b, 0x30, + 0x4f, 0xfc, 0xa3, 0x38, 0x76, 0x27, 0x0d, 0x96, 0x38, 0x58, 0x61, 0x31, 0x8d, 0xde, 0x84, 0xea, + 0x24, 0x98, 0xfa, 0x2c, 0x4e, 0xac, 0x5a, 0x1a, 0x51, 0x74, 0x45, 0xb0, 0x9a, 0x45, 0xd7, 0x60, + 0xd1, 0x15, 0x36, 0x88, 0xb3, 0xa7, 0xe5, 0xa4, 0x38, 0x15, 0x70, 0x1c, 0xcf, 0xa3, 0x8f, 0xf4, + 0xad, 0x53, 0xcf, 0xdc, 0x1b, 0x19, 0xa5, 0x16, 0x5e, 0x3d, 0xf7, 0xd2, 0x57, 0x0f, 0x08, 0x16, + 0xd7, 0x66, 0xb2, 0x38, 0xb9, 0xb4, 0xbc, 0x08, 0x10, 0x52, 0xef, 0xc8, 0x1b, 0x93, 0x11, 0x71, + 0x3b, 0x0d, 0xd1, 0xfb, 0x30, 0x20, 0xa2, 0x9f, 0xa6, 0xfa, 0x33, 0x36, 0x0d, 0x02, 0xf6, 0x34, + 0xea, 0x2c, 0xc9, 0x9e, 0x4c, 0x0c, 0xc6, 0x02, 0x8a, 0xce, 0x43, 0x8d, 0xd7, 0xf0, 0xc2, 0xa1, + 0x9a, 0xb2, 0x74, 0x18, 0x07, 0x23, 0xe1, 0x4f, 0x6b, 0xfc, 0xce, 0x76, 0x3d, 0xbf, 0xd3, 0x12, + 0x94, 0x72, 0xc0, 0x43, 0xb1, 0xf8, 0xb0, 0x03, 0x7f, 0x48, 0x3a, 0xcb, 0x62, 0xaa, 0x2e, 0x20, + 0x0f, 0xfd, 0xa1, 0xb8, 0x29, 0x18, 0x3b, 0xee, 0xb4, 0x05, 0x9c, 0x7f, 0xa2, 0xff, 0x8a, 0xf3, + 0xe2, 0x15, 0xe1, 0x28, 0x17, 0x66, 0xc4, 0x97, 0x57, 0xa6, 0x08, 0xfe, 0x53, 0x09, 0xd6, 0x77, + 0x44, 0xa6, 0x61, 0xc4, 0xbe, 0x33, 0x14, 0x71, 0xe8, 0x3d, 0x5d, 0x2d, 0x67, 0x2b, 0xae, 0xec, + 0x66, 0x15, 0x1e, 0xfa, 0x04, 0x5a, 0x31, 0x4f, 0x45, 0x59, 0xf9, 0xa1, 0x3a, 0xbb, 0x19, 0x99, + 0x43, 0xeb, 0x23, 0xd8, 0xc8, 0xc9, 0xac, 0xb2, 0x82, 0xcb, 0xb0, 0x94, 0x84, 0x16, 0x2d, 0x72, + 0x43, 0xc3, 0xf6, 0x5d, 0xeb, 0x36, 0xaf, 0xb6, 0x1d, 0xca, 0x72, 0x1b, 0x3e, 0x05, 0xad, 0x28, + 0xb5, 0xd3, 0xb4, 0xaa, 0x1a, 0x1e, 0xc0, 0x1a, 0x2f, 0xc2, 0x5f, 0x82, 0x29, 0x0f, 0x19, 0x7c, + 0xdb, 0xc1, 0x94, 0xa9, 0xd0, 0x1c, 0x0f, 0xad, 0x0d, 0xd9, 0x18, 0xc8, 0xaf, 0xf6, 0x21, 0xac, + 0xcb, 0xba, 0xfc, 0x65, 0x36, 0x71, 0x3e, 0xee, 0x0a, 0xe4, 0xf9, 0xfe, 0xba, 0x6c, 0xc4, 0xcc, + 0x19, 0x85, 0xc4, 0xbb, 0xe9, 0x42, 0x62, 0x23, 0x6f, 0xf0, 0x54, 0x72, 0x9b, 0x77, 0xa3, 0x4a, + 0x81, 0x1b, 0xe1, 0x5c, 0xb5, 0x31, 0x2f, 0x42, 0xc6, 0xdb, 0x79, 0xee, 0xff, 0xc1, 0x62, 0x63, + 0x5f, 0x16, 0x1b, 0x7a, 0x69, 0xdd, 0xe0, 0x78, 0x2f, 0x53, 0x6c, 0x74, 0x66, 0x89, 0xa9, 0x6b, + 0x8d, 0x5f, 0xcd, 0x43, 0x5d, 0xcf, 0xe5, 0x74, 0x9a, 0x57, 0x52, 0xb9, 0x40, 0x49, 0xe6, 0xed, + 0x55, 0x79, 0x99, 0xdb, 0x6b, 0xfe, 0x87, 0x6e, 0xaf, 0x4d, 0xa8, 0x8b, 0x0f, 0x9b, 0x92, 0xa7, + 0xea, 0x36, 0xaa, 0x09, 0x00, 0x26, 0x4f, 0x13, 0xc3, 0x57, 0x4f, 0x65, 0xf8, 0x74, 0x55, 0xb3, + 0x98, 0xad, 0x6a, 0xde, 0xd7, 0xf7, 0x8b, 0xbc, 0x89, 0x2e, 0xe6, 0xd9, 0x15, 0xde, 0x2c, 0x7b, + 0xe9, 0x9b, 0x45, 0x5e, 0x4e, 0xaf, 0x17, 0x10, 0xbf, 0xb2, 0x35, 0xcd, 0x7d, 0x59, 0xd3, 0x98, + 0x5e, 0xa5, 0xa2, 0xd7, 0x36, 0x80, 0x3e, 0xa8, 0x71, 0x61, 0x83, 0xf2, 0x5b, 0xc3, 0x06, 0x16, + 0x0f, 0x05, 0x29, 0xfd, 0x27, 0x5d, 0xb8, 0x53, 0x84, 0x82, 0x3f, 0x2e, 0x18, 0xe7, 0x7d, 0x46, + 0xbb, 0xea, 0xfd, 0x5c, 0x19, 0x7c, 0x3a, 0xaf, 0x7b, 0x37, 0x5d, 0x05, 0x9f, 0xcd, 0x5d, 0x72, + 0x45, 0xb0, 0xb8, 0x89, 0x1d, 0xaa, 0xa6, 0x65, 0xfd, 0x52, 0x57, 0x90, 0x1e, 0xe3, 0x89, 0xd4, + 0x53, 0xcf, 0xf7, 0xa2, 0x43, 0x39, 0x5f, 0x15, 0xf3, 0x10, 0x83, 0x7a, 0xe2, 0x5d, 0x89, 0xbc, + 0xf0, 0x98, 0x3d, 0x0c, 0x5c, 0x22, 0x9c, 0x71, 0x01, 0xd7, 0x38, 0x60, 0x27, 0x70, 0x49, 0x72, + 0x40, 0x6a, 0x67, 0x3a, 0x20, 0xf5, 0xcc, 0x01, 0x59, 0x87, 0x2a, 0x25, 0x4e, 0x14, 0xf8, 0x1d, + 0x90, 0xaf, 0x53, 0x72, 0xc4, 0x03, 0xfc, 0x84, 0x44, 0x11, 0x5f, 0xa0, 0x21, 0xb3, 0x0e, 0x35, + 0x34, 0x92, 0xac, 0xa5, 0x59, 0x49, 0xd6, 0x09, 0xfd, 0xb0, 0x4c, 0x92, 0xd5, 0x9c, 0x95, 0x64, + 0x9d, 0xa6, 0x1d, 0x66, 0xa4, 0x90, 0xad, 0x93, 0x52, 0xc8, 0x9f, 0xf2, 0xe0, 0xdc, 0x83, 0x8d, + 0x9c, 0xab, 0xab, 0x93, 0xf3, 0x5e, 0xa6, 0x6b, 0xd6, 0x99, 0xa5, 0x05, 0xdd, 0x34, 0xfb, 0x39, + 0x2c, 0xef, 0xbd, 0x20, 0xc3, 0xc1, 0xb1, 0x3f, 0x3c, 0xc3, 0x5d, 0xdd, 0x86, 0xca, 0x70, 0xe2, + 0xaa, 0xda, 0x93, 0x7f, 0x9a, 0xb7, 0x77, 0x25, 0x7d, 0x7b, 0xdb, 0xd0, 0x4e, 0x56, 0x50, 0x72, + 0xae, 0x73, 0x39, 0x5d, 0x8e, 0xcc, 0x99, 0x2f, 0x61, 0x35, 0x52, 0x70, 0x42, 0xa9, 0xd8, 0xb5, + 0x84, 0x13, 0x4a, 0xd3, 0x6e, 0x5b, 0x49, 0xbb, 0xad, 0xf5, 0x0c, 0x1a, 0x7c, 0x81, 0x1f, 0x25, + 0xbe, 0x4a, 0x61, 0x2b, 0x49, 0x0a, 0xab, 0x33, 0xe1, 0x79, 0x23, 0x13, 0xb6, 0xb6, 0x60, 0x49, + 0xae, 0xa5, 0x36, 0xc2, 0xab, 0x4b, 0x3a, 0x8e, 0xed, 0x36, 0xa5, 0x63, 0xeb, 0xff, 0xa0, 0xd9, + 0x63, 0xcc, 0x19, 0x1e, 0x9e, 0x41, 0x1e, 0xbd, 0x56, 0xd9, 0x5c, 0xcb, 0x82, 0x56, 0xcc, 0x69, + 0xe6, 0x6a, 0x7d, 0x40, 0x07, 0x01, 0x65, 0x9f, 0x05, 0xf4, 0x3b, 0x87, 0xba, 0x67, 0xcb, 0x59, + 0x11, 0xcc, 0xab, 0x07, 0xe8, 0xca, 0xd5, 0x05, 0x2c, 0xbe, 0xad, 0xb7, 0x60, 0x35, 0xc5, 0x6f, + 0xe6, 0xc2, 0xb7, 0xa0, 0x21, 0xa2, 0x82, 0xca, 0x8e, 0xae, 0x9a, 0xed, 0xa4, 0x93, 0x42, 0x07, + 0x2f, 0xa1, 0x79, 0xd8, 0x17, 0x70, 0x1d, 0xa3, 0xdf, 0xc9, 0x24, 0x12, 0x6b, 0x69, 0xfa, 0x4c, + 0x12, 0xf1, 0x1c, 0x16, 0x04, 0x38, 0x17, 0xa3, 0x37, 0xa1, 0x4e, 0x49, 0x18, 0xd8, 0xcc, 0x19, + 0xe9, 0x27, 0x7d, 0x0e, 0x78, 0xe4, 0x8c, 0x22, 0xf1, 0x47, 0x02, 0x9f, 0x74, 0xbd, 0x11, 0x89, + 0x58, 0xfc, 0xae, 0xdf, 0xe0, 0xb0, 0x5d, 0x09, 0xe2, 0x1a, 0x89, 0xbc, 0x5f, 0xc8, 0x04, 0x61, + 0x1e, 0x8b, 0x6f, 0xeb, 0x23, 0x40, 0xa6, 0xbc, 0x4a, 0x21, 0x6f, 0x42, 0x55, 0x6c, 0x27, 0xbe, + 0x9e, 0x5a, 0x69, 0x81, 0xb1, 0x9a, 0xb5, 0xee, 0x02, 0x92, 0x1a, 0x48, 0x5d, 0x49, 0xa7, 0xd7, + 0xd6, 0x87, 0xb0, 0x9a, 0xa2, 0xd7, 0x0f, 0x6a, 0x29, 0x06, 0xd9, 0xd5, 0x15, 0xf1, 0xdf, 0x4a, + 0x00, 0xbd, 0x29, 0x3b, 0x54, 0x55, 0x7f, 0x17, 0x6a, 0xd3, 0x88, 0x50, 0xa3, 0x57, 0xa1, 0xc7, + 0x7c, 0x2e, 0x74, 0xa2, 0xe8, 0xbb, 0x80, 0xc6, 0x39, 0x97, 0x1e, 0x8b, 0x8a, 0x7d, 0xca, 0x0e, + 0xe3, 0xce, 0x16, 0xff, 0x46, 0x57, 0xa0, 0x25, 0x7f, 0xa3, 0xb0, 0x1d, 0xd7, 0xa5, 0x24, 0x8a, + 0x54, 0x8b, 0xab, 0x29, 0xa1, 0x3d, 0x09, 0xe4, 0x68, 0x9e, 0x4b, 0x7c, 0xe6, 0xb1, 0x63, 0x9b, + 0x05, 0xcf, 0x89, 0xaf, 0xb2, 0xa9, 0x66, 0x0c, 0x7d, 0xc4, 0x81, 0x1c, 0x8d, 0x92, 0x91, 0x17, + 0x31, 0x1a, 0xa3, 0x55, 0x25, 0x5a, 0x0c, 0x15, 0x68, 0xd6, 0xf7, 0x25, 0x68, 0x1f, 0x4c, 0xc7, + 0x63, 0xb9, 0xc9, 0xb3, 0xea, 0x12, 0xbd, 0xa5, 0xf6, 0x91, 0x6d, 0x58, 0x25, 0x2a, 0x52, 0x9b, + 0xfb, 0xf1, 0xa5, 0xd9, 0x2a, 0xac, 0x18, 0x82, 0xaa, 0xaa, 0xe2, 0x2e, 0x20, 0x59, 0x70, 0xbc, + 0x9c, 0xfc, 0xd6, 0x39, 0x58, 0x4d, 0xd1, 0x2b, 0xb6, 0xd7, 0xa1, 0xa9, 0x5e, 0xa7, 0x94, 0x9d, + 0xcf, 0x43, 0x8d, 0x9f, 0xfe, 0xa1, 0xe7, 0xc6, 0x5d, 0xcb, 0xc5, 0x30, 0x70, 0x77, 0x3c, 0x97, + 0x5a, 0x7d, 0x68, 0x62, 0xc9, 0x5e, 0xe1, 0xde, 0x81, 0x96, 0x7a, 0xcb, 0xb2, 0x53, 0xaf, 0xbd, + 0x49, 0x4b, 0x2e, 0xc5, 0x1b, 0x37, 0x7d, 0x73, 0x68, 0x7d, 0x03, 0xdd, 0xc7, 0xa1, 0xcb, 0xd3, + 0x17, 0x93, 0x6b, 0xbc, 0xb5, 0x3b, 0x10, 0xff, 0xc9, 0x33, 0x8b, 0x79, 0x9a, 0xac, 0x49, 0xcd, + 0xa1, 0x75, 0x01, 0x36, 0x0b, 0x99, 0xcb, 0x7d, 0x5f, 0x7f, 0x0d, 0x6a, 0xf1, 0xff, 0x35, 0x68, + 0x11, 0x2a, 0x8f, 0x76, 0x0e, 0xda, 0x73, 0xfc, 0xe3, 0xf1, 0xee, 0x41, 0xbb, 0x74, 0xfd, 0x36, + 0x2c, 0x67, 0x9e, 0x19, 0xd0, 0x0a, 0x34, 0x07, 0xbd, 0xfe, 0xee, 0xa7, 0x0f, 0xbf, 0xb2, 0xf1, + 0x5e, 0x6f, 0xf7, 0xeb, 0xf6, 0x1c, 0x5a, 0x83, 0x76, 0x0c, 0xea, 0x3f, 0x7c, 0x24, 0xa1, 0xa5, + 0xeb, 0xcf, 0xa1, 0x95, 0x4e, 0xce, 0xd0, 0x39, 0x58, 0xd9, 0x79, 0xd8, 0x7f, 0xd4, 0xdb, 0xef, + 0xef, 0x61, 0x7b, 0x07, 0xef, 0xf5, 0x1e, 0xed, 0xed, 0xb6, 0xe7, 0xd2, 0x60, 0xfc, 0xb8, 0xdf, + 0xdf, 0xef, 0x7f, 0xde, 0x2e, 0x71, 0xae, 0x09, 0x78, 0xef, 0xab, 0x7d, 0x8e, 0x5c, 0x4e, 0x23, + 0x3f, 0xee, 0xdf, 0xeb, 0x3f, 0xfc, 0x59, 0xbf, 0x5d, 0xd9, 0xfe, 0x07, 0x40, 0x4b, 0x6d, 0x70, + 0x40, 0xa8, 0x68, 0x0c, 0xde, 0x85, 0xc5, 0xf8, 0xd7, 0xa7, 0x24, 0x5d, 0x4c, 0xff, 0xa7, 0xd5, + 0xed, 0xe4, 0x27, 0x94, 0x3f, 0xcc, 0xa1, 0x03, 0x61, 0x65, 0xe3, 0x49, 0xe7, 0x82, 0xa9, 0xf0, + 0xdc, 0x9b, 0x51, 0xf7, 0xe2, 0xac, 0x69, 0xcd, 0x71, 0x00, 0xad, 0xf4, 0xdb, 0x3a, 0x4a, 0x68, + 0x0a, 0xdf, 0xec, 0xbb, 0x97, 0x66, 0xce, 0x6b, 0xa6, 0x5f, 0x43, 0x3b, 0xfb, 0xaa, 0x8e, 0x92, + 0x4e, 0xf4, 0x8c, 0x17, 0xfb, 0xee, 0xe5, 0x13, 0x30, 0x4c, 0xd6, 0xb9, 0xf7, 0xe7, 0xad, 0xd9, + 0x2f, 0x88, 0x39, 0xd6, 0xb3, 0x9e, 0x25, 0xa5, 0x2a, 0xd2, 0x4f, 0x31, 0xc8, 0x7c, 0xf5, 0x2d, + 0x78, 0x92, 0x33, 0x54, 0x51, 0xfc, 0x86, 0x63, 0xcd, 0xa1, 0x2f, 0x61, 0x39, 0xd3, 0xca, 0x41, + 0x09, 0x55, 0x71, 0x63, 0xaa, 0xbb, 0x35, 0x1b, 0x21, 0x6d, 0x37, 0xb3, 0x51, 0x93, 0xb2, 0x5b, + 0x41, 0xf7, 0x27, 0x65, 0xb7, 0xc2, 0x0e, 0x8f, 0x70, 0xaf, 0x54, 0x3b, 0xc6, 0x70, 0xaf, 0xa2, + 0xde, 0x4f, 0xf7, 0xe2, 0xac, 0x69, 0x73, 0xfb, 0x99, 0x56, 0x8c, 0xb1, 0xfd, 0xe2, 0x0e, 0x4f, + 0x77, 0x6b, 0x36, 0x42, 0xd6, 0x56, 0x49, 0x89, 0x99, 0xb1, 0x55, 0xae, 0xa3, 0x91, 0xb1, 0x55, + 0xbe, 0x36, 0x55, 0xb6, 0xca, 0xd4, 0x8a, 0x97, 0x66, 0xa6, 0xd9, 0x79, 0x5b, 0x15, 0x67, 0xee, + 0xd6, 0x1c, 0xea, 0x41, 0x2d, 0xce, 0x93, 0x51, 0x72, 0xba, 0x33, 0xc9, 0x79, 0xf7, 0x7c, 0xc1, + 0x8c, 0x66, 0xf1, 0x3f, 0x30, 0xcf, 0xa1, 0x68, 0x2d, 0x85, 0x14, 0x93, 0x9e, 0xcb, 0x40, 0x35, + 0xd9, 0x87, 0x50, 0x95, 0x89, 0x26, 0x4a, 0x22, 0x73, 0x2a, 0x87, 0xed, 0x6e, 0xe4, 0xe0, 0x9a, + 0xf8, 0x0b, 0xf9, 0x3b, 0xa4, 0xca, 0x18, 0xd1, 0x66, 0xea, 0x87, 0xad, 0x74, 0x5e, 0xda, 0x7d, + 0xad, 0x78, 0x52, 0xf3, 0x7a, 0x02, 0xab, 0x05, 0x11, 0x1f, 0x25, 0x6d, 0x8d, 0xd9, 0x97, 0x4d, + 0xf7, 0x8d, 0x93, 0x91, 0xe2, 0x35, 0xb6, 0xff, 0x5c, 0x86, 0x25, 0x79, 0xb5, 0xaa, 0x68, 0xfb, + 0x39, 0x40, 0x92, 0xe0, 0xa1, 0x6e, 0xca, 0x01, 0x52, 0x59, 0x6a, 0x77, 0xb3, 0x70, 0xce, 0xd4, + 0x84, 0x91, 0xab, 0x19, 0x9a, 0xc8, 0x67, 0x80, 0x86, 0x26, 0x0a, 0xd2, 0x3b, 0x6b, 0x0e, 0xed, + 0x42, 0x5d, 0x27, 0x10, 0xc8, 0xc8, 0x3b, 0x32, 0xd9, 0x4f, 0xb7, 0x5b, 0x34, 0x65, 0x4a, 0x64, + 0x64, 0x0c, 0x86, 0x44, 0xf9, 0x3c, 0xc4, 0x90, 0xa8, 0x28, 0xc9, 0x98, 0xfb, 0x57, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xc4, 0xce, 0xc1, 0x50, 0x3f, 0x2c, 0x00, 0x00, } From e0f89a322b5fb6e9a8b4d8b66ada79d17e021651 Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Mon, 31 Oct 2016 16:05:02 +0800 Subject: [PATCH 3/3] CRI: Add devices implementation and moves GPU to devices --- pkg/kubelet/dockershim/docker_container.go | 12 ++++++- pkg/kubelet/dockertools/docker_manager.go | 18 +++++----- pkg/kubelet/kubelet_pods.go | 18 ++++++++++ pkg/kubelet/kubelet_pods_test.go | 34 +++++++++++++++++++ .../kuberuntime/kuberuntime_container.go | 17 ++++++++++ 5 files changed, 89 insertions(+), 10 deletions(-) diff --git a/pkg/kubelet/dockershim/docker_container.go b/pkg/kubelet/dockershim/docker_container.go index 682b44d5273..2921975b802 100644 --- a/pkg/kubelet/dockershim/docker_container.go +++ b/pkg/kubelet/dockershim/docker_container.go @@ -162,13 +162,23 @@ func (ds *dockerService) CreateContainer(podSandboxID string, config *runtimeApi CPUShares: rOpts.GetCpuShares(), CPUQuota: rOpts.GetCpuQuota(), CPUPeriod: rOpts.GetCpuPeriod(), - // TODO: Need to set devices. } hc.OomScoreAdj = int(rOpts.GetOomScoreAdj()) } // Note: ShmSize is handled in kube_docker_client.go } + // Set devices for container. + devices := make([]dockercontainer.DeviceMapping, len(config.Devices)) + for i, device := range config.Devices { + devices[i] = dockercontainer.DeviceMapping{ + PathOnHost: device.GetHostPath(), + PathInContainer: device.GetContainerPath(), + CgroupPermissions: device.GetPermissions(), + } + } + hc.Resources.Devices = devices + var err error hc.SecurityOpt, err = getContainerSecurityOpts(config.Metadata.GetName(), sandboxConfig, ds.seccompProfileRoot) if err != nil { diff --git a/pkg/kubelet/dockertools/docker_manager.go b/pkg/kubelet/dockertools/docker_manager.go index fc316758bca..9443106c835 100644 --- a/pkg/kubelet/dockertools/docker_manager.go +++ b/pkg/kubelet/dockertools/docker_manager.go @@ -619,7 +619,6 @@ func (dm *DockerManager) runContainer( memoryLimit := container.Resources.Limits.Memory().Value() cpuRequest := container.Resources.Requests.Cpu() cpuLimit := container.Resources.Limits.Cpu() - nvidiaGPULimit := container.Resources.Limits.NvidiaGPU() var cpuShares int64 // If request is not specified, but limit is, we want request to default to limit. // API server does this for new containers, but we repeat this logic in Kubelet @@ -631,17 +630,18 @@ func (dm *DockerManager) runContainer( // of CPU shares. cpuShares = milliCPUToShares(cpuRequest.MilliValue()) } - var devices []dockercontainer.DeviceMapping - if nvidiaGPULimit.Value() != 0 { - // Experimental. For now, we hardcode /dev/nvidia0 no matter what the user asks for - // (we only support one device per node). - devices = []dockercontainer.DeviceMapping{ - {PathOnHost: "/dev/nvidia0", PathInContainer: "/dev/nvidia0", CgroupPermissions: "mrw"}, - {PathOnHost: "/dev/nvidiactl", PathInContainer: "/dev/nvidiactl", CgroupPermissions: "mrw"}, - {PathOnHost: "/dev/nvidia-uvm", PathInContainer: "/dev/nvidia-uvm", CgroupPermissions: "mrw"}, + + // Set devices for container. + devices := make([]dockercontainer.DeviceMapping, len(opts.Devices)) + for i, device := range opts.Devices { + devices[i] = dockercontainer.DeviceMapping{ + PathOnHost: device.PathOnHost, + PathInContainer: device.PathInContainer, + CgroupPermissions: device.Permissions, } } binds := makeMountBindings(opts.Mounts) + // The reason we create and mount the log file in here (not in kubelet) is because // the file's location depends on the ID of the container, and we need to create and // mount the file before actually starting the container. diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index 02f5be6ef3f..a6746607f50 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -74,6 +74,23 @@ func (kl *Kubelet) getActivePods() []*api.Pod { return activePods } +// makeDevices determines the devices for the given container. +// Experimental. For now, we hardcode /dev/nvidia0 no matter what the user asks for +// (we only support one device per node). +// TODO: add support for more than 1 GPU after #28216. +func makeDevices(container *api.Container) []kubecontainer.DeviceInfo { + nvidiaGPULimit := container.Resources.Limits.NvidiaGPU() + if nvidiaGPULimit.Value() != 0 { + return []kubecontainer.DeviceInfo{ + {PathOnHost: "/dev/nvidia0", PathInContainer: "/dev/nvidia0", Permissions: "mrw"}, + {PathOnHost: "/dev/nvidiactl", PathInContainer: "/dev/nvidiactl", Permissions: "mrw"}, + {PathOnHost: "/dev/nvidia-uvm", PathInContainer: "/dev/nvidia-uvm", Permissions: "mrw"}, + } + } + + return nil +} + // makeMounts determines the mount points for the given container. func makeMounts(pod *api.Pod, podDir string, container *api.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap) ([]kubecontainer.Mount, error) { // Kubernetes only mounts on /etc/hosts if : @@ -252,6 +269,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *api.Pod, container *api.Cont volumes := kl.volumeManager.GetMountedVolumesForPod(podName) opts.PortMappings = makePortMappings(container) + opts.Devices = makeDevices(container) opts.Mounts, err = makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes) if err != nil { diff --git a/pkg/kubelet/kubelet_pods_test.go b/pkg/kubelet/kubelet_pods_test.go index 159e9c0c356..add58b6f3c4 100644 --- a/pkg/kubelet/kubelet_pods_test.go +++ b/pkg/kubelet/kubelet_pods_test.go @@ -28,6 +28,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/apimachinery/registered" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" @@ -1262,3 +1263,36 @@ func TestGetHostPortConflicts(t *testing.T) { pods = append(pods, expected) assert.True(t, hasHostPortConflicts(pods), "Should have port conflicts") } + +func TestMakeDevices(t *testing.T) { + testCases := []struct { + container *api.Container + devices []kubecontainer.DeviceInfo + test string + }{ + { + test: "no device", + container: &api.Container{}, + devices: nil, + }, + { + test: "gpu", + container: &api.Container{ + Resources: api.ResourceRequirements{ + Limits: map[api.ResourceName]resource.Quantity{ + api.ResourceNvidiaGPU: resource.MustParse("1000"), + }, + }, + }, + devices: []kubecontainer.DeviceInfo{ + {PathOnHost: "/dev/nvidia0", PathInContainer: "/dev/nvidia0", Permissions: "mrw"}, + {PathOnHost: "/dev/nvidiactl", PathInContainer: "/dev/nvidiactl", Permissions: "mrw"}, + {PathOnHost: "/dev/nvidia-uvm", PathInContainer: "/dev/nvidia-uvm", Permissions: "mrw"}, + }, + }, + } + + for _, test := range testCases { + assert.Equal(t, test.devices, makeDevices(test.container), "[test %q]", test.test) + } +} diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index 6b4566848e2..3b867387e2d 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -151,6 +151,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *api.Conta Labels: newContainerLabels(container, pod), Annotations: newContainerAnnotations(container, pod, restartCount), Mounts: m.makeMounts(opts, container, podHasSELinuxLabel), + Devices: makeDevices(opts), LogPath: &containerLogsPath, Stdin: &container.Stdin, StdinOnce: &container.StdinOnce, @@ -251,6 +252,22 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerConfig(container *api. return linuxConfig } +// makeDevices generates container devices for kubelet runtime api. +func makeDevices(opts *kubecontainer.RunContainerOptions) []*runtimeApi.Device { + devices := make([]*runtimeApi.Device, len(opts.Devices)) + + for idx := range opts.Devices { + device := opts.Devices[idx] + devices[idx] = &runtimeApi.Device{ + HostPath: &device.PathOnHost, + ContainerPath: &device.PathInContainer, + Permissions: &device.Permissions, + } + } + + return devices +} + // makeMounts generates container volume mounts for kubelet runtime api. func (m *kubeGenericRuntimeManager) makeMounts(opts *kubecontainer.RunContainerOptions, container *api.Container, podHasSELinuxLabel bool) []*runtimeApi.Mount { volumeMounts := []*runtimeApi.Mount{}