diff --git a/pkg/kubelet/api/testing/fake_runtime_service.go b/pkg/kubelet/api/testing/fake_runtime_service.go index c612299af12..60933d7e243 100644 --- a/pkg/kubelet/api/testing/fake_runtime_service.go +++ b/pkg/kubelet/api/testing/fake_runtime_service.go @@ -313,12 +313,14 @@ func (r *FakeRuntimeService) ListContainers(filter *runtimeApi.ContainerFilter) } result = append(result, &runtimeApi.Container{ - Id: s.Id, - Metadata: s.Metadata, - State: s.State, - Image: s.Image, - ImageRef: s.ImageRef, - Labels: s.Labels, + Id: s.Id, + CreatedAt: s.CreatedAt, + PodSandboxId: &s.SandboxID, + Metadata: s.Metadata, + State: s.State, + Image: s.Image, + ImageRef: s.ImageRef, + Labels: s.Labels, }) } diff --git a/pkg/kubelet/api/v1alpha1/runtime/api.pb.go b/pkg/kubelet/api/v1alpha1/runtime/api.pb.go index 6505a229407..05dbd69247b 100644 --- a/pkg/kubelet/api/v1alpha1/runtime/api.pb.go +++ b/pkg/kubelet/api/v1alpha1/runtime/api.pb.go @@ -1722,20 +1722,24 @@ type Container struct { // The ID of the container, used by the container runtime to identify // a container. Id *string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"` + // The id of the sandbox which this container belongs to. + PodSandboxId *string `protobuf:"bytes,2,opt,name=pod_sandbox_id,json=podSandboxId" json:"pod_sandbox_id,omitempty"` // The metadata of the container. - Metadata *ContainerMetadata `protobuf:"bytes,2,opt,name=metadata" json:"metadata,omitempty"` + Metadata *ContainerMetadata `protobuf:"bytes,3,opt,name=metadata" json:"metadata,omitempty"` // The spec of the image - Image *ImageSpec `protobuf:"bytes,3,opt,name=image" json:"image,omitempty"` + Image *ImageSpec `protobuf:"bytes,4,opt,name=image" json:"image,omitempty"` // Reference to the image in use. For most runtimes, this should be an // image ID. - ImageRef *string `protobuf:"bytes,4,opt,name=image_ref,json=imageRef" json:"image_ref,omitempty"` + ImageRef *string `protobuf:"bytes,5,opt,name=image_ref,json=imageRef" json:"image_ref,omitempty"` // State is the state of the container. - State *ContainerState `protobuf:"varint,5,opt,name=state,enum=runtime.ContainerState" json:"state,omitempty"` + State *ContainerState `protobuf:"varint,6,opt,name=state,enum=runtime.ContainerState" json:"state,omitempty"` + // Creation time of the container. + CreatedAt *int64 `protobuf:"varint,7,opt,name=created_at,json=createdAt" json:"created_at,omitempty"` // Labels are key value pairs that may be used to scope and select individual resources. - Labels map[string]string `protobuf:"bytes,6,rep,name=labels" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Labels map[string]string `protobuf:"bytes,8,rep,name=labels" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // Annotations is an unstructured key value map that may be set by external // tools to store and retrieve arbitrary metadata. - Annotations map[string]string `protobuf:"bytes,7,rep,name=annotations" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Annotations map[string]string `protobuf:"bytes,9,rep,name=annotations" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` XXX_unrecognized []byte `json:"-"` } @@ -1751,6 +1755,13 @@ func (m *Container) GetId() string { return "" } +func (m *Container) GetPodSandboxId() string { + if m != nil && m.PodSandboxId != nil { + return *m.PodSandboxId + } + return "" +} + func (m *Container) GetMetadata() *ContainerMetadata { if m != nil { return m.Metadata @@ -1779,6 +1790,13 @@ func (m *Container) GetState() ContainerState { return ContainerState_CREATED } +func (m *Container) GetCreatedAt() int64 { + if m != nil && m.CreatedAt != nil { + return *m.CreatedAt + } + return 0 +} + func (m *Container) GetLabels() map[string]string { if m != nil { return m.Labels @@ -3097,182 +3115,183 @@ var _ImageService_serviceDesc = grpc.ServiceDesc{ } var fileDescriptorApi = []byte{ - // 2828 bytes of a gzipped FileDescriptorProto + // 2842 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x5a, 0xcd, 0x72, 0x1b, 0xc7, - 0xf1, 0x17, 0x08, 0x82, 0x04, 0x1a, 0x04, 0x08, 0x8e, 0x28, 0x12, 0x82, 0x64, 0x89, 0x5a, 0xdb, - 0xff, 0xbf, 0x44, 0x5b, 0x2c, 0x85, 0x49, 0x49, 0x91, 0x6c, 0xcb, 0xa6, 0x49, 0x46, 0x45, 0x4b, - 0xa2, 0x98, 0x85, 0xa4, 0x58, 0x27, 0xd4, 0x0a, 0x3b, 0x24, 0x57, 0x02, 0x76, 0xd7, 0xbb, 0x0b, - 0x46, 0xcc, 0x35, 0xd7, 0x5c, 0x72, 0xcb, 0x03, 0xb8, 0x2a, 0xa7, 0x5c, 0x52, 0x95, 0x4b, 0x6e, - 0xa9, 0x1c, 0x53, 0x79, 0x90, 0x3c, 0x41, 0x72, 0x4c, 0xcf, 0xc7, 0xce, 0xce, 0x7e, 0x51, 0xa4, - 0x5c, 0x15, 0xeb, 0xb6, 0xd3, 0xdd, 0x33, 0xd3, 0xd3, 0xdd, 0xd3, 0xd3, 0xbf, 0x99, 0x85, 0x86, - 0xe5, 0x3b, 0x6b, 0x7e, 0xe0, 0x45, 0x1e, 0x99, 0x0d, 0x26, 0x6e, 0xe4, 0x8c, 0xa9, 0xb1, 0x0a, - 0xed, 0xe7, 0x34, 0x08, 0x1d, 0xcf, 0x35, 0xe9, 0x77, 0x13, 0x1a, 0x46, 0xa4, 0x0b, 0xb3, 0x47, - 0x82, 0xd2, 0xad, 0xac, 0x54, 0xae, 0x37, 0xcc, 0xb8, 0x69, 0xfc, 0xb1, 0x02, 0xf3, 0x4a, 0x38, - 0xf4, 0x3d, 0x37, 0xa4, 0xe5, 0xd2, 0xe4, 0x1a, 0xcc, 0xc9, 0x49, 0x06, 0xae, 0x35, 0xa6, 0xdd, - 0x29, 0xce, 0x6e, 0x4a, 0xda, 0x2e, 0x92, 0xc8, 0xff, 0xc3, 0x7c, 0x2c, 0x12, 0x0f, 0x52, 0xe5, - 0x52, 0x6d, 0x49, 0x96, 0xb3, 0x91, 0x35, 0x38, 0x1f, 0x0b, 0xe2, 0x1a, 0x94, 0xf0, 0x34, 0x17, - 0x5e, 0x90, 0xac, 0x0d, 0xdf, 0x91, 0xf2, 0xc6, 0x06, 0x34, 0xb6, 0x76, 0xfb, 0x4f, 0xfc, 0x88, - 0x75, 0x46, 0x15, 0x43, 0x1a, 0xb0, 0x3e, 0xa8, 0x62, 0x95, 0xa9, 0x28, 0x9b, 0xa4, 0x07, 0xf5, - 0x90, 0x5a, 0xc1, 0xf0, 0x90, 0x86, 0xa8, 0x1e, 0x63, 0xa9, 0xb6, 0xf1, 0xa7, 0x0a, 0x34, 0xf7, - 0xbc, 0x20, 0x7a, 0x6c, 0xf9, 0xbe, 0xe3, 0x1e, 0x10, 0x02, 0xd3, 0x7c, 0x19, 0x62, 0x95, 0xfc, - 0x9b, 0xdc, 0x84, 0x3a, 0x37, 0xe7, 0xd0, 0x1b, 0xf1, 0xe5, 0xb5, 0xd7, 0x17, 0xd6, 0xa4, 0x32, - 0x6b, 0x7b, 0x92, 0x61, 0x2a, 0x11, 0xf2, 0x31, 0xb4, 0x87, 0x9e, 0x1b, 0x59, 0x8e, 0x4b, 0x83, - 0x81, 0x8f, 0x63, 0xf3, 0xd5, 0xd6, 0xcc, 0x96, 0xa2, 0xb2, 0x09, 0xc9, 0x25, 0x68, 0x1c, 0x7a, - 0x61, 0x24, 0x24, 0xa6, 0xb9, 0x44, 0x9d, 0x11, 0x38, 0x73, 0x19, 0x66, 0x39, 0xd3, 0xf1, 0xbb, - 0x35, 0xae, 0xc9, 0x0c, 0x6b, 0xee, 0xf8, 0xc6, 0xf7, 0x15, 0xa8, 0x3d, 0xf6, 0x70, 0xf2, 0x42, - 0x4d, 0xd3, 0x53, 0x5b, 0xd1, 0xa1, 0x74, 0x87, 0x36, 0x35, 0x12, 0x93, 0xa9, 0x99, 0x84, 0x70, - 0x85, 0x98, 0x9a, 0x31, 0xd1, 0x5a, 0x01, 0xb5, 0x6c, 0xcf, 0x1d, 0x1d, 0x73, 0xb5, 0xea, 0xa6, - 0x6a, 0x33, 0x4f, 0x86, 0x74, 0xe4, 0xb8, 0x93, 0x37, 0x83, 0x80, 0x8e, 0xac, 0x97, 0x74, 0xc4, - 0xd5, 0xab, 0x9b, 0x6d, 0x49, 0x36, 0x05, 0xd5, 0x78, 0x05, 0xf3, 0xcc, 0xf5, 0xa1, 0x6f, 0x0d, - 0xa9, 0xf4, 0x0f, 0x06, 0x0a, 0x9f, 0xd4, 0xa5, 0xd1, 0xaf, 0xbd, 0xe0, 0x35, 0xd7, 0xbb, 0x6e, - 0x36, 0x19, 0x6d, 0x57, 0x90, 0xc8, 0x45, 0xa8, 0x0b, 0xbd, 0x1c, 0x9b, 0x2b, 0x5e, 0x37, 0xb9, - 0x15, 0xf6, 0x1c, 0x5b, 0xb1, 0x1c, 0x7f, 0xc8, 0x35, 0x96, 0xac, 0x1d, 0x7f, 0x68, 0xfc, 0xb6, - 0x02, 0x17, 0x1e, 0xb1, 0xc9, 0xf7, 0x3c, 0xbb, 0x6f, 0xb9, 0xf6, 0x4b, 0xef, 0xcd, 0xa6, 0xe7, - 0xee, 0x3b, 0x07, 0xe4, 0x43, 0x68, 0x0d, 0x0f, 0x02, 0x6f, 0xe2, 0xe3, 0x4a, 0x03, 0xea, 0x46, - 0xd2, 0x56, 0x73, 0x82, 0xb8, 0xc7, 0x69, 0x64, 0x1b, 0x16, 0xdc, 0x58, 0xd5, 0x81, 0xc7, 0x75, - 0x0d, 0xf9, 0xec, 0xcd, 0xf5, 0xae, 0x72, 0x73, 0x66, 0x31, 0x66, 0xc7, 0x4d, 0x13, 0x42, 0x23, - 0x00, 0x92, 0xcc, 0xff, 0x98, 0x46, 0x96, 0x6d, 0x45, 0x56, 0xa1, 0x93, 0x3a, 0x50, 0x9d, 0xc8, - 0x05, 0x36, 0x4c, 0xf6, 0x49, 0x2e, 0x43, 0x43, 0x8d, 0x27, 0xfd, 0x91, 0x10, 0x58, 0x60, 0x5b, - 0x51, 0x44, 0xc7, 0xbe, 0x08, 0x93, 0x96, 0x19, 0x37, 0x8d, 0xbf, 0x4e, 0x43, 0x27, 0xb7, 0xe8, - 0x3b, 0x50, 0x1f, 0xcb, 0xe9, 0xf9, 0xb4, 0xcd, 0xf5, 0x4b, 0x49, 0xb4, 0xe6, 0x34, 0x34, 0x95, - 0x30, 0x73, 0x3c, 0x33, 0xa9, 0xb6, 0x8b, 0x55, 0x9b, 0x59, 0x72, 0xe4, 0x1d, 0x0c, 0x6c, 0x27, - 0xa0, 0xc3, 0xc8, 0x0b, 0x8e, 0xa5, 0x96, 0x73, 0x48, 0xdc, 0x8a, 0x69, 0xe4, 0xa7, 0xd0, 0xb4, - 0xdd, 0x50, 0xd9, 0x70, 0x9a, 0x4f, 0x4e, 0xd4, 0xe4, 0x6a, 0xab, 0x9a, 0x80, 0x62, 0xd2, 0x6e, - 0xe4, 0x2e, 0xb4, 0xd8, 0x0e, 0x18, 0x8c, 0xc5, 0x06, 0x0c, 0x31, 0xa0, 0xaa, 0xd8, 0x6d, 0x51, - 0xd3, 0x59, 0xed, 0x4e, 0x73, 0xce, 0x4f, 0x1a, 0x21, 0xf9, 0x02, 0x66, 0x78, 0xb4, 0x85, 0xdd, - 0x19, 0xde, 0xe7, 0xe3, 0x82, 0x75, 0x0a, 0xa3, 0xac, 0x3d, 0xe2, 0x72, 0xdb, 0x6e, 0x14, 0x1c, - 0x9b, 0xb2, 0x13, 0x79, 0x04, 0x4d, 0xcb, 0x75, 0xbd, 0xc8, 0x12, 0xea, 0xce, 0xf2, 0x31, 0x56, - 0xcb, 0xc7, 0xd8, 0x48, 0x84, 0xc5, 0x40, 0x7a, 0x77, 0xf2, 0x33, 0xa8, 0xf1, 0x1d, 0xd0, 0xad, - 0xf3, 0x65, 0x5f, 0x51, 0xe3, 0x14, 0x86, 0xa6, 0x29, 0x84, 0x7b, 0x77, 0xa1, 0xa9, 0xa9, 0xc6, - 0x42, 0xe3, 0x35, 0x3d, 0x96, 0xd1, 0xc2, 0x3e, 0xc9, 0x22, 0xd4, 0x8e, 0xac, 0xd1, 0x24, 0xf6, - 0x88, 0x68, 0xdc, 0x9b, 0xfa, 0x79, 0xa5, 0x77, 0x1f, 0x3a, 0x59, 0x8d, 0xce, 0xd2, 0xdf, 0xd8, - 0x81, 0x45, 0x73, 0xe2, 0x26, 0x8a, 0xc5, 0x07, 0xc3, 0x4f, 0x60, 0x66, 0xc8, 0x75, 0x94, 0xd1, - 0x73, 0xb1, 0xd4, 0x22, 0xa6, 0x14, 0x34, 0xbe, 0x80, 0x0b, 0x99, 0xa1, 0xe4, 0xb1, 0xf1, 0x11, - 0xb4, 0x7d, 0xcf, 0x1e, 0x84, 0x82, 0x3c, 0xc0, 0xa8, 0x97, 0x3b, 0xd0, 0x57, 0xb2, 0x3b, 0x36, - 0xeb, 0xde, 0x8f, 0x3c, 0x3f, 0xaf, 0xca, 0xe9, 0xba, 0x77, 0x61, 0x29, 0xdb, 0x5d, 0x4c, 0x6f, - 0x7c, 0x09, 0xcb, 0x26, 0x1d, 0x7b, 0x47, 0xf4, 0x5d, 0x87, 0xee, 0x41, 0x37, 0x3f, 0x40, 0x32, - 0x78, 0x42, 0xed, 0xa3, 0x1b, 0x26, 0xe1, 0xd9, 0x06, 0xbf, 0xa1, 0x0f, 0x20, 0x53, 0xa0, 0x18, - 0x87, 0xb4, 0x61, 0x0a, 0x33, 0xbf, 0xe8, 0x84, 0x5f, 0xc6, 0x0b, 0x68, 0xec, 0xea, 0xf9, 0x40, - 0xcf, 0xa1, 0x78, 0xd0, 0xc9, 0x26, 0x59, 0x87, 0xd9, 0xd3, 0x26, 0xb0, 0x58, 0xd0, 0x78, 0x98, - 0x4b, 0x9e, 0x52, 0x87, 0x75, 0x00, 0x95, 0x83, 0x42, 0x19, 0x0b, 0x24, 0x3f, 0x9e, 0xa9, 0x49, - 0x19, 0xdf, 0xa7, 0x12, 0x92, 0xb6, 0x18, 0x5b, 0x2d, 0xc6, 0x4e, 0x25, 0xa8, 0xa9, 0xb3, 0x24, - 0xa8, 0x35, 0xa8, 0x85, 0x38, 0xa4, 0x48, 0x91, 0x6d, 0x6d, 0x71, 0xb2, 0xd7, 0xd7, 0x62, 0x4a, - 0x6a, 0x0a, 0x31, 0xf2, 0x01, 0xc0, 0x10, 0x8f, 0xae, 0x88, 0xda, 0x03, 0x4b, 0xe4, 0xce, 0xaa, - 0xd9, 0x90, 0x94, 0x8d, 0x88, 0xdc, 0x4b, 0xec, 0x58, 0xe3, 0x6a, 0xac, 0x14, 0xa8, 0x91, 0xf2, - 0x4b, 0x62, 0x69, 0xb5, 0xdb, 0x67, 0x4e, 0xde, 0xed, 0xb2, 0x9f, 0x10, 0xd6, 0x12, 0xd6, 0x6c, - 0x69, 0xc2, 0x12, 0x3d, 0x4e, 0x93, 0xb0, 0xea, 0xa5, 0x09, 0x4b, 0x8e, 0x71, 0x62, 0xc2, 0xfa, - 0x31, 0x53, 0xcf, 0x63, 0xe8, 0xe6, 0xb7, 0x8e, 0x4c, 0x19, 0x98, 0x7e, 0x42, 0x4e, 0x39, 0x21, - 0xfd, 0xc8, 0x2e, 0x52, 0xd0, 0xf8, 0x57, 0x45, 0x8f, 0xba, 0x5f, 0x38, 0xa3, 0x88, 0x06, 0xb9, - 0xa8, 0x53, 0xc1, 0x33, 0x75, 0xba, 0xe0, 0xe9, 0x43, 0x9b, 0x9b, 0x7d, 0x80, 0x95, 0x0d, 0x3f, - 0xdf, 0x30, 0xea, 0x98, 0xbd, 0x3f, 0x2d, 0xd0, 0x47, 0x4c, 0x29, 0x7c, 0xd6, 0x97, 0xe2, 0xc2, - 0xe2, 0xad, 0x91, 0x4e, 0xeb, 0x7d, 0x05, 0x24, 0x2f, 0x74, 0x26, 0xd3, 0x7d, 0xc3, 0xb6, 0x2b, - 0x2b, 0x12, 0x0b, 0xd2, 0xf6, 0x3e, 0x57, 0xe3, 0x04, 0xbb, 0x09, 0x3d, 0x4d, 0x29, 0x68, 0xfc, - 0x61, 0x0a, 0x20, 0x61, 0xbe, 0xb7, 0xfb, 0xf4, 0x8e, 0xda, 0x35, 0xa2, 0x34, 0xb8, 0x5a, 0xa0, - 0x45, 0xd1, 0x7e, 0xf9, 0x01, 0x11, 0x6e, 0x6c, 0xc2, 0x52, 0xd6, 0xcc, 0x32, 0x3e, 0x6f, 0x40, - 0xcd, 0xc1, 0xea, 0x4b, 0x80, 0x8c, 0xe6, 0xfa, 0xf9, 0x02, 0x65, 0x4c, 0x21, 0x61, 0x5c, 0x83, - 0xc6, 0xce, 0xd8, 0x3a, 0xa0, 0x7d, 0x9f, 0x0e, 0xd9, 0x5c, 0x0e, 0x6b, 0xc8, 0xf9, 0x45, 0xc3, - 0x58, 0x87, 0xfa, 0x43, 0x7a, 0xfc, 0x9c, 0xcd, 0x7b, 0x5a, 0xfd, 0x8c, 0x7f, 0x54, 0x60, 0x99, - 0xa7, 0x99, 0xcd, 0xb8, 0xa8, 0x47, 0xe5, 0xbc, 0x49, 0x80, 0x09, 0x98, 0x9b, 0xd2, 0x9f, 0x0c, - 0x7c, 0x1a, 0x38, 0x9e, 0xf0, 0x25, 0x33, 0xa5, 0x3f, 0xd9, 0xe3, 0x04, 0x56, 0xf8, 0x33, 0xf6, - 0x77, 0x13, 0x4f, 0xfa, 0xb4, 0x6a, 0xd6, 0x91, 0xf0, 0x4b, 0xd6, 0x8e, 0xfb, 0x86, 0x87, 0x58, - 0x17, 0x87, 0xdc, 0x77, 0xa2, 0x6f, 0x9f, 0x13, 0x30, 0xc0, 0x2e, 0x8c, 0xf1, 0x2c, 0x0c, 0x8e, - 0x07, 0x23, 0x67, 0xec, 0x60, 0x25, 0xee, 0x0e, 0x5e, 0x1e, 0x47, 0x34, 0x94, 0x0e, 0x23, 0x82, - 0xf9, 0x88, 0xf1, 0x76, 0xdc, 0xaf, 0x19, 0x87, 0x18, 0xd0, 0xf2, 0xbc, 0xf1, 0x20, 0x1c, 0x7a, - 0x01, 0x22, 0x3a, 0xfb, 0x15, 0xcf, 0xb3, 0x55, 0xb3, 0x89, 0xc4, 0x3e, 0xa3, 0x6d, 0xd8, 0xaf, - 0x0c, 0x0b, 0x5a, 0xfd, 0x6d, 0xbe, 0x1c, 0x89, 0x13, 0xb0, 0x64, 0x9e, 0x84, 0x32, 0x8c, 0xb1, - 0x64, 0x66, 0xdf, 0x8c, 0x16, 0x78, 0xa3, 0xd8, 0x0e, 0xfc, 0x9b, 0xd1, 0xa2, 0x63, 0x3f, 0xae, - 0x97, 0xf9, 0x37, 0x33, 0xd8, 0x88, 0x1e, 0x21, 0x2a, 0x11, 0x90, 0x51, 0x34, 0x0c, 0x1b, 0x60, - 0xd3, 0xf2, 0xad, 0x97, 0xce, 0xc8, 0x89, 0x8e, 0xd1, 0x81, 0x1d, 0xcb, 0xb6, 0x07, 0xc3, 0x98, - 0xe2, 0xd0, 0x18, 0x30, 0xce, 0x23, 0x7d, 0x53, 0x23, 0x93, 0x4f, 0x60, 0xc1, 0x0e, 0x3c, 0x3f, - 0x2d, 0x2b, 0x10, 0x64, 0x87, 0x31, 0x74, 0x61, 0xe3, 0x3f, 0x15, 0x58, 0x4c, 0xbb, 0x45, 0x16, - 0xe4, 0xf7, 0xa1, 0x11, 0xc4, 0x0e, 0x92, 0x9b, 0x73, 0x25, 0x7d, 0x5e, 0xe4, 0x1d, 0x69, 0x26, - 0x5d, 0x30, 0xfe, 0xe7, 0x32, 0x0a, 0x54, 0x52, 0x81, 0x97, 0xac, 0xcd, 0x4c, 0x09, 0x92, 0x2f, - 0x13, 0xb4, 0x16, 0x97, 0x05, 0x55, 0xde, 0x77, 0x49, 0xf5, 0x4d, 0x99, 0x5e, 0xa1, 0xb8, 0xb8, - 0x36, 0xff, 0x3f, 0xe9, 0x8a, 0x6c, 0x25, 0xcf, 0xfb, 0x3c, 0x43, 0x8e, 0x70, 0x8f, 0xf1, 0x2d, - 0x34, 0x14, 0x29, 0x86, 0x37, 0x22, 0xf6, 0x38, 0xbc, 0x41, 0xca, 0x81, 0x04, 0x3c, 0x48, 0xc1, - 0x4f, 0x86, 0x23, 0xd1, 0xd6, 0x0e, 0x9b, 0xc5, 0x1a, 0x0d, 0x90, 0x12, 0xf2, 0xec, 0x5a, 0x35, - 0xdb, 0x09, 0xf9, 0x01, 0x52, 0x11, 0xe1, 0x2f, 0x28, 0xe3, 0x9c, 0x08, 0xaa, 0x34, 0x90, 0x34, - 0x95, 0x06, 0x49, 0xff, 0xae, 0xc1, 0x7c, 0xd6, 0x25, 0xb7, 0x73, 0x18, 0xa9, 0x97, 0x98, 0x33, - 0x3b, 0x9f, 0x96, 0xd9, 0xae, 0xc7, 0x9b, 0x78, 0x2a, 0x63, 0x11, 0xb5, 0xcf, 0xe5, 0xc6, 0x66, - 0xfa, 0x0c, 0xbd, 0xf1, 0x18, 0x13, 0x02, 0x5f, 0x19, 0x16, 0x69, 0xb2, 0xc9, 0xb4, 0xb7, 0x82, - 0x03, 0xb6, 0x6d, 0x18, 0x99, 0x7f, 0x93, 0xab, 0xd0, 0x64, 0x65, 0x05, 0xa2, 0x1a, 0x06, 0xb1, - 0x24, 0xe4, 0x07, 0x49, 0x42, 0x80, 0x85, 0xc0, 0x7e, 0x9a, 0xba, 0x47, 0x31, 0xd0, 0x49, 0xae, - 0x1f, 0xe2, 0xe4, 0x61, 0x72, 0x36, 0x3a, 0x6c, 0x66, 0xcc, 0x2e, 0x07, 0xe2, 0x02, 0xa3, 0xad, - 0x04, 0xf9, 0x9d, 0x81, 0x29, 0xb9, 0xe4, 0x73, 0x95, 0x52, 0x45, 0x11, 0xf1, 0x51, 0x7e, 0xf5, - 0x27, 0x00, 0xa7, 0x87, 0xe9, 0x3a, 0xa4, 0xc1, 0x87, 0xb8, 0x51, 0x3a, 0xc4, 0xc9, 0xb8, 0xe9, - 0x0a, 0x80, 0x1f, 0x38, 0x47, 0xce, 0x88, 0x1e, 0x50, 0xbb, 0x0b, 0x1c, 0xda, 0x6b, 0x14, 0x7e, - 0x79, 0x24, 0xaf, 0x1f, 0x06, 0x81, 0xe7, 0x45, 0xfb, 0x61, 0xb7, 0x29, 0xae, 0x1c, 0x62, 0xb2, - 0xc9, 0xa9, 0xec, 0x86, 0x80, 0x41, 0x54, 0x7e, 0xa7, 0x31, 0x27, 0xea, 0x62, 0x6c, 0xf3, 0x2b, - 0x8d, 0x45, 0x76, 0x20, 0xd9, 0x8e, 0xdb, 0x6d, 0xf1, 0x9e, 0xa2, 0xc1, 0xf2, 0x1d, 0xff, 0x18, - 0x78, 0x2e, 0xc2, 0xee, 0x36, 0x67, 0x35, 0x38, 0xe5, 0x09, 0x12, 0x58, 0xd4, 0x46, 0xd1, 0x71, - 0x77, 0x9e, 0xd3, 0xd9, 0x27, 0xe2, 0x5b, 0x59, 0xf4, 0x75, 0xb8, 0xf7, 0x3f, 0x28, 0xd9, 0xc4, - 0xef, 0x0d, 0xc2, 0xfb, 0x73, 0x05, 0x96, 0x36, 0xf9, 0x31, 0xaa, 0x25, 0x98, 0x33, 0x20, 0x14, - 0x72, 0x4b, 0x41, 0xc1, 0x2c, 0x9c, 0xc8, 0x2e, 0x56, 0xca, 0x91, 0xaf, 0xa0, 0x1d, 0x8f, 0x29, - 0x7b, 0x56, 0xdf, 0x06, 0x22, 0x5b, 0xa1, 0xde, 0x34, 0x3e, 0x87, 0xe5, 0x9c, 0xce, 0xf2, 0xe8, - 0xbd, 0x86, 0x89, 0x50, 0xdd, 0x6e, 0x29, 0x95, 0x9b, 0x8a, 0x86, 0x98, 0xea, 0x1e, 0x83, 0x92, - 0x56, 0x10, 0xe5, 0x16, 0x7c, 0x8a, 0xbe, 0x1c, 0x47, 0xa6, 0xfb, 0x4a, 0xa8, 0xd7, 0x87, 0x45, - 0x86, 0x30, 0xdf, 0x61, 0x50, 0x96, 0x07, 0xd8, 0xb2, 0xbd, 0x49, 0x24, 0xf3, 0x5f, 0xdc, 0x34, - 0x96, 0x05, 0xea, 0xcd, 0xcf, 0xf6, 0x19, 0x2c, 0x09, 0xd0, 0xf9, 0x2e, 0x8b, 0xb8, 0x18, 0x43, - 0xde, 0xfc, 0xb8, 0xbf, 0x9b, 0xd2, 0x12, 0x61, 0x49, 0x95, 0x7c, 0x33, 0x5d, 0x25, 0x2f, 0xe7, - 0x1d, 0x9e, 0xaa, 0xdc, 0xf2, 0x61, 0x54, 0x2d, 0x08, 0x23, 0x33, 0x57, 0x4a, 0x4f, 0xf3, 0x94, - 0xf1, 0x49, 0x7e, 0xf4, 0xff, 0x61, 0x25, 0xbd, 0x23, 0x2a, 0x69, 0x35, 0xb5, 0x42, 0xef, 0xb7, - 0x32, 0x95, 0x74, 0xb7, 0x4c, 0x4d, 0x55, 0x48, 0xff, 0xbd, 0x0a, 0x0d, 0xc5, 0xcb, 0xd9, 0xf4, - 0x76, 0xae, 0x8e, 0x3e, 0xe3, 0x61, 0x53, 0x7d, 0xdb, 0x61, 0x83, 0x65, 0x1d, 0xff, 0x18, 0x04, - 0x74, 0x5f, 0x96, 0x3e, 0x75, 0x4e, 0x30, 0xe9, 0x7e, 0xe2, 0xd2, 0xda, 0xa9, 0x5c, 0x7a, 0x3b, - 0x73, 0xa9, 0x76, 0x25, 0x2f, 0x5f, 0x78, 0x28, 0x6c, 0x17, 0xdd, 0xa6, 0x7d, 0x58, 0xd0, 0xf9, - 0xbd, 0x45, 0xa5, 0x8f, 0x44, 0xcd, 0xaf, 0x07, 0x84, 0x4c, 0x3c, 0xeb, 0x58, 0x19, 0x2b, 0xaa, - 0x2c, 0xfc, 0x49, 0x7e, 0x69, 0xa6, 0x26, 0xc5, 0x76, 0x71, 0xca, 0xc0, 0xc9, 0xed, 0xd0, 0x29, - 0x76, 0xf1, 0xef, 0xf5, 0x9a, 0xa5, 0xe4, 0x1a, 0xe5, 0x5d, 0xc3, 0xea, 0x66, 0x1a, 0x9d, 0xbd, - 0x2d, 0x1e, 0xde, 0x02, 0xce, 0xf8, 0x21, 0x8a, 0x49, 0x53, 0xb0, 0x45, 0x7d, 0xdf, 0x90, 0x14, - 0x64, 0x63, 0x61, 0xb3, 0xef, 0xb8, 0x4e, 0x78, 0x28, 0xf8, 0x33, 0x9c, 0x0f, 0x31, 0x69, 0x83, - 0xbf, 0x82, 0xd0, 0x37, 0x88, 0x26, 0x86, 0x9e, 0x4d, 0x31, 0x68, 0xf8, 0x2b, 0x08, 0x23, 0x6c, - 0x62, 0x3b, 0xd9, 0x01, 0xf5, 0x33, 0xed, 0x80, 0x46, 0x66, 0x07, 0x2c, 0xc1, 0x0c, 0xea, 0x1b, - 0x7a, 0x2e, 0x2f, 0x2f, 0x1a, 0xa6, 0x6c, 0x69, 0x55, 0x50, 0xb3, 0xac, 0x0a, 0x3a, 0xe1, 0x36, - 0x26, 0x53, 0x05, 0xcd, 0x95, 0x55, 0x41, 0xa7, 0xb9, 0x8c, 0xd1, 0x0a, 0xb7, 0xd6, 0x49, 0x85, - 0xdb, 0x8f, 0xb9, 0x3d, 0x1e, 0xe2, 0xc1, 0x9c, 0x0d, 0x68, 0xb9, 0x3f, 0x6e, 0x65, 0xee, 0x6c, - 0xba, 0x65, 0x56, 0x50, 0x57, 0x36, 0xaf, 0xa0, 0xb9, 0xfd, 0x06, 0xdd, 0x77, 0xfa, 0x83, 0x14, - 0x55, 0x1d, 0x8e, 0x6d, 0x89, 0xbe, 0xd8, 0x67, 0x5c, 0xa0, 0x55, 0x93, 0x02, 0x4d, 0xd5, 0x79, - 0x2c, 0x4c, 0xe7, 0x64, 0x9d, 0x67, 0xdc, 0x87, 0x39, 0x31, 0x97, 0xd4, 0x76, 0x89, 0x69, 0x6b, - 0xb3, 0x13, 0xb9, 0xc2, 0xc5, 0x64, 0x4b, 0xd2, 0x69, 0x10, 0xf0, 0xb5, 0x0b, 0x3a, 0xb6, 0x8c, - 0x3b, 0xd0, 0xe4, 0xf1, 0x26, 0x8f, 0xcc, 0xeb, 0x3a, 0x90, 0x3f, 0x29, 0x28, 0x19, 0x78, 0x61, - 0x09, 0x85, 0xd3, 0xd5, 0xee, 0xff, 0x34, 0x73, 0xba, 0x2c, 0xa6, 0xfb, 0x67, 0x4e, 0x96, 0xd7, - 0x50, 0xe3, 0xe4, 0xdc, 0xee, 0xbf, 0xc4, 0x40, 0xa5, 0xef, 0x0d, 0x22, 0xeb, 0x40, 0x3d, 0x6a, - 0x32, 0xc2, 0x53, 0x6c, 0xf3, 0x37, 0x59, 0xc6, 0xb4, 0x1d, 0x9c, 0x38, 0x0a, 0x25, 0x02, 0x69, - 0x32, 0xda, 0x96, 0x20, 0x31, 0x14, 0x12, 0x3a, 0xbf, 0xa1, 0xdc, 0x52, 0xd3, 0x26, 0xff, 0xc6, - 0xd2, 0x8b, 0xe8, 0xfa, 0x4a, 0x73, 0x61, 0x68, 0xf2, 0xe5, 0xc4, 0x89, 0xaf, 0x9d, 0x56, 0xd8, - 0x94, 0x5c, 0x34, 0x33, 0x11, 0x16, 0x48, 0x25, 0xbb, 0xd3, 0x5b, 0xeb, 0x33, 0x38, 0x9f, 0xea, - 0xaf, 0x9e, 0x10, 0x52, 0x03, 0x64, 0x67, 0x97, 0x9d, 0xff, 0x59, 0x01, 0xd8, 0x98, 0x44, 0x87, - 0x12, 0xdf, 0xf5, 0xa0, 0xce, 0x80, 0xa9, 0x86, 0x12, 0x55, 0x9b, 0xf1, 0x7c, 0x2b, 0x0c, 0x11, - 0x5c, 0xc5, 0x6f, 0x70, 0xaa, 0xcd, 0xb1, 0xd9, 0x44, 0xbd, 0x89, 0xf2, 0x6f, 0xf6, 0xa6, 0x2a, - 0x1e, 0x92, 0x07, 0x88, 0x4d, 0x11, 0x95, 0x87, 0xf2, 0x84, 0x6d, 0x09, 0xea, 0x86, 0x20, 0x32, - 0x31, 0xc7, 0xa6, 0xa8, 0x5a, 0x74, 0x3c, 0x88, 0xbc, 0xd7, 0xd4, 0x95, 0x28, 0xae, 0x15, 0x53, - 0x9f, 0x32, 0x22, 0x13, 0x0b, 0xe8, 0x01, 0x5a, 0x39, 0x88, 0xc5, 0x66, 0x84, 0x58, 0x4c, 0xe5, - 0x62, 0xec, 0x0d, 0xbe, 0xb3, 0x37, 0x19, 0x8d, 0xc4, 0x22, 0xcf, 0x6a, 0x4b, 0x04, 0x4d, 0x62, - 0x1d, 0xd9, 0xab, 0x82, 0xc4, 0x44, 0x72, 0x71, 0x3f, 0xbc, 0x5e, 0x3f, 0x0f, 0x0b, 0x9a, 0xa2, - 0xb2, 0xd4, 0xc4, 0x58, 0x10, 0x55, 0xe8, 0xbb, 0xe9, 0x6f, 0x5c, 0x80, 0xf3, 0xa9, 0xfe, 0x62, - 0xd8, 0xd5, 0xcb, 0x50, 0x8f, 0xdf, 0xdb, 0xc9, 0x2c, 0x54, 0x9f, 0x6e, 0xee, 0x75, 0xce, 0xb1, - 0x8f, 0x67, 0x5b, 0x7b, 0x9d, 0xca, 0xea, 0x2a, 0xcc, 0x67, 0x2e, 0x18, 0x49, 0x03, 0x6a, 0xe6, - 0xf6, 0xc6, 0xd6, 0x0b, 0x14, 0x9b, 0x83, 0xfa, 0xee, 0x93, 0xa7, 0xa2, 0x55, 0x59, 0xdd, 0x84, - 0x76, 0xfa, 0xb8, 0x23, 0x4d, 0x98, 0xdd, 0x44, 0xee, 0xd3, 0xed, 0x2d, 0x14, 0xc6, 0x86, 0xf9, - 0x6c, 0x77, 0x77, 0x67, 0xf7, 0x41, 0xa7, 0x42, 0x00, 0x66, 0xb6, 0xbf, 0xdd, 0x61, 0x8c, 0x29, - 0xc6, 0x78, 0xb6, 0xfb, 0x70, 0xf7, 0xc9, 0xaf, 0x76, 0x3b, 0xd5, 0xf5, 0xbf, 0xd5, 0xa1, 0x6d, - 0x8a, 0x25, 0xf4, 0x31, 0x16, 0x1c, 0x04, 0x8d, 0xf7, 0x61, 0x36, 0xfe, 0x99, 0x21, 0x39, 0x58, - 0xd3, 0x7f, 0x5e, 0xf4, 0xba, 0x79, 0x86, 0x34, 0xdb, 0x39, 0xb2, 0x07, 0xad, 0xd4, 0x4b, 0x1a, - 0x49, 0x40, 0x66, 0xd1, 0x63, 0x5d, 0xef, 0x4a, 0x19, 0x5b, 0x8d, 0xd8, 0x87, 0x76, 0xfa, 0x75, - 0x8c, 0x24, 0x7d, 0x0a, 0x5f, 0xdd, 0x7a, 0x57, 0x4b, 0xf9, 0x6a, 0xd0, 0x17, 0xd0, 0xc9, 0xbe, - 0x8b, 0x91, 0xe4, 0x4e, 0xab, 0xe4, 0xcd, 0xad, 0x77, 0xed, 0x04, 0x09, 0x7d, 0xe8, 0xdc, 0x0b, - 0xd2, 0x4a, 0xf9, 0x1b, 0x40, 0x6e, 0xe8, 0xb2, 0x87, 0x05, 0x61, 0x8a, 0xf4, 0xa5, 0x2e, 0xd1, - 0xdf, 0x6d, 0x0a, 0x2e, 0xd5, 0x35, 0x53, 0x14, 0xdf, 0x06, 0xe3, 0xa0, 0xcf, 0xb1, 0x52, 0x4b, - 0xe3, 0x55, 0x92, 0xf4, 0x2a, 0x46, 0xdf, 0xbd, 0x95, 0x72, 0x81, 0xb4, 0xdf, 0x74, 0x34, 0x9a, - 0xf2, 0x5b, 0x01, 0xc4, 0x4d, 0xf9, 0xad, 0x10, 0xc6, 0xf2, 0xf0, 0x4a, 0x61, 0x4e, 0x2d, 0xbc, - 0x8a, 0x00, 0x6e, 0xef, 0x4a, 0x19, 0x5b, 0x5f, 0x7e, 0x06, 0x6f, 0x6a, 0xcb, 0x2f, 0x86, 0xb1, - 0xbd, 0x95, 0x72, 0x81, 0xac, 0xaf, 0x92, 0x62, 0x3c, 0xe3, 0xab, 0x1c, 0x6c, 0xcb, 0xf8, 0x2a, - 0x5f, 0xc5, 0x4b, 0x5f, 0x65, 0xaa, 0xea, 0xab, 0xa5, 0xa5, 0x4a, 0xde, 0x57, 0xc5, 0xd5, 0x0f, - 0x8e, 0x7b, 0x17, 0xa6, 0x59, 0x85, 0x41, 0x92, 0xb3, 0x5c, 0x2b, 0x6e, 0x7a, 0x17, 0x32, 0xd4, - 0xb8, 0xdb, 0xf5, 0xca, 0xad, 0xca, 0xfa, 0x5f, 0xa6, 0x60, 0x4e, 0xa4, 0x3f, 0x99, 0x41, 0x1e, - 0x00, 0x24, 0x87, 0x30, 0xe9, 0xa5, 0x16, 0x95, 0xaa, 0x24, 0x7a, 0x97, 0x0a, 0x79, 0x4a, 0xa9, - 0x6f, 0x64, 0xd9, 0x22, 0x17, 0x7a, 0x29, 0x93, 0x6d, 0x53, 0x8b, 0xbc, 0x5c, 0xcc, 0x54, 0x63, - 0x6d, 0x41, 0x43, 0x25, 0x79, 0xa2, 0x9d, 0x0d, 0x99, 0x13, 0xaa, 0xd7, 0x2b, 0x62, 0xe9, 0x1a, - 0x69, 0x59, 0x5d, 0xd3, 0x28, 0x7f, 0x56, 0x68, 0x1a, 0x15, 0x1c, 0x04, 0xc6, 0xb9, 0xff, 0x06, - 0x00, 0x00, 0xff, 0xff, 0xe6, 0xbd, 0x3a, 0x14, 0xe5, 0x26, 0x00, 0x00, + 0x11, 0x16, 0x00, 0x82, 0x04, 0x1a, 0x04, 0x08, 0x8e, 0x28, 0x12, 0x82, 0x64, 0x89, 0x5a, 0xdb, + 0x89, 0x44, 0x5b, 0x2c, 0x85, 0x49, 0x49, 0x91, 0x6c, 0xcb, 0xa6, 0x49, 0x46, 0x45, 0x4b, 0xa2, + 0x98, 0x85, 0xa4, 0x58, 0x27, 0xd4, 0x0a, 0x3b, 0x24, 0x57, 0x02, 0x76, 0xd7, 0xbb, 0x0b, 0x46, + 0xcc, 0x35, 0x97, 0x1c, 0x72, 0xc9, 0x2d, 0x0f, 0xe0, 0xaa, 0x9c, 0x72, 0x49, 0x55, 0x2e, 0xb9, + 0xe5, 0x9c, 0xca, 0x83, 0xe4, 0x09, 0x92, 0x63, 0x7a, 0x7e, 0x76, 0x76, 0xf6, 0x07, 0x34, 0x28, + 0x57, 0xd9, 0xba, 0xed, 0x74, 0xf7, 0xcc, 0xf4, 0x74, 0xf7, 0xf4, 0xf4, 0x37, 0xb3, 0x50, 0xb7, + 0x7c, 0x67, 0xdd, 0x0f, 0xbc, 0xc8, 0x23, 0x73, 0xc1, 0xd8, 0x8d, 0x9c, 0x11, 0x35, 0xd6, 0xa0, + 0xf5, 0x9c, 0x06, 0xa1, 0xe3, 0xb9, 0x26, 0xfd, 0x66, 0x4c, 0xc3, 0x88, 0x74, 0x60, 0xee, 0x58, + 0x50, 0x3a, 0xa5, 0xd5, 0xd2, 0xf5, 0xba, 0x19, 0x37, 0x8d, 0xbf, 0x94, 0x60, 0x41, 0x09, 0x87, + 0xbe, 0xe7, 0x86, 0x74, 0xb2, 0x34, 0xb9, 0x06, 0xf3, 0x72, 0x92, 0xbe, 0x6b, 0x8d, 0x68, 0xa7, + 0xcc, 0xd9, 0x0d, 0x49, 0xdb, 0x43, 0x12, 0xf9, 0x29, 0x2c, 0xc4, 0x22, 0xf1, 0x20, 0x15, 0x2e, + 0xd5, 0x92, 0x64, 0x39, 0x1b, 0x59, 0x87, 0xf3, 0xb1, 0x20, 0xae, 0x41, 0x09, 0xcf, 0x70, 0xe1, + 0x45, 0xc9, 0xda, 0xf4, 0x1d, 0x29, 0x6f, 0x6c, 0x42, 0x7d, 0x7b, 0xaf, 0xf7, 0xc4, 0x8f, 0x58, + 0x67, 0x54, 0x31, 0xa4, 0x01, 0xeb, 0x83, 0x2a, 0x56, 0x98, 0x8a, 0xb2, 0x49, 0xba, 0x50, 0x0b, + 0xa9, 0x15, 0x0c, 0x8e, 0x68, 0x88, 0xea, 0x31, 0x96, 0x6a, 0x1b, 0x7f, 0x2d, 0x41, 0x63, 0xdf, + 0x0b, 0xa2, 0xc7, 0x96, 0xef, 0x3b, 0xee, 0x21, 0x21, 0x30, 0xc3, 0x97, 0x21, 0x56, 0xc9, 0xbf, + 0xc9, 0x4d, 0xa8, 0x71, 0x73, 0x0e, 0xbc, 0x21, 0x5f, 0x5e, 0x6b, 0x63, 0x71, 0x5d, 0x2a, 0xb3, + 0xbe, 0x2f, 0x19, 0xa6, 0x12, 0x21, 0x1f, 0x42, 0x6b, 0xe0, 0xb9, 0x91, 0xe5, 0xb8, 0x34, 0xe8, + 0xfb, 0x38, 0x36, 0x5f, 0x6d, 0xd5, 0x6c, 0x2a, 0x2a, 0x9b, 0x90, 0x5c, 0x82, 0xfa, 0x91, 0x17, + 0x46, 0x42, 0x62, 0x86, 0x4b, 0xd4, 0x18, 0x81, 0x33, 0x57, 0x60, 0x8e, 0x33, 0x1d, 0xbf, 0x53, + 0xe5, 0x9a, 0xcc, 0xb2, 0xe6, 0xae, 0x6f, 0x7c, 0x5b, 0x82, 0xea, 0x63, 0x0f, 0x27, 0x2f, 0xd4, + 0x34, 0x3d, 0xb5, 0x15, 0x1d, 0x49, 0x77, 0x68, 0x53, 0x23, 0x31, 0x99, 0x9a, 0x49, 0x08, 0x57, + 0x88, 0xa9, 0x19, 0x13, 0xad, 0x15, 0x50, 0xcb, 0xf6, 0xdc, 0xe1, 0x09, 0x57, 0xab, 0x66, 0xaa, + 0x36, 0xf3, 0x64, 0x48, 0x87, 0x8e, 0x3b, 0x7e, 0xd3, 0x0f, 0xe8, 0xd0, 0x7a, 0x49, 0x87, 0x5c, + 0xbd, 0x9a, 0xd9, 0x92, 0x64, 0x53, 0x50, 0x8d, 0x57, 0xb0, 0xc0, 0x5c, 0x1f, 0xfa, 0xd6, 0x80, + 0x4a, 0xff, 0x60, 0xa0, 0xf0, 0x49, 0x5d, 0x1a, 0xfd, 0xd6, 0x0b, 0x5e, 0x73, 0xbd, 0x6b, 0x66, + 0x83, 0xd1, 0xf6, 0x04, 0x89, 0x5c, 0x84, 0x9a, 0xd0, 0xcb, 0xb1, 0xb9, 0xe2, 0x35, 0x93, 0x5b, + 0x61, 0xdf, 0xb1, 0x15, 0xcb, 0xf1, 0x07, 0x5c, 0x63, 0xc9, 0xda, 0xf5, 0x07, 0xc6, 0xef, 0x4b, + 0x70, 0xe1, 0x11, 0x9b, 0x7c, 0xdf, 0xb3, 0x7b, 0x96, 0x6b, 0xbf, 0xf4, 0xde, 0x6c, 0x79, 0xee, + 0x81, 0x73, 0x48, 0xde, 0x87, 0xe6, 0xe0, 0x30, 0xf0, 0xc6, 0x3e, 0xae, 0x34, 0xa0, 0x6e, 0x24, + 0x6d, 0x35, 0x2f, 0x88, 0xfb, 0x9c, 0x46, 0x76, 0x60, 0xd1, 0x8d, 0x55, 0xed, 0x7b, 0x5c, 0xd7, + 0x90, 0xcf, 0xde, 0xd8, 0xe8, 0x28, 0x37, 0x67, 0x16, 0x63, 0xb6, 0xdd, 0x34, 0x21, 0x34, 0x02, + 0x20, 0xc9, 0xfc, 0x8f, 0x69, 0x64, 0xd9, 0x56, 0x64, 0x15, 0x3a, 0xa9, 0x0d, 0x95, 0xb1, 0x5c, + 0x60, 0xdd, 0x64, 0x9f, 0xe4, 0x32, 0xd4, 0xd5, 0x78, 0xd2, 0x1f, 0x09, 0x81, 0x05, 0xb6, 0x15, + 0x45, 0x74, 0xe4, 0x8b, 0x30, 0x69, 0x9a, 0x71, 0xd3, 0xf8, 0xc7, 0x0c, 0xb4, 0x73, 0x8b, 0xbe, + 0x03, 0xb5, 0x91, 0x9c, 0x9e, 0x4f, 0xdb, 0xd8, 0xb8, 0x94, 0x44, 0x6b, 0x4e, 0x43, 0x53, 0x09, + 0x33, 0xc7, 0x33, 0x93, 0x6a, 0xbb, 0x58, 0xb5, 0x99, 0x25, 0x87, 0xde, 0x61, 0xdf, 0x76, 0x02, + 0x3a, 0x88, 0xbc, 0xe0, 0x44, 0x6a, 0x39, 0x8f, 0xc4, 0xed, 0x98, 0x46, 0x7e, 0x0e, 0x0d, 0xdb, + 0x0d, 0x95, 0x0d, 0x67, 0xf8, 0xe4, 0x44, 0x4d, 0xae, 0xb6, 0xaa, 0x09, 0x28, 0x26, 0xed, 0x46, + 0xee, 0x42, 0x93, 0xed, 0x80, 0xfe, 0x48, 0x6c, 0xc0, 0x10, 0x03, 0xaa, 0x82, 0xdd, 0x96, 0x34, + 0x9d, 0xd5, 0xee, 0x34, 0xe7, 0xfd, 0xa4, 0x11, 0x92, 0xcf, 0x60, 0x96, 0x47, 0x5b, 0xd8, 0x99, + 0xe5, 0x7d, 0x3e, 0x2c, 0x58, 0xa7, 0x30, 0xca, 0xfa, 0x23, 0x2e, 0xb7, 0xe3, 0x46, 0xc1, 0x89, + 0x29, 0x3b, 0x91, 0x47, 0xd0, 0xb0, 0x5c, 0xd7, 0x8b, 0x2c, 0xa1, 0xee, 0x1c, 0x1f, 0x63, 0x6d, + 0xf2, 0x18, 0x9b, 0x89, 0xb0, 0x18, 0x48, 0xef, 0x4e, 0x7e, 0x01, 0x55, 0xbe, 0x03, 0x3a, 0x35, + 0xbe, 0xec, 0x2b, 0x6a, 0x9c, 0xc2, 0xd0, 0x34, 0x85, 0x70, 0xf7, 0x2e, 0x34, 0x34, 0xd5, 0x58, + 0x68, 0xbc, 0xa6, 0x27, 0x32, 0x5a, 0xd8, 0x27, 0x59, 0x82, 0xea, 0xb1, 0x35, 0x1c, 0xc7, 0x1e, + 0x11, 0x8d, 0x7b, 0xe5, 0x5f, 0x96, 0xba, 0xf7, 0xa1, 0x9d, 0xd5, 0xe8, 0x2c, 0xfd, 0x8d, 0x5d, + 0x58, 0x32, 0xc7, 0x6e, 0xa2, 0x58, 0x7c, 0x30, 0xfc, 0x0c, 0x66, 0x07, 0x5c, 0x47, 0x19, 0x3d, + 0x17, 0x27, 0x5a, 0xc4, 0x94, 0x82, 0xc6, 0x67, 0x70, 0x21, 0x33, 0x94, 0x3c, 0x36, 0x3e, 0x80, + 0x96, 0xef, 0xd9, 0xfd, 0x50, 0x90, 0xfb, 0x18, 0xf5, 0x72, 0x07, 0xfa, 0x4a, 0x76, 0xd7, 0x66, + 0xdd, 0x7b, 0x91, 0xe7, 0xe7, 0x55, 0x99, 0xae, 0x7b, 0x07, 0x96, 0xb3, 0xdd, 0xc5, 0xf4, 0xc6, + 0xe7, 0xb0, 0x62, 0xd2, 0x91, 0x77, 0x4c, 0xdf, 0x76, 0xe8, 0x2e, 0x74, 0xf2, 0x03, 0x24, 0x83, + 0x27, 0xd4, 0x1e, 0xba, 0x61, 0x1c, 0x9e, 0x6d, 0xf0, 0x1b, 0xfa, 0x00, 0x32, 0x05, 0x8a, 0x71, + 0x48, 0x0b, 0xca, 0x98, 0xf9, 0x45, 0x27, 0xfc, 0x32, 0x5e, 0x40, 0x7d, 0x4f, 0xcf, 0x07, 0x7a, + 0x0e, 0xc5, 0x83, 0x4e, 0x36, 0xc9, 0x06, 0xcc, 0x4d, 0x9b, 0xc0, 0x62, 0x41, 0xe3, 0x61, 0x2e, + 0x79, 0x4a, 0x1d, 0x36, 0x00, 0x54, 0x0e, 0x0a, 0x65, 0x2c, 0x90, 0xfc, 0x78, 0xa6, 0x26, 0x65, + 0x7c, 0x9b, 0x4a, 0x48, 0xda, 0x62, 0x6c, 0xb5, 0x18, 0x3b, 0x95, 0xa0, 0xca, 0x67, 0x49, 0x50, + 0xeb, 0x50, 0x0d, 0x71, 0x48, 0x91, 0x22, 0x5b, 0xda, 0xe2, 0x64, 0xaf, 0x2f, 0xc5, 0x94, 0xd4, + 0x14, 0x62, 0xe4, 0x3d, 0x80, 0x01, 0x1e, 0x5d, 0x11, 0xb5, 0xfb, 0x96, 0xc8, 0x9d, 0x15, 0xb3, + 0x2e, 0x29, 0x9b, 0x11, 0xb9, 0x97, 0xd8, 0xb1, 0xca, 0xd5, 0x58, 0x2d, 0x50, 0x23, 0xe5, 0x97, + 0xc4, 0xd2, 0x6a, 0xb7, 0xcf, 0x9e, 0xbe, 0xdb, 0x65, 0x3f, 0x21, 0xac, 0x25, 0xac, 0xb9, 0x89, + 0x09, 0x4b, 0xf4, 0x98, 0x26, 0x61, 0xd5, 0x26, 0x26, 0x2c, 0x39, 0xc6, 0xa9, 0x09, 0xeb, 0xc7, + 0x4c, 0x3d, 0x8f, 0xa1, 0x93, 0xdf, 0x3a, 0x32, 0x65, 0x60, 0xfa, 0x09, 0x39, 0xe5, 0x94, 0xf4, + 0x23, 0xbb, 0x48, 0x41, 0xe3, 0x3f, 0x25, 0x3d, 0xea, 0x7e, 0xe5, 0x0c, 0x23, 0x1a, 0xe4, 0xa2, + 0x4e, 0x05, 0x4f, 0x79, 0xba, 0xe0, 0xe9, 0x41, 0x8b, 0x9b, 0xbd, 0x8f, 0x95, 0x0d, 0x3f, 0xdf, + 0x30, 0xea, 0x98, 0xbd, 0x3f, 0x2e, 0xd0, 0x47, 0x4c, 0x29, 0x7c, 0xd6, 0x93, 0xe2, 0xc2, 0xe2, + 0xcd, 0xa1, 0x4e, 0xeb, 0x7e, 0x01, 0x24, 0x2f, 0x74, 0x26, 0xd3, 0x7d, 0xc5, 0xb6, 0x2b, 0x2b, + 0x12, 0x0b, 0xd2, 0xf6, 0x01, 0x57, 0xe3, 0x14, 0xbb, 0x09, 0x3d, 0x4d, 0x29, 0x68, 0xfc, 0xb9, + 0x0c, 0x90, 0x30, 0xdf, 0xd9, 0x7d, 0x7a, 0x47, 0xed, 0x1a, 0x51, 0x1a, 0x5c, 0x2d, 0xd0, 0xa2, + 0x68, 0xbf, 0x7c, 0x8f, 0x08, 0x37, 0xb6, 0x60, 0x39, 0x6b, 0x66, 0x19, 0x9f, 0x37, 0xa0, 0xea, + 0x60, 0xf5, 0x25, 0x40, 0x46, 0x63, 0xe3, 0x7c, 0x81, 0x32, 0xa6, 0x90, 0x30, 0xae, 0x41, 0x7d, + 0x77, 0x64, 0x1d, 0xd2, 0x9e, 0x4f, 0x07, 0x6c, 0x2e, 0x87, 0x35, 0xe4, 0xfc, 0xa2, 0x61, 0x6c, + 0x40, 0xed, 0x21, 0x3d, 0x79, 0xce, 0xe6, 0x9d, 0x56, 0x3f, 0xe3, 0x5f, 0x25, 0x58, 0xe1, 0x69, + 0x66, 0x2b, 0x2e, 0xea, 0x51, 0x39, 0x6f, 0x1c, 0x60, 0x02, 0xe6, 0xa6, 0xf4, 0xc7, 0x7d, 0x9f, + 0x06, 0x8e, 0x27, 0x7c, 0xc9, 0x4c, 0xe9, 0x8f, 0xf7, 0x39, 0x81, 0x15, 0xfe, 0x8c, 0xfd, 0xcd, + 0xd8, 0x93, 0x3e, 0xad, 0x98, 0x35, 0x24, 0xfc, 0x9a, 0xb5, 0xe3, 0xbe, 0xe1, 0x11, 0xd6, 0xc5, + 0x21, 0xf7, 0x9d, 0xe8, 0xdb, 0xe3, 0x04, 0x0c, 0xb0, 0x0b, 0x23, 0x3c, 0x0b, 0x83, 0x93, 0xfe, + 0xd0, 0x19, 0x39, 0x58, 0x89, 0xbb, 0xfd, 0x97, 0x27, 0x11, 0x0d, 0xa5, 0xc3, 0x88, 0x60, 0x3e, + 0x62, 0xbc, 0x5d, 0xf7, 0x4b, 0xc6, 0x21, 0x06, 0x34, 0x3d, 0x6f, 0xd4, 0x0f, 0x07, 0x5e, 0x80, + 0x88, 0xce, 0x7e, 0xc5, 0xf3, 0x6c, 0xc5, 0x6c, 0x20, 0xb1, 0xc7, 0x68, 0x9b, 0xf6, 0x2b, 0xc3, + 0x82, 0x66, 0x6f, 0x87, 0x2f, 0x47, 0xe2, 0x04, 0x2c, 0x99, 0xc7, 0xa1, 0x0c, 0x63, 0x2c, 0x99, + 0xd9, 0x37, 0xa3, 0x05, 0xde, 0x30, 0xb6, 0x03, 0xff, 0x66, 0xb4, 0xe8, 0xc4, 0x8f, 0xeb, 0x65, + 0xfe, 0xcd, 0x0c, 0x36, 0xa4, 0xc7, 0x88, 0x4a, 0x04, 0x64, 0x14, 0x0d, 0xc3, 0x06, 0xd8, 0xb2, + 0x7c, 0xeb, 0xa5, 0x33, 0x74, 0xa2, 0x13, 0x74, 0x60, 0xdb, 0xb2, 0xed, 0xfe, 0x20, 0xa6, 0x38, + 0x34, 0x06, 0x8c, 0x0b, 0x48, 0xdf, 0xd2, 0xc8, 0xe4, 0x23, 0x58, 0xb4, 0x03, 0xcf, 0x4f, 0xcb, + 0x0a, 0x04, 0xd9, 0x66, 0x0c, 0x5d, 0xd8, 0xf8, 0x5f, 0x09, 0x96, 0xd2, 0x6e, 0x91, 0x05, 0xf9, + 0x7d, 0xa8, 0x07, 0xb1, 0x83, 0xe4, 0xe6, 0x5c, 0x4d, 0x9f, 0x17, 0x79, 0x47, 0x9a, 0x49, 0x17, + 0x8c, 0xff, 0xf9, 0x8c, 0x02, 0xa5, 0x54, 0xe0, 0x25, 0x6b, 0x33, 0x53, 0x82, 0xe4, 0xf3, 0x04, + 0xad, 0xc5, 0x65, 0x41, 0x85, 0xf7, 0x5d, 0x56, 0x7d, 0x53, 0xa6, 0x57, 0x28, 0x2e, 0xae, 0xcd, + 0x7f, 0x22, 0x5d, 0x91, 0xad, 0xe4, 0x79, 0x9f, 0x67, 0xc8, 0x11, 0xee, 0x31, 0xbe, 0x86, 0xba, + 0x22, 0xc5, 0xf0, 0x46, 0xc4, 0x1e, 0x87, 0x37, 0x48, 0x39, 0x94, 0x80, 0x07, 0x29, 0xf8, 0xc9, + 0x70, 0x24, 0xda, 0xda, 0x61, 0xb3, 0x58, 0xc3, 0x3e, 0x52, 0x42, 0x9e, 0x5d, 0x2b, 0x66, 0x2b, + 0x21, 0x3f, 0x40, 0x2a, 0x22, 0xfc, 0x45, 0x65, 0x9c, 0x53, 0x41, 0x95, 0x06, 0x92, 0xca, 0x69, + 0x90, 0xf4, 0xdf, 0x2a, 0x2c, 0x64, 0x5d, 0x72, 0x3b, 0x87, 0x91, 0xba, 0x89, 0x39, 0xb3, 0xf3, + 0x69, 0x99, 0xed, 0x7a, 0xbc, 0x89, 0xcb, 0x19, 0x8b, 0xa8, 0x7d, 0x2e, 0x37, 0x36, 0xd3, 0x67, + 0xe0, 0x8d, 0x46, 0x98, 0x10, 0xf8, 0xca, 0xb0, 0x48, 0x93, 0x4d, 0xa6, 0xbd, 0x15, 0x1c, 0xb2, + 0x6d, 0xc3, 0xc8, 0xfc, 0x9b, 0x5c, 0x85, 0x06, 0x2b, 0x2b, 0x10, 0xd5, 0x30, 0x88, 0x25, 0x21, + 0x3f, 0x48, 0x12, 0x02, 0x2c, 0x04, 0xf6, 0x33, 0xd4, 0x3d, 0x8e, 0x81, 0x4e, 0x72, 0xfd, 0x10, + 0x27, 0x0f, 0x93, 0xb3, 0xd1, 0x61, 0xb3, 0x23, 0x76, 0x39, 0x10, 0x17, 0x18, 0x2d, 0x25, 0xc8, + 0xef, 0x0c, 0x4c, 0xc9, 0x25, 0x9f, 0xaa, 0x94, 0x2a, 0x8a, 0x88, 0x0f, 0xf2, 0xab, 0x3f, 0x05, + 0x38, 0x3d, 0x4c, 0xd7, 0x21, 0x75, 0x3e, 0xc4, 0x8d, 0x89, 0x43, 0x9c, 0x8e, 0x9b, 0xae, 0x00, + 0xf8, 0x81, 0x73, 0xec, 0x0c, 0xe9, 0x21, 0xb5, 0x3b, 0xc0, 0xa1, 0xbd, 0x46, 0xe1, 0x97, 0x47, + 0xf2, 0xfa, 0xa1, 0x1f, 0x78, 0x5e, 0x74, 0x10, 0x76, 0x1a, 0xe2, 0xca, 0x21, 0x26, 0x9b, 0x9c, + 0xca, 0x6e, 0x08, 0x18, 0x44, 0xe5, 0x77, 0x1a, 0xf3, 0xa2, 0x2e, 0xc6, 0x36, 0xbf, 0xd2, 0x58, + 0x62, 0x07, 0x92, 0xed, 0xb8, 0x9d, 0x26, 0xef, 0x29, 0x1a, 0x2c, 0xdf, 0xf1, 0x8f, 0xbe, 0xe7, + 0x22, 0xec, 0x6e, 0x71, 0x56, 0x9d, 0x53, 0x9e, 0x20, 0x81, 0x45, 0x6d, 0x14, 0x9d, 0x74, 0x16, + 0x38, 0x9d, 0x7d, 0x22, 0xbe, 0x95, 0x45, 0x5f, 0x9b, 0x7b, 0xff, 0xbd, 0x09, 0x9b, 0xf8, 0x9d, + 0x41, 0x78, 0x7f, 0x2b, 0xc1, 0xf2, 0x16, 0x3f, 0x46, 0xb5, 0x04, 0x73, 0x06, 0x84, 0x42, 0x6e, + 0x29, 0x28, 0x98, 0x85, 0x13, 0xd9, 0xc5, 0x4a, 0x39, 0xf2, 0x05, 0xb4, 0xe2, 0x31, 0x65, 0xcf, + 0xca, 0x77, 0x81, 0xc8, 0x66, 0xa8, 0x37, 0x8d, 0x4f, 0x61, 0x25, 0xa7, 0xb3, 0x3c, 0x7a, 0xaf, + 0x61, 0x22, 0x54, 0xb7, 0x5b, 0x4a, 0xe5, 0x86, 0xa2, 0x21, 0xa6, 0xba, 0xc7, 0xa0, 0xa4, 0x15, + 0x44, 0xb9, 0x05, 0x4f, 0xd1, 0x97, 0xe3, 0xc8, 0x74, 0x5f, 0x09, 0xf5, 0x7a, 0xb0, 0xc4, 0x10, + 0xe6, 0x5b, 0x0c, 0xca, 0xf2, 0x00, 0x5b, 0xb6, 0x37, 0x8e, 0x64, 0xfe, 0x8b, 0x9b, 0xc6, 0x8a, + 0x40, 0xbd, 0xf9, 0xd9, 0x3e, 0x81, 0x65, 0x01, 0x3a, 0xdf, 0x66, 0x11, 0x17, 0x63, 0xc8, 0x9b, + 0x1f, 0xf7, 0x8f, 0x65, 0x2d, 0x11, 0x4e, 0xa8, 0x92, 0x6f, 0xa6, 0xab, 0xe4, 0x95, 0xbc, 0xc3, + 0x53, 0x95, 0x5b, 0x3e, 0x8c, 0x2a, 0x05, 0x61, 0x64, 0xe6, 0x4a, 0xe9, 0x19, 0x9e, 0x32, 0x3e, + 0xca, 0x8f, 0xfe, 0x03, 0x56, 0xd2, 0xbb, 0xa2, 0x92, 0x56, 0x53, 0x2b, 0xf4, 0x7e, 0x2b, 0x53, + 0x49, 0x77, 0x26, 0xa9, 0xa9, 0x0a, 0xe9, 0x3f, 0xcc, 0x40, 0x5d, 0xf1, 0x72, 0x36, 0xcd, 0x1b, + 0xa9, 0x5c, 0x60, 0x24, 0xfd, 0x48, 0xaa, 0xbc, 0xcd, 0x91, 0x34, 0xf3, 0x5d, 0x47, 0x12, 0x16, + 0x7f, 0xfc, 0xa3, 0x1f, 0xd0, 0x03, 0x79, 0xc4, 0xd4, 0x38, 0xc1, 0xa4, 0x07, 0x89, 0xe3, 0x67, + 0xa7, 0x72, 0x7c, 0xba, 0x64, 0x9f, 0xcb, 0x96, 0xec, 0xb7, 0x33, 0xe7, 0xcb, 0x95, 0xfc, 0x70, + 0x85, 0x27, 0xcb, 0x4e, 0xd1, 0xc9, 0xf2, 0x7e, 0x41, 0xe7, 0x77, 0x16, 0xda, 0x3e, 0x12, 0xc0, + 0x41, 0x8f, 0x2a, 0x99, 0xbd, 0x36, 0xd0, 0x64, 0x8a, 0x2a, 0xd1, 0x03, 0xc9, 0x2f, 0xcd, 0xd4, + 0xa4, 0x58, 0x2a, 0x48, 0xd9, 0x3f, 0xb9, 0x62, 0x9a, 0x22, 0x15, 0xfc, 0x49, 0x2f, 0x7c, 0x26, + 0xdc, 0xc5, 0xdc, 0xce, 0x61, 0xbc, 0xe9, 0xa2, 0xee, 0x66, 0x1a, 0xe2, 0x9d, 0x2d, 0x5c, 0x72, + 0x08, 0x8f, 0x9f, 0xc4, 0x98, 0x79, 0x05, 0x5b, 0x80, 0x84, 0xba, 0xa4, 0x20, 0x1b, 0xab, 0xa3, + 0x03, 0xc7, 0x75, 0xc2, 0x23, 0xc1, 0x9f, 0xe5, 0x7c, 0x88, 0x49, 0x9b, 0xfc, 0x29, 0x85, 0xbe, + 0x41, 0x48, 0x32, 0xf0, 0x6c, 0xca, 0x83, 0xb1, 0x6a, 0xd6, 0x18, 0x61, 0x0b, 0xdb, 0xc9, 0x06, + 0xa9, 0x9d, 0x69, 0x83, 0xd4, 0x33, 0x1b, 0x64, 0x19, 0x66, 0x51, 0xdf, 0xd0, 0x73, 0x79, 0x8d, + 0x52, 0x37, 0x65, 0x4b, 0x2b, 0xa5, 0x1a, 0x93, 0x4a, 0xa9, 0x53, 0xae, 0x74, 0x32, 0xa5, 0xd4, + 0xfc, 0xa4, 0x52, 0x6a, 0x9a, 0x1b, 0x1d, 0xad, 0xfa, 0x6b, 0x9e, 0x56, 0xfd, 0xfd, 0x98, 0xdb, + 0xe3, 0x21, 0x9e, 0xee, 0xd9, 0x80, 0x96, 0xfb, 0xe3, 0x56, 0xe6, 0xe2, 0xa7, 0x33, 0xc9, 0x0a, + 0xea, 0xde, 0xe7, 0x15, 0x34, 0x76, 0xde, 0xa0, 0xfb, 0xa6, 0x3f, 0x8d, 0x51, 0xd5, 0xc1, 0xc8, + 0x96, 0x10, 0x8e, 0x7d, 0xc6, 0x55, 0x5e, 0x25, 0xa9, 0xf2, 0x54, 0xb1, 0xc8, 0xc2, 0x74, 0x5e, + 0x16, 0x8b, 0xc6, 0x7d, 0x98, 0x17, 0x73, 0x49, 0x6d, 0x97, 0x99, 0xb6, 0x36, 0x3b, 0xd6, 0x4b, + 0x5c, 0x4c, 0xb6, 0x24, 0x9d, 0x06, 0x01, 0x5f, 0xbb, 0xa0, 0x63, 0xcb, 0xb8, 0x03, 0x0d, 0x1e, + 0x6f, 0xf2, 0xdc, 0xbd, 0xae, 0xdf, 0x06, 0x9c, 0x16, 0x94, 0x0c, 0x01, 0xb1, 0x84, 0xc2, 0xe9, + 0x6a, 0xf7, 0x7f, 0x9c, 0x39, 0xa2, 0x96, 0xd2, 0xfd, 0x33, 0xc7, 0xd3, 0x6b, 0xa8, 0x72, 0x72, + 0x6e, 0xf7, 0x5f, 0x62, 0xc8, 0xd4, 0xf7, 0xfa, 0x91, 0x75, 0xa8, 0x5e, 0x46, 0x19, 0xe1, 0x29, + 0xb6, 0xf9, 0xc3, 0x2e, 0x63, 0xda, 0x0e, 0x4e, 0x1c, 0x85, 0x12, 0xc6, 0x34, 0x18, 0x6d, 0x5b, + 0x90, 0x18, 0x94, 0x09, 0x9d, 0xdf, 0x89, 0xa3, 0x67, 0xc6, 0xe4, 0xdf, 0x58, 0xbf, 0x11, 0x5d, + 0x5f, 0x69, 0x2e, 0x0c, 0x4d, 0xbe, 0x9c, 0x38, 0xf1, 0xb5, 0xd2, 0x0a, 0x9b, 0x92, 0x8b, 0x66, + 0x26, 0xc2, 0x02, 0xa9, 0x64, 0x37, 0xbd, 0xb5, 0x3e, 0x81, 0xf3, 0xa9, 0xfe, 0xea, 0x1d, 0x22, + 0x35, 0x40, 0x76, 0x76, 0xd9, 0xf9, 0xdf, 0x25, 0x80, 0xcd, 0x71, 0x74, 0x24, 0x41, 0x62, 0x17, + 0x6a, 0x0c, 0xdd, 0x6a, 0x50, 0x53, 0xb5, 0x19, 0xcf, 0xb7, 0xc2, 0x10, 0x11, 0x5a, 0x7c, 0x9a, + 0xab, 0x36, 0x07, 0x78, 0x63, 0xf5, 0xb0, 0xca, 0xbf, 0xd9, 0xc3, 0xac, 0x78, 0x8d, 0xee, 0x23, + 0xc0, 0x45, 0x68, 0x1f, 0xca, 0x1b, 0x8a, 0xa6, 0xa0, 0x6e, 0x0a, 0x22, 0x13, 0x73, 0x6c, 0x8a, + 0xaa, 0x45, 0x27, 0xfd, 0xc8, 0x7b, 0x4d, 0x5d, 0x79, 0x4e, 0x37, 0x63, 0xea, 0x53, 0x46, 0x64, + 0x62, 0x01, 0x3d, 0x44, 0x2b, 0x07, 0xb1, 0xd8, 0xac, 0x10, 0x8b, 0xa9, 0x5c, 0x8c, 0x3d, 0xe4, + 0xb7, 0xf7, 0xc7, 0xc3, 0xa1, 0x58, 0xe4, 0x59, 0x6d, 0x89, 0xc8, 0x4b, 0xac, 0x23, 0x7b, 0xdf, + 0x90, 0x98, 0x48, 0x2e, 0xee, 0xfb, 0x17, 0xfd, 0xe7, 0x61, 0x51, 0x53, 0x54, 0xd6, 0xab, 0x18, + 0x0b, 0xa2, 0x94, 0x7d, 0x3b, 0xfd, 0x8d, 0x0b, 0x70, 0x3e, 0xd5, 0x5f, 0x0c, 0xbb, 0x76, 0x19, + 0x6a, 0xf1, 0xa3, 0x3d, 0x99, 0x83, 0xca, 0xd3, 0xad, 0xfd, 0xf6, 0x39, 0xf6, 0xf1, 0x6c, 0x7b, + 0xbf, 0x5d, 0x5a, 0x5b, 0x83, 0x85, 0xcc, 0x2d, 0x25, 0xa9, 0x43, 0xd5, 0xdc, 0xd9, 0xdc, 0x7e, + 0x81, 0x62, 0xf3, 0x50, 0xdb, 0x7b, 0xf2, 0x54, 0xb4, 0x4a, 0x6b, 0x5b, 0xd0, 0x4a, 0x1f, 0x77, + 0xa4, 0x01, 0x73, 0x5b, 0xc8, 0x7d, 0xba, 0xb3, 0x8d, 0xc2, 0xd8, 0x30, 0x9f, 0xed, 0xed, 0xed, + 0xee, 0x3d, 0x68, 0x97, 0x08, 0xc0, 0xec, 0xce, 0xd7, 0xbb, 0x8c, 0x51, 0x66, 0x8c, 0x67, 0x7b, + 0x0f, 0xf7, 0x9e, 0xfc, 0x66, 0xaf, 0x5d, 0xd9, 0xf8, 0x67, 0x0d, 0x5a, 0xa6, 0x58, 0x42, 0x0f, + 0x63, 0xc1, 0x41, 0xe4, 0x79, 0x1f, 0xe6, 0xe2, 0x3f, 0x22, 0x92, 0x83, 0x35, 0xfd, 0xfb, 0x46, + 0xb7, 0x93, 0x67, 0x48, 0xb3, 0x9d, 0x23, 0xfb, 0xd0, 0x4c, 0x3d, 0xc7, 0x91, 0x04, 0xa9, 0x16, + 0xbd, 0xf8, 0x75, 0xaf, 0x4c, 0x62, 0xab, 0x11, 0x7b, 0xd0, 0x4a, 0x3f, 0xb1, 0x91, 0xa4, 0x4f, + 0xe1, 0xd3, 0x5d, 0xf7, 0xea, 0x44, 0xbe, 0x1a, 0xf4, 0x05, 0xb4, 0xb3, 0x8f, 0x6b, 0x24, 0xb9, + 0x18, 0x9b, 0xf0, 0x70, 0xd7, 0xbd, 0x76, 0x8a, 0x84, 0x3e, 0x74, 0xee, 0x19, 0x6a, 0x75, 0xf2, + 0x43, 0x42, 0x6e, 0xe8, 0x49, 0xaf, 0x13, 0xc2, 0x14, 0xe9, 0x9b, 0x61, 0xa2, 0x3f, 0xfe, 0x14, + 0xdc, 0xcc, 0x6b, 0xa6, 0x28, 0xbe, 0x52, 0xc6, 0x41, 0x9f, 0x63, 0xa5, 0x96, 0x06, 0xbd, 0x24, + 0xe9, 0x55, 0x0c, 0xe1, 0xbb, 0xab, 0x93, 0x05, 0xd2, 0x7e, 0xd3, 0x21, 0x6d, 0xca, 0x6f, 0x05, + 0x38, 0x39, 0xe5, 0xb7, 0x42, 0x2c, 0xcc, 0xc3, 0x2b, 0x05, 0x5c, 0xb5, 0xf0, 0x2a, 0x42, 0xc9, + 0xdd, 0x2b, 0x93, 0xd8, 0xfa, 0xf2, 0x33, 0xa0, 0x55, 0x5b, 0x7e, 0x31, 0x16, 0xee, 0xae, 0x4e, + 0x16, 0xc8, 0xfa, 0x2a, 0x29, 0xc6, 0x33, 0xbe, 0xca, 0x61, 0xbf, 0x8c, 0xaf, 0xf2, 0x55, 0xbc, + 0xf4, 0x55, 0xa6, 0xaa, 0xbe, 0x3a, 0xb1, 0x54, 0xc9, 0xfb, 0xaa, 0xb8, 0xfa, 0xc1, 0x71, 0xef, + 0xc2, 0x0c, 0xab, 0x30, 0x48, 0x72, 0x96, 0x6b, 0xc5, 0x4d, 0xf7, 0x42, 0x86, 0x1a, 0x77, 0xbb, + 0x5e, 0xba, 0x55, 0xda, 0xf8, 0x7b, 0x19, 0xe6, 0x45, 0xfa, 0x93, 0x19, 0xe4, 0x01, 0x40, 0x72, + 0x08, 0x93, 0x6e, 0x6a, 0x51, 0xa9, 0x4a, 0xa2, 0x7b, 0xa9, 0x90, 0xa7, 0x94, 0xfa, 0x4a, 0x96, + 0x2d, 0x72, 0xa1, 0x97, 0x32, 0xd9, 0x36, 0xb5, 0xc8, 0xcb, 0xc5, 0x4c, 0x35, 0xd6, 0x36, 0xd4, + 0x55, 0x92, 0x27, 0xda, 0xd9, 0x90, 0x39, 0xa1, 0xba, 0xdd, 0x22, 0x96, 0xae, 0x91, 0x96, 0xd5, + 0x35, 0x8d, 0xf2, 0x67, 0x85, 0xa6, 0x51, 0xc1, 0x41, 0x60, 0x9c, 0xfb, 0x7f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x22, 0x65, 0x58, 0x85, 0x2a, 0x27, 0x00, 0x00, } diff --git a/pkg/kubelet/api/v1alpha1/runtime/api.proto b/pkg/kubelet/api/v1alpha1/runtime/api.proto index cfd1bac7bfe..26ef4b79529 100644 --- a/pkg/kubelet/api/v1alpha1/runtime/api.proto +++ b/pkg/kubelet/api/v1alpha1/runtime/api.proto @@ -515,20 +515,24 @@ message Container { // The ID of the container, used by the container runtime to identify // a container. optional string id = 1; + // The id of the sandbox which this container belongs to. + optional string pod_sandbox_id = 2; // The metadata of the container. - optional ContainerMetadata metadata = 2; + optional ContainerMetadata metadata = 3; // The spec of the image - optional ImageSpec image = 3; + optional ImageSpec image = 4; // Reference to the image in use. For most runtimes, this should be an // image ID. - optional string image_ref = 4; + optional string image_ref = 5; // State is the state of the container. - optional ContainerState state = 5; + optional ContainerState state = 6; + // Creation time of the container. + optional int64 created_at = 7; // Labels are key value pairs that may be used to scope and select individual resources. - map labels = 6; + map labels = 8; // Annotations is an unstructured key value map that may be set by external // tools to store and retrieve arbitrary metadata. - map annotations = 7; + map annotations = 9; } message ListContainersResponse { diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 6a19c3c2b12..4f67912e8bf 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -619,6 +619,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub kubecontainer.FilterEventRecorder(kubeDeps.Recorder), klet.livenessManager, containerRefManager, + klet.podManager, kubeDeps.OSInterface, klet.networkPlugin, klet, diff --git a/pkg/kubelet/kuberuntime/doc.go b/pkg/kubelet/kuberuntime/doc.go index 14cf60f19ae..c1284ee28c0 100644 --- a/pkg/kubelet/kuberuntime/doc.go +++ b/pkg/kubelet/kuberuntime/doc.go @@ -15,5 +15,5 @@ limitations under the License. */ // Package kuberuntime contains an implementation of kubecontainer.Runtime using -// the interface in pkg/kubelet/api/services.go. +// the interface in pkg/kubelet/api. package kuberuntime diff --git a/pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go b/pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go index 3a3edcf6362..3e026d9ab08 100644 --- a/pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/fake_kuberuntime_manager.go @@ -30,6 +30,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/network" proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results" + "k8s.io/kubernetes/pkg/types" kubetypes "k8s.io/kubernetes/pkg/types" "k8s.io/kubernetes/pkg/util/flowcontrol" ) @@ -76,6 +77,19 @@ func (f *fakeRuntimeHelper) GetExtraSupplementalGroupsForPod(pod *api.Pod) []int return nil } +type fakePodGetter struct { + pods map[types.UID]*api.Pod +} + +func newFakePodGetter() *fakePodGetter { + return &fakePodGetter{make(map[types.UID]*api.Pod)} +} + +func (f *fakePodGetter) GetPodByUID(uid types.UID) (*api.Pod, bool) { + pod, found := f.pods[uid] + return pod, found +} + func NewFakeKubeRuntimeManager(runtimeService internalApi.RuntimeService, imageService internalApi.ImageManagerService, networkPlugin network.NetworkPlugin, osInterface kubecontainer.OSInterface) (*kubeGenericRuntimeManager, error) { recorder := &record.FakeRecorder{} kubeRuntimeManager := &kubeGenericRuntimeManager{ @@ -96,6 +110,7 @@ func NewFakeKubeRuntimeManager(runtimeService internalApi.RuntimeService, imageS return nil, err } + kubeRuntimeManager.containerGC = NewContainerGC(runtimeService, newFakePodGetter(), kubeRuntimeManager) kubeRuntimeManager.runtimeName = typedVersion.GetRuntimeName() kubeRuntimeManager.imagePuller = images.NewImageManager( kubecontainer.FilterEventRecorder(recorder), diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index 61df94a2fd6..48d4ec76885 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -239,6 +239,31 @@ func makeUID() string { return fmt.Sprintf("%08x", rand.Uint32()) } +// getTerminationMessage gets termination message of the container. +func getTerminationMessage(status *runtimeApi.ContainerStatus, kubeStatus *kubecontainer.ContainerStatus, terminationMessagePath string) string { + message := "" + + if !kubeStatus.FinishedAt.IsZero() || kubeStatus.ExitCode != 0 { + if terminationMessagePath == "" { + return "" + } + + for _, mount := range status.Mounts { + if mount.GetContainerPath() == terminationMessagePath { + path := mount.GetHostPath() + if data, err := ioutil.ReadFile(path); err != nil { + message = fmt.Sprintf("Error on reading termination-log %s: %v", path, err) + } else { + message = string(data) + } + break + } + } + } + + return message +} + // getKubeletContainerStatuses gets all containers' status for the pod sandbox. func (m *kubeGenericRuntimeManager) getKubeletContainerStatuses(podSandboxID string) ([]*kubecontainer.ContainerStatus, error) { containers, err := m.runtimeService.ListContainers(&runtimeApi.ContainerFilter{ @@ -282,23 +307,7 @@ func (m *kubeGenericRuntimeManager) getKubeletContainerStatuses(podSandboxID str cStatus.FinishedAt = time.Unix(status.GetFinishedAt(), 0) } - message := "" - if !cStatus.FinishedAt.IsZero() || cStatus.ExitCode != 0 { - if annotatedInfo.TerminationMessagePath != "" { - for _, mount := range status.Mounts { - if mount.GetContainerPath() == annotatedInfo.TerminationMessagePath { - path := mount.GetHostPath() - if data, err := ioutil.ReadFile(path); err != nil { - message = fmt.Sprintf("Error on reading termination-log %s: %v", path, err) - } else { - message = string(data) - } - break - } - } - } - } - cStatus.Message = message + cStatus.Message = getTerminationMessage(status, cStatus, annotatedInfo.TerminationMessagePath) statuses[i] = cStatus } diff --git a/pkg/kubelet/kuberuntime/kuberuntime_gc.go b/pkg/kubelet/kuberuntime/kuberuntime_gc.go new file mode 100644 index 00000000000..6ed5413433f --- /dev/null +++ b/pkg/kubelet/kuberuntime/kuberuntime_gc.go @@ -0,0 +1,280 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kuberuntime + +import ( + "sort" + "time" + + "github.com/golang/glog" + internalApi "k8s.io/kubernetes/pkg/kubelet/api" + runtimeApi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/types" +) + +// containerGC is the manager of garbage collection. +type containerGC struct { + client internalApi.RuntimeService + manager *kubeGenericRuntimeManager + podGetter podGetter +} + +// NewContainerGC creates a new containerGC. +func NewContainerGC(client internalApi.RuntimeService, podGetter podGetter, manager *kubeGenericRuntimeManager) *containerGC { + return &containerGC{ + client: client, + manager: manager, + podGetter: podGetter, + } +} + +// containerGCInfo is the internal information kept for containers being considered for GC. +type containerGCInfo struct { + // The ID of the container. + id string + // The name of the container. + name string + // The sandbox ID which this container belongs to + sandboxID string + // Creation time for the container. + createTime time.Time +} + +// evictUnit is considered for eviction as units of (UID, container name) pair. +type evictUnit struct { + // UID of the pod. + uid types.UID + // Name of the container in the pod. + name string +} + +type containersByEvictUnit map[evictUnit][]containerGCInfo + +// NumContainers returns the number of containers in this map. +func (cu containersByEvictUnit) NumContainers() int { + num := 0 + for key := range cu { + num += len(cu[key]) + } + return num +} + +// NumEvictUnits returns the number of pod in this map. +func (cu containersByEvictUnit) NumEvictUnits() int { + return len(cu) +} + +// Newest first. +type byCreated []containerGCInfo + +func (a byCreated) Len() int { return len(a) } +func (a byCreated) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byCreated) Less(i, j int) bool { return a[i].createTime.After(a[j].createTime) } + +// enforceMaxContainersPerEvictUnit enforces MaxPerPodContainer for each evictUnit. +func (cgc *containerGC) enforceMaxContainersPerEvictUnit(evictUnits containersByEvictUnit, MaxContainers int) { + for key := range evictUnits { + toRemove := len(evictUnits[key]) - MaxContainers + + if toRemove > 0 { + evictUnits[key] = cgc.removeOldestN(evictUnits[key], toRemove) + } + } +} + +// removeOldestN removes the oldest toRemove containers and returns the resulting slice. +func (cgc *containerGC) removeOldestN(containers []containerGCInfo, toRemove int) []containerGCInfo { + // Remove from oldest to newest (last to first). + numToKeep := len(containers) - toRemove + for i := numToKeep; i < len(containers); i++ { + cgc.removeContainer(containers[i].id, containers[i].name) + } + + // Assume we removed the containers so that we're not too aggressive. + return containers[:numToKeep] +} + +// removeContainer removes the container by containerID. +func (cgc *containerGC) removeContainer(containerID, containerName string) { + glog.V(4).Infof("Removing container %q name %q", containerID, containerName) + if err := cgc.client.RemoveContainer(containerID); err != nil { + glog.Warningf("Failed to remove container %q: %v", containerID, err) + } +} + +// removeSandbox removes the sandbox by sandboxID. +func (cgc *containerGC) removeSandbox(sandboxID string) { + glog.V(4).Infof("Removing sandbox %q", sandboxID) + if err := cgc.client.RemovePodSandbox(sandboxID); err != nil { + glog.Warningf("Failed to remove sandbox %q: %v", sandboxID, err) + } +} + +// evictableContainers gets all containers that are evictable. Evictable containers are: not running +// and created more than MinAge ago. +func (cgc *containerGC) evictableContainers(minAge time.Duration) (containersByEvictUnit, error) { + containers, err := cgc.manager.getKubeletContainers(true) + if err != nil { + return containersByEvictUnit{}, err + } + + evictUnits := make(containersByEvictUnit) + newestGCTime := time.Now().Add(-minAge) + for _, container := range containers { + // Prune out running containers. + if container.GetState() == runtimeApi.ContainerState_RUNNING { + continue + } + + createdAt := time.Unix(container.GetCreatedAt(), 0) + if newestGCTime.Before(createdAt) { + continue + } + + labeledInfo := getContainerInfoFromLabels(container.Labels) + containerInfo := containerGCInfo{ + id: container.GetId(), + name: container.Metadata.GetName(), + createTime: createdAt, + sandboxID: container.GetPodSandboxId(), + } + key := evictUnit{ + uid: labeledInfo.PodUID, + name: containerInfo.name, + } + evictUnits[key] = append(evictUnits[key], containerInfo) + } + + // Sort the containers by age. + for uid := range evictUnits { + sort.Sort(byCreated(evictUnits[uid])) + } + + return evictUnits, nil +} + +// evictableSandboxes gets all sandboxes that are evictable. Evictable sandboxes are: not running +// and contains no containers at all. +func (cgc *containerGC) evictableSandboxes() ([]string, error) { + containers, err := cgc.manager.getKubeletContainers(true) + if err != nil { + return nil, err + } + + sandboxes, err := cgc.manager.getKubeletSandboxes(true) + if err != nil { + return nil, err + } + + evictSandboxes := make([]string, 0) + for _, sandbox := range sandboxes { + // Prune out ready sandboxes. + if sandbox.GetState() == runtimeApi.PodSandBoxState_READY { + continue + } + + // Prune out sandboxes that still have containers. + found := false + sandboxID := sandbox.GetId() + for _, container := range containers { + if container.GetPodSandboxId() == sandboxID { + found = true + break + } + } + if found { + continue + } + + evictSandboxes = append(evictSandboxes, sandboxID) + } + + return evictSandboxes, nil +} + +// isPodDeleted returns true if the pod is already deleted. +func (cgc *containerGC) isPodDeleted(podUID types.UID) bool { + _, found := cgc.podGetter.GetPodByUID(podUID) + return !found +} + +// GarbageCollect removes dead containers using the specified container gc policy. +// Note that gc policy is not applied to sandboxes. Sandboxes are only removed when they are +// not ready and containing no containers. +// +// GarbageCollect consists of the following steps: +// * gets evictable containers which are not active and created more than gcPolicy.MinAge ago. +// * removes oldest dead containers for each pod by enforcing gcPolicy.MaxPerPodContainer. +// * removes oldest dead containers by enforcing gcPolicy.MaxContainers. +// * gets evictable sandboxes which are not ready and contains no containers. +// * removes evictable sandboxes. +func (cgc *containerGC) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool) error { + // Separate containers by evict units. + evictUnits, err := cgc.evictableContainers(gcPolicy.MinAge) + if err != nil { + return err + } + + // Remove deleted pod containers if all sources are ready. + if allSourcesReady { + for key, unit := range evictUnits { + if cgc.isPodDeleted(key.uid) { + cgc.removeOldestN(unit, len(unit)) // Remove all. + delete(evictUnits, key) + } + } + } + + // Enforce max containers per evict unit. + if gcPolicy.MaxPerPodContainer >= 0 { + cgc.enforceMaxContainersPerEvictUnit(evictUnits, gcPolicy.MaxPerPodContainer) + } + + // Enforce max total number of containers. + if gcPolicy.MaxContainers >= 0 && evictUnits.NumContainers() > gcPolicy.MaxContainers { + // Leave an equal number of containers per evict unit (min: 1). + numContainersPerEvictUnit := gcPolicy.MaxContainers / evictUnits.NumEvictUnits() + if numContainersPerEvictUnit < 1 { + numContainersPerEvictUnit = 1 + } + cgc.enforceMaxContainersPerEvictUnit(evictUnits, numContainersPerEvictUnit) + + // If we still need to evict, evict oldest first. + numContainers := evictUnits.NumContainers() + if numContainers > gcPolicy.MaxContainers { + flattened := make([]containerGCInfo, 0, numContainers) + for key := range evictUnits { + flattened = append(flattened, evictUnits[key]...) + } + sort.Sort(byCreated(flattened)) + + cgc.removeOldestN(flattened, numContainers-gcPolicy.MaxContainers) + } + } + + // Remove sandboxes with zero containers + evictSandboxes, err := cgc.evictableSandboxes() + if err != nil { + return err + } + for _, sandbox := range evictSandboxes { + cgc.removeSandbox(sandbox) + } + + return nil +} diff --git a/pkg/kubelet/kuberuntime/kuberuntime_gc_test.go b/pkg/kubelet/kuberuntime/kuberuntime_gc_test.go new file mode 100644 index 00000000000..6d5adc6c422 --- /dev/null +++ b/pkg/kubelet/kuberuntime/kuberuntime_gc_test.go @@ -0,0 +1,141 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kuberuntime + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "k8s.io/kubernetes/pkg/api" + apitest "k8s.io/kubernetes/pkg/kubelet/api/testing" + runtimeApi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime" + kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/types" +) + +type apiPodWithCreatedAt struct { + apiPod *api.Pod + createdAt int64 +} + +func makeAndSetFakeEvictablePod(m *kubeGenericRuntimeManager, fakeRuntime *apitest.FakeRuntimeService, pods []*apiPodWithCreatedAt) error { + sandboxes := make([]*apitest.FakePodSandbox, 0) + containers := make([]*apitest.FakeContainer, 0) + for _, pod := range pods { + fakePodSandbox, err := makeFakePodSandbox(m, pod.apiPod, pod.createdAt) + if err != nil { + return err + } + + fakeContainers, err := makeFakeContainers(m, pod.apiPod, pod.apiPod.Spec.Containers, pod.createdAt) + if err != nil { + return err + } + + // Set sandbox to not ready state + sandboxNotReady := runtimeApi.PodSandBoxState_NOTREADY + fakePodSandbox.State = &sandboxNotReady + sandboxes = append(sandboxes, fakePodSandbox) + + // Set containers to exited state + containerExited := runtimeApi.ContainerState_EXITED + for _, c := range fakeContainers { + c.State = &containerExited + containers = append(containers, c) + } + + } + + fakeRuntime.SetFakeSandboxes(sandboxes) + fakeRuntime.SetFakeContainers(containers) + return nil +} + +func makeTestContainer(name, image string) api.Container { + return api.Container{ + Name: name, + Image: image, + } +} + +func makeTestPod(podName, podNamespace, podUID string, containers []api.Container, createdAt int64) *apiPodWithCreatedAt { + return &apiPodWithCreatedAt{ + createdAt: createdAt, + apiPod: &api.Pod{ + ObjectMeta: api.ObjectMeta{ + UID: types.UID(podUID), + Name: podName, + Namespace: podNamespace, + }, + Spec: api.PodSpec{ + Containers: containers, + }, + }, + } +} + +func TestGarbageCollect(t *testing.T) { + fakeRuntime, _, m, err := createTestRuntimeManager() + assert.NoError(t, err) + + pods := []*apiPodWithCreatedAt{ + makeTestPod("123456", "foo1", "new", []api.Container{ + makeTestContainer("foo1", "busybox"), + makeTestContainer("foo2", "busybox"), + makeTestContainer("foo3", "busybox"), + }, 1), + makeTestPod("1234567", "foo2", "new", []api.Container{ + makeTestContainer("foo4", "busybox"), + makeTestContainer("foo5", "busybox"), + }, 2), + makeTestPod("12345678", "foo3", "new", []api.Container{ + makeTestContainer("foo6", "busybox"), + }, 3), + } + err = makeAndSetFakeEvictablePod(m, fakeRuntime, pods) + assert.NoError(t, err) + assert.NoError(t, m.GarbageCollect(kubecontainer.ContainerGCPolicy{ + MinAge: time.Second, + MaxPerPodContainer: 1, + MaxContainers: 3, + }, false)) + assert.Equal(t, 3, len(fakeRuntime.Containers)) + assert.Equal(t, 2, len(fakeRuntime.Sandboxes)) + + // no containers should be removed. + err = makeAndSetFakeEvictablePod(m, fakeRuntime, pods) + assert.NoError(t, err) + assert.NoError(t, m.GarbageCollect(kubecontainer.ContainerGCPolicy{ + MinAge: time.Second, + MaxPerPodContainer: 10, + MaxContainers: 100, + }, false)) + assert.Equal(t, 6, len(fakeRuntime.Containers)) + assert.Equal(t, 3, len(fakeRuntime.Sandboxes)) + + // all containers should be removed. + err = makeAndSetFakeEvictablePod(m, fakeRuntime, pods) + assert.NoError(t, err) + assert.NoError(t, m.GarbageCollect(kubecontainer.ContainerGCPolicy{ + MinAge: time.Second, + MaxPerPodContainer: 0, + MaxContainers: 0, + }, false)) + assert.Equal(t, 0, len(fakeRuntime.Containers)) + assert.Equal(t, 0, len(fakeRuntime.Sandboxes)) +} diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index 9cf25f44074..910d4e3998a 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -55,12 +55,20 @@ var ( ErrVersionNotSupported = errors.New("Runtime api version is not supported") ) +// A subset of the pod.Manager interface extracted for garbage collection purposes. +type podGetter interface { + GetPodByUID(kubetypes.UID) (*api.Pod, bool) +} + type kubeGenericRuntimeManager struct { runtimeName string recorder record.EventRecorder osInterface kubecontainer.OSInterface containerRefManager *kubecontainer.RefManager + // Container GC manager + containerGC *containerGC + // Keyring for pulling images keyring credentialprovider.DockerKeyring @@ -92,6 +100,7 @@ func NewKubeGenericRuntimeManager( recorder record.EventRecorder, livenessManager proberesults.Manager, containerRefManager *kubecontainer.RefManager, + podGetter podGetter, osInterface kubecontainer.OSInterface, networkPlugin network.NetworkPlugin, runtimeHelper kubecontainer.RuntimeHelper, @@ -151,6 +160,7 @@ func NewKubeGenericRuntimeManager( imageBackOff, serializeImagePulls) kubeRuntimeManager.runner = lifecycle.NewHandlerRunner(httpClient, kubeRuntimeManager, kubeRuntimeManager) + kubeRuntimeManager.containerGC = NewContainerGC(runtimeService, podGetter, kubeRuntimeManager) return kubeRuntimeManager, nil } @@ -475,6 +485,11 @@ func (m *kubeGenericRuntimeManager) GetNetNS(sandboxID kubecontainer.ContainerID return "", fmt.Errorf("not supported") } +// GarbageCollect removes dead containers using the specified container gc policy. +func (m *kubeGenericRuntimeManager) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool) error { + return m.containerGC.GarbageCollect(gcPolicy, allSourcesReady) +} + // GetPodContainerID gets pod sandbox ID func (m *kubeGenericRuntimeManager) GetPodContainerID(pod *kubecontainer.Pod) (kubecontainer.ContainerID, error) { podFullName := kubecontainer.BuildPodFullName(pod.Name, pod.Namespace) @@ -491,8 +506,3 @@ func (m *kubeGenericRuntimeManager) GetPodContainerID(pod *kubecontainer.Pod) (k func (m *kubeGenericRuntimeManager) PortForward(pod *kubecontainer.Pod, port uint16, stream io.ReadWriteCloser) error { return fmt.Errorf("not implemented") } - -// GarbageCollect removes dead containers using the specified container gc policy -func (m *kubeGenericRuntimeManager) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool) error { - return fmt.Errorf("not implemented") -} diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go index dc9a8fecbd3..bba07b073d7 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go @@ -54,12 +54,12 @@ func createTestRuntimeManager() (*apitest.FakeRuntimeService, *apitest.FakeImage } func makeAndSetFakePod(m *kubeGenericRuntimeManager, fakeRuntime *apitest.FakeRuntimeService, pod *api.Pod) (*apitest.FakePodSandbox, []*apitest.FakeContainer, error) { - fakePodSandbox, err := makeFakePodSandbox(m, pod) + fakePodSandbox, err := makeFakePodSandbox(m, pod, fakeCreatedAt) if err != nil { return nil, nil, err } - fakeContainers, err := makeFakeContainers(m, pod, pod.Spec.Containers) + fakeContainers, err := makeFakeContainers(m, pod, pod.Spec.Containers, fakeCreatedAt) if err != nil { return nil, nil, err } @@ -69,7 +69,7 @@ func makeAndSetFakePod(m *kubeGenericRuntimeManager, fakeRuntime *apitest.FakeRu return fakePodSandbox, fakeContainers, nil } -func makeFakePodSandbox(m *kubeGenericRuntimeManager, pod *api.Pod) (*apitest.FakePodSandbox, error) { +func makeFakePodSandbox(m *kubeGenericRuntimeManager, pod *api.Pod, createdAt int64) (*apitest.FakePodSandbox, error) { config, err := m.generatePodSandboxConfig(pod, "", 0) if err != nil { return nil, err @@ -82,13 +82,13 @@ func makeFakePodSandbox(m *kubeGenericRuntimeManager, pod *api.Pod) (*apitest.Fa Id: &podSandboxID, Metadata: config.Metadata, State: &readyState, - CreatedAt: &fakeCreatedAt, + CreatedAt: &createdAt, Labels: config.Labels, }, }, nil } -func makeFakeContainer(m *kubeGenericRuntimeManager, pod *api.Pod, container api.Container, sandboxConfig *runtimeApi.PodSandboxConfig) (*apitest.FakeContainer, error) { +func makeFakeContainer(m *kubeGenericRuntimeManager, pod *api.Pod, container api.Container, sandboxConfig *runtimeApi.PodSandboxConfig, createdAt int64) (*apitest.FakeContainer, error) { containerConfig, err := m.generateContainerConfig(&container, pod, 0, "") if err != nil { return nil, err @@ -104,7 +104,7 @@ func makeFakeContainer(m *kubeGenericRuntimeManager, pod *api.Pod, container api Metadata: containerConfig.Metadata, Image: containerConfig.Image, ImageRef: &imageRef, - CreatedAt: &fakeCreatedAt, + CreatedAt: &createdAt, State: &runningState, Labels: containerConfig.Labels, Annotations: containerConfig.Annotations, @@ -113,7 +113,7 @@ func makeFakeContainer(m *kubeGenericRuntimeManager, pod *api.Pod, container api }, nil } -func makeFakeContainers(m *kubeGenericRuntimeManager, pod *api.Pod, containers []api.Container) ([]*apitest.FakeContainer, error) { +func makeFakeContainers(m *kubeGenericRuntimeManager, pod *api.Pod, containers []api.Container, createdAt int64) ([]*apitest.FakeContainer, error) { sandboxConfig, err := m.generatePodSandboxConfig(pod, "", 0) if err != nil { return nil, err @@ -121,7 +121,7 @@ func makeFakeContainers(m *kubeGenericRuntimeManager, pod *api.Pod, containers [ result := make([]*apitest.FakeContainer, len(containers)) for idx, c := range containers { - containerWithState, err := makeFakeContainer(m, pod, c, sandboxConfig) + containerWithState, err := makeFakeContainer(m, pod, c, sandboxConfig, createdAt) if err != nil { return nil, err }