github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-08-30 14:25:43 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1152 from Pennyzct/memory_hotplug

Browse Source 
runtime: support memory hotplug via probe interface on aarch64
This commit is contained in:
Julio Montes 2019-04-08 08:43:03 -05:00 committed by GitHub
commit 34e2064b39
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 580 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Gopkg.lock generated
View File

@ -388,7 +388,7 @@
revision = "78d079db6d1f3e32e3ed7578a54baa6257b058a7"
[[projects]]
digest = "1:590bfb6f8d5741fa38bb3022f55588fb7e06389f6b7b6692a965e0e5a4363fb1"
digest = "1:2f9680b194232a3f8744fbfb2c353360c497ed59840f7cbee34e903b09cb00b6"
name = "github.com/kata-containers/agent"
packages = [
"pkg/types",
@ -396,7 +396,7 @@
"protocols/grpc",
]
pruneopts = "NUT"
revision = "8e48125fa2a793706e5e660e95bdc9755f1cf474"
revision = "17192be88b2725b0e7f482a8b7616935e1b1ddf4"
[[projects]]
digest = "1:04054595e5c5a35d1553a7f3464d18577caf597445d643992998643df56d4afd"

2
Gopkg.toml
View File

@ -56,7 +56,7 @@
[[constraint]]
name = "github.com/kata-containers/agent"
revision = "8e48125fa2a793706e5e660e95bdc9755f1cf474"
revision = "17192be88b2725b0e7f482a8b7616935e1b1ddf4"
[[constraint]]
name = "github.com/containerd/cri-containerd"

645
vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go generated vendored
View File

@ -55,6 +55,7 @@
AgentDetails
GuestDetailsRequest
GuestDetailsResponse
MemHotplugByProbeRequest
SetGuestDateTimeRequest
Storage
Device
@ -1353,6 +1354,10 @@ type GuestDetailsRequest struct {
// for memory hotplug alignment. Typically the server returns what's in
// /sys/devices/system/memory/block_size_bytes.
MemBlockSize bool `protobuf:"varint,1,opt,name=mem_block_size,json=memBlockSize,proto3" json:"mem_block_size,omitempty"`
// MemoryHotplugProbe asks server to return whether guest kernel supports memory hotplug
// via probeinterface. Typically the server will check if the path
// /sys/devices/system/memory/probe exists.
MemHotplugProbe bool `protobuf:"varint,2,opt,name=mem_hotplug_probe,json=memHotplugProbe,proto3" json:"mem_hotplug_probe,omitempty"`
}
func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} }
@ -1367,10 +1372,18 @@ func (m *GuestDetailsRequest) GetMemBlockSize() bool {
return false
}
func (m *GuestDetailsRequest) GetMemHotplugProbe() bool {
if m != nil {
return m.MemHotplugProbe
}
return false
}
type GuestDetailsResponse struct {
// MemBlockSizeBytes returns the system memory block size in bytes.
MemBlockSizeBytes uint64 `protobuf:"varint,1,opt,name=mem_block_size_bytes,json=memBlockSizeBytes,proto3" json:"mem_block_size_bytes,omitempty"`
AgentDetails *AgentDetails `protobuf:"bytes,2,opt,name=agent_details,json=agentDetails" json:"agent_details,omitempty"`
MemBlockSizeBytes uint64 `protobuf:"varint,1,opt,name=mem_block_size_bytes,json=memBlockSizeBytes,proto3" json:"mem_block_size_bytes,omitempty"`
AgentDetails *AgentDetails `protobuf:"bytes,2,opt,name=agent_details,json=agentDetails" json:"agent_details,omitempty"`
SupportMemHotplugProbe bool `protobuf:"varint,3,opt,name=support_mem_hotplug_probe,json=supportMemHotplugProbe,proto3" json:"support_mem_hotplug_probe,omitempty"`
}
func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} }
@ -1392,6 +1405,31 @@ func (m *GuestDetailsResponse) GetAgentDetails() *AgentDetails {
return nil
}
func (m *GuestDetailsResponse) GetSupportMemHotplugProbe() bool {
if m != nil {
return m.SupportMemHotplugProbe
}
return false
}
type MemHotplugByProbeRequest struct {
// server needs to send the value of memHotplugProbeAddr into file /sys/devices/system/memory/probe,
// in order to notify the guest kernel about hot-add memory event
MemHotplugProbeAddr []uint64 `protobuf:"varint,1,rep,packed,name=memHotplugProbeAddr" json:"memHotplugProbeAddr,omitempty"`
}
func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} }
func (m *MemHotplugByProbeRequest) String() string { return proto.CompactTextString(m) }
func (*MemHotplugByProbeRequest) ProtoMessage() {}
func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} }
func (m *MemHotplugByProbeRequest) GetMemHotplugProbeAddr() []uint64 {
if m != nil {
return m.MemHotplugProbeAddr
}
return nil
}
type SetGuestDateTimeRequest struct {
// Sec the second since the Epoch.
Sec int64 `protobuf:"varint,1,opt,name=Sec,proto3" json:"Sec,omitempty"`
@ -1402,7 +1440,7 @@ type SetGuestDateTimeRequest struct {
func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} }
func (m *SetGuestDateTimeRequest) String() string { return proto.CompactTextString(m) }
func (*SetGuestDateTimeRequest) ProtoMessage() {}
func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} }
func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} }
func (m *SetGuestDateTimeRequest) GetSec() int64 {
if m != nil {
@ -1451,7 +1489,7 @@ type Storage struct {
func (m *Storage) Reset() { *m = Storage{} }
func (m *Storage) String() string { return proto.CompactTextString(m) }
func (*Storage) ProtoMessage() {}
func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} }
func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} }
func (m *Storage) GetDriver() string {
if m != nil {
@ -1534,7 +1572,7 @@ type Device struct {
func (m *Device) Reset() { *m = Device{} }
func (m *Device) String() string { return proto.CompactTextString(m) }
func (*Device) ProtoMessage() {}
func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} }
func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} }
func (m *Device) GetId() string {
if m != nil {
@ -1580,7 +1618,7 @@ type StringUser struct {
func (m *StringUser) Reset() { *m = StringUser{} }
func (m *StringUser) String() string { return proto.CompactTextString(m) }
func (*StringUser) ProtoMessage() {}
func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} }
func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} }
func (m *StringUser) GetUid() string {
if m != nil {
@ -1628,7 +1666,7 @@ type CopyFileRequest struct {
func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} }
func (m *CopyFileRequest) String() string { return proto.CompactTextString(m) }
func (*CopyFileRequest) ProtoMessage() {}
func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} }
func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} }
func (m *CopyFileRequest) GetPath() string {
if m != nil {
@ -1732,6 +1770,7 @@ func init() {
proto.RegisterType((*AgentDetails)(nil), "grpc.AgentDetails")
proto.RegisterType((*GuestDetailsRequest)(nil), "grpc.GuestDetailsRequest")
proto.RegisterType((*GuestDetailsResponse)(nil), "grpc.GuestDetailsResponse")
proto.RegisterType((*MemHotplugByProbeRequest)(nil), "grpc.MemHotplugByProbeRequest")
proto.RegisterType((*SetGuestDateTimeRequest)(nil), "grpc.SetGuestDateTimeRequest")
proto.RegisterType((*Storage)(nil), "grpc.Storage")
proto.RegisterType((*Device)(nil), "grpc.Device")
@ -1787,6 +1826,7 @@ type AgentServiceClient interface {
OnlineCPUMem(ctx context.Context, in *OnlineCPUMemRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
ReseedRandomDev(ctx context.Context, in *ReseedRandomDevRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
GetGuestDetails(ctx context.Context, in *GuestDetailsRequest, opts ...grpc1.CallOption) (*GuestDetailsResponse, error)
MemHotplugByProbe(ctx context.Context, in *MemHotplugByProbeRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
SetGuestDateTime(ctx context.Context, in *SetGuestDateTimeRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
CopyFile(ctx context.Context, in *CopyFileRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error)
}
@ -2042,6 +2082,15 @@ func (c *agentServiceClient) GetGuestDetails(ctx context.Context, in *GuestDetai
return out, nil
}
func (c *agentServiceClient) MemHotplugByProbe(ctx context.Context, in *MemHotplugByProbeRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) {
out := new(google_protobuf2.Empty)
err := grpc1.Invoke(ctx, "/grpc.AgentService/MemHotplugByProbe", in, out, c.cc, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *agentServiceClient) SetGuestDateTime(ctx context.Context, in *SetGuestDateTimeRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) {
out := new(google_protobuf2.Empty)
err := grpc1.Invoke(ctx, "/grpc.AgentService/SetGuestDateTime", in, out, c.cc, opts...)
@ -2100,6 +2149,7 @@ type AgentServiceServer interface {
OnlineCPUMem(context.Context, *OnlineCPUMemRequest) (*google_protobuf2.Empty, error)
ReseedRandomDev(context.Context, *ReseedRandomDevRequest) (*google_protobuf2.Empty, error)
GetGuestDetails(context.Context, *GuestDetailsRequest) (*GuestDetailsResponse, error)
MemHotplugByProbe(context.Context, *MemHotplugByProbeRequest) (*google_protobuf2.Empty, error)
SetGuestDateTime(context.Context, *SetGuestDateTimeRequest) (*google_protobuf2.Empty, error)
CopyFile(context.Context, *CopyFileRequest) (*google_protobuf2.Empty, error)
}
@ -2594,6 +2644,24 @@ func _AgentService_GetGuestDetails_Handler(srv interface{}, ctx context.Context,
return interceptor(ctx, in, info, handler)
}
func _AgentService_MemHotplugByProbe_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) {
in := new(MemHotplugByProbeRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AgentServiceServer).MemHotplugByProbe(ctx, in)
}
info := &grpc1.UnaryServerInfo{
Server: srv,
FullMethod: "/grpc.AgentService/MemHotplugByProbe",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AgentServiceServer).MemHotplugByProbe(ctx, req.(*MemHotplugByProbeRequest))
}
return interceptor(ctx, in, info, handler)
}
func _AgentService_SetGuestDateTime_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) {
in := new(SetGuestDateTimeRequest)
if err := dec(in); err != nil {
@ -2742,6 +2810,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{
MethodName: "GetGuestDetails",
Handler: _AgentService_GetGuestDetails_Handler,
},
{
MethodName: "MemHotplugByProbe",
Handler: _AgentService_MemHotplugByProbe_Handler,
},
{
MethodName: "SetGuestDateTime",
Handler: _AgentService_SetGuestDateTime_Handler,
@ -4424,6 +4496,16 @@ func (m *GuestDetailsRequest) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.MemHotplugProbe {
dAtA[i] = 0x10
i++
if m.MemHotplugProbe {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
return i, nil
}
@ -4457,6 +4539,51 @@ func (m *GuestDetailsResponse) MarshalTo(dAtA []byte) (int, error) {
}
i += n23
}
if m.SupportMemHotplugProbe {
dAtA[i] = 0x18
i++
if m.SupportMemHotplugProbe {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
return i, nil
}
func (m *MemHotplugByProbeRequest) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *MemHotplugByProbeRequest) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.MemHotplugProbeAddr) > 0 {
dAtA25 := make([]byte, len(m.MemHotplugProbeAddr)*10)
var j24 int
for _, num := range m.MemHotplugProbeAddr {
for num >= 1<<7 {
dAtA25[j24] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
j24++
}
dAtA25[j24] = uint8(num)
j24++
}
dAtA[i] = 0xa
i++
i = encodeVarintAgent(dAtA, i, uint64(j24))
i += copy(dAtA[i:], dAtA25[:j24])
}
return i, nil
}
@ -5435,6 +5562,9 @@ func (m *GuestDetailsRequest) Size() (n int) {
if m.MemBlockSize {
n += 2
}
if m.MemHotplugProbe {
n += 2
}
return n
}
@ -5448,6 +5578,22 @@ func (m *GuestDetailsResponse) Size() (n int) {
l = m.AgentDetails.Size()
n += 1 + l + sovAgent(uint64(l))
}
if m.SupportMemHotplugProbe {
n += 2
}
return n
}
func (m *MemHotplugByProbeRequest) Size() (n int) {
var l int
_ = l
if len(m.MemHotplugProbeAddr) > 0 {
l = 0
for _, e := range m.MemHotplugProbeAddr {
l += sovAgent(uint64(e))
}
n += 1 + sovAgent(uint64(l)) + l
}
return n
}
@ -10845,6 +10991,26 @@ func (m *GuestDetailsRequest) Unmarshal(dAtA []byte) error {
}
}
m.MemBlockSize = bool(v != 0)
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field MemHotplugProbe", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowAgent
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.MemHotplugProbe = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipAgent(dAtA[iNdEx:])
@ -10947,6 +11113,138 @@ func (m *GuestDetailsResponse) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field SupportMemHotplugProbe", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowAgent
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.SupportMemHotplugProbe = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipAgent(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthAgent
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *MemHotplugByProbeRequest) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowAgent
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: MemHotplugByProbeRequest: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: MemHotplugByProbeRequest: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType == 0 {
var v uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowAgent
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.MemHotplugProbeAddr = append(m.MemHotplugProbeAddr, v)
} else if wireType == 2 {
var packedLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowAgent
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
packedLen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if packedLen < 0 {
return ErrInvalidLengthAgent
}
postIndex := iNdEx + packedLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
for iNdEx < postIndex {
var v uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowAgent
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.MemHotplugProbeAddr = append(m.MemHotplugProbeAddr, v)
}
} else {
return fmt.Errorf("proto: wrong wireType = %d for field MemHotplugProbeAddr", wireType)
}
default:
iNdEx = preIndex
skippy, err := skipAgent(dAtA[iNdEx:])
@ -11944,168 +12242,173 @@ var (
func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) }
var fileDescriptorAgent = []byte{
// 2596 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0x5f, 0x6f, 0x24, 0x47,
0x11, 0xd7, 0xfe, 0xf1, 0x7a, 0xb7, 0x76, 0xd7, 0xeb, 0x6d, 0xfb, 0x7c, 0x9b, 0xbd, 0x24, 0x5c,
0x26, 0xe1, 0xe2, 0x10, 0xb2, 0x26, 0x97, 0x88, 0x84, 0x9c, 0xc2, 0xe9, 0xce, 0x67, 0xce, 0x26,
0x39, 0xce, 0x8c, 0x63, 0x05, 0x09, 0xa1, 0xd1, 0x78, 0xa6, 0xbd, 0xee, 0x78, 0x67, 0x7a, 0xd2,
0xdd, 0xe3, 0xbb, 0x0d, 0x12, 0xe2, 0x89, 0x6f, 0xc1, 0x17, 0x40, 0xbc, 0xf1, 0x0d, 0x10, 0x0f,
0x11, 0x4f, 0xbc, 0x23, 0x21, 0x94, 0x8f, 0xc0, 0x27, 0x40, 0xfd, 0x6f, 0xfe, 0xec, 0x1f, 0x5f,
0x70, 0x2c, 0xf1, 0x32, 0x9a, 0xaa, 0xae, 0xfe, 0x75, 0x55, 0x75, 0x77, 0x75, 0x55, 0x41, 0xdb,
0x1f, 0xe3, 0x58, 0x8c, 0x12, 0x46, 0x05, 0x45, 0xf5, 0x31, 0x4b, 0x82, 0x61, 0x8b, 0x06, 0x44,
0x33, 0x86, 0x3f, 0x1e, 0x13, 0x71, 0x96, 0x9e, 0x8c, 0x02, 0x1a, 0xed, 0x9c, 0xfb, 0xc2, 0x7f,
0x27, 0xa0, 0xb1, 0xf0, 0x49, 0x8c, 0x19, 0xdf, 0x51, 0x13, 0x77, 0x92, 0xf3, 0xf1, 0x8e, 0x98,
0x26, 0x98, 0xeb, 0xaf, 0x99, 0x77, 0x6b, 0x4c, 0xe9, 0x78, 0x82, 0x77, 0x14, 0x75, 0x92, 0x9e,
0xee, 0xe0, 0x28, 0x11, 0x53, 0x3d, 0xe8, 0xfc, 0xb1, 0x0a, 0x5b, 0xbb, 0x0c, 0xfb, 0x02, 0xef,
0x5a, 0x34, 0x17, 0x7f, 0x99, 0x62, 0x2e, 0xd0, 0x6b, 0xd0, 0xc9, 0x56, 0xf0, 0x48, 0x38, 0xa8,
0xdc, 0xae, 0x6c, 0xb7, 0xdc, 0x76, 0xc6, 0x3b, 0x08, 0xd1, 0x4d, 0x58, 0xc5, 0xcf, 0x71, 0x20,
0x47, 0xab, 0x6a, 0xb4, 0x21, 0xc9, 0x83, 0x10, 0xbd, 0x0b, 0x6d, 0x2e, 0x18, 0x89, 0xc7, 0x5e,
0xca, 0x31, 0x1b, 0xd4, 0x6e, 0x57, 0xb6, 0xdb, 0x77, 0xd7, 0x47, 0xd2, 0xa4, 0xd1, 0x91, 0x1a,
0x38, 0xe6, 0x98, 0xb9, 0xc0, 0xb3, 0x7f, 0x74, 0x07, 0x56, 0x43, 0x7c, 0x41, 0x02, 0xcc, 0x07,
0xf5, 0xdb, 0xb5, 0xed, 0xf6, 0xdd, 0x8e, 0x16, 0x7f, 0xa4, 0x98, 0xae, 0x1d, 0x44, 0x6f, 0x41,
0x93, 0x0b, 0xca, 0xfc, 0x31, 0xe6, 0x83, 0x15, 0x25, 0xd8, 0xb5, 0xb8, 0x8a, 0xeb, 0x66, 0xc3,
0xe8, 0x65, 0xa8, 0x3d, 0xdd, 0x3d, 0x18, 0x34, 0xd4, 0xea, 0x60, 0xa4, 0x12, 0x1c, 0xb8, 0x92,
0x8d, 0x5e, 0x87, 0x2e, 0xf7, 0xe3, 0xf0, 0x84, 0x3e, 0xf7, 0x12, 0x12, 0xc6, 0x7c, 0xb0, 0x7a,
0xbb, 0xb2, 0xdd, 0x74, 0x3b, 0x86, 0x79, 0x28, 0x79, 0xce, 0x47, 0x70, 0xe3, 0x48, 0xf8, 0x4c,
0x5c, 0xc1, 0x3b, 0xce, 0x31, 0x6c, 0xb9, 0x38, 0xa2, 0x17, 0x57, 0x72, 0xed, 0x00, 0x56, 0x05,
0x89, 0x30, 0x4d, 0x85, 0x72, 0x6d, 0xd7, 0xb5, 0xa4, 0xf3, 0xe7, 0x0a, 0xa0, 0xbd, 0xe7, 0x38,
0x38, 0x64, 0x34, 0xc0, 0x9c, 0xff, 0x9f, 0xb6, 0xeb, 0x4d, 0x58, 0x4d, 0xb4, 0x02, 0x83, 0xba,
0x12, 0x37, 0xbb, 0x60, 0xb5, 0xb2, 0xa3, 0xce, 0x17, 0xb0, 0x79, 0x44, 0xc6, 0xb1, 0x3f, 0xb9,
0x46, 0x7d, 0xb7, 0xa0, 0xc1, 0x15, 0xa6, 0x52, 0xb5, 0xeb, 0x1a, 0xca, 0x39, 0x04, 0xf4, 0xb9,
0x4f, 0xc4, 0xf5, 0xad, 0xe4, 0xbc, 0x03, 0x1b, 0x25, 0x44, 0x9e, 0xd0, 0x98, 0x63, 0xa5, 0x80,
0xf0, 0x45, 0xca, 0x15, 0xd8, 0x8a, 0x6b, 0x28, 0x07, 0xc3, 0xe6, 0xa7, 0x84, 0x5b, 0x71, 0xfc,
0xbf, 0xa8, 0xb0, 0x05, 0x8d, 0x53, 0xca, 0x22, 0x5f, 0x58, 0x0d, 0x34, 0x85, 0x10, 0xd4, 0x7d,
0x36, 0xe6, 0x83, 0xda, 0xed, 0xda, 0x76, 0xcb, 0x55, 0xff, 0xf2, 0x54, 0xce, 0x2c, 0x63, 0xf4,
0x7a, 0x0d, 0x3a, 0xc6, 0xef, 0xde, 0x84, 0x70, 0xa1, 0xd6, 0xe9, 0xb8, 0x6d, 0xc3, 0x93, 0x73,
0x1c, 0x0a, 0x5b, 0xc7, 0x49, 0x78, 0xc5, 0x0b, 0x7f, 0x17, 0x5a, 0x0c, 0x73, 0x9a, 0x32, 0x79,
0x4d, 0xab, 0x6a, 0xdf, 0x37, 0xf5, 0xbe, 0x7f, 0x4a, 0xe2, 0xf4, 0xb9, 0x6b, 0xc7, 0xdc, 0x5c,
0xcc, 0x5c, 0x21, 0xc1, 0xaf, 0x72, 0x85, 0x3e, 0x82, 0x1b, 0x87, 0x7e, 0xca, 0xaf, 0xa2, 0xab,
0x73, 0x4f, 0x5e, 0x3f, 0x9e, 0x46, 0x57, 0x9a, 0xfc, 0xa7, 0x0a, 0x34, 0x77, 0x93, 0xf4, 0x98,
0xfb, 0x63, 0x8c, 0xbe, 0x07, 0x6d, 0x41, 0x85, 0x3f, 0xf1, 0x52, 0x49, 0x2a, 0xf1, 0xba, 0x0b,
0x8a, 0xa5, 0x05, 0xa4, 0xdb, 0x31, 0x0b, 0x92, 0xd4, 0x48, 0x54, 0x6f, 0xd7, 0xb6, 0xeb, 0x6e,
0x5b, 0xf3, 0xb4, 0xc8, 0x08, 0x36, 0xd4, 0x98, 0x47, 0x62, 0xef, 0x1c, 0xb3, 0x18, 0x4f, 0x22,
0x1a, 0x62, 0x75, 0x7e, 0xeb, 0x6e, 0x5f, 0x0d, 0x1d, 0xc4, 0x9f, 0x64, 0x03, 0xe8, 0x07, 0xd0,
0xcf, 0xe4, 0xe5, 0xa5, 0x54, 0xd2, 0x75, 0x25, 0xdd, 0x33, 0xd2, 0xc7, 0x86, 0xed, 0xfc, 0x0e,
0xd6, 0x3e, 0x3b, 0x63, 0x54, 0x88, 0x09, 0x89, 0xc7, 0x8f, 0x7c, 0xe1, 0xcb, 0xe8, 0x91, 0x60,
0x46, 0x68, 0xc8, 0x8d, 0xb6, 0x96, 0x44, 0x6f, 0x43, 0x5f, 0x68, 0x59, 0x1c, 0x7a, 0x56, 0xa6,
0xaa, 0x64, 0xd6, 0xb3, 0x81, 0x43, 0x23, 0xfc, 0x7d, 0x58, 0xcb, 0x85, 0x65, 0xfc, 0x31, 0xfa,
0x76, 0x33, 0xee, 0x67, 0x24, 0xc2, 0xce, 0x85, 0xf2, 0x95, 0xda, 0x64, 0xf4, 0x36, 0xb4, 0x72,
0x3f, 0x54, 0xd4, 0x09, 0x59, 0xd3, 0x27, 0xc4, 0xba, 0xd3, 0x6d, 0x66, 0x4e, 0xf9, 0x18, 0x7a,
0x22, 0x53, 0xdc, 0x0b, 0x7d, 0xe1, 0x97, 0x0f, 0x55, 0xd9, 0x2a, 0x77, 0x4d, 0x94, 0x68, 0xe7,
0x1e, 0xb4, 0x0e, 0x49, 0xc8, 0xf5, 0xc2, 0x03, 0x58, 0x0d, 0x52, 0xc6, 0x70, 0x2c, 0xac, 0xc9,
0x86, 0x44, 0x9b, 0xb0, 0x32, 0x21, 0x11, 0x11, 0xc6, 0x4c, 0x4d, 0x38, 0x14, 0xe0, 0x09, 0x8e,
0x28, 0x9b, 0x2a, 0x87, 0x6d, 0xc2, 0x4a, 0x71, 0x73, 0x35, 0x81, 0x6e, 0x41, 0x2b, 0xf2, 0x9f,
0x67, 0x9b, 0x2a, 0x47, 0x9a, 0x91, 0xff, 0x5c, 0x2b, 0x3f, 0x80, 0xd5, 0x53, 0x9f, 0x4c, 0x82,
0x58, 0x18, 0xaf, 0x58, 0x32, 0x5f, 0xb0, 0x5e, 0x5c, 0xf0, 0x6f, 0x55, 0x68, 0xeb, 0x15, 0xb5,
0xc2, 0x9b, 0xb0, 0x12, 0xf8, 0xc1, 0x59, 0xb6, 0xa4, 0x22, 0xd0, 0x1d, 0xab, 0x48, 0xb5, 0x18,
0x84, 0x73, 0x4d, 0xad, 0x6a, 0x3b, 0x00, 0xfc, 0x99, 0x9f, 0x18, 0xdd, 0x6a, 0x4b, 0x84, 0x5b,
0x52, 0x46, 0xab, 0xfb, 0x1e, 0x74, 0xf4, 0xb9, 0x33, 0x53, 0xea, 0x4b, 0xa6, 0xb4, 0xb5, 0x94,
0x9e, 0xf4, 0x3a, 0x74, 0x53, 0x8e, 0xbd, 0x33, 0x82, 0x99, 0xcf, 0x82, 0xb3, 0xe9, 0x60, 0x45,
0xbf, 0x91, 0x29, 0xc7, 0xfb, 0x96, 0x87, 0xee, 0xc2, 0x8a, 0x0c, 0x7f, 0x7c, 0xd0, 0x50, 0xcf,
0xf1, 0xcb, 0x45, 0x48, 0x65, 0xea, 0x48, 0x7d, 0xf7, 0x62, 0xc1, 0xa6, 0xae, 0x16, 0x1d, 0x7e,
0x08, 0x90, 0x33, 0xd1, 0x3a, 0xd4, 0xce, 0xf1, 0xd4, 0xdc, 0x43, 0xf9, 0x2b, 0x9d, 0x73, 0xe1,
0x4f, 0x52, 0xeb, 0x75, 0x4d, 0x7c, 0x54, 0xfd, 0xb0, 0xe2, 0x04, 0xd0, 0x7b, 0x38, 0x39, 0x27,
0xb4, 0x30, 0x7d, 0x13, 0x56, 0x22, 0xff, 0x0b, 0xca, 0xac, 0x27, 0x15, 0xa1, 0xb8, 0x24, 0xa6,
0xcc, 0x42, 0x28, 0x02, 0xad, 0x41, 0x95, 0x26, 0xca, 0x5f, 0x2d, 0xb7, 0x4a, 0x93, 0x7c, 0xa1,
0x7a, 0x61, 0x21, 0xe7, 0x5f, 0x75, 0x80, 0x7c, 0x15, 0xe4, 0xc2, 0x90, 0x50, 0x8f, 0x63, 0x26,
0x53, 0x10, 0xef, 0x64, 0x2a, 0x30, 0xf7, 0x18, 0x0e, 0x52, 0xc6, 0xc9, 0x85, 0xdc, 0x3f, 0x69,
0xf6, 0x0d, 0x6d, 0xf6, 0x8c, 0x6e, 0xee, 0x4d, 0x42, 0x8f, 0xf4, 0xbc, 0x87, 0x72, 0x9a, 0x6b,
0x67, 0xa1, 0x03, 0xb8, 0x91, 0x63, 0x86, 0x05, 0xb8, 0xea, 0x65, 0x70, 0x1b, 0x19, 0x5c, 0x98,
0x43, 0xed, 0xc1, 0x06, 0xa1, 0xde, 0x97, 0x29, 0x4e, 0x4b, 0x40, 0xb5, 0xcb, 0x80, 0xfa, 0x84,
0xfe, 0x52, 0x4d, 0xc8, 0x61, 0x0e, 0xe1, 0xa5, 0x82, 0x95, 0xf2, 0xba, 0x17, 0xc0, 0xea, 0x97,
0x81, 0x6d, 0x65, 0x5a, 0xc9, 0x78, 0x90, 0x23, 0xfe, 0x1c, 0xb6, 0x08, 0xf5, 0x9e, 0xf9, 0x44,
0xcc, 0xc2, 0xad, 0xbc, 0xc0, 0x48, 0xf9, 0xe8, 0x96, 0xb1, 0xb4, 0x91, 0x11, 0x66, 0xe3, 0x92,
0x91, 0x8d, 0x17, 0x18, 0xf9, 0x44, 0x4d, 0xc8, 0x61, 0x1e, 0x40, 0x9f, 0xd0, 0x59, 0x6d, 0x56,
0x2f, 0x03, 0xe9, 0x11, 0x5a, 0xd6, 0xe4, 0x21, 0xf4, 0x39, 0x0e, 0x04, 0x65, 0xc5, 0x43, 0xd0,
0xbc, 0x0c, 0x62, 0xdd, 0xc8, 0x67, 0x18, 0xce, 0xaf, 0xa1, 0xb3, 0x9f, 0x8e, 0xb1, 0x98, 0x9c,
0x64, 0xc1, 0xe0, 0xda, 0xe2, 0x8f, 0xf3, 0x9f, 0x2a, 0xb4, 0x77, 0xc7, 0x8c, 0xa6, 0x49, 0x29,
0x26, 0xeb, 0x4b, 0x3a, 0x1b, 0x93, 0x95, 0x88, 0x8a, 0xc9, 0x5a, 0xf8, 0x7d, 0xe8, 0x44, 0xea,
0xea, 0x1a, 0x79, 0x1d, 0x87, 0xfa, 0x73, 0x97, 0xda, 0x6d, 0x47, 0x85, 0x60, 0x36, 0x02, 0x48,
0x48, 0xc8, 0xcd, 0x1c, 0x1d, 0x8e, 0x7a, 0x26, 0x23, 0xb4, 0x21, 0xda, 0x6d, 0x25, 0x59, 0xb4,
0x7e, 0x17, 0xda, 0x27, 0xd2, 0x49, 0x66, 0x42, 0x29, 0x18, 0xe5, 0xde, 0x73, 0xe1, 0x24, 0xbf,
0x84, 0xfb, 0xd0, 0x3d, 0xd3, 0x2e, 0x33, 0x93, 0xf4, 0x19, 0x7a, 0xdd, 0x58, 0x92, 0xdb, 0x3b,
0x2a, 0x7a, 0x56, 0x6f, 0x40, 0xe7, 0xac, 0xc0, 0x1a, 0x1e, 0x41, 0x7f, 0x4e, 0x64, 0x41, 0x0c,
0xda, 0x2e, 0xc6, 0xa0, 0xf6, 0x5d, 0xa4, 0x17, 0x2a, 0xce, 0x2c, 0xc6, 0xa5, 0x5f, 0xc0, 0xd6,
0x6c, 0x9a, 0x63, 0x92, 0xb2, 0xf7, 0xa1, 0x13, 0x28, 0xed, 0x4a, 0x3b, 0xd0, 0x9f, 0xd3, 0xdb,
0x6d, 0x07, 0x39, 0xe1, 0x84, 0x80, 0x3e, 0x67, 0x44, 0xe0, 0x23, 0xc1, 0xb0, 0x1f, 0x5d, 0x47,
0xd6, 0x8c, 0xa0, 0xae, 0x9e, 0xd8, 0x9a, 0x4a, 0x0a, 0xd5, 0xbf, 0xf3, 0x26, 0x6c, 0x94, 0x56,
0x31, 0x2a, 0xaf, 0x43, 0x6d, 0x82, 0x63, 0x85, 0xde, 0x75, 0xe5, 0xaf, 0xe3, 0x43, 0xdf, 0xc5,
0x7e, 0x78, 0x7d, 0xda, 0x98, 0x25, 0x6a, 0xf9, 0x12, 0xdb, 0x80, 0x8a, 0x4b, 0x18, 0x55, 0xac,
0xd6, 0x95, 0x82, 0xd6, 0x4f, 0xa1, 0xbf, 0x3b, 0xa1, 0x1c, 0x1f, 0x89, 0x90, 0xc4, 0xd7, 0x91,
0xe6, 0xff, 0x16, 0x36, 0x3e, 0x13, 0xd3, 0xcf, 0x25, 0x18, 0x27, 0x5f, 0xe1, 0x6b, 0xb2, 0x8f,
0xd1, 0x67, 0xd6, 0x3e, 0x46, 0x9f, 0xc9, 0x0c, 0x3f, 0xa0, 0x93, 0x34, 0x8a, 0xd5, 0x71, 0xef,
0xba, 0x86, 0x72, 0xfe, 0x59, 0x81, 0x4d, 0x5d, 0x83, 0x1f, 0xe9, 0xd2, 0xd3, 0x2e, 0x3f, 0x84,
0xe6, 0x19, 0xe5, 0x22, 0xf6, 0x23, 0x6c, 0x96, 0xce, 0x68, 0x09, 0x2f, 0x6b, 0xd6, 0xaa, 0xaa,
0x0a, 0xe4, 0x6f, 0xa9, 0x30, 0xae, 0x5d, 0x5e, 0x18, 0xcf, 0x95, 0xbe, 0xf5, 0xf9, 0xd2, 0x17,
0xbd, 0x02, 0x60, 0x85, 0x48, 0xa8, 0x1e, 0xfe, 0x96, 0xdb, 0x32, 0x9c, 0x83, 0x10, 0xdd, 0x81,
0xde, 0x58, 0x6a, 0xe9, 0x9d, 0x51, 0x7a, 0xee, 0x25, 0xbe, 0x38, 0x53, 0x85, 0x76, 0xcb, 0xed,
0x2a, 0xf6, 0x3e, 0xa5, 0xe7, 0x87, 0xbe, 0x38, 0x73, 0x6e, 0xc2, 0x8d, 0x47, 0x98, 0x0b, 0x46,
0xa7, 0x65, 0xeb, 0x9c, 0x9f, 0x02, 0x1c, 0xc4, 0x02, 0xb3, 0x53, 0x5f, 0x96, 0xf5, 0x3f, 0x2a,
0x52, 0xe6, 0x49, 0x5d, 0x1f, 0xe9, 0x3e, 0x46, 0x36, 0xe0, 0x16, 0x64, 0x9c, 0x11, 0x34, 0x5c,
0x9a, 0x0a, 0xcc, 0xd1, 0x1b, 0xf6, 0xcf, 0xcc, 0xeb, 0x98, 0x79, 0x8a, 0xe9, 0x9a, 0x31, 0x67,
0xdf, 0x16, 0x3e, 0x39, 0x9c, 0xf1, 0xf3, 0x08, 0x5a, 0xc4, 0xf2, 0xcc, 0xed, 0x9c, 0x5f, 0x3a,
0x17, 0x71, 0xf6, 0x60, 0xe3, 0x41, 0x18, 0x7e, 0x67, 0x98, 0x7d, 0xdb, 0x1f, 0xf8, 0xce, 0x48,
0xf7, 0x60, 0x43, 0x9b, 0xa6, 0x4d, 0xb5, 0x30, 0x6f, 0x40, 0x83, 0x59, 0xbf, 0x54, 0xf2, 0x8e,
0x8a, 0x11, 0x32, 0x63, 0x72, 0x83, 0x64, 0x61, 0x98, 0x7b, 0xd6, 0x6e, 0xd0, 0x06, 0xf4, 0xe5,
0x40, 0x09, 0xd3, 0xf9, 0x0d, 0x6c, 0x3c, 0x8d, 0x27, 0x24, 0xc6, 0xbb, 0x87, 0xc7, 0x4f, 0x70,
0x16, 0x09, 0x10, 0xd4, 0xe5, 0x33, 0xaf, 0x16, 0x6a, 0xba, 0xea, 0x5f, 0x5e, 0x8d, 0xf8, 0xc4,
0x0b, 0x92, 0x94, 0x9b, 0x16, 0x46, 0x23, 0x3e, 0xd9, 0x4d, 0x52, 0x8e, 0x5e, 0x02, 0xf9, 0xdc,
0x78, 0x34, 0x9e, 0x4c, 0xd5, 0xfd, 0x68, 0xba, 0xab, 0x41, 0x92, 0x3e, 0x8d, 0x27, 0x53, 0xe7,
0x87, 0xaa, 0x68, 0xc3, 0x38, 0x74, 0xfd, 0x38, 0xa4, 0xd1, 0x23, 0x7c, 0x51, 0x58, 0x21, 0x2b,
0x10, 0x6c, 0x1c, 0xf8, 0xba, 0x02, 0x9d, 0x07, 0x63, 0x1c, 0x8b, 0x47, 0x58, 0xf8, 0x64, 0xa2,
0x8a, 0x80, 0x0b, 0xcc, 0x38, 0xa1, 0xb1, 0xb9, 0x30, 0x96, 0x94, 0x35, 0x1c, 0x89, 0x89, 0xf0,
0x42, 0x1f, 0x47, 0x34, 0x56, 0x28, 0x4d, 0x17, 0x24, 0xeb, 0x91, 0xe2, 0xa0, 0x37, 0xa1, 0xa7,
0x5b, 0x4c, 0xde, 0x99, 0x1f, 0x87, 0x13, 0xcc, 0x6c, 0xc9, 0xbd, 0xa6, 0xd9, 0xfb, 0x86, 0x8b,
0xde, 0x82, 0x75, 0x73, 0x91, 0x72, 0xc9, 0xba, 0x92, 0xec, 0x19, 0x7e, 0x49, 0x34, 0x4d, 0x12,
0xca, 0x04, 0xf7, 0x38, 0x0e, 0x02, 0x1a, 0x25, 0x26, 0x83, 0xee, 0x59, 0xfe, 0x91, 0x66, 0xcb,
0x2d, 0x7c, 0x2c, 0xed, 0x34, 0x96, 0xe4, 0x5b, 0xb8, 0x16, 0xe1, 0xc8, 0x3b, 0x99, 0xd0, 0xe0,
0xdc, 0x93, 0xa1, 0xc9, 0x78, 0x58, 0xbe, 0xd1, 0x0f, 0x25, 0xf3, 0x88, 0x7c, 0x85, 0x9d, 0xdf,
0x57, 0x60, 0xb3, 0x3c, 0xdb, 0x04, 0xcf, 0x1d, 0xd8, 0x2c, 0x4f, 0xd7, 0xc9, 0xab, 0xc9, 0x32,
0xfa, 0x45, 0x10, 0x95, 0x9e, 0xa2, 0x0f, 0xa0, 0xab, 0x7a, 0x89, 0x5e, 0xa8, 0x91, 0xca, 0x6f,
0x5f, 0xd1, 0xd7, 0x6e, 0xc7, 0x2f, 0x50, 0xce, 0x7d, 0xb8, 0x79, 0x84, 0x85, 0x56, 0xc2, 0x17,
0x26, 0x0f, 0xd4, 0x36, 0xac, 0x43, 0xed, 0x08, 0x07, 0x6a, 0xcd, 0x9a, 0x2b, 0x7f, 0xe5, 0x5e,
0x1e, 0x73, 0x1c, 0x28, 0xf0, 0x9a, 0xab, 0xfe, 0x9d, 0xbf, 0x54, 0x60, 0xd5, 0x44, 0x2a, 0x19,
0x29, 0x43, 0x46, 0x2e, 0x30, 0x33, 0xbb, 0x68, 0x28, 0x59, 0x8f, 0xea, 0x3f, 0x8f, 0x26, 0x82,
0xd0, 0x2c, 0xfe, 0x75, 0x35, 0xf7, 0xa9, 0x66, 0xaa, 0xee, 0x8c, 0x6a, 0x3e, 0x98, 0x3c, 0xdf,
0x50, 0xaa, 0xc5, 0xc2, 0xe5, 0x35, 0x52, 0xf1, 0xae, 0xe5, 0x1a, 0x4a, 0x9e, 0x1a, 0x8b, 0xb7,
0xa2, 0xf0, 0x2c, 0x29, 0x4f, 0x4d, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0x01, 0x0e,
0x14, 0xeb, 0x50, 0x72, 0x9c, 0x3f, 0x54, 0xa0, 0xa1, 0x3b, 0x94, 0xb2, 0xb2, 0xc8, 0x9e, 0x88,
0x2a, 0x51, 0xcf, 0xad, 0x5a, 0x4b, 0x3f, 0x0b, 0xea, 0x5f, 0x5e, 0x89, 0x8b, 0x48, 0x07, 0x4b,
0xa3, 0xda, 0x45, 0x24, 0xa3, 0xa4, 0xb4, 0x2c, 0x7f, 0x69, 0xd4, 0xb8, 0x56, 0xb1, 0x9b, 0x71,
0x95, 0xd8, 0x52, 0x4d, 0x9d, 0x5f, 0xc9, 0x82, 0x2a, 0xeb, 0xce, 0xad, 0x43, 0x2d, 0xcd, 0x94,
0x91, 0xbf, 0x92, 0x33, 0xce, 0xde, 0x28, 0xf9, 0x8b, 0xee, 0xc0, 0x9a, 0x1f, 0x86, 0x44, 0x4e,
0xf7, 0x27, 0x8f, 0x49, 0x98, 0x9d, 0xf7, 0x32, 0xd7, 0xf9, 0x7b, 0x05, 0x7a, 0xbb, 0x34, 0x99,
0xfe, 0x8c, 0x4c, 0x70, 0xe1, 0x32, 0x2a, 0x25, 0xf5, 0x02, 0xea, 0x5f, 0x26, 0xab, 0xa7, 0x64,
0x82, 0xf5, 0x29, 0xd5, 0x3b, 0xdb, 0x94, 0x0c, 0x79, 0xb8, 0xb2, 0xc1, 0xac, 0xe9, 0xd1, 0xd5,
0x83, 0x4f, 0x68, 0x88, 0x65, 0x3c, 0x08, 0x09, 0xf3, 0xb2, 0x16, 0x47, 0xd7, 0x5d, 0x0d, 0x09,
0x53, 0x43, 0xc6, 0x90, 0x15, 0xd5, 0x65, 0x2b, 0x1a, 0xd2, 0xd0, 0x1c, 0x69, 0xc8, 0x16, 0x34,
0xe8, 0xe9, 0x29, 0xc7, 0x42, 0x75, 0x70, 0x6b, 0xae, 0xa1, 0xb2, 0x88, 0xd1, 0xcc, 0x23, 0xc6,
0xdd, 0xbf, 0xf6, 0x4c, 0xc4, 0x30, 0xf5, 0x0a, 0x7a, 0x0c, 0xbd, 0x99, 0xfe, 0x37, 0x32, 0x05,
0xec, 0xe2, 0xb6, 0xf8, 0x70, 0x6b, 0xa4, 0xfb, 0xe9, 0x23, 0xdb, 0x4f, 0x1f, 0xed, 0x45, 0x89,
0x98, 0xa2, 0x3d, 0x58, 0x2b, 0x77, 0x8a, 0xd1, 0x2d, 0xfb, 0xfc, 0x2e, 0xe8, 0x1f, 0x2f, 0x85,
0x79, 0x0c, 0xbd, 0x99, 0xa6, 0xb1, 0xd5, 0x67, 0x71, 0x2f, 0x79, 0x29, 0xd0, 0x7d, 0x68, 0x17,
0xba, 0xc4, 0x68, 0xa0, 0x41, 0xe6, 0x1b, 0xc7, 0x4b, 0x01, 0x76, 0xa1, 0x5b, 0x6a, 0xdc, 0xa2,
0xa1, 0xb1, 0x67, 0x41, 0x37, 0x77, 0x29, 0xc8, 0x43, 0x68, 0x17, 0xfa, 0xa7, 0x56, 0x8b, 0xf9,
0x26, 0xed, 0xf0, 0xa5, 0x05, 0x23, 0x26, 0x88, 0xed, 0x43, 0xb7, 0xd4, 0xed, 0xb4, 0x8a, 0x2c,
0xea, 0xb4, 0x0e, 0x6f, 0x2d, 0x1c, 0x33, 0x48, 0x8f, 0xa1, 0x37, 0xd3, 0xfb, 0xb4, 0xce, 0x5d,
0xdc, 0x12, 0x5d, 0x6a, 0xd6, 0x27, 0x6a, 0xb3, 0x0b, 0xc9, 0x7e, 0x61, 0xb3, 0xe7, 0x3b, 0x9d,
0xc3, 0x97, 0x17, 0x0f, 0x1a, 0xad, 0xf6, 0x60, 0xad, 0xdc, 0xe4, 0xb4, 0x60, 0x0b, 0x5b, 0x9f,
0x97, 0x9f, 0x9c, 0x52, 0xbf, 0x33, 0x3f, 0x39, 0x8b, 0xda, 0xa0, 0x4b, 0x81, 0x1e, 0x00, 0x98,
0x9a, 0x20, 0x24, 0x71, 0xb6, 0x65, 0x73, 0xb5, 0x48, 0xb6, 0x65, 0x0b, 0xea, 0x87, 0xfb, 0x00,
0x3a, 0x95, 0x0f, 0x69, 0x2a, 0xd0, 0x4d, 0xab, 0xc6, 0x4c, 0xfd, 0x30, 0x1c, 0xcc, 0x0f, 0xcc,
0x01, 0x60, 0xc6, 0xae, 0x02, 0xf0, 0x31, 0x40, 0x5e, 0x22, 0x58, 0x80, 0xb9, 0xa2, 0xe1, 0x12,
0x1f, 0x74, 0x8a, 0x05, 0x01, 0x32, 0xb6, 0x2e, 0x28, 0x12, 0x96, 0x42, 0xdc, 0x83, 0x4e, 0x31,
0x4b, 0xb4, 0x10, 0x0b, 0x32, 0xc7, 0xe1, 0x5c, 0x72, 0x87, 0x1e, 0xd8, 0x93, 0x9a, 0xb3, 0x4a,
0x27, 0xf5, 0xdb, 0x41, 0xcc, 0xa4, 0x97, 0xe5, 0x48, 0xf2, 0x2d, 0x20, 0x3e, 0x80, 0x4e, 0x31,
0xaf, 0xb4, 0x26, 0x2c, 0xc8, 0x35, 0x87, 0xa5, 0xdc, 0x12, 0xdd, 0x87, 0xb5, 0x72, 0x4e, 0x89,
0x0a, 0xf7, 0x72, 0x2e, 0xd3, 0x1c, 0x9a, 0xc2, 0xbf, 0x20, 0xfe, 0x1e, 0x40, 0x9e, 0x7b, 0xda,
0xed, 0x9b, 0xcb, 0x46, 0x67, 0x56, 0xdd, 0x85, 0x6e, 0xa9, 0x8e, 0xb2, 0x81, 0x62, 0x51, 0x71,
0x75, 0x59, 0x1c, 0x2f, 0xd7, 0x2b, 0x56, 0xf5, 0x85, 0x55, 0xcc, 0x65, 0x07, 0xa8, 0x98, 0x27,
0x5b, 0xd7, 0x2d, 0xc8, 0x9d, 0x5f, 0x70, 0xa1, 0x8b, 0xb9, 0x70, 0xe1, 0x42, 0x2f, 0x48, 0x91,
0x97, 0x02, 0xed, 0x43, 0xef, 0xb1, 0xcd, 0xcd, 0x4c, 0xa2, 0x6c, 0xd4, 0x59, 0x90, 0x72, 0x0e,
0x87, 0x8b, 0x86, 0xcc, 0xad, 0x3a, 0x80, 0xf5, 0xd9, 0x2c, 0x0f, 0xbd, 0x62, 0x82, 0xdb, 0xe2,
0xec, 0x6f, 0xa9, 0x52, 0x3f, 0x81, 0xa6, 0xcd, 0x2a, 0x90, 0x69, 0x9b, 0xcd, 0x64, 0x19, 0xcb,
0xa6, 0x3e, 0xec, 0x7c, 0xfd, 0xcd, 0xab, 0x95, 0x7f, 0x7c, 0xf3, 0x6a, 0xe5, 0xdf, 0xdf, 0xbc,
0x5a, 0x39, 0x69, 0xa8, 0xd1, 0xf7, 0xfe, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x44, 0x3b, 0xc9, 0x9d,
0x3e, 0x1f, 0x00, 0x00,
// 2675 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x19, 0xdb, 0x6e, 0x1b, 0xc7,
0x15, 0xbc, 0x88, 0x22, 0x0f, 0x49, 0x51, 0x1c, 0xc9, 0x32, 0x4d, 0x27, 0xae, 0xb3, 0x49, 0x1d,
0xa5, 0x69, 0xa8, 0xc4, 0x09, 0x9a, 0x8b, 0x91, 0x1a, 0x96, 0xac, 0x5a, 0x6a, 0xe2, 0x5a, 0x5d,
0x45, 0x48, 0x81, 0xa2, 0x58, 0xac, 0x76, 0x47, 0xe4, 0x44, 0xdc, 0x9d, 0xcd, 0xce, 0xac, 0x6c,
0xa6, 0x40, 0x1f, 0xfb, 0xda, 0x2f, 0xe8, 0x0f, 0x14, 0x7d, 0xeb, 0x63, 0x5f, 0xfb, 0x10, 0xf4,
0xa9, 0xef, 0x05, 0x8a, 0x22, 0x9f, 0xd0, 0x2f, 0x28, 0xe6, 0xb6, 0x17, 0x72, 0xa9, 0xa4, 0x8a,
0x80, 0xbe, 0x10, 0x73, 0x2e, 0x73, 0x6e, 0x33, 0x73, 0xf6, 0x9c, 0x43, 0x68, 0xbb, 0x63, 0x1c,
0xf2, 0x51, 0x14, 0x53, 0x4e, 0x51, 0x7d, 0x1c, 0x47, 0xde, 0xb0, 0x45, 0x3d, 0xa2, 0x10, 0xc3,
0x9f, 0x8c, 0x09, 0x9f, 0x24, 0xa7, 0x23, 0x8f, 0x06, 0x3b, 0xe7, 0x2e, 0x77, 0xdf, 0xf2, 0x68,
0xc8, 0x5d, 0x12, 0xe2, 0x98, 0xed, 0xc8, 0x8d, 0x3b, 0xd1, 0xf9, 0x78, 0x87, 0xcf, 0x22, 0xcc,
0xd4, 0xaf, 0xde, 0x77, 0x7b, 0x4c, 0xe9, 0x78, 0x8a, 0x77, 0x24, 0x74, 0x9a, 0x9c, 0xed, 0xe0,
0x20, 0xe2, 0x33, 0x45, 0xb4, 0xfe, 0x58, 0x85, 0xad, 0xbd, 0x18, 0xbb, 0x1c, 0xef, 0x19, 0x69,
0x36, 0xfe, 0x32, 0xc1, 0x8c, 0xa3, 0x57, 0xa0, 0x93, 0x6a, 0x70, 0x88, 0x3f, 0xa8, 0xdc, 0xad,
0x6c, 0xb7, 0xec, 0x76, 0x8a, 0x3b, 0xf4, 0xd1, 0x4d, 0x58, 0xc5, 0x2f, 0xb0, 0x27, 0xa8, 0x55,
0x49, 0x6d, 0x08, 0xf0, 0xd0, 0x47, 0xef, 0x40, 0x9b, 0xf1, 0x98, 0x84, 0x63, 0x27, 0x61, 0x38,
0x1e, 0xd4, 0xee, 0x56, 0xb6, 0xdb, 0xf7, 0xd7, 0x47, 0xc2, 0xa5, 0xd1, 0xb1, 0x24, 0x9c, 0x30,
0x1c, 0xdb, 0xc0, 0xd2, 0x35, 0xba, 0x07, 0xab, 0x3e, 0xbe, 0x20, 0x1e, 0x66, 0x83, 0xfa, 0xdd,
0xda, 0x76, 0xfb, 0x7e, 0x47, 0xb1, 0x3f, 0x96, 0x48, 0xdb, 0x10, 0xd1, 0x1b, 0xd0, 0x64, 0x9c,
0xc6, 0xee, 0x18, 0xb3, 0xc1, 0x8a, 0x64, 0xec, 0x1a, 0xb9, 0x12, 0x6b, 0xa7, 0x64, 0xf4, 0x12,
0xd4, 0x9e, 0xed, 0x1d, 0x0e, 0x1a, 0x52, 0x3b, 0x68, 0xae, 0x08, 0x7b, 0xb6, 0x40, 0xa3, 0x57,
0xa1, 0xcb, 0xdc, 0xd0, 0x3f, 0xa5, 0x2f, 0x9c, 0x88, 0xf8, 0x21, 0x1b, 0xac, 0xde, 0xad, 0x6c,
0x37, 0xed, 0x8e, 0x46, 0x1e, 0x09, 0x9c, 0xf5, 0x11, 0xdc, 0x38, 0xe6, 0x6e, 0xcc, 0xaf, 0x10,
0x1d, 0xeb, 0x04, 0xb6, 0x6c, 0x1c, 0xd0, 0x8b, 0x2b, 0x85, 0x76, 0x00, 0xab, 0x9c, 0x04, 0x98,
0x26, 0x5c, 0x86, 0xb6, 0x6b, 0x1b, 0xd0, 0xfa, 0x73, 0x05, 0xd0, 0xfe, 0x0b, 0xec, 0x1d, 0xc5,
0xd4, 0xc3, 0x8c, 0xfd, 0x9f, 0x8e, 0xeb, 0x75, 0x58, 0x8d, 0x94, 0x01, 0x83, 0xba, 0x64, 0xd7,
0xa7, 0x60, 0xac, 0x32, 0x54, 0xeb, 0x0b, 0xd8, 0x3c, 0x26, 0xe3, 0xd0, 0x9d, 0x5e, 0xa3, 0xbd,
0x5b, 0xd0, 0x60, 0x52, 0xa6, 0x34, 0xb5, 0x6b, 0x6b, 0xc8, 0x3a, 0x02, 0xf4, 0xb9, 0x4b, 0xf8,
0xf5, 0x69, 0xb2, 0xde, 0x82, 0x8d, 0x82, 0x44, 0x16, 0xd1, 0x90, 0x61, 0x69, 0x00, 0x77, 0x79,
0xc2, 0xa4, 0xb0, 0x15, 0x5b, 0x43, 0x16, 0x86, 0xcd, 0x4f, 0x09, 0x33, 0xec, 0xf8, 0x7f, 0x31,
0x61, 0x0b, 0x1a, 0x67, 0x34, 0x0e, 0x5c, 0x6e, 0x2c, 0x50, 0x10, 0x42, 0x50, 0x77, 0xe3, 0x31,
0x1b, 0xd4, 0xee, 0xd6, 0xb6, 0x5b, 0xb6, 0x5c, 0x8b, 0x5b, 0x39, 0xa7, 0x46, 0xdb, 0xf5, 0x0a,
0x74, 0x74, 0xdc, 0x9d, 0x29, 0x61, 0x5c, 0xea, 0xe9, 0xd8, 0x6d, 0x8d, 0x13, 0x7b, 0x2c, 0x0a,
0x5b, 0x27, 0x91, 0x7f, 0xc5, 0x07, 0x7f, 0x1f, 0x5a, 0x31, 0x66, 0x34, 0x89, 0xc5, 0x33, 0xad,
0xca, 0x73, 0xdf, 0x54, 0xe7, 0xfe, 0x29, 0x09, 0x93, 0x17, 0xb6, 0xa1, 0xd9, 0x19, 0x9b, 0x7e,
0x42, 0x9c, 0x5d, 0xe5, 0x09, 0x7d, 0x04, 0x37, 0x8e, 0xdc, 0x84, 0x5d, 0xc5, 0x56, 0xeb, 0x81,
0x78, 0x7e, 0x2c, 0x09, 0xae, 0xb4, 0xf9, 0x4f, 0x15, 0x68, 0xee, 0x45, 0xc9, 0x09, 0x73, 0xc7,
0x18, 0xfd, 0x00, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94, 0xec, 0x75, 0x1b, 0x24, 0x4a,
0x31, 0x88, 0xb0, 0xe3, 0xd8, 0x8b, 0x12, 0xcd, 0x51, 0xbd, 0x5b, 0xdb, 0xae, 0xdb, 0x6d, 0x85,
0x53, 0x2c, 0x23, 0xd8, 0x90, 0x34, 0x87, 0x84, 0xce, 0x39, 0x8e, 0x43, 0x3c, 0x0d, 0xa8, 0x8f,
0xe5, 0xfd, 0xad, 0xdb, 0x7d, 0x49, 0x3a, 0x0c, 0x3f, 0x49, 0x09, 0xe8, 0x47, 0xd0, 0x4f, 0xf9,
0xc5, 0xa3, 0x94, 0xdc, 0x75, 0xc9, 0xdd, 0xd3, 0xdc, 0x27, 0x1a, 0x6d, 0xfd, 0x0e, 0xd6, 0x3e,
0x9b, 0xc4, 0x94, 0xf3, 0x29, 0x09, 0xc7, 0x8f, 0x5d, 0xee, 0x8a, 0xec, 0x11, 0xe1, 0x98, 0x50,
0x9f, 0x69, 0x6b, 0x0d, 0x88, 0xde, 0x84, 0x3e, 0x57, 0xbc, 0xd8, 0x77, 0x0c, 0x4f, 0x55, 0xf2,
0xac, 0xa7, 0x84, 0x23, 0xcd, 0xfc, 0x43, 0x58, 0xcb, 0x98, 0x45, 0xfe, 0xd1, 0xf6, 0x76, 0x53,
0xec, 0x67, 0x24, 0xc0, 0xd6, 0x85, 0x8c, 0x95, 0x3c, 0x64, 0xf4, 0x26, 0xb4, 0xb2, 0x38, 0x54,
0xe4, 0x0d, 0x59, 0x53, 0x37, 0xc4, 0x84, 0xd3, 0x6e, 0xa6, 0x41, 0xf9, 0x18, 0x7a, 0x3c, 0x35,
0xdc, 0xf1, 0x5d, 0xee, 0x16, 0x2f, 0x55, 0xd1, 0x2b, 0x7b, 0x8d, 0x17, 0x60, 0xeb, 0x01, 0xb4,
0x8e, 0x88, 0xcf, 0x94, 0xe2, 0x01, 0xac, 0x7a, 0x49, 0x1c, 0xe3, 0x90, 0x1b, 0x97, 0x35, 0x88,
0x36, 0x61, 0x65, 0x4a, 0x02, 0xc2, 0xb5, 0x9b, 0x0a, 0xb0, 0x28, 0xc0, 0x53, 0x1c, 0xd0, 0x78,
0x26, 0x03, 0xb6, 0x09, 0x2b, 0xf9, 0xc3, 0x55, 0x00, 0xba, 0x0d, 0xad, 0xc0, 0x7d, 0x91, 0x1e,
0xaa, 0xa0, 0x34, 0x03, 0xf7, 0x85, 0x32, 0x7e, 0x00, 0xab, 0x67, 0x2e, 0x99, 0x7a, 0x21, 0xd7,
0x51, 0x31, 0x60, 0xa6, 0xb0, 0x9e, 0x57, 0xf8, 0xb7, 0x2a, 0xb4, 0x95, 0x46, 0x65, 0xf0, 0x26,
0xac, 0x78, 0xae, 0x37, 0x49, 0x55, 0x4a, 0x00, 0xdd, 0x33, 0x86, 0x54, 0xf3, 0x49, 0x38, 0xb3,
0xd4, 0x98, 0xb6, 0x03, 0xc0, 0x9e, 0xbb, 0x91, 0xb6, 0xad, 0xb6, 0x84, 0xb9, 0x25, 0x78, 0x94,
0xb9, 0xef, 0x42, 0x47, 0xdd, 0x3b, 0xbd, 0xa5, 0xbe, 0x64, 0x4b, 0x5b, 0x71, 0xa9, 0x4d, 0xaf,
0x42, 0x37, 0x61, 0xd8, 0x99, 0x10, 0x1c, 0xbb, 0xb1, 0x37, 0x99, 0x0d, 0x56, 0xd4, 0x37, 0x32,
0x61, 0xf8, 0xc0, 0xe0, 0xd0, 0x7d, 0x58, 0x11, 0xe9, 0x8f, 0x0d, 0x1a, 0xf2, 0x73, 0xfc, 0x52,
0x5e, 0xa4, 0x74, 0x75, 0x24, 0x7f, 0xf7, 0x43, 0x1e, 0xcf, 0x6c, 0xc5, 0x3a, 0xfc, 0x00, 0x20,
0x43, 0xa2, 0x75, 0xa8, 0x9d, 0xe3, 0x99, 0x7e, 0x87, 0x62, 0x29, 0x82, 0x73, 0xe1, 0x4e, 0x13,
0x13, 0x75, 0x05, 0x7c, 0x54, 0xfd, 0xa0, 0x62, 0x79, 0xd0, 0xdb, 0x9d, 0x9e, 0x13, 0x9a, 0xdb,
0xbe, 0x09, 0x2b, 0x81, 0xfb, 0x05, 0x8d, 0x4d, 0x24, 0x25, 0x20, 0xb1, 0x24, 0xa4, 0xb1, 0x11,
0x21, 0x01, 0xb4, 0x06, 0x55, 0x1a, 0xc9, 0x78, 0xb5, 0xec, 0x2a, 0x8d, 0x32, 0x45, 0xf5, 0x9c,
0x22, 0xeb, 0x5f, 0x75, 0x80, 0x4c, 0x0b, 0xb2, 0x61, 0x48, 0xa8, 0xc3, 0x70, 0x2c, 0x4a, 0x10,
0xe7, 0x74, 0xc6, 0x31, 0x73, 0x62, 0xec, 0x25, 0x31, 0x23, 0x17, 0xe2, 0xfc, 0x84, 0xdb, 0x37,
0x94, 0xdb, 0x73, 0xb6, 0xd9, 0x37, 0x09, 0x3d, 0x56, 0xfb, 0x76, 0xc5, 0x36, 0xdb, 0xec, 0x42,
0x87, 0x70, 0x23, 0x93, 0xe9, 0xe7, 0xc4, 0x55, 0x2f, 0x13, 0xb7, 0x91, 0x8a, 0xf3, 0x33, 0x51,
0xfb, 0xb0, 0x41, 0xa8, 0xf3, 0x65, 0x82, 0x93, 0x82, 0xa0, 0xda, 0x65, 0x82, 0xfa, 0x84, 0xfe,
0x52, 0x6e, 0xc8, 0xc4, 0x1c, 0xc1, 0xad, 0x9c, 0x97, 0xe2, 0xb9, 0xe7, 0x84, 0xd5, 0x2f, 0x13,
0xb6, 0x95, 0x5a, 0x25, 0xf2, 0x41, 0x26, 0xf1, 0xe7, 0xb0, 0x45, 0xa8, 0xf3, 0xdc, 0x25, 0x7c,
0x5e, 0xdc, 0xca, 0xb7, 0x38, 0x29, 0x3e, 0xba, 0x45, 0x59, 0xca, 0xc9, 0x00, 0xc7, 0xe3, 0x82,
0x93, 0x8d, 0x6f, 0x71, 0xf2, 0xa9, 0xdc, 0x90, 0x89, 0x79, 0x04, 0x7d, 0x42, 0xe7, 0xad, 0x59,
0xbd, 0x4c, 0x48, 0x8f, 0xd0, 0xa2, 0x25, 0xbb, 0xd0, 0x67, 0xd8, 0xe3, 0x34, 0xce, 0x5f, 0x82,
0xe6, 0x65, 0x22, 0xd6, 0x35, 0x7f, 0x2a, 0xc3, 0xfa, 0x35, 0x74, 0x0e, 0x92, 0x31, 0xe6, 0xd3,
0xd3, 0x34, 0x19, 0x5c, 0x5b, 0xfe, 0xb1, 0xfe, 0x53, 0x85, 0xf6, 0xde, 0x38, 0xa6, 0x49, 0x54,
0xc8, 0xc9, 0xea, 0x91, 0xce, 0xe7, 0x64, 0xc9, 0x22, 0x73, 0xb2, 0x62, 0x7e, 0x0f, 0x3a, 0x81,
0x7c, 0xba, 0x9a, 0x5f, 0xe5, 0xa1, 0xfe, 0xc2, 0xa3, 0xb6, 0xdb, 0x41, 0x2e, 0x99, 0x8d, 0x00,
0x22, 0xe2, 0x33, 0xbd, 0x47, 0xa5, 0xa3, 0x9e, 0xae, 0x08, 0x4d, 0x8a, 0xb6, 0x5b, 0x51, 0x9a,
0xad, 0xdf, 0x81, 0xf6, 0xa9, 0x08, 0x92, 0xde, 0x50, 0x48, 0x46, 0x59, 0xf4, 0x6c, 0x38, 0xcd,
0x1e, 0xe1, 0x01, 0x74, 0x27, 0x2a, 0x64, 0x7a, 0x93, 0xba, 0x43, 0xaf, 0x6a, 0x4f, 0x32, 0x7f,
0x47, 0xf9, 0xc8, 0xaa, 0x03, 0xe8, 0x4c, 0x72, 0xa8, 0xe1, 0x31, 0xf4, 0x17, 0x58, 0x4a, 0x72,
0xd0, 0x76, 0x3e, 0x07, 0xb5, 0xef, 0x23, 0xa5, 0x28, 0xbf, 0x33, 0x9f, 0x97, 0x7e, 0x01, 0x5b,
0xf3, 0x65, 0x8e, 0x2e, 0xca, 0xde, 0x83, 0x8e, 0x27, 0xad, 0x2b, 0x9c, 0x40, 0x7f, 0xc1, 0x6e,
0xbb, 0xed, 0x65, 0x80, 0xe5, 0x03, 0xfa, 0x3c, 0x26, 0x1c, 0x1f, 0xf3, 0x18, 0xbb, 0xc1, 0x75,
0x54, 0xcd, 0x08, 0xea, 0xf2, 0x13, 0x5b, 0x93, 0x45, 0xa1, 0x5c, 0x5b, 0xaf, 0xc3, 0x46, 0x41,
0x8b, 0x36, 0x79, 0x1d, 0x6a, 0x53, 0x1c, 0x4a, 0xe9, 0x5d, 0x5b, 0x2c, 0x2d, 0x17, 0xfa, 0x36,
0x76, 0xfd, 0xeb, 0xb3, 0x46, 0xab, 0xa8, 0x65, 0x2a, 0xb6, 0x01, 0xe5, 0x55, 0x68, 0x53, 0x8c,
0xd5, 0x95, 0x9c, 0xd5, 0xcf, 0xa0, 0xbf, 0x37, 0xa5, 0x0c, 0x1f, 0x73, 0x9f, 0x84, 0xd7, 0x51,
0xe6, 0xff, 0x16, 0x36, 0x3e, 0xe3, 0xb3, 0xcf, 0x85, 0x30, 0x46, 0xbe, 0xc2, 0xd7, 0xe4, 0x5f,
0x4c, 0x9f, 0x1b, 0xff, 0x62, 0xfa, 0x5c, 0x54, 0xf8, 0x1e, 0x9d, 0x26, 0x41, 0x28, 0xaf, 0x7b,
0xd7, 0xd6, 0x90, 0xf5, 0xcf, 0x0a, 0x6c, 0xaa, 0x1e, 0xfc, 0x58, 0xb5, 0x9e, 0x46, 0xfd, 0x10,
0x9a, 0x13, 0xca, 0x78, 0xe8, 0x06, 0x58, 0xab, 0x4e, 0x61, 0x21, 0x5e, 0xf4, 0xac, 0x55, 0xd9,
0x15, 0x88, 0x65, 0xa1, 0x31, 0xae, 0x5d, 0xde, 0x18, 0x2f, 0xb4, 0xbe, 0xf5, 0xc5, 0xd6, 0x17,
0xbd, 0x0c, 0x60, 0x98, 0x88, 0x2f, 0x3f, 0xfc, 0x2d, 0xbb, 0xa5, 0x31, 0x87, 0x3e, 0xba, 0x07,
0xbd, 0xb1, 0xb0, 0xd2, 0x99, 0x50, 0x7a, 0xee, 0x44, 0x2e, 0x9f, 0xc8, 0x46, 0xbb, 0x65, 0x77,
0x25, 0xfa, 0x80, 0xd2, 0xf3, 0x23, 0x97, 0x4f, 0xac, 0x9b, 0x70, 0xe3, 0x31, 0x66, 0x3c, 0xa6,
0xb3, 0xa2, 0x77, 0xd6, 0x4f, 0x01, 0x0e, 0x43, 0x8e, 0xe3, 0x33, 0x57, 0xb4, 0xf5, 0x6f, 0xe7,
0x21, 0xfd, 0x49, 0x5d, 0x1f, 0xa9, 0x39, 0x46, 0x4a, 0xb0, 0x73, 0x3c, 0xd6, 0x08, 0x1a, 0x36,
0x4d, 0x38, 0x66, 0xe8, 0x35, 0xb3, 0xd2, 0xfb, 0x3a, 0x7a, 0x9f, 0x44, 0xda, 0x9a, 0x66, 0x1d,
0x98, 0xc6, 0x27, 0x13, 0xa7, 0xe3, 0x3c, 0x82, 0x16, 0x31, 0x38, 0xfd, 0x3a, 0x17, 0x55, 0x67,
0x2c, 0xd6, 0x3e, 0x6c, 0x3c, 0xf2, 0xfd, 0xef, 0x2d, 0xe6, 0xc0, 0xcc, 0x07, 0xbe, 0xb7, 0xa4,
0x07, 0xb0, 0xa1, 0x5c, 0x53, 0xae, 0x1a, 0x31, 0xaf, 0x41, 0x23, 0x36, 0x71, 0xa9, 0x64, 0x13,
0x15, 0xcd, 0xa4, 0x69, 0xe2, 0x80, 0x44, 0x63, 0x98, 0x45, 0xd6, 0x1c, 0xd0, 0x06, 0xf4, 0x05,
0xa1, 0x20, 0xd3, 0xfa, 0x0d, 0x6c, 0x3c, 0x0b, 0xa7, 0x24, 0xc4, 0x7b, 0x47, 0x27, 0x4f, 0x71,
0x9a, 0x09, 0x10, 0xd4, 0xc5, 0x67, 0x5e, 0x2a, 0x6a, 0xda, 0x72, 0x2d, 0x9e, 0x46, 0x78, 0xea,
0x78, 0x51, 0xc2, 0xf4, 0x08, 0xa3, 0x11, 0x9e, 0xee, 0x45, 0x09, 0x43, 0xb7, 0x40, 0x7c, 0x6e,
0x1c, 0x1a, 0x4e, 0x67, 0xf2, 0x7d, 0x34, 0xed, 0x55, 0x2f, 0x4a, 0x9e, 0x85, 0xd3, 0x99, 0xf5,
0x63, 0xd9, 0xb4, 0x61, 0xec, 0xdb, 0x6e, 0xe8, 0xd3, 0xe0, 0x31, 0xbe, 0xc8, 0x69, 0x48, 0x1b,
0x04, 0x93, 0x07, 0xbe, 0xae, 0x40, 0xe7, 0xd1, 0x18, 0x87, 0xfc, 0x31, 0xe6, 0x2e, 0x99, 0xca,
0x26, 0xe0, 0x02, 0xc7, 0x8c, 0xd0, 0x50, 0x3f, 0x18, 0x03, 0x8a, 0x1e, 0x8e, 0x84, 0x84, 0x3b,
0xbe, 0x8b, 0x03, 0x1a, 0x4a, 0x29, 0x4d, 0x1b, 0x04, 0xea, 0xb1, 0xc4, 0xa0, 0xd7, 0xa1, 0xa7,
0x46, 0x4c, 0xce, 0xc4, 0x0d, 0xfd, 0x29, 0x8e, 0x4d, 0xcb, 0xbd, 0xa6, 0xd0, 0x07, 0x1a, 0x8b,
0xde, 0x80, 0x75, 0xfd, 0x90, 0x32, 0xce, 0xba, 0xe4, 0xec, 0x69, 0x7c, 0x81, 0x35, 0x89, 0x22,
0x1a, 0x73, 0xe6, 0x30, 0xec, 0x79, 0x34, 0x88, 0x74, 0x05, 0xdd, 0x33, 0xf8, 0x63, 0x85, 0xb6,
0xc6, 0xb0, 0xf1, 0x44, 0xf8, 0xa9, 0x3d, 0xc9, 0x8e, 0x70, 0x2d, 0xc0, 0x81, 0x73, 0x3a, 0xa5,
0xde, 0xb9, 0x23, 0x52, 0x93, 0x8e, 0xb0, 0xf8, 0x46, 0xef, 0x0a, 0xe4, 0x31, 0xf9, 0x4a, 0x36,
0x8b, 0x82, 0x6b, 0x42, 0x79, 0x34, 0x4d, 0xc6, 0x4e, 0x14, 0xd3, 0x53, 0xac, 0x5d, 0xec, 0x05,
0x38, 0x38, 0x50, 0xf8, 0x23, 0x81, 0xb6, 0xfe, 0x5a, 0x81, 0xcd, 0xa2, 0x26, 0x9d, 0x68, 0x77,
0x60, 0xb3, 0xa8, 0x4a, 0x15, 0xba, 0xba, 0x22, 0xe9, 0xe7, 0x15, 0xca, 0x52, 0x16, 0xbd, 0x0f,
0x5d, 0x39, 0x77, 0x74, 0x7c, 0x25, 0xa9, 0xf8, 0x9d, 0xcc, 0x9f, 0x8b, 0xdd, 0x71, 0xf3, 0xa7,
0xf4, 0x21, 0xdc, 0xd2, 0xee, 0x3b, 0x8b, 0x66, 0xab, 0x0b, 0xb1, 0xa5, 0x19, 0x9e, 0xce, 0x59,
0xff, 0x29, 0x0c, 0x32, 0xd4, 0xee, 0x4c, 0x22, 0x4d, 0xac, 0xde, 0x86, 0x8d, 0x39, 0x67, 0x1f,
0xf9, 0x7e, 0x2c, 0x73, 0x42, 0xdd, 0x2e, 0x23, 0x59, 0x0f, 0xe1, 0xe6, 0x31, 0xe6, 0x2a, 0x1a,
0x2e, 0xd7, 0xc5, 0xab, 0x12, 0xb6, 0x0e, 0xb5, 0x63, 0xec, 0x49, 0xe7, 0x6b, 0xb6, 0x58, 0x8a,
0x0b, 0x78, 0xc2, 0xb0, 0x27, 0xbd, 0xac, 0xd9, 0x72, 0x6d, 0xfd, 0xa5, 0x02, 0xab, 0x3a, 0xbd,
0x8a, 0xf4, 0xee, 0xc7, 0xe4, 0x02, 0xc7, 0xfa, 0xea, 0x69, 0x48, 0x34, 0xd1, 0x6a, 0xe5, 0xd0,
0x88, 0x13, 0x9a, 0x26, 0xed, 0xae, 0xc2, 0x3e, 0x53, 0x48, 0x39, 0x52, 0x92, 0x13, 0x13, 0xdd,
0x9c, 0x68, 0x48, 0xce, 0x85, 0x98, 0x78, 0xfb, 0x32, 0x49, 0xb7, 0x6c, 0x0d, 0x89, 0xab, 0x6e,
0xe4, 0xad, 0x48, 0x79, 0x06, 0x14, 0x57, 0x3d, 0xa0, 0x49, 0xc8, 0x9d, 0x88, 0x92, 0x90, 0xeb,
0xac, 0x0c, 0x12, 0x75, 0x24, 0x30, 0xd6, 0xef, 0x2b, 0xd0, 0x50, 0x63, 0x55, 0xd1, 0x0e, 0xa5,
0xdf, 0xb5, 0x2a, 0x91, 0x35, 0x82, 0xd4, 0xa5, 0xbe, 0x65, 0x72, 0x2d, 0xde, 0xf1, 0x45, 0xa0,
0x32, 0xbc, 0x36, 0xed, 0x22, 0x10, 0xa9, 0x5d, 0x78, 0x96, 0x7d, 0x1e, 0x25, 0x5d, 0x99, 0xd8,
0x4d, 0xb1, 0x92, 0x6d, 0xa9, 0xa5, 0xd6, 0xaf, 0x44, 0x17, 0x98, 0x8e, 0x14, 0xd7, 0xa1, 0x96,
0xa4, 0xc6, 0x88, 0xa5, 0xc0, 0x8c, 0xd3, 0x0f, 0xab, 0x58, 0xa2, 0x7b, 0xb0, 0xe6, 0xfa, 0x3e,
0x11, 0xdb, 0xdd, 0xe9, 0x13, 0xe2, 0xa7, 0x8f, 0xb4, 0x88, 0xb5, 0xfe, 0x5e, 0x81, 0xde, 0x1e,
0x8d, 0x66, 0x3f, 0x23, 0x53, 0x9c, 0xcb, 0x20, 0xd2, 0x48, 0xa5, 0x40, 0xae, 0x45, 0x85, 0x7d,
0x46, 0xa6, 0x58, 0x3d, 0x2d, 0x75, 0xb2, 0x4d, 0x81, 0x90, 0xcf, 0xca, 0x10, 0xd3, 0x49, 0x4d,
0x57, 0x11, 0x9f, 0x52, 0x1f, 0x8b, 0x24, 0xe6, 0x93, 0xd8, 0x49, 0xe7, 0x32, 0x5d, 0x7b, 0xd5,
0x27, 0xb1, 0x24, 0x69, 0x47, 0x56, 0xe4, 0x68, 0x30, 0xef, 0x48, 0x43, 0x61, 0x84, 0x23, 0x5b,
0xd0, 0xa0, 0x67, 0x67, 0x0c, 0x73, 0x39, 0x76, 0xae, 0xd9, 0x1a, 0x4a, 0xd3, 0x5c, 0x33, 0x4b,
0x73, 0xf7, 0xff, 0xb0, 0xae, 0xd3, 0x9c, 0x6e, 0xb2, 0xd0, 0x13, 0xe8, 0xcd, 0x0d, 0xed, 0x91,
0xee, 0xba, 0xcb, 0x67, 0xf9, 0xc3, 0xad, 0x91, 0xfa, 0x13, 0x60, 0x64, 0xfe, 0x04, 0x18, 0xed,
0x07, 0x11, 0x9f, 0xa1, 0x7d, 0x58, 0x2b, 0x8e, 0xb7, 0xd1, 0x6d, 0x53, 0x33, 0x94, 0x0c, 0xbd,
0x97, 0x8a, 0x79, 0x02, 0xbd, 0xb9, 0x49, 0xb7, 0xb1, 0xa7, 0x7c, 0x00, 0xbe, 0x54, 0xd0, 0x43,
0x68, 0xe7, 0x46, 0xdb, 0x68, 0xa0, 0x84, 0x2c, 0x4e, 0xbb, 0x97, 0x0a, 0xd8, 0x83, 0x6e, 0x61,
0xda, 0x8c, 0x86, 0xda, 0x9f, 0x92, 0x11, 0xf4, 0x52, 0x21, 0xbb, 0xd0, 0xce, 0x0d, 0x7d, 0x8d,
0x15, 0x8b, 0x93, 0xe5, 0xe1, 0xad, 0x12, 0x8a, 0xce, 0xa6, 0x07, 0xd0, 0x2d, 0x8c, 0x68, 0x8d,
0x21, 0x65, 0xe3, 0xe1, 0xe1, 0xed, 0x52, 0x9a, 0x96, 0xf4, 0x04, 0x7a, 0x73, 0x03, 0x5b, 0x13,
0xdc, 0xf2, 0x39, 0xee, 0x52, 0xb7, 0x3e, 0x91, 0x87, 0x9d, 0xeb, 0x50, 0x72, 0x87, 0xbd, 0x38,
0x9e, 0x1d, 0xbe, 0x54, 0x4e, 0xd4, 0x56, 0xed, 0xc3, 0x5a, 0x71, 0x32, 0x6b, 0x84, 0x95, 0xce,
0x6b, 0x2f, 0xbf, 0x39, 0x85, 0x21, 0x6d, 0x76, 0x73, 0xca, 0x66, 0xb7, 0x4b, 0x05, 0x3d, 0x02,
0xd0, 0x8d, 0x8c, 0x4f, 0xc2, 0xf4, 0xc8, 0x16, 0x1a, 0xa8, 0xf4, 0xc8, 0x4a, 0x9a, 0x9e, 0x87,
0x00, 0xaa, 0xff, 0xf0, 0x69, 0xc2, 0xd1, 0x4d, 0x63, 0xc6, 0x5c, 0xd3, 0x33, 0x1c, 0x2c, 0x12,
0x16, 0x04, 0xe0, 0x38, 0xbe, 0x8a, 0x80, 0x8f, 0x01, 0xb2, 0xbe, 0xc6, 0x08, 0x58, 0xe8, 0x74,
0x2e, 0x89, 0x41, 0x27, 0xdf, 0xc5, 0x20, 0xed, 0x6b, 0x49, 0x67, 0xb3, 0x54, 0xc4, 0x03, 0xe8,
0xe4, 0x4b, 0x5b, 0x23, 0xa2, 0xa4, 0xdc, 0x1d, 0x2e, 0x54, 0xa4, 0xe8, 0x91, 0xb9, 0xa9, 0x19,
0xaa, 0x70, 0x53, 0xbf, 0x9b, 0x88, 0xb9, 0x9a, 0xb8, 0x98, 0x49, 0xbe, 0x83, 0x88, 0xf7, 0xa1,
0x93, 0x2f, 0x86, 0x8d, 0x0b, 0x25, 0x05, 0xf2, 0xb0, 0x50, 0x10, 0xa3, 0x87, 0xb0, 0x56, 0x2c,
0x84, 0x51, 0xee, 0x5d, 0x2e, 0x94, 0xc7, 0x43, 0x3d, 0xad, 0xc8, 0xb1, 0xbf, 0x0b, 0x90, 0x15,
0xcc, 0xe6, 0xf8, 0x16, 0x4a, 0xe8, 0x39, 0xad, 0x7b, 0xd0, 0x2d, 0x34, 0x7f, 0x26, 0x51, 0x94,
0x75, 0x84, 0x97, 0xe5, 0xf1, 0x62, 0x93, 0x65, 0x4c, 0x2f, 0x6d, 0xbd, 0x2e, 0xbb, 0x40, 0xf9,
0xe2, 0xde, 0x84, 0xae, 0xa4, 0xe0, 0xff, 0x96, 0x07, 0x9d, 0x2f, 0xe0, 0x73, 0x0f, 0xba, 0xa4,
0xae, 0x5f, 0x2a, 0xe8, 0x00, 0x7a, 0x4f, 0x4c, 0x6d, 0xa6, 0xeb, 0x46, 0x6d, 0x4e, 0x49, 0x9d,
0x3c, 0x1c, 0x96, 0x91, 0xf4, 0xab, 0xfa, 0x04, 0xfa, 0x0b, 0x35, 0x23, 0xba, 0x93, 0x0e, 0xb4,
0x4a, 0x8b, 0xc9, 0xa5, 0x66, 0x1d, 0xc2, 0xfa, 0x7c, 0xc9, 0x88, 0x5e, 0xd6, 0x99, 0xb2, 0xbc,
0x94, 0x5c, 0x2a, 0xea, 0x43, 0x68, 0x9a, 0x12, 0x05, 0xe9, 0xc1, 0xe1, 0x5c, 0xc9, 0xb2, 0x6c,
0xeb, 0x6e, 0xe7, 0xeb, 0x6f, 0xee, 0x54, 0xfe, 0xf1, 0xcd, 0x9d, 0xca, 0xbf, 0xbf, 0xb9, 0x53,
0x39, 0x6d, 0x48, 0xea, 0xbb, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xbe, 0xbd, 0x6e, 0x1d, 0x40,
0x20, 0x00, 0x00,
}

6
virtcontainers/agent.go
View File

@ -216,6 +216,12 @@ type agent interface {
// cpuOnly specifies that we should online cpu or online memory or both
onlineCPUMem(cpus uint32, cpuOnly bool) error
// memHotplugByProbe will notify the guest kernel about memory hotplug event through
// probe interface.
// This function should be called after hot adding Memory and before online memory.
// addr specifies the address of the recently hotplugged or unhotplugged memory device.
memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error
// statsContainer will tell the agent to get stats from a container related to a Sandbox
statsContainer(sandbox *Sandbox, c Container) (*ContainerStats, error)

4
virtcontainers/fc.go
View File

@ -683,8 +683,8 @@ func (fc *firecracker) hypervisorConfig() HypervisorConfig {
return fc.config
}
func (fc *firecracker) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) {
return 0, nil
func (fc *firecracker) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) {
return 0, memoryDevice{}, nil
}
func (fc *firecracker) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) {

5
virtcontainers/hyperstart_agent.go
View File

@ -891,6 +891,11 @@ func (h *hyper) sendCmd(proxyCmd hyperstartProxyCmd) (interface{}, error) {
return h.client.HyperWithTokens(proxyCmd.cmd, tokens, proxyCmd.message)
}
func (h *hyper) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error {
// hyperstart-agent does not support notify memory hotplug event via probe interface
return nil
}
func (h *hyper) onlineCPUMem(cpus uint32, cpuOnly bool) error {
// hyperstart-agent uses udev to online CPUs automatically
return nil

4
virtcontainers/hypervisor.go
View File

@ -91,6 +91,8 @@ const (
type memoryDevice struct {
slot int
sizeMB int
addr uint64
probe bool
}
// Set sets an hypervisor type based on the input string.
@ -592,7 +594,7 @@ type hypervisor interface {
addDevice(devInfo interface{}, devType deviceType) error
hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error)
hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error)
resizeMemory(memMB uint32, memoryBlockSizeMB uint32) (uint32, error)
resizeMemory(memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error)
resizeVCPUs(vcpus uint32) (uint32, uint32, error)
getSandboxConsole(sandboxID string) (string, error)
disconnect()

27
virtcontainers/kata_agent.go
View File

@ -1346,6 +1346,30 @@ func (k *kataAgent) resumeContainer(sandbox *Sandbox, c Container) error {
return err
}
func (k *kataAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error {
if memorySectionSizeMB == uint32(0) {
return fmt.Errorf("memorySectionSizeMB couldn't be zero")
}
// hot-added memory device should be sliced into the size of memory section, which is the basic unit for
// memory hotplug
numSection := uint64(sizeMB / memorySectionSizeMB)
var addrList []uint64
index := uint64(0)
for index < numSection {
k.Logger().WithFields(logrus.Fields{
"addr": fmt.Sprintf("0x%x", addr+(index*uint64(memorySectionSizeMB))<<20),
}).Debugf("notify guest kernel the address of memory device")
addrList = append(addrList, addr+(index*uint64(memorySectionSizeMB))<<20)
index++
}
req := &grpc.MemHotplugByProbeRequest{
MemHotplugProbeAddr: addrList,
}
_, err := k.sendReq(req)
return err
}
func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
req := &grpc.OnlineCPUMemRequest{
Wait: false,
@ -1574,6 +1598,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
k.reqHandlers["grpc.GuestDetailsRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
return k.client.GetGuestDetails(ctx, req.(*grpc.GuestDetailsRequest), opts...)
}
k.reqHandlers["grpc.MemHotplugByProbeRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
return k.client.MemHotplugByProbe(ctx, req.(*grpc.MemHotplugByProbeRequest), opts...)
}
k.reqHandlers["grpc.CopyFileRequest"] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) {
return k.client.CopyFile(ctx, req.(*grpc.CopyFileRequest), opts...)
}

4
virtcontainers/kata_agent_test.go
View File

@ -256,6 +256,10 @@ func (p *gRPCProxy) CopyFile(ctx context.Context, req *pb.CopyFileRequest) (*gpb
return &gpb.Empty{}, nil
}
func (p *gRPCProxy) MemHotplugByProbe(ctx context.Context, req *pb.MemHotplugByProbeRequest) (*gpb.Empty, error) {
return &gpb.Empty{}, nil
}
func gRPCRegister(s *grpc.Server, srv interface{}) {
switch g := srv.(type) {
case *gRPCProxy:

4
virtcontainers/mock_hypervisor.go
View File

@ -84,8 +84,8 @@ func (m *mockHypervisor) getSandboxConsole(sandboxID string) (string, error) {
return "", nil
}
func (m *mockHypervisor) resizeMemory(memMB uint32, memorySectionSizeMB uint32) (uint32, error) {
return 0, nil
func (m *mockHypervisor) resizeMemory(memMB uint32, memorySectionSizeMB uint32, probe bool) (uint32, memoryDevice, error) {
return 0, memoryDevice{}, nil
}
func (m *mockHypervisor) resizeVCPUs(cpus uint32) (uint32, uint32, error) {
return 0, 0, nil

5
virtcontainers/noop_agent.go
View File

@ -91,6 +91,11 @@ func (n *noopAgent) updateContainer(sandbox *Sandbox, c Container, resources spe
return nil
}
// memHotplugByProbe is the Noop agent notify meomory hotplug event via probe interface implementation. It does nothing.
func (n *noopAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionSizeMB uint32) error {
return nil
}
// onlineCPUMem is the Noop agent Container online CPU and Memory implementation. It does nothing.
func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error {
return nil

51
virtcontainers/qemu.go
View File

@ -1240,7 +1240,19 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) {
q.Logger().WithError(err).Error("hotplug memory")
return 0, err
}
// if guest kernel only supports memory hotplug via probe interface, we need to get address of hot-add memory device
if memDev.probe {
memoryDevices, err := q.qmpMonitorCh.qmp.ExecQueryMemoryDevices(q.qmpMonitorCh.ctx)
if err != nil {
return 0, fmt.Errorf("failed to query memory devices: %v", err)
}
if len(memoryDevices) != 0 {
q.Logger().WithField("addr", fmt.Sprintf("0x%x", memoryDevices[len(memoryDevices)-1].Data.Addr)).Debug("recently hot-add memory device")
memDev.addr = memoryDevices[len(memoryDevices)-1].Data.Addr
} else {
return 0, fmt.Errorf("failed to probe address of recently hot-add memory device, no device exists")
}
}
q.state.HotpluggedMemory += memDev.sizeMB
return memDev.sizeMB, q.store.Store(store.Hypervisor, q.state)
}
@ -1370,32 +1382,33 @@ func (q *qemu) disconnect() {
// the memory to remove has to be at least the size of one slot.
// To return memory back we are resizing the VM memory balloon.
// A longer term solution is evaluate solutions like virtio-mem
func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) {
func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) {
currentMemory := q.config.MemorySize + uint32(q.state.HotpluggedMemory)
err := q.qmpSetup()
if err != nil {
return 0, err
return 0, memoryDevice{}, err
}
var addMemDevice memoryDevice
switch {
case currentMemory < reqMemMB:
//hotplug
addMemMB := reqMemMB - currentMemory
memHotplugMB, err := calcHotplugMemMiBSize(addMemMB, memoryBlockSizeMB)
if err != nil {
return currentMemory, err
return currentMemory, memoryDevice{}, err
}
addMemDevice := &memoryDevice{
sizeMB: int(memHotplugMB),
}
data, err := q.hotplugAddDevice(addMemDevice, memoryDev)
addMemDevice.sizeMB = int(memHotplugMB)
addMemDevice.probe = probe
data, err := q.hotplugAddDevice(&addMemDevice, memoryDev)
if err != nil {
return currentMemory, err
return currentMemory, addMemDevice, err
}
memoryAdded, ok := data.(int)
if !ok {
return currentMemory, fmt.Errorf("Could not get the memory added, got %+v", data)
return currentMemory, addMemDevice, fmt.Errorf("Could not get the memory added, got %+v", data)
}
currentMemory += uint32(memoryAdded)
case currentMemory > reqMemMB:
@ -1403,31 +1416,31 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32,
addMemMB := currentMemory - reqMemMB
memHotunplugMB, err := calcHotplugMemMiBSize(addMemMB, memoryBlockSizeMB)
if err != nil {
return currentMemory, err
return currentMemory, memoryDevice{}, err
}
addMemDevice := &memoryDevice{
sizeMB: int(memHotunplugMB),
}
data, err := q.hotplugRemoveDevice(addMemDevice, memoryDev)
addMemDevice.sizeMB = int(memHotunplugMB)
addMemDevice.probe = probe
data, err := q.hotplugRemoveDevice(&addMemDevice, memoryDev)
if err != nil {
return currentMemory, err
return currentMemory, addMemDevice, err
}
memoryRemoved, ok := data.(int)
if !ok {
return currentMemory, fmt.Errorf("Could not get the memory removed, got %+v", data)
return currentMemory, addMemDevice, fmt.Errorf("Could not get the memory removed, got %+v", data)
}
//FIXME: This is to check memory hotplugRemoveDevice reported 0, as this is not supported.
// In the future if this is implemented this validation should be removed.
if memoryRemoved != 0 {
return currentMemory, fmt.Errorf("memory hot unplug is not supported, something went wrong")
return currentMemory, addMemDevice, fmt.Errorf("memory hot unplug is not supported, something went wrong")
}
currentMemory -= uint32(memoryRemoved)
}
// currentMemory is the current memory (updated) of the VM, return to caller to allow verify
// the current VM memory state.
return currentMemory, nil
return currentMemory, addMemDevice, nil
}
// genericAppendBridges appends to devices the given bridges

4
virtcontainers/qemu_test.go
View File

@ -393,9 +393,9 @@ func TestHotplugUnsupportedDeviceType(t *testing.T) {
}
q.store = vcStore
_, err = q.hotplugAddDevice(&memoryDevice{0, 128}, fsDev)
_, err = q.hotplugAddDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev)
assert.Error(err)
_, err = q.hotplugRemoveDevice(&memoryDevice{0, 128}, fsDev)
_, err = q.hotplugRemoveDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev)
assert.Error(err)
}

13
virtcontainers/sandbox.go
View File

@ -416,7 +416,8 @@ func createAssets(ctx context.Context, sandboxConfig *SandboxConfig) error {
func (s *Sandbox) getAndStoreGuestDetails() error {
guestDetailRes, err := s.agent.getGuestDetails(&grpc.GuestDetailsRequest{
MemBlockSize: true,
MemBlockSize: true,
MemHotplugProbe: true,
})
if err != nil {
return err
@ -427,6 +428,7 @@ func (s *Sandbox) getAndStoreGuestDetails() error {
if guestDetailRes.AgentDetails != nil {
s.seccompSupported = guestDetailRes.AgentDetails.SupportsSeccomp
}
s.state.GuestMemoryHotplugProbe = guestDetailRes.SupportMemHotplugProbe
if err = s.store.Store(store.State, s.state); err != nil {
return err
@ -1671,11 +1673,18 @@ func (s *Sandbox) updateResources() error {
// Update Memory
s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory")
newMemory, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB)
newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe)
if err != nil {
return err
}
s.Logger().Debugf("Sandbox memory size: %d Byte", newMemory)
if s.state.GuestMemoryHotplugProbe && updatedMemoryDevice.addr != 0 {
//notify the guest kernel about memory hot-add event, before onlining them
s.Logger().Debugf("notify guest kernel memory hot-add event via probe interface, memory device located at 0x%x", updatedMemoryDevice.addr)
if err := s.agent.memHotplugByProbe(updatedMemoryDevice.addr, uint32(updatedMemoryDevice.sizeMB), s.state.GuestMemoryBlockSizeMB); err != nil {
return err
}
}
if err := s.agent.onlineCPUMem(0, false); err != nil {
return err
}

3
virtcontainers/types/sandbox.go
View File

@ -45,6 +45,9 @@ type State struct {
// GuestMemoryBlockSizeMB is the size of memory block of guestos
GuestMemoryBlockSizeMB uint32 `json:"guestMemoryBlockSize"`
// GuestMemoryHotplugProbe determines whether guest kernel supports memory hotplug probe interface
GuestMemoryHotplugProbe bool `json:"guestMemoryHotplugProbe"`
// CgroupPath is the cgroup hierarchy where sandbox's processes
// including the hypervisor are placed.
CgroupPath string `json:"cgroupPath,omitempty"`

2
virtcontainers/vm.go
View File

@ -373,7 +373,7 @@ func (v *VM) AddCPUs(num uint32) error {
func (v *VM) AddMemory(numMB uint32) error {
if numMB > 0 {
v.logger().Infof("hot adding %d MB memory", numMB)
dev := &memoryDevice{1, int(numMB)}
dev := &memoryDevice{1, int(numMB), 0, false}
if _, err := v.hypervisor.hotplugAddDevice(dev, memoryDev); err != nil {
return err
}