From a1bd33f510db303d23933c92b6192274e93921d1 Mon Sep 17 00:00:00 2001 From: Minhan Xia Date: Tue, 17 May 2016 16:55:04 -0700 Subject: [PATCH] promote sourceRange into service spec --- pkg/api/service/util.go | 36 ++++++++++----- pkg/api/service/util_test.go | 44 +++++++++++++++++-- pkg/api/types.go | 7 ++- pkg/api/v1/types.go | 6 +++ pkg/api/validation/validation.go | 24 +++++++--- pkg/api/validation/validation_test.go | 36 ++++++++++++++- pkg/cloudprovider/cloud.go | 2 +- pkg/cloudprovider/providers/aws/aws.go | 10 ++--- pkg/cloudprovider/providers/aws/aws_test.go | 2 +- pkg/cloudprovider/providers/fake/fake.go | 2 +- pkg/cloudprovider/providers/gce/gce.go | 6 +-- .../providers/openstack/openstack.go | 6 +-- pkg/controller/service/servicecontroller.go | 2 +- pkg/kubectl/cmd/create.go | 13 ++++++ 14 files changed, 160 insertions(+), 36 deletions(-) diff --git a/pkg/api/service/util.go b/pkg/api/service/util.go index a77e5b9c70b..3e00957c331 100644 --- a/pkg/api/service/util.go +++ b/pkg/api/service/util.go @@ -20,6 +20,7 @@ import ( "fmt" "strings" + "k8s.io/kubernetes/pkg/api" netsets "k8s.io/kubernetes/pkg/util/net/sets" ) @@ -37,18 +38,31 @@ func IsAllowAll(ipnets netsets.IPNet) bool { return false } -// GetLoadBalancerSourceRanges verifies and parses the AnnotationLoadBalancerSourceRangesKey annotation from a service, +// GetLoadBalancerSourceRanges first try to parse and verify LoadBalancerSourceRanges field from a service. +// If field not specified, turn to verifies and parses the AnnotationLoadBalancerSourceRangesKey annotation from a service, // extracting the source ranges to allow, and if not present returns a default (allow-all) value. -func GetLoadBalancerSourceRanges(annotations map[string]string) (netsets.IPNet, error) { - val := annotations[AnnotationLoadBalancerSourceRangesKey] - val = strings.TrimSpace(val) - if val == "" { - val = defaultLoadBalancerSourceRanges - } - specs := strings.Split(val, ",") - ipnets, err := netsets.ParseIPNets(specs...) - if err != nil { - return nil, fmt.Errorf("Service annotation %s:%s is not valid. Expecting a comma-separated list of source IP ranges. For example, 10.0.0.0/24,192.168.2.0/24", AnnotationLoadBalancerSourceRangesKey, val) +func GetLoadBalancerSourceRanges(service *api.Service) (netsets.IPNet, error) { + var ipnets netsets.IPNet + var err error + // if SourceRange field is specified, ignore sourceRange annotation + if len(service.Spec.LoadBalancerSourceRanges) > 0 { + specs := service.Spec.LoadBalancerSourceRanges + ipnets, err = netsets.ParseIPNets(specs...) + + if err != nil { + return nil, fmt.Errorf("service.Spec.LoadBalancerSourceRanges: %v is not valid. Expecting a list of IP ranges. For example, 10.0.0.0/24. Error msg: %v", specs, err) + } + } else { + val := service.Annotations[AnnotationLoadBalancerSourceRangesKey] + val = strings.TrimSpace(val) + if val == "" { + val = defaultLoadBalancerSourceRanges + } + specs := strings.Split(val, ",") + ipnets, err = netsets.ParseIPNets(specs...) + if err != nil { + return nil, fmt.Errorf("%s: %s is not valid. Expecting a comma-separated list of source IP ranges. For example, 10.0.0.0/24,192.168.2.0/24", AnnotationLoadBalancerSourceRangesKey, val) + } } return ipnets, nil } diff --git a/pkg/api/service/util_test.go b/pkg/api/service/util_test.go index c77d4f25906..a13f1e588d5 100644 --- a/pkg/api/service/util_test.go +++ b/pkg/api/service/util_test.go @@ -19,14 +19,24 @@ package service import ( "testing" + "k8s.io/kubernetes/pkg/api" netsets "k8s.io/kubernetes/pkg/util/net/sets" + "strings" ) func TestGetLoadBalancerSourceRanges(t *testing.T) { checkError := func(v string) { annotations := make(map[string]string) annotations[AnnotationLoadBalancerSourceRangesKey] = v - _, err := GetLoadBalancerSourceRanges(annotations) + svc := api.Service{} + svc.Annotations = annotations + _, err := GetLoadBalancerSourceRanges(&svc) + if err == nil { + t.Errorf("Expected error parsing: %q", v) + } + svc = api.Service{} + svc.Spec.LoadBalancerSourceRanges = strings.Split(v, ",") + _, err = GetLoadBalancerSourceRanges(&svc) if err == nil { t.Errorf("Expected error parsing: %q", v) } @@ -41,7 +51,15 @@ func TestGetLoadBalancerSourceRanges(t *testing.T) { checkOK := func(v string) netsets.IPNet { annotations := make(map[string]string) annotations[AnnotationLoadBalancerSourceRangesKey] = v - cidrs, err := GetLoadBalancerSourceRanges(annotations) + svc := api.Service{} + svc.Annotations = annotations + cidrs, err := GetLoadBalancerSourceRanges(&svc) + if err != nil { + t.Errorf("Unexpected error parsing: %q", v) + } + svc = api.Service{} + svc.Spec.LoadBalancerSourceRanges = strings.Split(v, ",") + cidrs, err = GetLoadBalancerSourceRanges(&svc) if err != nil { t.Errorf("Unexpected error parsing: %q", v) } @@ -63,7 +81,27 @@ func TestGetLoadBalancerSourceRanges(t *testing.T) { if len(cidrs) != 2 { t.Errorf("Expected two CIDRs: %v", cidrs.StringSlice()) } - cidrs = checkOK("") + // check LoadBalancerSourceRanges not specified + svc := api.Service{} + cidrs, err := GetLoadBalancerSourceRanges(&svc) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if len(cidrs) != 1 { + t.Errorf("Expected exactly one CIDR: %v", cidrs.StringSlice()) + } + if !IsAllowAll(cidrs) { + t.Errorf("Expected default to be allow-all: %v", cidrs.StringSlice()) + } + // check SourceRanges annotation is empty + annotations := make(map[string]string) + annotations[AnnotationLoadBalancerSourceRangesKey] = "" + svc = api.Service{} + svc.Annotations = annotations + cidrs, err = GetLoadBalancerSourceRanges(&svc) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } if len(cidrs) != 1 { t.Errorf("Expected exactly one CIDR: %v", cidrs.StringSlice()) } diff --git a/pkg/api/types.go b/pkg/api/types.go index 96d187d9cfc..57cc66de4d8 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -1751,8 +1751,13 @@ type ServiceSpec struct { // This field will be ignored if the cloud-provider does not support the feature. LoadBalancerIP string `json:"loadBalancerIP,omitempty"` - // Required: Supports "ClientIP" and "None". Used to maintain session affinity. + // Optional: Supports "ClientIP" and "None". Used to maintain session affinity. SessionAffinity ServiceAffinity `json:"sessionAffinity,omitempty"` + + // Optional: If specified and supported by the platform, this will restrict traffic through the cloud-provider + // load-balancer will be restricted to the specified client IPs. This field will be ignored if the + // cloud-provider does not support the feature." + LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty"` } type ServicePort struct { diff --git a/pkg/api/v1/types.go b/pkg/api/v1/types.go index b45b56b9a69..25eb5ee711a 100644 --- a/pkg/api/v1/types.go +++ b/pkg/api/v1/types.go @@ -2092,6 +2092,12 @@ type ServiceSpec struct { // the loadBalancerIP when a load balancer is created. // This field will be ignored if the cloud-provider does not support the feature. LoadBalancerIP string `json:"loadBalancerIP,omitempty" protobuf:"bytes,8,opt,name=loadBalancerIP"` + + // If specified and supported by the platform, this will restrict traffic through the cloud-provider + // load-balancer will be restricted to the specified client IPs. This field will be ignored if the + // cloud-provider does not support the feature." + // More info: http://releases.k8s.io/HEAD/docs/user-guide/services-firewalls.md + LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty" protobuf:"bytes,9,opt,name=loadBalancerSourceRanges"` } // ServicePort contains information on service's port. diff --git a/pkg/api/validation/validation.go b/pkg/api/validation/validation.go index d494f4ec1fb..31461b2a2c5 100644 --- a/pkg/api/validation/validation.go +++ b/pkg/api/validation/validation.go @@ -2055,12 +2055,26 @@ func ValidateService(service *api.Service) field.ErrorList { nodePorts[key] = true } - _, err := apiservice.GetLoadBalancerSourceRanges(service.Annotations) - if err != nil { - v := service.Annotations[apiservice.AnnotationLoadBalancerSourceRangesKey] - allErrs = append(allErrs, field.Invalid(field.NewPath("metadata", "annotations").Key(apiservice.AnnotationLoadBalancerSourceRangesKey), v, "must be a comma separated list of CIDRs e.g. 192.168.0.0/16,10.0.0.0/8")) + // Validate SourceRange field and annotation + _, ok := service.Annotations[apiservice.AnnotationLoadBalancerSourceRangesKey] + if len(service.Spec.LoadBalancerSourceRanges) > 0 || ok { + var fieldPath *field.Path + var val string + if len(service.Spec.LoadBalancerSourceRanges) > 0 { + fieldPath = specPath.Child("LoadBalancerSourceRanges") + val = fmt.Sprintf("%v", service.Spec.LoadBalancerSourceRanges) + } else { + fieldPath = field.NewPath("metadata", "annotations").Key(apiservice.AnnotationLoadBalancerSourceRangesKey) + val = service.Annotations[apiservice.AnnotationLoadBalancerSourceRangesKey] + } + if service.Spec.Type != api.ServiceTypeLoadBalancer { + allErrs = append(allErrs, field.Invalid(fieldPath, "", "may only be used when `type` is 'LoadBalancer'")) + } + _, err := apiservice.GetLoadBalancerSourceRanges(service) + if err != nil { + allErrs = append(allErrs, field.Invalid(fieldPath, val, "must be a list of IP ranges. For example, 10.240.0.0/24,10.250.0.0/24 ")) + } } - return allErrs } diff --git a/pkg/api/validation/validation_test.go b/pkg/api/validation/validation_test.go index 2da1d313277..59773d44542 100644 --- a/pkg/api/validation/validation_test.go +++ b/pkg/api/validation/validation_test.go @@ -3487,6 +3487,7 @@ func TestValidateService(t *testing.T) { { name: "valid LoadBalancer source range annotation", tweakSvc: func(s *api.Service) { + s.Spec.Type = api.ServiceTypeLoadBalancer s.Annotations[service.AnnotationLoadBalancerSourceRangesKey] = "1.2.3.4/8, 5.6.7.8/16" }, numErrs: 0, @@ -3494,6 +3495,7 @@ func TestValidateService(t *testing.T) { { name: "empty LoadBalancer source range annotation", tweakSvc: func(s *api.Service) { + s.Spec.Type = api.ServiceTypeLoadBalancer s.Annotations[service.AnnotationLoadBalancerSourceRangesKey] = "" }, numErrs: 0, @@ -3503,15 +3505,47 @@ func TestValidateService(t *testing.T) { tweakSvc: func(s *api.Service) { s.Annotations[service.AnnotationLoadBalancerSourceRangesKey] = "foo.bar" }, - numErrs: 1, + numErrs: 2, }, { name: "invalid LoadBalancer source range annotation (invalid CIDR)", tweakSvc: func(s *api.Service) { + s.Spec.Type = api.ServiceTypeLoadBalancer s.Annotations[service.AnnotationLoadBalancerSourceRangesKey] = "1.2.3.4/33" }, numErrs: 1, }, + { + name: "invalid source range for non LoadBalancer type service", + tweakSvc: func(s *api.Service) { + s.Spec.LoadBalancerSourceRanges = []string{"1.2.3.4/8", "5.6.7.8/16"} + }, + numErrs: 1, + }, + { + name: "valid LoadBalancer source range", + tweakSvc: func(s *api.Service) { + s.Spec.Type = api.ServiceTypeLoadBalancer + s.Spec.LoadBalancerSourceRanges = []string{"1.2.3.4/8", "5.6.7.8/16"} + }, + numErrs: 0, + }, + { + name: "empty LoadBalancer source range", + tweakSvc: func(s *api.Service) { + s.Spec.Type = api.ServiceTypeLoadBalancer + s.Spec.LoadBalancerSourceRanges = []string{" "} + }, + numErrs: 1, + }, + { + name: "invalid LoadBalancer source range", + tweakSvc: func(s *api.Service) { + s.Spec.Type = api.ServiceTypeLoadBalancer + s.Spec.LoadBalancerSourceRanges = []string{"foo.bar"} + }, + numErrs: 1, + }, } for _, tc := range testCases { diff --git a/pkg/cloudprovider/cloud.go b/pkg/cloudprovider/cloud.go index cda6db22d73..0190ed50ac7 100644 --- a/pkg/cloudprovider/cloud.go +++ b/pkg/cloudprovider/cloud.go @@ -84,7 +84,7 @@ type LoadBalancer interface { GetLoadBalancer(service *api.Service) (status *api.LoadBalancerStatus, exists bool, err error) // EnsureLoadBalancer creates a new load balancer 'name', or updates the existing one. Returns the status of the balancer // Implementations must treat the *api.Service parameter as read-only and not modify it. - EnsureLoadBalancer(service *api.Service, hosts []string, annotations map[string]string) (*api.LoadBalancerStatus, error) + EnsureLoadBalancer(service *api.Service, hosts []string) (*api.LoadBalancerStatus, error) // UpdateLoadBalancer updates hosts under the specified load balancer. // Implementations must treat the *api.Service parameter as read-only and not modify it. UpdateLoadBalancer(service *api.Service, hosts []string) error diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index 3ea6dfcfd19..064b19f1508 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -2120,9 +2120,9 @@ func buildListener(port api.ServicePort, annotations map[string]string) (*elb.Li } // EnsureLoadBalancer implements LoadBalancer.EnsureLoadBalancer -func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, annotations map[string]string) (*api.LoadBalancerStatus, error) { +func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string) (*api.LoadBalancerStatus, error) { glog.V(2).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)", - apiService.Namespace, apiService.Name, s.region, apiService.Spec.LoadBalancerIP, apiService.Spec.Ports, hosts, annotations) + apiService.Namespace, apiService.Name, s.region, apiService.Spec.LoadBalancerIP, apiService.Spec.Ports, hosts, apiService.Annotations) if apiService.Spec.SessionAffinity != api.ServiceAffinityNone { // ELB supports sticky sessions, but only when configured for HTTP/HTTPS @@ -2143,7 +2143,7 @@ func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, a glog.Errorf("Ignoring port without NodePort defined: %v", port) continue } - listener, err := buildListener(port, annotations) + listener, err := buildListener(port, apiService.Annotations) if err != nil { return nil, err } @@ -2159,14 +2159,14 @@ func (s *AWSCloud) EnsureLoadBalancer(apiService *api.Service, hosts []string, a return nil, err } - sourceRanges, err := service.GetLoadBalancerSourceRanges(annotations) + sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService) if err != nil { return nil, err } // Determine if this is tagged as an Internal ELB internalELB := false - internalAnnotation := annotations[ServiceAnnotationLoadBalancerInternal] + internalAnnotation := apiService.Annotations[ServiceAnnotationLoadBalancerInternal] if internalAnnotation != "" { if internalAnnotation != "0.0.0.0/0" { return nil, fmt.Errorf("annotation %q=%q detected, but the only value supported currently is 0.0.0.0/0", ServiceAnnotationLoadBalancerInternal, internalAnnotation) diff --git a/pkg/cloudprovider/providers/aws/aws_test.go b/pkg/cloudprovider/providers/aws/aws_test.go index 9e8df8103eb..eaa21b4fcd6 100644 --- a/pkg/cloudprovider/providers/aws/aws_test.go +++ b/pkg/cloudprovider/providers/aws/aws_test.go @@ -1197,7 +1197,7 @@ func TestDescribeLoadBalancerOnEnsure(t *testing.T) { c, _ := newAWSCloud(strings.NewReader("[global]"), awsServices) awsServices.elb.expectDescribeLoadBalancers("aid") - c.EnsureLoadBalancer(&api.Service{ObjectMeta: api.ObjectMeta{Name: "myservice", UID: "id"}}, []string{}, map[string]string{}) + c.EnsureLoadBalancer(&api.Service{ObjectMeta: api.ObjectMeta{Name: "myservice", UID: "id"}}, []string{}) } func TestBuildListener(t *testing.T) { diff --git a/pkg/cloudprovider/providers/fake/fake.go b/pkg/cloudprovider/providers/fake/fake.go index 6bc0a0e761b..10c58991c5f 100644 --- a/pkg/cloudprovider/providers/fake/fake.go +++ b/pkg/cloudprovider/providers/fake/fake.go @@ -130,7 +130,7 @@ func (f *FakeCloud) GetLoadBalancer(service *api.Service) (*api.LoadBalancerStat // EnsureLoadBalancer is a test-spy implementation of LoadBalancer.EnsureLoadBalancer. // It adds an entry "create" into the internal method call record. -func (f *FakeCloud) EnsureLoadBalancer(service *api.Service, hosts []string, annotations map[string]string) (*api.LoadBalancerStatus, error) { +func (f *FakeCloud) EnsureLoadBalancer(service *api.Service, hosts []string) (*api.LoadBalancerStatus, error) { f.addCall("create") if f.Balancers == nil { f.Balancers = make(map[string]FakeBalancer) diff --git a/pkg/cloudprovider/providers/gce/gce.go b/pkg/cloudprovider/providers/gce/gce.go index 20505d7098e..aabb8a48bd3 100644 --- a/pkg/cloudprovider/providers/gce/gce.go +++ b/pkg/cloudprovider/providers/gce/gce.go @@ -480,7 +480,7 @@ func isHTTPErrorCode(err error, code int) bool { // Due to an interesting series of design decisions, this handles both creating // new load balancers and updating existing load balancers, recognizing when // each is needed. -func (gce *GCECloud) EnsureLoadBalancer(apiService *api.Service, hostNames []string, annotations map[string]string) (*api.LoadBalancerStatus, error) { +func (gce *GCECloud) EnsureLoadBalancer(apiService *api.Service, hostNames []string) (*api.LoadBalancerStatus, error) { if len(hostNames) == 0 { return nil, fmt.Errorf("Cannot EnsureLoadBalancer() with no hosts") } @@ -501,7 +501,7 @@ func (gce *GCECloud) EnsureLoadBalancer(apiService *api.Service, hostNames []str affinityType := apiService.Spec.SessionAffinity serviceName := types.NamespacedName{Namespace: apiService.Namespace, Name: apiService.Name} - glog.V(2).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)", loadBalancerName, gce.region, loadBalancerIP, portStr, hosts, serviceName, annotations) + glog.V(2).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v, %v)", loadBalancerName, gce.region, loadBalancerIP, portStr, hosts, serviceName, apiService.Annotations) // Check if the forwarding rule exists, and if so, what its IP is. fwdRuleExists, fwdRuleNeedsUpdate, fwdRuleIP, err := gce.forwardingRuleNeedsUpdate(loadBalancerName, gce.region, loadBalancerIP, ports) @@ -611,7 +611,7 @@ func (gce *GCECloud) EnsureLoadBalancer(apiService *api.Service, hostNames []str // is because the forwarding rule is used as the indicator that the load // balancer is fully created - it's what getLoadBalancer checks for. // Check if user specified the allow source range - sourceRanges, err := service.GetLoadBalancerSourceRanges(annotations) + sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService) if err != nil { return nil, err } diff --git a/pkg/cloudprovider/providers/openstack/openstack.go b/pkg/cloudprovider/providers/openstack/openstack.go index 8c5b8ac4cb4..30c79055059 100644 --- a/pkg/cloudprovider/providers/openstack/openstack.go +++ b/pkg/cloudprovider/providers/openstack/openstack.go @@ -666,8 +666,8 @@ func (lb *LoadBalancer) GetLoadBalancer(service *api.Service) (*api.LoadBalancer // a list of regions (from config) and query/create loadbalancers in // each region. -func (lb *LoadBalancer) EnsureLoadBalancer(apiService *api.Service, hosts []string, annotations map[string]string) (*api.LoadBalancerStatus, error) { - glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v)", apiService.Namespace, apiService.Name, apiService.Spec.LoadBalancerIP, apiService.Spec.Ports, hosts, annotations) +func (lb *LoadBalancer) EnsureLoadBalancer(apiService *api.Service, hosts []string) (*api.LoadBalancerStatus, error) { + glog.V(4).Infof("EnsureLoadBalancer(%v, %v, %v, %v, %v, %v)", apiService.Namespace, apiService.Name, apiService.Spec.LoadBalancerIP, apiService.Spec.Ports, hosts, apiService.Annotations) ports := apiService.Spec.Ports if len(ports) > 1 { @@ -693,7 +693,7 @@ func (lb *LoadBalancer) EnsureLoadBalancer(apiService *api.Service, hosts []stri return nil, fmt.Errorf("unsupported load balancer affinity: %v", affinity) } - sourceRanges, err := service.GetLoadBalancerSourceRanges(annotations) + sourceRanges, err := service.GetLoadBalancerSourceRanges(apiService) if err != nil { return nil, err } diff --git a/pkg/controller/service/servicecontroller.go b/pkg/controller/service/servicecontroller.go index 6803d8c894e..0fb1820869e 100644 --- a/pkg/controller/service/servicecontroller.go +++ b/pkg/controller/service/servicecontroller.go @@ -404,7 +404,7 @@ func (s *ServiceController) createLoadBalancer(service *api.Service) error { // - Only one protocol supported per service // - Not all cloud providers support all protocols and the next step is expected to return // an error for unsupported protocols - status, err := s.balancer.EnsureLoadBalancer(service, hostsFromNodeList(&nodes), service.ObjectMeta.Annotations) + status, err := s.balancer.EnsureLoadBalancer(service, hostsFromNodeList(&nodes)) if err != nil { return err } else { diff --git a/pkg/kubectl/cmd/create.go b/pkg/kubectl/cmd/create.go index 05cc2e3de29..24dfc74bddf 100644 --- a/pkg/kubectl/cmd/create.go +++ b/pkg/kubectl/cmd/create.go @@ -24,6 +24,7 @@ import ( "github.com/spf13/cobra" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/service" "k8s.io/kubernetes/pkg/api/unversioned" "k8s.io/kubernetes/pkg/kubectl" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" @@ -167,6 +168,18 @@ See http://releases.k8s.io/HEAD/docs/user-guide/services-firewalls.md for more d makePortsString(obj.Spec.Ports, true)) out.Write([]byte(msg)) } + + _, ok := obj.Annotations[service.AnnotationLoadBalancerSourceRangesKey] + if ok { + msg := fmt.Sprintf( + `You are using service annotation [service.beta.kubernetes.io/load-balancer-source-ranges]. +It has been promoted to field [loadBalancerSourceRanges] in service spec. This annotation will be deprecated in the future. +Please use the loadBalancerSourceRanges field instead. + +See http://releases.k8s.io/HEAD/docs/user-guide/services-firewalls.md for more details. +`) + out.Write([]byte(msg)) + } } }