From b8959bf146a8d7bbbd2bbd7fa676f2e3fe711c3a Mon Sep 17 00:00:00 2001 From: Brian Grant Date: Fri, 24 Oct 2014 00:03:23 +0000 Subject: [PATCH] Initial cut of simple config generation/transformation tools --- contrib/enscope/README.md | 214 +++++++++++++++++++++ contrib/enscope/enscope.go | 193 +++++++++++++++++++ contrib/flags2yaml/README.md | 8 + contrib/flags2yaml/flags2yaml.go | 41 ++++ contrib/simplegen/README.md | 61 ++++++ contrib/simplegen/simplegen.go | 266 ++++++++++++++++++++++++++ contrib/srvexpand/README.md | 216 +++++++++++++++++++++ contrib/srvexpand/srvexpand.go | 313 +++++++++++++++++++++++++++++++ 8 files changed, 1312 insertions(+) create mode 100644 contrib/enscope/README.md create mode 100644 contrib/enscope/enscope.go create mode 100644 contrib/flags2yaml/README.md create mode 100644 contrib/flags2yaml/flags2yaml.go create mode 100644 contrib/simplegen/README.md create mode 100644 contrib/simplegen/simplegen.go create mode 100644 contrib/srvexpand/README.md create mode 100644 contrib/srvexpand/srvexpand.go diff --git a/contrib/enscope/README.md b/contrib/enscope/README.md new file mode 100644 index 00000000000..78b4dcba8e3 --- /dev/null +++ b/contrib/enscope/README.md @@ -0,0 +1,214 @@ +# enscope + +Typically a configuration is comprised of a set of objects (e.g., a simple service, replication controller, and template). Within that configuration, objects may refer to each other by object reference (as with replication controller to template, as of v1beta3) and/or by label selector (as with service and replication controller to pods generated from the pod template). + +If one wants to create multiple instances of that configuration, such as for dev and prod deployments (aka horizontal composition) or to embed in composite macro-services (aka hierarchical composition), the names must be uniquified and the label selectors must be scoped to just one instance of the configuration, by adding deployment-specific labels and label selector requirements (e.g., env=prod, app==coolapp). + +Enscope is a standalone minimally schema-aware transformation pass for this purpose. It identifies all names, references, label sets, and label selectors that must be uniquified/scoped. An alternative would be to use a generic templating mechanism, such as [Mustache](http://mustache.github.io), but the scoping mechanism would need to be reimplemented in every templating language, and it would also make configurations more complex. + +Currently targets only v1beta3, which isn't yet fully implemented. + +## Usage +``` +$ enscope specFilename configFilename +``` + +## Scope schema +``` +type EnscopeSpec struct { + NameSuffix string `json:"nameSuffix,omitempty" yaml:"nameSuffix,omitempty"` + Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` +} +``` + +## Example +The following name suffix and labels applied to the output from the [contrib/srvexpand example](../srvexpand/README.md): +``` +nameSuffix: -coolapp-prod +labels: + app: coolapp + env: prod +``` +Output: +``` +- apiVersion: v1beta3 + kind: Service + metadata: + creationTimestamp: "null" + labels: + app: coolapp + env: prod + service: foo + name: foo-coolapp-prod + spec: + containerPort: 8080 + port: 80 + selector: + app: coolapp + env: prod + service: foo + status: {} +- apiVersion: v1beta3 + kind: PodTemplate + metadata: + creationTimestamp: "null" + labels: + app: coolapp + env: prod + service: foo + track: canary + name: foo-canary-coolapp-prod + spec: + metadata: + creationTimestamp: "null" + labels: + app: coolapp + env: prod + service: foo + track: canary + spec: + containers: + - image: me/coolappserver:canary + imagePullPolicy: "" + name: web + restartPolicy: {} + volumes: [] +- apiVersion: v1beta3 + kind: ReplicationController + metadata: + creationTimestamp: "null" + labels: + app: coolapp + env: prod + service: foo + track: canary + name: foo-canary-coolapp-prod + spec: + replicas: 2 + selector: + app: coolapp + env: prod + service: foo + track: canary + template: + apiVersion: v1beta3 + kind: PodTemplate + name: foo-canary-coolapp-prod + status: + replicas: 0 +- apiVersion: v1beta3 + kind: PodTemplate + metadata: + creationTimestamp: "null" + labels: + app: coolapp + env: prod + service: foo + track: stable + name: foo-stable-coolapp-prod + spec: + metadata: + creationTimestamp: "null" + labels: + app: coolapp + env: prod + service: foo + track: stable + spec: + containers: + - image: me/coolappserver:stable + imagePullPolicy: "" + name: web + restartPolicy: {} + volumes: [] +- apiVersion: v1beta3 + kind: ReplicationController + metadata: + creationTimestamp: "null" + labels: + app: coolapp + env: prod + service: foo + track: stable + name: foo-stable-coolapp-prod + spec: + replicas: 10 + selector: + app: coolapp + env: prod + service: foo + track: stable + template: + apiVersion: v1beta3 + kind: PodTemplate + name: foo-stable-coolapp-prod + status: + replicas: 0 +- apiVersion: v1beta3 + kind: Service + metadata: + creationTimestamp: "null" + labels: + app: coolapp + env: prod + service: bar + name: bar-coolapp-prod + spec: + containerPort: 3306 + port: 3306 + selector: + app: coolapp + env: prod + service: bar + status: {} +- apiVersion: v1beta3 + kind: PodTemplate + metadata: + creationTimestamp: "null" + labels: + app: coolapp + env: prod + service: bar + track: solo + name: bar-solo-coolapp-prod + spec: + metadata: + creationTimestamp: "null" + labels: + app: coolapp + env: prod + service: bar + track: solo + spec: + containers: + - image: mysql + imagePullPolicy: "" + name: db + restartPolicy: {} + volumes: + - name: dbdir + source: null +- apiVersion: v1beta3 + kind: ReplicationController + metadata: + creationTimestamp: "null" + labels: + app: coolapp + env: prod + service: bar + track: solo + name: bar-solo-coolapp-prod + spec: + replicas: 1 + selector: + app: coolapp + env: prod + service: bar + track: solo + template: + apiVersion: v1beta3 + kind: PodTemplate + name: bar-solo-coolapp-prod + status: + replicas: 0 +``` \ No newline at end of file diff --git a/contrib/enscope/enscope.go b/contrib/enscope/enscope.go new file mode 100644 index 00000000000..dfebbb50283 --- /dev/null +++ b/contrib/enscope/enscope.go @@ -0,0 +1,193 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + + "github.com/golang/glog" + "gopkg.in/v1/yaml" +) + +const usage = "usage: enscope specFilename configFilename" + +func checkErr(err error) { + if err != nil { + glog.Fatalf("%v", err) + } +} + +// TODO: If name suffix is not specified, deterministically generate it by hashing the labels. + +type EnscopeSpec struct { + NameSuffix string `json:"nameSuffix,omitempty" yaml:"nameSuffix,omitempty"` + Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` +} + +func main() { + if len(os.Args) != 3 { + checkErr(fmt.Errorf(usage)) + } + specFilename := os.Args[1] + configFilename := os.Args[2] + + specData, err := ReadConfigData(specFilename) + checkErr(err) + + spec := EnscopeSpec{} + err = yaml.Unmarshal(specData, &spec) + checkErr(err) + + configData, err := ReadConfigData(configFilename) + checkErr(err) + + var data interface{} + + err = yaml.Unmarshal([]byte(configData), &data) + checkErr(err) + + xData, err := enscope("", spec, data) + checkErr(err) + + out, err := yaml.Marshal(xData) + checkErr(err) + + fmt.Print(string(out)) +} + +func enscope(parent string, spec EnscopeSpec, in interface{}) (out interface{}, err error) { + var ok bool + switch in.(type) { + case map[interface{}]interface{}: + o := make(map[interface{}]interface{}) + for k, v := range in.(map[interface{}]interface{}) { + var kstring string + if kstring, ok = k.(string); !ok { + kstring = parent + } + v, err = enscope(kstring, spec, v) + if err != nil { + return nil, err + } + o[k] = v + } + var ifc interface{} + var name string + // TODO: Figure out a more general way to identify references + if parent == "metadata" || parent == "template" { + if ifc, ok = o["name"]; ok { + if name, ok = ifc.(string); ok { + o["name"] = name + spec.NameSuffix + } + } + if ifc, ok = o["labels"]; ok { + var labels map[interface{}]interface{} + if labels, ok = ifc.(map[interface{}]interface{}); ok { + for k, v := range spec.Labels { + labels[k] = v + } + o["labels"] = labels + } + } + } + if parent == "spec" { + // Note that nodeSelector doesn't match, so we won't modify it + if ifc, ok = o["selector"]; ok { + var selector map[interface{}]interface{} + if selector, ok = ifc.(map[interface{}]interface{}); ok { + for k, v := range spec.Labels { + selector[k] = v + } + o["selector"] = selector + } + } + } + return o, nil + case []interface{}: + in1 := in.([]interface{}) + len1 := len(in1) + o := make([]interface{}, len1) + for i := 0; i < len1; i++ { + o[i], err = enscope(parent, spec, in1[i]) + if err != nil { + return nil, err + } + } + return o, nil + default: + return in, nil + } + return in, nil +} + +////////////////////////////////////////////////////////////////////// + +// Client tool utility functions copied from kubectl, kubecfg, and podex. +// This should probably be a separate package, but the right solution is +// to refactor the copied code and delete it from here. + +func ReadConfigData(location string) ([]byte, error) { + if len(location) == 0 { + return nil, fmt.Errorf("Location given but empty") + } + + if location == "-" { + // Read from stdin. + data, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return nil, err + } + + if len(data) == 0 { + return nil, fmt.Errorf(`Read from stdin specified ("-") but no data found`) + } + + return data, nil + } + + // Use the location as a file path or URL. + return readConfigDataFromLocation(location) +} + +func readConfigDataFromLocation(location string) ([]byte, error) { + // we look for http:// or https:// to determine if valid URL, otherwise do normal file IO + if strings.Index(location, "http://") == 0 || strings.Index(location, "https://") == 0 { + resp, err := http.Get(location) + if err != nil { + return nil, fmt.Errorf("Unable to access URL %s: %v\n", location, err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("Unable to read URL, server reported %d %s", resp.StatusCode, resp.Status) + } + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("Unable to read URL %s: %v\n", location, err) + } + return data, nil + } else { + data, err := ioutil.ReadFile(location) + if err != nil { + return nil, fmt.Errorf("Unable to read %s: %v\n", location, err) + } + return data, nil + } +} diff --git a/contrib/flags2yaml/README.md b/contrib/flags2yaml/README.md new file mode 100644 index 00000000000..6eff0432d4e --- /dev/null +++ b/contrib/flags2yaml/README.md @@ -0,0 +1,8 @@ +# flags2yaml + +`flags2yaml` is a command-line tool to generate flat YAML from command-line flags + +### Usage +``` +$ flags2yaml image=dockerfile/nginx | simplegen - | cluster/kubectl.sh createall -f - +``` diff --git a/contrib/flags2yaml/flags2yaml.go b/contrib/flags2yaml/flags2yaml.go new file mode 100644 index 00000000000..bd63620dbee --- /dev/null +++ b/contrib/flags2yaml/flags2yaml.go @@ -0,0 +1,41 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// flags2yaml is a tool to generate flat YAML from command-line flags +// +// $ flags2yaml name=foo image=busybox | simplegen - | kubectl apply - + +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/golang/glog" +) + +const usage = "usage: flags2yaml key1=value1 [key2=value2 ...]" + +func main() { + for i := 1; i < len(os.Args); i++ { + pieces := strings.Split(os.Args[i], "=") + if len(pieces) != 2 { + glog.Fatalf("Bad arg: %s", os.Args[i]) + } + fmt.Printf("%s: %s\n", pieces[0], pieces[1]) + } +} diff --git a/contrib/simplegen/README.md b/contrib/simplegen/README.md new file mode 100644 index 00000000000..f2031756f78 --- /dev/null +++ b/contrib/simplegen/README.md @@ -0,0 +1,61 @@ +# Simple configuration generation tool + +`simplegen` is a command-line tool to expand a simple container +description into Kubernetes API objects, such as for consumption by +kubectl or other tools. + +Currently targets only v1beta1. + +### Usage +``` +$ simplegen myservice.json +$ simplegen myservice.yaml +$ simplegen - +$ simplegen http://some.blog.site.com/k8s-example.yaml +``` + +### Schema +``` +// Optional: Defaults to image base name if not specified +Name string `yaml:"name,omitempty" json:"name,omitempty"` +// Required. +Image string `yaml:"image" json:"image"` +// Optional: Defaults to one +Replicas int `yaml:"replicas,omitempty" json:"replicas,omitempty"` +// Optional: Creates a service if specified: servicePort:containerPort +PortSpec string `yaml:"portSpec,omitempty" json:"portSpec,omitempty"` +``` + +### Example +``` +redismaster.yaml: +name: redismaster +image: dockerfile/redis +portSpec: 6379:6379 + +redisslave.yaml: +name: redisslave +image: brendanburns/redis-slave +replicas: 2 +portSpec: 10001:6379 +``` +Output: +``` +$ simplegen redismaster.yaml | cluster/kubectl.sh createall -f - +$ simplegen redisslave.yaml | cluster/kubectl.sh createall -f - +$ cluster/kubectl.sh get services +NAME LABELS SELECTOR IP PORT +kubernetes-ro component=apiserver,provider=kubernetes 10.0.0.2 80 +kubernetes component=apiserver,provider=kubernetes 10.0.0.1 443 +redismaster simpleservice=redismaster simpleservice=redismaster 10.0.0.3 6379 +redisslave simpleservice=redisslave simpleservice=redisslave 10.0.0.4 10001 +$ cluster/kubectl.sh get replicationcontrollers +NAME IMAGE(S) SELECTOR REPLICAS +redismaster dockerfile/redis simpleservice=redismaster 1 +redisslave brendanburns/redis-slave simpleservice=redisslave 2 +$ cluster/kubectl.sh get pods +NAME IMAGE(S) HOST LABELS STATUS +89adf546-6457-11e4-9f97-42010af0d824 dockerfile/redis kubernetes-minion-3/146.148.79.186 simpleservice=redismaster Running +93a555ac-6457-11e4-9f97-42010af0d824 brendanburns/redis-slave kubernetes-minion-4/130.211.186.4 simpleservice=redisslave Running +93a862d1-6457-11e4-9f97-42010af0d824 brendanburns/redis-slave kubernetes-minion-1/130.211.117.14 simpleservice=redisslave Running +``` diff --git a/contrib/simplegen/simplegen.go b/contrib/simplegen/simplegen.go new file mode 100644 index 00000000000..ea17cb4967f --- /dev/null +++ b/contrib/simplegen/simplegen.go @@ -0,0 +1,266 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// simplegen is a tool to generate simple services from a simple description +// +// $ simplegen myservice.json | kubectl createall -f - +// $ simplegen myservice.yaml | kubectl createall -f - +// +// This is completely separate from kubectl at the moment, until we figure out +// what the right integration approach is. + +package main + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "strconv" + "strings" + + // TODO: handle multiple versions correctly + "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta1" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/golang/glog" + "gopkg.in/v1/yaml" +) + +// TODO: Also handle lists of simple services, and multiple input files + +const usage = "usage: simplegen filename" + +type SimpleService struct { + // Optional: Defaults to image base name if not specified + Name string `yaml:"name,omitempty" json:"name,omitempty"` + // Required. + Image string `yaml:"image" json:"image"` + // Optional: Defaults to one + Replicas int `yaml:"replicas,omitempty" json:"replicas,omitempty"` + // Optional: Creates a service if specified: servicePort:containerPort + PortSpec string `yaml:"portSpec,omitempty" json:"portSpec,omitempty"` +} + +func checkErr(err error) { + if err != nil { + glog.Fatalf("%v", err) + } +} + +func main() { + if len(os.Args) != 2 { + checkErr(fmt.Errorf(usage)) + } + filename := os.Args[1] + + simpleService := readSimpleService(filename) + + var servicePort, containerPort int + var err error + var ports []v1beta1.Port + if simpleService.PortSpec != "" { + servicePort, containerPort, err = portsFromString(simpleService.PortSpec) + checkErr(err) + + generateService(simpleService.Name, servicePort, containerPort) + + // For replication controller + ports = []v1beta1.Port{{Name: "main", ContainerPort: containerPort}} + } + + generateReplicationController(simpleService.Name, simpleService.Image, simpleService.Replicas, ports) +} + +func generateService(name string, servicePort int, containerPort int) { + svc := []v1beta1.Service{{ + TypeMeta: v1beta1.TypeMeta{APIVersion: "v1beta1", Kind: "Service", ID: name}, + Port: servicePort, + ContainerPort: util.NewIntOrStringFromInt(containerPort), + Labels: map[string]string{ + "simpleservice": name, + }, + Selector: map[string]string{ + "simpleservice": name, + }, + }} + + svcOutData, err := yaml.Marshal(svc) + checkErr(err) + + fmt.Print(string(svcOutData)) +} + +func generateReplicationController(name string, image string, replicas int, ports []v1beta1.Port) { + controller := []v1beta1.ReplicationController{{ + TypeMeta: v1beta1.TypeMeta{APIVersion: "v1beta1", Kind: "ReplicationController", ID: name}, + DesiredState: v1beta1.ReplicationControllerState{ + Replicas: replicas, + ReplicaSelector: map[string]string{ + "simpleservice": name, + }, + PodTemplate: v1beta1.PodTemplate{ + DesiredState: v1beta1.PodState{ + Manifest: v1beta1.ContainerManifest{ + Version: "v1beta2", + Containers: []v1beta1.Container{ + { + Name: name, + Image: image, + Ports: ports, + }, + }, + }, + }, + Labels: map[string]string{ + "simpleservice": name, + }, + }, + }, + Labels: map[string]string{ + "simpleservice": name, + }, + }} + controllerOutData, err := yaml.Marshal(controller) + checkErr(err) + + fmt.Print(string(controllerOutData)) +} + +func readSimpleService(filename string) SimpleService { + inData, err := ReadConfigData(filename) + checkErr(err) + + simpleService := SimpleService{} + err = yaml.Unmarshal(inData, &simpleService) + checkErr(err) + + if simpleService.Name == "" { + _, simpleService.Name = ParseDockerImage(simpleService.Image) + // TODO: encode/scrub the name + } + simpleService.Name = strings.ToLower(simpleService.Name) + + // TODO: Validate the image name and extract exposed ports + + // TODO: Do more validation + if !util.IsDNSLabel(simpleService.Name) { + checkErr(fmt.Errorf("name (%s) is not a valid DNS label", simpleService.Name)) + } + + if simpleService.Replicas == 0 { + simpleService.Replicas = 1 + } + + return simpleService +} + +// TODO: what defaults make the most sense? +func portsFromString(spec string) (servicePort int, containerPort int, err error) { + if spec == "" { + return 0, 0, fmt.Errorf("empty port spec") + } + pieces := strings.Split(spec, ":") + if len(pieces) != 2 { + glog.Infof("Bad port spec: %s", spec) + return 0, 0, fmt.Errorf("Bad port spec: %s", spec) + } + servicePort, err = strconv.Atoi(pieces[0]) + if err != nil { + glog.Errorf("Service port is not integer: %s %v", pieces[0], err) + return 0, 0, err + } + if servicePort < 1 { + glog.Errorf("Service port is not valid: %d", servicePort) + return 0, 0, err + } + containerPort, err = strconv.Atoi(pieces[1]) + if err != nil { + glog.Errorf("Container port is not integer: %s %v", pieces[1], err) + return 0, 0, err + } + if containerPort < 1 { + glog.Errorf("Container port is not valid: %d", containerPort) + return 0, 0, err + } + + return +} + +////////////////////////////////////////////////////////////////////// + +// Client tool utility functions copied from kubectl, kubecfg, and podex. +// This should probably be a separate package, but the right solution is +// to refactor the copied code and delete it from here. + +func ReadConfigData(location string) ([]byte, error) { + if len(location) == 0 { + return nil, fmt.Errorf("Location given but empty") + } + + if location == "-" { + // Read from stdin. + data, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return nil, err + } + + if len(data) == 0 { + return nil, fmt.Errorf(`Read from stdin specified ("-") but no data found`) + } + + return data, nil + } + + // Use the location as a file path or URL. + return readConfigDataFromLocation(location) +} + +func readConfigDataFromLocation(location string) ([]byte, error) { + // we look for http:// or https:// to determine if valid URL, otherwise do normal file IO + if strings.Index(location, "http://") == 0 || strings.Index(location, "https://") == 0 { + resp, err := http.Get(location) + if err != nil { + return nil, fmt.Errorf("Unable to access URL %s: %v\n", location, err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("Unable to read URL, server reported %d %s", resp.StatusCode, resp.Status) + } + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("Unable to read URL %s: %v\n", location, err) + } + return data, nil + } else { + data, err := ioutil.ReadFile(location) + if err != nil { + return nil, fmt.Errorf("Unable to read %s: %v\n", location, err) + } + return data, nil + } +} + +// ParseDockerImage split a docker image name of the form [REGISTRYHOST/][USERNAME/]NAME[:TAG] +// TODO: handle the TAG +// Returns array of images name parts and base image name +func ParseDockerImage(imageName string) (parts []string, baseName string) { + // Parse docker image name + // IMAGE: [REGISTRYHOST/][USERNAME/]NAME[:TAG] + // NAME: [a-z0-9-_.] + parts = strings.Split(imageName, "/") + baseName = parts[len(parts)-1] + return +} diff --git a/contrib/srvexpand/README.md b/contrib/srvexpand/README.md new file mode 100644 index 00000000000..7e1588fc66f --- /dev/null +++ b/contrib/srvexpand/README.md @@ -0,0 +1,216 @@ +# srvexpand + +srvexpand is a tool to generate non-trivial but regular services +from a description free of most boilerplate. + +Currently targets only v1beta3, which isn't yet fully implemented. + +## Usage +``` +$ srvexpand myservice.json +$ srvexpand myservice.yaml +``` + +## Schema +``` +type HierarchicalController struct { + // Optional: Defaults to one + Replicas int `yaml:"replicas,omitempty" json:"replicas,omitempty"` + // Spec defines the behavior of a pod. + Spec v1beta3.PodSpec `json:"spec,omitempty" yaml:"spec,omitempty"` +} + +type ControllerMap map[string]HierarchicalController + +type HierarchicalService struct { + // Optional: Creates a service if specified: servicePort:containerPort + // TODO: Support multiple protocols + PortSpec string `yaml:"portSpec,omitempty" json:"portSpec,omitempty"` + // Map of replication controllers to create + ControllerMap ControllerMap `json:"controllers,omitempty" yaml:"controllers,omitempty"` +} + +type ServiceMap map[string]HierarchicalService +``` + +## Example +``` +foo: + portSpec: 80:8080 + controllers: + canary: + replicas: 2 + spec: + containers: + - name: web + image: me/myappserver:canary + stable: + replicas: 10 + spec: + containers: + - name: web + image: me/myappserver:stable +bar: + portSpec: 3306:3306 + controllers: + solo: + replicas: 1 + spec: + containers: + - name: db + image: mysql + volumes: + - name: dbdir +``` +Output: +``` +- kind: Service + apiVersion: v1beta3 + metadata: + name: foo + creationTimestamp: "null" + labels: + service: foo + spec: + port: 80 + selector: + service: foo + containerPort: 8080 + status: {} +- kind: PodTemplate + apiVersion: v1beta3 + metadata: + name: foo-canary + creationTimestamp: "null" + labels: + service: foo + track: canary + spec: + metadata: + creationTimestamp: "null" + labels: + service: foo + track: canary + spec: + volumes: [] + containers: + - name: web + image: me/myappserver:canary + imagePullPolicy: "" + restartPolicy: {} +- kind: ReplicationController + apiVersion: v1beta3 + metadata: + name: foo-canary + creationTimestamp: "null" + labels: + service: foo + track: canary + spec: + replicas: 2 + selector: + service: foo + track: canary + template: + kind: PodTemplate + name: foo-canary + apiVersion: v1beta3 + status: + replicas: 0 +- kind: PodTemplate + apiVersion: v1beta3 + metadata: + name: foo-stable + creationTimestamp: "null" + labels: + service: foo + track: stable + spec: + metadata: + creationTimestamp: "null" + labels: + service: foo + track: stable + spec: + volumes: [] + containers: + - name: web + image: me/myappserver:stable + imagePullPolicy: "" + restartPolicy: {} +- kind: ReplicationController + apiVersion: v1beta3 + metadata: + name: foo-stable + creationTimestamp: "null" + labels: + service: foo + track: stable + spec: + replicas: 10 + selector: + service: foo + track: stable + template: + kind: PodTemplate + name: foo-stable + apiVersion: v1beta3 + status: + replicas: 0 +- kind: Service + apiVersion: v1beta3 + metadata: + name: bar + creationTimestamp: "null" + labels: + service: bar + spec: + port: 3306 + selector: + service: bar + containerPort: 3306 + status: {} +- kind: PodTemplate + apiVersion: v1beta3 + metadata: + name: bar-solo + creationTimestamp: "null" + labels: + service: bar + track: solo + spec: + metadata: + creationTimestamp: "null" + labels: + service: bar + track: solo + spec: + volumes: + - name: dbdir + source: null + containers: + - name: db + image: mysql + imagePullPolicy: "" + restartPolicy: {} +- kind: ReplicationController + apiVersion: v1beta3 + metadata: + name: bar-solo + creationTimestamp: "null" + labels: + service: bar + track: solo + spec: + replicas: 1 + selector: + service: bar + track: solo + template: + kind: PodTemplate + name: bar-solo + apiVersion: v1beta3 + status: + replicas: 0 + +``` \ No newline at end of file diff --git a/contrib/srvexpand/srvexpand.go b/contrib/srvexpand/srvexpand.go new file mode 100644 index 00000000000..a842961a396 --- /dev/null +++ b/contrib/srvexpand/srvexpand.go @@ -0,0 +1,313 @@ +/* +Copyright 2014 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// srvexpand is a tool to generate non-trivial but regular services +// from a description free of most boilerplate +// +// $ srvexpand myservice.json | kubectl createall -f - +// $ srvexpand myservice.yaml | kubectl createall -f - +// +// This is completely separate from kubectl at the moment, until we figure out +// what the right integration approach is. +// +// Whether this type of wrapper should be encouraged is debatable. It eliminates +// some boilerplate, at the cost of needing to be updated whenever the generated +// API objects change. For instance, this initial version does not expose the +// protocol and createExternalLoadBalancer fields of Service. It's likely that we +// should support boilerplate elimination in the API itself, such as with more +// intelligent defaults, and generic transformations such as map keys to names. + +package main + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "strconv" + "strings" + + // TODO: handle multiple versions correctly. Targeting v1beta3 because + // v1beta1 is too much of a mess. Once we do support multiple versions, + // it should be possible to specify the version for the whole map. + "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta3" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/golang/glog" + "gopkg.in/v1/yaml" +) + +const usage = "usage: srvexpand filename" + +// Hierarchical service structures are a common pattern and allows omission +// of kind fields on input. + +// TODO: Enable apiversion and namespace to be provided for the whole map. +// Note that I don't provide a way to specify labels and annotations to be +// propagated to all the objects (except those required to distinguish and +// connect the objects read in) because I expect that to be done as a +// separate pass. + +type HierarchicalController struct { + // Optional: Defaults to one + Replicas int `yaml:"replicas,omitempty" json:"replicas,omitempty"` + // Spec defines the behavior of a pod. + Spec v1beta3.PodSpec `json:"spec,omitempty" yaml:"spec,omitempty"` +} + +type ControllerMap map[string]HierarchicalController + +type HierarchicalService struct { + // Optional: Creates a service if specified: servicePort:containerPort + // TODO: Support multiple protocols + PortSpec string `yaml:"portSpec,omitempty" json:"portSpec,omitempty"` + // Map of replication controllers to create + ControllerMap ControllerMap `json:"controllers,omitempty" yaml:"controllers,omitempty"` +} + +type ServiceMap map[string]HierarchicalService + +func checkErr(err error) { + if err != nil { + glog.Fatalf("%v", err) + } +} + +func main() { + if len(os.Args) != 2 { + checkErr(fmt.Errorf(usage)) + } + filename := os.Args[1] + + serviceMap := readServiceMap(filename) + + expandServiceMap(serviceMap) +} + +func readServiceMap(filename string) ServiceMap { + inData, err := ReadConfigData(filename) + checkErr(err) + + serviceMap := ServiceMap{} + err = yaml.Unmarshal(inData, &serviceMap) + checkErr(err) + + return serviceMap +} + +func canonicalizeName(name *string) { + *name = strings.ToLower(*name) + if !util.IsDNSLabel(*name) { + checkErr(fmt.Errorf("name (%s) is not a valid DNS label", *name)) + } +} + +func expandServiceMap(serviceMap ServiceMap) { + for name, service := range serviceMap { + canonicalizeName(&name) + + generateService(name, service.PortSpec) + generateReplicationControllers(name, service.ControllerMap) + } +} + +func generateService(name string, portSpec string) { + if portSpec == "" { + return + } + + servicePort, containerPort, err := portsFromString(portSpec) + checkErr(err) + + svc := []v1beta3.Service{{ + TypeMeta: v1beta3.TypeMeta{APIVersion: "v1beta3", Kind: "Service"}, + Metadata: v1beta3.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "service": name, + }, + }, + Spec: v1beta3.ServiceSpec{ + Port: servicePort, + ContainerPort: util.NewIntOrStringFromInt(containerPort), + Selector: map[string]string{ + "service": name, + }, + }, + }} + + svcOutData, err := yaml.Marshal(svc) + checkErr(err) + + fmt.Print(string(svcOutData)) +} + +func generateReplicationControllers(sname string, controllerMap ControllerMap) { + for cname, controller := range controllerMap { + canonicalizeName(&cname) + + generatePodTemplate(sname, cname, controller.Spec) + generateReplicationController(sname, cname, controller.Replicas) + } +} + +func generatePodTemplate(sname string, cname string, podSpec v1beta3.PodSpec) { + name := fmt.Sprintf("%s-%s", sname, cname) + pt := []v1beta3.PodTemplate{{ + TypeMeta: v1beta3.TypeMeta{APIVersion: "v1beta3", Kind: "PodTemplate"}, + Metadata: v1beta3.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "service": sname, + "track": cname, + }, + }, + Spec: v1beta3.PodTemplateSpec{ + Metadata: v1beta3.ObjectMeta{ + Labels: map[string]string{ + "service": sname, + "track": cname, + }, + }, + Spec: podSpec, + }, + }} + + ptOutData, err := yaml.Marshal(pt) + checkErr(err) + + fmt.Print(string(ptOutData)) +} + +func generateReplicationController(sname string, cname string, replicas int) { + if replicas < 1 { + replicas = 1 + } + + name := fmt.Sprintf("%s-%s", sname, cname) + rc := []v1beta3.ReplicationController{{ + TypeMeta: v1beta3.TypeMeta{APIVersion: "v1beta3", Kind: "ReplicationController"}, + Metadata: v1beta3.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "service": sname, + "track": cname, + }, + }, + Spec: v1beta3.ReplicationControllerSpec{ + Replicas: replicas, + Selector: map[string]string{ + "service": sname, + "track": cname, + }, + Template: v1beta3.ObjectReference{ + Kind: "PodTemplate", + Name: name, + APIVersion: "v1beta3", + }, + }, + }} + + rcOutData, err := yaml.Marshal(rc) + checkErr(err) + + fmt.Print(string(rcOutData)) +} + +// TODO: what defaults make the most sense? +func portsFromString(spec string) (servicePort int, containerPort int, err error) { + if spec == "" { + return 0, 0, fmt.Errorf("empty port spec") + } + pieces := strings.Split(spec, ":") + if len(pieces) != 2 { + glog.Infof("Bad port spec: %s", spec) + return 0, 0, fmt.Errorf("Bad port spec: %s", spec) + } + servicePort, err = strconv.Atoi(pieces[0]) + if err != nil { + glog.Errorf("Service port is not integer: %s %v", pieces[0], err) + return 0, 0, err + } + if servicePort < 1 { + glog.Errorf("Service port is not valid: %d", servicePort) + return 0, 0, err + } + containerPort, err = strconv.Atoi(pieces[1]) + if err != nil { + glog.Errorf("Container port is not integer: %s %v", pieces[1], err) + return 0, 0, err + } + if containerPort < 1 { + glog.Errorf("Container port is not valid: %d", containerPort) + return 0, 0, err + } + + return +} + +////////////////////////////////////////////////////////////////////// + +// Client tool utility functions copied from kubectl, kubecfg, and podex. +// This should probably be a separate package, but the right solution is +// to refactor the copied code and delete it from here. + +func ReadConfigData(location string) ([]byte, error) { + if len(location) == 0 { + return nil, fmt.Errorf("Location given but empty") + } + + if location == "-" { + // Read from stdin. + data, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return nil, err + } + + if len(data) == 0 { + return nil, fmt.Errorf(`Read from stdin specified ("-") but no data found`) + } + + return data, nil + } + + // Use the location as a file path or URL. + return readConfigDataFromLocation(location) +} + +func readConfigDataFromLocation(location string) ([]byte, error) { + // we look for http:// or https:// to determine if valid URL, otherwise do normal file IO + if strings.Index(location, "http://") == 0 || strings.Index(location, "https://") == 0 { + resp, err := http.Get(location) + if err != nil { + return nil, fmt.Errorf("Unable to access URL %s: %v\n", location, err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("Unable to read URL, server reported %d %s", resp.StatusCode, resp.Status) + } + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("Unable to read URL %s: %v\n", location, err) + } + return data, nil + } else { + data, err := ioutil.ReadFile(location) + if err != nil { + return nil, fmt.Errorf("Unable to read %s: %v\n", location, err) + } + return data, nil + } +}