From fddfd0b3ec5ac643884bd2ea226957a4e314ca6a Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Thu, 6 Apr 2017 15:14:25 +0100 Subject: [PATCH 1/4] makefile: Allow a bin/moby to be provided This is needed for CI to be able to run `moby run $platform` with a known good version of moby. For example, the one generated when building the `master`. This protects CI from any nefarious changes to the `moby` tool and allows `moby run` to be used safely to test PRs. Signed-off-by: Dave Tucker --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a8979370a..674db1e96 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ GIT_COMMIT=$(shell git rev-list -1 HEAD) GO_COMPILE=mobylinux/go-compile:a2ff853b00d687f845d0f67189fa645a567c006e@sha256:09fff8a5c022fc9ead35b2779209c043196b09193c6e61d98603d402c0971f03 +MOBY?=bin/moby GOOS=$(shell uname -s | tr '[:upper:]' '[:lower:]') GOARCH=amd64 ifneq ($(GOOS),linux) @@ -27,7 +28,7 @@ bin/infrakit-instance-hyperkit: $(INFRAKIT_DEPS) | bin tar cf - vendor -C src/cmd/infrakit-instance-hyperkit . | docker run --rm --net=none --log-driver=none -i $(CROSS) $(GO_COMPILE) --package github.com/docker/moby -o $@ | tar xf - touch $@ -test-initrd.img: bin/moby test/test.yml +test-initrd.img: $(MOBY) test/test.yml bin/moby build test/test.yml test-bzImage: test-initrd.img @@ -46,9 +47,9 @@ define check_test_log endef .PHONY: hyperkit-test -hyperkit-test: bin/moby test-initrd.img test-bzImage test-cmdline +hyperkit-test: $(MOBY) test-initrd.img test-bzImage test-cmdline rm -f disk.img - script -q /dev/null bin/moby run test | tee test.log + script -q /dev/null $(MOBY) run test | tee test.log $(call check_test_log, test.log) .PHONY: test From a4980c8bfb26c3347e42c1dc0d6c822733fbd684 Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Thu, 6 Apr 2017 15:23:23 +0100 Subject: [PATCH 2/4] makefile: Add targets to test GCP Signed-off-by: Dave Tucker --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 674db1e96..838a48e97 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,11 @@ hyperkit-test: $(MOBY) test-initrd.img test-bzImage test-cmdline script -q /dev/null $(MOBY) run test | tee test.log $(call check_test_log, test.log) +.PHONY: test-gcp +test-gcp: $(MOBY) test.img.tar.gz + script -q /dev/null $(MOBY) run gcp test.img.tar.gz | tee test-gcp.log + $(call check_test_log, test-gcp.log) + .PHONY: test test: test-initrd.img test-bzImage test-cmdline tar cf - $^ | ./scripts/qemu.sh 2>&1 | tee test.log From c50e0de0c134f6b82dff89cbe164d80f62ed5e47 Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Fri, 7 Apr 2017 13:25:30 +0100 Subject: [PATCH 3/4] moby: Allow combination of env vars and flags for gcp Signed-off-by: Dave Tucker --- src/cmd/moby/run_gcp.go | 49 +++++++++++++++++++++++++----------- src/cmd/moby/util.go | 55 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 15 deletions(-) create mode 100644 src/cmd/moby/util.go diff --git a/src/cmd/moby/run_gcp.go b/src/cmd/moby/run_gcp.go index 32cedf4dc..856944a00 100644 --- a/src/cmd/moby/run_gcp.go +++ b/src/cmd/moby/run_gcp.go @@ -9,6 +9,18 @@ import ( log "github.com/Sirupsen/logrus" ) +const ( + defaultZone = "europe-west1-d" + defaultMachine = "g1-small" + zoneVar = "MOBY_GCP_ZONE" + machineVar = "MOBY_GCP_MACHINE" + keysVar = "MOBY_GCP_KEYS" + projectVar = "MOBY_GCP_PROJECT" + bucketVar = "MOBY_GCP_BUCKET" + familyVar = "MOBY_GCP_FAMILY" + publicVar = "MOBY_GCP_PUBLIC" +) + // Process the run arguments and execute run func runGcp(args []string) { gcpCmd := flag.NewFlagSet("gcp", flag.ExitOnError) @@ -20,14 +32,13 @@ func runGcp(args []string) { fmt.Printf("Options:\n\n") gcpCmd.PrintDefaults() } - zone := gcpCmd.String("zone", "europe-west1-d", "GCP Zone") - machine := gcpCmd.String("machine", "g1-small", "GCP Machine Type") - keys := gcpCmd.String("keys", "", "Path to Service Account JSON key file") - project := gcpCmd.String("project", "", "GCP Project Name") - bucket := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required* when 'prefix' is a filename") - public := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional* when 'prefix' is a filename") - family := gcpCmd.String("family", "", "GCP Image Family. A group of images where the family name points to the most recent image. *Optional* when 'prefix' is a filename") - + zoneFlag := gcpCmd.String("zone", defaultZone, "GCP Zone") + machineFlag := gcpCmd.String("machine", defaultMachine, "GCP Machine Type") + keysFlag := gcpCmd.String("keys", "", "Path to Service Account JSON key file") + projectFlag := gcpCmd.String("project", "", "GCP Project Name") + bucketFlag := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required* when 'prefix' is a filename") + publicFlag := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional* when 'prefix' is a filename") + familyFlag := gcpCmd.String("family", "", "GCP Image Family. A group of images where the family name points to the most recent image. *Optional* when 'prefix' is a filename") gcpCmd.Parse(args) remArgs := gcpCmd.Args() if len(remArgs) == 0 { @@ -37,7 +48,15 @@ func runGcp(args []string) { } prefix := remArgs[0] - client, err := NewGCPClient(*keys, *project) + zone := getStringValue(zoneVar, *zoneFlag, defaultZone) + machine := getStringValue(machineVar, *machineFlag, defaultMachine) + keys := getStringValue(keysVar, *keysFlag, "") + project := getStringValue(projectVar, *projectFlag, "") + bucket := getStringValue(bucketVar, *bucketFlag, "") + public := getBoolValue(publicVar, *publicFlag) + family := getStringValue(familyVar, *familyFlag, "") + + client, err := NewGCPClient(keys, project) if err != nil { log.Fatalf("Unable to connect to GCP") } @@ -46,28 +65,28 @@ func runGcp(args []string) { if strings.HasSuffix(prefix, suffix) { filename := prefix prefix = prefix[:len(prefix)-len(suffix)] - if *bucket == "" { + if bucket == "" { log.Fatalf("No bucket specified. Please provide one using the -bucket flag") } - err = client.UploadFile(filename, *bucket, *public) + err = client.UploadFile(filename, bucket, public) if err != nil { log.Fatalf("Error copying to Google Storage: %v", err) } - err = client.CreateImage(prefix, "https://storage.googleapis.com/"+*bucket+"/"+prefix+".img.tar.gz", *family, true) + err = client.CreateImage(prefix, "https://storage.googleapis.com/"+bucket+"/"+prefix+".img.tar.gz", family, true) if err != nil { log.Fatalf("Error creating Google Compute Image: %v", err) } } - if err = client.CreateInstance(prefix, *zone, *machine, true); err != nil { + if err = client.CreateInstance(prefix, zone, machine, true); err != nil { log.Fatal(err) } - if err = client.ConnectToInstanceSerialPort(prefix, *zone); err != nil { + if err = client.ConnectToInstanceSerialPort(prefix, zone); err != nil { log.Fatal(err) } - if err = client.DeleteInstance(prefix, *zone, true); err != nil { + if err = client.DeleteInstance(prefix, zone, true); err != nil { log.Fatal(err) } } diff --git a/src/cmd/moby/util.go b/src/cmd/moby/util.go new file mode 100644 index 000000000..e43c3f493 --- /dev/null +++ b/src/cmd/moby/util.go @@ -0,0 +1,55 @@ +package main + +import "os" + +func getStringValue(envKey string, flagVal string, defaultVal string) string { + var res string + + // If defined, take the env variable + if _, ok := os.LookupEnv(envKey); ok { + res = os.Getenv(envKey) + } + + // If a flag is specified, this value takes precedence + // Ignore cases where the flag carries the default value + if flagVal != "" && flagVal != defaultVal { + res = flagVal + } + + // if we still don't have a value, use the default + if res == "" { + res = defaultVal + } + return res +} + +func getBoolValue(envKey string, flagVal bool) bool { + var res bool + + // If defined, take the env variable + if _, ok := os.LookupEnv(envKey); ok { + switch os.Getenv(envKey) { + case "": + res = false + case "0": + res = false + case "false": + res = false + case "FALSE": + res = false + case "1": + res = true + default: + // catches "true", "TRUE" or anything else + res = true + + } + } + + // If a flag is specified, this value takes precedence + if res != flagVal { + res = flagVal + } + + return res +} From e2da49c7e27610552def1b26d087a32fe29db4b4 Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Fri, 7 Apr 2017 14:39:13 +0100 Subject: [PATCH 4/4] moby: Add -img-name flag to moby run gcp This allows overriding the name used of the file in google storage, image name or instance name. This will vary depending on how much `moby run` is doing which is goverened by whether the positional argument contains an `.img.tar.gz` or not. For example: `moby run gcp -img-name test-ea34d1 test` creates an instance called `test-ea34d1` from the image `test` `moby run gcp -img-name test-ea34d1` test.img.tar.gz` will upload the file as `test-ea34d1.tar.gz`, create image `test-ea34d1` and create an instance called `test-ea34d1`. The use case for this is for CI to be able to spawn many concurrent test machines and provide it's own name for them. Signed-off-by: Dave Tucker --- src/cmd/moby/gcp.go | 34 +++++++++++++++++----------------- src/cmd/moby/output.go | 4 ++-- src/cmd/moby/run_gcp.go | 25 +++++++++++++++++++------ 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/cmd/moby/gcp.go b/src/cmd/moby/gcp.go index 80d2f435f..d3e3b9ca9 100644 --- a/src/cmd/moby/gcp.go +++ b/src/cmd/moby/gcp.go @@ -99,15 +99,15 @@ func NewGCPClient(keys, projectName string) (*GCPClient, error) { } // UploadFile uploads a file to Google Storage -func (g GCPClient) UploadFile(filename, bucketName string, public bool) error { - log.Infof("Uploading file %s to Google Storage", filename) - f, err := os.Open(filename) +func (g GCPClient) UploadFile(src, dst, bucketName string, public bool) error { + log.Infof("Uploading file %s to Google Storage as %s", src, dst) + f, err := os.Open(src) if err != nil { return err } defer f.Close() - objectCall := g.storage.Objects.Insert(bucketName, &storage.Object{Name: filename}).Media(f) + objectCall := g.storage.Objects.Insert(bucketName, &storage.Object{Name: dst}).Media(f) if public { objectCall.PredefinedAcl("publicRead") @@ -118,24 +118,24 @@ func (g GCPClient) UploadFile(filename, bucketName string, public bool) error { return err } log.Infof("Upload Complete!") - fmt.Println("gs://" + bucketName + "/" + filename) + fmt.Println("gs://" + bucketName + "/" + dst) return nil } // CreateImage creates a GCP image using the a source from Google Storage -func (g GCPClient) CreateImage(filename, storageURL, family string, replace bool) error { +func (g GCPClient) CreateImage(name, storageURL, family string, replace bool) error { if replace { - if err := g.DeleteImage(filename); err != nil { + if err := g.DeleteImage(name); err != nil { return err } } - log.Infof("Creating image: %s", filename) + log.Infof("Creating image: %s", name) imgObj := &compute.Image{ RawDisk: &compute.ImageRawDisk{ Source: storageURL, }, - Name: filename, + Name: name, } if family != "" { @@ -150,14 +150,14 @@ func (g GCPClient) CreateImage(filename, storageURL, family string, replace bool if err := g.pollOperationStatus(op.Name); err != nil { return err } - log.Infof("Image %s created", filename) + log.Infof("Image %s created", name) return nil } // DeleteImage deletes and image -func (g GCPClient) DeleteImage(filename string) error { +func (g GCPClient) DeleteImage(name string) error { var notFound bool - op, err := g.compute.Images.Delete(g.projectName, filename).Do() + op, err := g.compute.Images.Delete(g.projectName, name).Do() if err != nil { if err.(*googleapi.Error).Code != 404 { return err @@ -169,20 +169,20 @@ func (g GCPClient) DeleteImage(filename string) error { if err := g.pollOperationStatus(op.Name); err != nil { return err } - log.Infof("Image %s deleted", filename) + log.Infof("Image %s deleted", name) } return nil } // CreateInstance creates and starts an instance on GCP -func (g GCPClient) CreateInstance(image, zone, machineType string, replace bool) error { +func (g GCPClient) CreateInstance(name, image, zone, machineType string, replace bool) error { if replace { - if err := g.DeleteInstance(image, zone, true); err != nil { + if err := g.DeleteInstance(name, zone, true); err != nil { return err } } - log.Infof("Creating instance %s", image) + log.Infof("Creating instance %s from image %s", name, image) enabled := new(string) *enabled = "1" @@ -195,7 +195,7 @@ func (g GCPClient) CreateInstance(image, zone, machineType string, replace bool) instanceObj := &compute.Instance{ MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", zone, machineType), - Name: image, + Name: name, Disks: []*compute.AttachedDisk{ { AutoDelete: true, diff --git a/src/cmd/moby/output.go b/src/cmd/moby/output.go index 5d7bf1d5a..a51d49444 100644 --- a/src/cmd/moby/output.go +++ b/src/cmd/moby/output.go @@ -55,7 +55,7 @@ func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error { if err != nil { return fmt.Errorf("Unable to connect to GCP") } - err = gClient.UploadFile(base+".img.tar.gz", o.Bucket, o.Public) + err = gClient.UploadFile(base+".img.tar.gz", base+".img.tar.gz", o.Bucket, o.Public) if err != nil { return fmt.Errorf("Error copying to Google Storage: %v", err) } @@ -71,7 +71,7 @@ func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error { if err != nil { return fmt.Errorf("Unable to connect to GCP") } - err = gClient.UploadFile(base+".img.tar.gz", o.Bucket, o.Public) + err = gClient.UploadFile(base+".img.tar.gz", base+".img.tar.gz", o.Bucket, o.Public) if err != nil { return fmt.Errorf("Error copying to Google Storage: %v", err) } diff --git a/src/cmd/moby/run_gcp.go b/src/cmd/moby/run_gcp.go index 856944a00..9b88ebf95 100644 --- a/src/cmd/moby/run_gcp.go +++ b/src/cmd/moby/run_gcp.go @@ -19,6 +19,7 @@ const ( bucketVar = "MOBY_GCP_BUCKET" familyVar = "MOBY_GCP_FAMILY" publicVar = "MOBY_GCP_PUBLIC" + nameVar = "MOBY_GCP_IMAGE_NAME" ) // Process the run arguments and execute run @@ -39,7 +40,9 @@ func runGcp(args []string) { bucketFlag := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required* when 'prefix' is a filename") publicFlag := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional* when 'prefix' is a filename") familyFlag := gcpCmd.String("family", "", "GCP Image Family. A group of images where the family name points to the most recent image. *Optional* when 'prefix' is a filename") + nameFlag := gcpCmd.String("img-name", "", "Overrides the Name used to identify the file in Google Storage, Image and Instance. Defaults to [name]") gcpCmd.Parse(args) + remArgs := gcpCmd.Args() if len(remArgs) == 0 { fmt.Printf("Please specify the prefix to the image to boot\n") @@ -55,6 +58,7 @@ func runGcp(args []string) { bucket := getStringValue(bucketVar, *bucketFlag, "") public := getBoolValue(publicVar, *publicFlag) family := getStringValue(familyVar, *familyFlag, "") + name := getStringValue(nameVar, *nameFlag, "") client, err := NewGCPClient(keys, project) if err != nil { @@ -63,12 +67,16 @@ func runGcp(args []string) { suffix := ".img.tar.gz" if strings.HasSuffix(prefix, suffix) { - filename := prefix - prefix = prefix[:len(prefix)-len(suffix)] + src := prefix + if name != "" { + prefix = name + } else { + prefix = prefix[:len(prefix)-len(suffix)] + } if bucket == "" { log.Fatalf("No bucket specified. Please provide one using the -bucket flag") } - err = client.UploadFile(filename, bucket, public) + err = client.UploadFile(src, prefix+suffix, bucket, public) if err != nil { log.Fatalf("Error copying to Google Storage: %v", err) } @@ -78,15 +86,20 @@ func runGcp(args []string) { } } - if err = client.CreateInstance(prefix, zone, machine, true); err != nil { + // If no name was supplied, use the prefix + if name == "" { + name = prefix + } + + if err = client.CreateInstance(name, prefix, zone, machine, true); err != nil { log.Fatal(err) } - if err = client.ConnectToInstanceSerialPort(prefix, zone); err != nil { + if err = client.ConnectToInstanceSerialPort(name, zone); err != nil { log.Fatal(err) } - if err = client.DeleteInstance(prefix, zone, true); err != nil { + if err = client.DeleteInstance(name, zone, true); err != nil { log.Fatal(err) } }