Merge pull request #1535 from dave-tucker/make

Allow CI to use moby run gcp

Makefile

@@ -7,6 +7,7 @@ GIT_COMMIT=$(shell git rev-list -1 HEAD)
 
 GO_COMPILE=mobylinux/go-compile:a2ff853b00d687f845d0f67189fa645a567c006e@sha256:09fff8a5c022fc9ead35b2779209c043196b09193c6e61d98603d402c0971f03
 
+MOBY?=bin/moby
 GOOS=$(shell uname -s | tr '[:upper:]' '[:lower:]')
 GOARCH=amd64
 ifneq ($(GOOS),linux)
@@ -27,7 +28,7 @@ bin/infrakit-instance-hyperkit: $(INFRAKIT_DEPS) | bin
 	tar cf - vendor -C src/cmd/infrakit-instance-hyperkit . | docker run --rm --net=none --log-driver=none -i $(CROSS) $(GO_COMPILE) --package github.com/docker/moby -o $@ | tar xf -
 	touch $@
 
-test-initrd.img: bin/moby test/test.yml
+test-initrd.img: $(MOBY) test/test.yml
 	bin/moby build test/test.yml
 
 test-bzImage: test-initrd.img
@@ -46,11 +47,16 @@ define check_test_log
 endef
 
 .PHONY: hyperkit-test
-hyperkit-test: bin/moby test-initrd.img test-bzImage test-cmdline
+hyperkit-test: $(MOBY) test-initrd.img test-bzImage test-cmdline
 	rm -f disk.img
-	script -q /dev/null bin/moby run test | tee test.log
+	script -q /dev/null $(MOBY) run test | tee test.log
 	$(call check_test_log, test.log)
 
+.PHONY: test-gcp
+test-gcp: $(MOBY) test.img.tar.gz
+	script -q /dev/null $(MOBY) run gcp test.img.tar.gz | tee test-gcp.log
+	$(call check_test_log, test-gcp.log)
+
 .PHONY: test
 test: test-initrd.img test-bzImage test-cmdline
 	tar cf - $^ | ./scripts/qemu.sh 2>&1 | tee test.log

src/cmd/moby/gcp.go

@@ -99,15 +99,15 @@ func NewGCPClient(keys, projectName string) (*GCPClient, error) {
 }
 
 // UploadFile uploads a file to Google Storage
-func (g GCPClient) UploadFile(filename, bucketName string, public bool) error {
+func (g GCPClient) UploadFile(src, dst, bucketName string, public bool) error {
-	log.Infof("Uploading file %s to Google Storage", filename)
+	log.Infof("Uploading file %s to Google Storage as %s", src, dst)
-	f, err := os.Open(filename)
+	f, err := os.Open(src)
 	if err != nil {
 		return err
 	}
 	defer f.Close()
 
-	objectCall := g.storage.Objects.Insert(bucketName, &storage.Object{Name: filename}).Media(f)
+	objectCall := g.storage.Objects.Insert(bucketName, &storage.Object{Name: dst}).Media(f)
 
 	if public {
 		objectCall.PredefinedAcl("publicRead")
@@ -118,24 +118,24 @@ func (g GCPClient) UploadFile(filename, bucketName string, public bool) error {
 		return err
 	}
 	log.Infof("Upload Complete!")
-	fmt.Println("gs://" + bucketName + "/" + filename)
+	fmt.Println("gs://" + bucketName + "/" + dst)
 	return nil
 }
 
 // CreateImage creates a GCP image using the a source from Google Storage
-func (g GCPClient) CreateImage(filename, storageURL, family string, replace bool) error {
+func (g GCPClient) CreateImage(name, storageURL, family string, replace bool) error {
 	if replace {
-		if err := g.DeleteImage(filename); err != nil {
+		if err := g.DeleteImage(name); err != nil {
 			return err
 		}
 	}
 
-	log.Infof("Creating image: %s", filename)
+	log.Infof("Creating image: %s", name)
 	imgObj := &compute.Image{
 		RawDisk: &compute.ImageRawDisk{
 			Source: storageURL,
 		},
-		Name: filename,
+		Name: name,
 	}
 
 	if family != "" {
@@ -150,14 +150,14 @@ func (g GCPClient) CreateImage(filename, storageURL, family string, replace bool
 	if err := g.pollOperationStatus(op.Name); err != nil {
 		return err
 	}
-	log.Infof("Image %s created", filename)
+	log.Infof("Image %s created", name)
 	return nil
 }
 
 // DeleteImage deletes and image
-func (g GCPClient) DeleteImage(filename string) error {
+func (g GCPClient) DeleteImage(name string) error {
 	var notFound bool
-	op, err := g.compute.Images.Delete(g.projectName, filename).Do()
+	op, err := g.compute.Images.Delete(g.projectName, name).Do()
 	if err != nil {
 		if err.(*googleapi.Error).Code != 404 {
 			return err
@@ -169,20 +169,20 @@ func (g GCPClient) DeleteImage(filename string) error {
 		if err := g.pollOperationStatus(op.Name); err != nil {
 			return err
 		}
-		log.Infof("Image %s deleted", filename)
+		log.Infof("Image %s deleted", name)
 	}
 	return nil
 }
 
 // CreateInstance creates and starts an instance on GCP
-func (g GCPClient) CreateInstance(image, zone, machineType string, replace bool) error {
+func (g GCPClient) CreateInstance(name, image, zone, machineType string, replace bool) error {
 	if replace {
-		if err := g.DeleteInstance(image, zone, true); err != nil {
+		if err := g.DeleteInstance(name, zone, true); err != nil {
 			return err
 		}
 	}
 
-	log.Infof("Creating instance %s", image)
+	log.Infof("Creating instance %s from image %s", name, image)
 	enabled := new(string)
 	*enabled = "1"
 
@@ -195,7 +195,7 @@ func (g GCPClient) CreateInstance(image, zone, machineType string, replace bool)
 
 	instanceObj := &compute.Instance{
 		MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", zone, machineType),
-		Name:        image,
+		Name:        name,
 		Disks: []*compute.AttachedDisk{
 			{
 				AutoDelete: true,

src/cmd/moby/output.go

@@ -55,7 +55,7 @@ func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error {
 			if err != nil {
 				return fmt.Errorf("Unable to connect to GCP")
 			}
-			err = gClient.UploadFile(base+".img.tar.gz", o.Bucket, o.Public)
+			err = gClient.UploadFile(base+".img.tar.gz", base+".img.tar.gz", o.Bucket, o.Public)
 			if err != nil {
 				return fmt.Errorf("Error copying to Google Storage: %v", err)
 			}
@@ -71,7 +71,7 @@ func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error {
 			if err != nil {
 				return fmt.Errorf("Unable to connect to GCP")
 			}
-			err = gClient.UploadFile(base+".img.tar.gz", o.Bucket, o.Public)
+			err = gClient.UploadFile(base+".img.tar.gz", base+".img.tar.gz", o.Bucket, o.Public)
 			if err != nil {
 				return fmt.Errorf("Error copying to Google Storage: %v", err)
 			}

src/cmd/moby/run_gcp.go

@@ -9,6 +9,19 @@ import (
 	log "github.com/Sirupsen/logrus"
 )
 
+const (
+	defaultZone    = "europe-west1-d"
+	defaultMachine = "g1-small"
+	zoneVar        = "MOBY_GCP_ZONE"
+	machineVar     = "MOBY_GCP_MACHINE"
+	keysVar        = "MOBY_GCP_KEYS"
+	projectVar     = "MOBY_GCP_PROJECT"
+	bucketVar      = "MOBY_GCP_BUCKET"
+	familyVar      = "MOBY_GCP_FAMILY"
+	publicVar      = "MOBY_GCP_PUBLIC"
+	nameVar        = "MOBY_GCP_IMAGE_NAME"
+)
+
 // Process the run arguments and execute run
 func runGcp(args []string) {
 	gcpCmd := flag.NewFlagSet("gcp", flag.ExitOnError)
@@ -20,15 +33,16 @@ func runGcp(args []string) {
 		fmt.Printf("Options:\n\n")
 		gcpCmd.PrintDefaults()
 	}
-	zone := gcpCmd.String("zone", "europe-west1-d", "GCP Zone")
+	zoneFlag := gcpCmd.String("zone", defaultZone, "GCP Zone")
-	machine := gcpCmd.String("machine", "g1-small", "GCP Machine Type")
+	machineFlag := gcpCmd.String("machine", defaultMachine, "GCP Machine Type")
-	keys := gcpCmd.String("keys", "", "Path to Service Account JSON key file")
+	keysFlag := gcpCmd.String("keys", "", "Path to Service Account JSON key file")
-	project := gcpCmd.String("project", "", "GCP Project Name")
+	projectFlag := gcpCmd.String("project", "", "GCP Project Name")
-	bucket := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required* when 'prefix' is a filename")
+	bucketFlag := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required* when 'prefix' is a filename")
-	public := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional* when 'prefix' is a filename")
+	publicFlag := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional* when 'prefix' is a filename")
-	family := gcpCmd.String("family", "", "GCP Image Family. A group of images where the family name points to the most recent image. *Optional* when 'prefix' is a filename")
+	familyFlag := gcpCmd.String("family", "", "GCP Image Family. A group of images where the family name points to the most recent image. *Optional* when 'prefix' is a filename")
-
+	nameFlag := gcpCmd.String("img-name", "", "Overrides the Name used to identify the file in Google Storage, Image and Instance. Defaults to [name]")
 	gcpCmd.Parse(args)
+
 	remArgs := gcpCmd.Args()
 	if len(remArgs) == 0 {
 		fmt.Printf("Please specify the prefix to the image to boot\n")
@@ -37,37 +51,55 @@ func runGcp(args []string) {
 	}
 	prefix := remArgs[0]
 
-	client, err := NewGCPClient(*keys, *project)
+	zone := getStringValue(zoneVar, *zoneFlag, defaultZone)
+	machine := getStringValue(machineVar, *machineFlag, defaultMachine)
+	keys := getStringValue(keysVar, *keysFlag, "")
+	project := getStringValue(projectVar, *projectFlag, "")
+	bucket := getStringValue(bucketVar, *bucketFlag, "")
+	public := getBoolValue(publicVar, *publicFlag)
+	family := getStringValue(familyVar, *familyFlag, "")
+	name := getStringValue(nameVar, *nameFlag, "")
+
+	client, err := NewGCPClient(keys, project)
 	if err != nil {
 		log.Fatalf("Unable to connect to GCP")
 	}
 
 	suffix := ".img.tar.gz"
 	if strings.HasSuffix(prefix, suffix) {
-		filename := prefix
+		src := prefix
+		if name != "" {
+			prefix = name
+		} else {
 			prefix = prefix[:len(prefix)-len(suffix)]
-		if *bucket == "" {
+		}
+		if bucket == "" {
 			log.Fatalf("No bucket specified. Please provide one using the -bucket flag")
 		}
-		err = client.UploadFile(filename, *bucket, *public)
+		err = client.UploadFile(src, prefix+suffix, bucket, public)
 		if err != nil {
 			log.Fatalf("Error copying to Google Storage: %v", err)
 		}
-		err = client.CreateImage(prefix, "https://storage.googleapis.com/"+*bucket+"/"+prefix+".img.tar.gz", *family, true)
+		err = client.CreateImage(prefix, "https://storage.googleapis.com/"+bucket+"/"+prefix+".img.tar.gz", family, true)
 		if err != nil {
 			log.Fatalf("Error creating Google Compute Image: %v", err)
 		}
 	}
 
-	if err = client.CreateInstance(prefix, *zone, *machine, true); err != nil {
+	// If no name was supplied, use the prefix
+	if name == "" {
+		name = prefix
+	}
+
+	if err = client.CreateInstance(name, prefix, zone, machine, true); err != nil {
 		log.Fatal(err)
 	}
 
-	if err = client.ConnectToInstanceSerialPort(prefix, *zone); err != nil {
+	if err = client.ConnectToInstanceSerialPort(name, zone); err != nil {
 		log.Fatal(err)
 	}
 
-	if err = client.DeleteInstance(prefix, *zone, true); err != nil {
+	if err = client.DeleteInstance(name, zone, true); err != nil {
 		log.Fatal(err)
 	}
 }

src/cmd/moby/util.go

@@ -0,0 +1,55 @@
+package main
+
+import "os"
+
+func getStringValue(envKey string, flagVal string, defaultVal string) string {
+	var res string
+
+	// If defined, take the env variable
+	if _, ok := os.LookupEnv(envKey); ok {
+		res = os.Getenv(envKey)
+	}
+
+	// If a flag is specified, this value takes precedence
+	// Ignore cases where the flag carries the default value
+	if flagVal != "" && flagVal != defaultVal {
+		res = flagVal
+	}
+
+	// if we still don't have a value, use the default
+	if res == "" {
+		res = defaultVal
+	}
+	return res
+}
+
+func getBoolValue(envKey string, flagVal bool) bool {
+	var res bool
+
+	// If defined, take the env variable
+	if _, ok := os.LookupEnv(envKey); ok {
+		switch os.Getenv(envKey) {
+		case "":
+			res = false
+		case "0":
+			res = false
+		case "false":
+			res = false
+		case "FALSE":
+			res = false
+		case "1":
+			res = true
+		default:
+			// catches "true", "TRUE" or anything else
+			res = true
+
+		}
+	}
+
+	// If a flag is specified, this value takes precedence
+	if res != flagVal {
+		res = flagVal
+	}
+
+	return res
+}