Merge pull request #1693 from justincormack/push

Split build and push, and remove push code from run

Makefile

@@ -63,7 +63,8 @@ test-hyperkit: $(MOBY) test-initrd.img test-bzImage test-cmdline
 
 .PHONY: test-gcp
 test-gcp: $(MOBY) test.img.tar.gz
-	$(MOBY) run gcp test.img.tar.gz | tee test-gcp.log
+	$(MOBY) push gcp test.img.tar.gz
+	$(MOBY) run gcp test | tee test-gcp.log
 	$(call check_test_log, test-gcp.log)
 
 .PHONY: test
@@ -76,7 +77,8 @@ test-ltp.img.tar.gz: $(MOBY) test/ltp/test-ltp.yml
 
 .PHONY: test-ltp
 test-ltp: $(MOBY) test-ltp.img.tar.gz
-	$(MOBY) run gcp -skip-cleanup -machine n1-highcpu-4 test-ltp.img.tar.gz | tee test-ltp.log
+	$(MOBY) push gcp test-ltp.img.tar.gz
+	$(MOBY) run gcp -skip-cleanup -machine n1-highcpu-4 test-ltp | tee test-ltp.log
 	$(call check_test_log, test-ltp.log)
 
 .PHONY: ci ci-tag ci-pr

docs/gcp.md

@@ -22,7 +22,7 @@ brew cask install google-cloud-sdk
 Or via source code:
 
 ```shell
-wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-151.0.0-darwin-x86_64.tar.gz
+curl -SsL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-151.0.0-darwin-x86_64.tar.gz
 tar xzvf google-cloud-sdk-151.0.0-darwin-x86_64.tar.gz
 ./google-cloud-sdk/install.sh
 ```
@@ -50,12 +50,16 @@ Make sure to download the credentials in JSON format and store them somewhere sa
 
 ## Build an image
 
-Add a `gcp` output line to your yaml config, see the example in `examples/gcp.yml`.
+Add a `gcp-img` output line to your yaml config, see the example in `examples/gcp.yml`.
 
 Then do `moby build myfile.yml`
 
-This will create a local `myfile.img.tar.gz` compressed image file, upload it to the
+This will create a local `myfile.img.tar.gz` compressed image file.
-specified bucket, and create a bootable image.
+
+## Push image
+
+Do `moby push gcp -project myproject-1234 -bucket bucketname myfile.img.tar.gz` to upload it to the
+specified bucket, and create a bootable image from the stored image.
 
 ## Create an instance and connect to it
 

examples/gcp.yml

@@ -65,8 +65,4 @@ trust:
     - linuxkit/kernel
 outputs:
   - format: kernel+initrd
-  - format: gcp
+  - format: gcp-img
-    project: moby
-    bucket: mobytestjustin
-    family: moby-dev
-    replace: true

src/cmd/moby/config.go

@@ -35,13 +35,7 @@ type Moby struct {
 		Contents  string
 	}
 	Outputs []struct {
-		Format  string
+		Format string
-		Project string
-		Bucket  string
-		Family  string
-		Keys    string
-		Public  bool
-		Replace bool
 	}
 }
 

src/cmd/moby/main.go

@@ -42,7 +42,8 @@ func main() {
 		fmt.Printf("USAGE: %s [options] COMMAND\n\n", filepath.Base(os.Args[0]))
 		fmt.Printf("Commands:\n")
 		fmt.Printf("  build       Build a Moby image from a YAML file\n")
-		fmt.Printf("  run         Run a Moby image on a local hypervisor or remote cloud\n")
+		fmt.Printf("  push        Push a VM image to a cloud or image store\n")
+		fmt.Printf("  run         Run a VM image on a local hypervisor or remote cloud\n")
 		fmt.Printf("  version     Print version information\n")
 		fmt.Printf("  help        Print this message\n")
 		fmt.Printf("\n")
@@ -81,6 +82,8 @@ func main() {
 	switch args[0] {
 	case "build":
 		build(args[1:])
+	case "push":
+		push(args[1:])
 	case "run":
 		run(args[1:])
 	case "version":

src/cmd/moby/output.go

@@ -43,42 +43,6 @@ func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error {
 			if err != nil {
 				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
 			}
-		case "gcp-storage":
-			err := outputImg(gcp, base+".img.tar.gz", bzimage, initrd, m.Kernel.Cmdline)
-			if err != nil {
-				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
-			}
-			if o.Bucket == "" {
-				return fmt.Errorf("No bucket specified for GCP output")
-			}
-			gClient, err := NewGCPClient(o.Keys, o.Project)
-			if err != nil {
-				return fmt.Errorf("Unable to connect to GCP")
-			}
-			err = gClient.UploadFile(base+".img.tar.gz", base+".img.tar.gz", o.Bucket, o.Public)
-			if err != nil {
-				return fmt.Errorf("Error copying to Google Storage: %v", err)
-			}
-		case "gcp":
-			err := outputImg(gcp, base+".img.tar.gz", bzimage, initrd, m.Kernel.Cmdline)
-			if err != nil {
-				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
-			}
-			if o.Bucket == "" {
-				return fmt.Errorf("No bucket specified for GCP output")
-			}
-			gClient, err := NewGCPClient(o.Keys, o.Project)
-			if err != nil {
-				return fmt.Errorf("Unable to connect to GCP")
-			}
-			err = gClient.UploadFile(base+".img.tar.gz", base+".img.tar.gz", o.Bucket, o.Public)
-			if err != nil {
-				return fmt.Errorf("Error copying to Google Storage: %v", err)
-			}
-			err = gClient.CreateImage(base, "https://storage.googleapis.com/"+o.Bucket+"/"+base+".img.tar.gz", o.Family, o.Replace)
-			if err != nil {
-				return fmt.Errorf("Error creating Google Compute Image: %v", err)
-			}
 		case "qcow", "qcow2":
 			err := outputImg(qcow, base+".qcow2", bzimage, initrd, m.Kernel.Cmdline)
 			if err != nil {

src/cmd/moby/push.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+func pushUsage() {
+	fmt.Printf("USAGE: %s push [backend] [options] [prefix]\n\n", os.Args[0])
+
+	fmt.Printf("'backend' specifies the push backend.\n")
+	fmt.Printf("Supported backends are\n")
+	fmt.Printf("  gcp\n")
+	fmt.Printf("\n")
+	fmt.Printf("'options' are the backend specific options.\n")
+	fmt.Printf("See 'moby push [backend] --help' for details.\n\n")
+	fmt.Printf("'prefix' specifies the path to the VM image.\n")
+}
+
+func push(args []string) {
+	if len(args) < 1 {
+		runUsage()
+		os.Exit(1)
+	}
+
+	switch args[0] {
+	case "help", "-h", "-help", "--help":
+		pushUsage()
+		os.Exit(0)
+	case "gcp":
+		pushGcp(args[1:])
+	default:
+		log.Errorf("No 'push' backend specified.")
+	}
+}

src/cmd/moby/push_gcp.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+// Process the run arguments and execute run
+func pushGcp(args []string) {
+	gcpCmd := flag.NewFlagSet("gcp", flag.ExitOnError)
+	gcpCmd.Usage = func() {
+		fmt.Printf("USAGE: %s push gcp [options] [name]\n\n", os.Args[0])
+		fmt.Printf("'name' specifies the full path of an image file which will be uploaded\n")
+		fmt.Printf("Options:\n\n")
+		gcpCmd.PrintDefaults()
+	}
+	keysFlag := gcpCmd.String("keys", "", "Path to Service Account JSON key file")
+	projectFlag := gcpCmd.String("project", "", "GCP Project Name")
+	bucketFlag := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required*")
+	publicFlag := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional*")
+	familyFlag := gcpCmd.String("family", "", "GCP Image Family. A group of images where the family name points to the most recent image. *Optional*")
+	nameFlag := gcpCmd.String("img-name", "", "Overrides the Name used to identify the file in Google Storage and Image. Defaults to [name]")
+
+	if err := gcpCmd.Parse(args); err != nil {
+		log.Fatal("Unable to parse args")
+	}
+
+	remArgs := gcpCmd.Args()
+	if len(remArgs) == 0 {
+		fmt.Printf("Please specify the prefix to the image to push\n")
+		gcpCmd.Usage()
+		os.Exit(1)
+	}
+	prefix := remArgs[0]
+
+	keys := getStringValue(keysVar, *keysFlag, "")
+	project := getStringValue(projectVar, *projectFlag, "")
+	bucket := getStringValue(bucketVar, *bucketFlag, "")
+	public := getBoolValue(publicVar, *publicFlag)
+	family := getStringValue(familyVar, *familyFlag, "")
+	name := getStringValue(nameVar, *nameFlag, "")
+
+	client, err := NewGCPClient(keys, project)
+	if err != nil {
+		log.Fatalf("Unable to connect to GCP")
+	}
+
+	suffix := ".img.tar.gz"
+	src := prefix
+	if strings.HasSuffix(prefix, suffix) {
+		prefix = prefix[:len(prefix)-len(suffix)]
+	} else {
+		src = prefix + suffix
+	}
+	if name != "" {
+		prefix = name
+	}
+	if bucket == "" {
+		log.Fatalf("No bucket specified. Please provide one using the -bucket flag")
+	}
+	err = client.UploadFile(src, prefix+suffix, bucket, public)
+	if err != nil {
+		log.Fatalf("Error copying to Google Storage: %v", err)
+	}
+	err = client.CreateImage(prefix, "https://storage.googleapis.com/"+bucket+"/"+prefix+".img.tar.gz", family, true)
+	if err != nil {
+		log.Fatalf("Error creating Google Compute Image: %v", err)
+	}
+}

src/cmd/moby/run_gcp.go

@@ -4,7 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"strings"
 
 	log "github.com/Sirupsen/logrus"
 )
@@ -40,10 +39,6 @@ func runGcp(args []string) {
 	machineFlag := gcpCmd.String("machine", defaultMachine, "GCP Machine Type")
 	keysFlag := gcpCmd.String("keys", "", "Path to Service Account JSON key file")
 	projectFlag := gcpCmd.String("project", "", "GCP Project Name")
-	bucketFlag := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required* when 'prefix' is a filename")
-	publicFlag := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional* when 'prefix' is a filename")
-	familyFlag := gcpCmd.String("family", "", "GCP Image Family. A group of images where the family name points to the most recent image. *Optional* when 'prefix' is a filename")
-	nameFlag := gcpCmd.String("img-name", "", "Overrides the Name used to identify the file in Google Storage, Image and Instance. Defaults to [name]")
 	diskSizeFlag := gcpCmd.Int("disk-size", 0, "Size of system disk in GB")
 	skipCleanup := gcpCmd.Bool("skip-cleanup", false, "Don't remove images or VMs")
 
@@ -53,20 +48,16 @@ func runGcp(args []string) {
 
 	remArgs := gcpCmd.Args()
 	if len(remArgs) == 0 {
-		fmt.Printf("Please specify the prefix to the image to boot\n")
+		fmt.Printf("Please specify the name of the image to boot\n")
 		gcpCmd.Usage()
 		os.Exit(1)
 	}
-	prefix := remArgs[0]
+	name := remArgs[0]
 
 	zone := getStringValue(zoneVar, *zoneFlag, defaultZone)
 	machine := getStringValue(machineVar, *machineFlag, defaultMachine)
 	keys := getStringValue(keysVar, *keysFlag, "")
 	project := getStringValue(projectVar, *projectFlag, "")
-	bucket := getStringValue(bucketVar, *bucketFlag, "")
-	public := getBoolValue(publicVar, *publicFlag)
-	family := getStringValue(familyVar, *familyFlag, "")
-	name := getStringValue(nameVar, *nameFlag, "")
 	diskSize := getIntValue(diskSizeVar, *diskSizeFlag, defaultDiskSize)
 
 	client, err := NewGCPClient(keys, project)
@@ -74,33 +65,7 @@ func runGcp(args []string) {
 		log.Fatalf("Unable to connect to GCP")
 	}
 
-	suffix := ".img.tar.gz"
+	if err = client.CreateInstance(name, name, zone, machine, diskSize, true); err != nil {
-	if strings.HasSuffix(prefix, suffix) {
-		src := prefix
-		if name != "" {
-			prefix = name
-		} else {
-			prefix = prefix[:len(prefix)-len(suffix)]
-		}
-		if bucket == "" {
-			log.Fatalf("No bucket specified. Please provide one using the -bucket flag")
-		}
-		err = client.UploadFile(src, prefix+suffix, bucket, public)
-		if err != nil {
-			log.Fatalf("Error copying to Google Storage: %v", err)
-		}
-		err = client.CreateImage(prefix, "https://storage.googleapis.com/"+bucket+"/"+prefix+".img.tar.gz", family, true)
-		if err != nil {
-			log.Fatalf("Error creating Google Compute Image: %v", err)
-		}
-	}
-
-	// If no name was supplied, use the prefix
-	if name == "" {
-		name = prefix
-	}
-
-	if err = client.CreateInstance(name, prefix, zone, machine, diskSize, true); err != nil {
 		log.Fatal(err)
 	}
 

src/cmd/moby/schema.go

@@ -31,13 +31,7 @@ var schema = string(`
       "type": "object",
       "additionalProperties": false,
       "properties": {
-        "format": {"type": "string"},
+        "format": {"type": "string"}
-        "project": {"type": "string"},
-        "bucket": {"type": "string"},
-        "family": {"type": "string"},
-        "keys": {"type": "string"},
-        "public": {"type": "boolean"},
-        "replace": {"type": "boolean"}
       }
     },
     "outputs": {