From 2709ee88bcd8060ed6142b0deb032671e130a109 Mon Sep 17 00:00:00 2001 From: Justin Cormack Date: Mon, 24 Apr 2017 16:34:56 +0100 Subject: [PATCH] Split build and push, and remove push code from run This currently only changes the `gcp` target, but is the new model - the `build` command will only do things locally, then you need to `push` to an image store such as GCP or other ones in order to `run` for platforms that cannot boot directly from a local image. Fix #1618 Signed-off-by: Justin Cormack --- Makefile | 6 ++-- docs/gcp.md | 12 ++++--- examples/gcp.yml | 6 +--- src/cmd/moby/config.go | 8 +---- src/cmd/moby/main.go | 5 ++- src/cmd/moby/output.go | 36 -------------------- src/cmd/moby/push.go | 37 ++++++++++++++++++++ src/cmd/moby/push_gcp.go | 73 ++++++++++++++++++++++++++++++++++++++++ src/cmd/moby/run_gcp.go | 41 ++-------------------- src/cmd/moby/schema.go | 8 +---- 10 files changed, 132 insertions(+), 100 deletions(-) create mode 100644 src/cmd/moby/push.go create mode 100644 src/cmd/moby/push_gcp.go diff --git a/Makefile b/Makefile index 9119ac49e..5955be38e 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,8 @@ test-hyperkit: $(MOBY) test-initrd.img test-bzImage test-cmdline .PHONY: test-gcp test-gcp: $(MOBY) test.img.tar.gz - $(MOBY) run gcp test.img.tar.gz | tee test-gcp.log + $(MOBY) push gcp test.img.tar.gz + $(MOBY) run gcp test | tee test-gcp.log $(call check_test_log, test-gcp.log) .PHONY: test @@ -76,7 +77,8 @@ test-ltp.img.tar.gz: $(MOBY) test/ltp/test-ltp.yml .PHONY: test-ltp test-ltp: $(MOBY) test-ltp.img.tar.gz - $(MOBY) run gcp -skip-cleanup -machine n1-highcpu-4 test-ltp.img.tar.gz | tee test-ltp.log + $(MOBY) push gcp test-ltp.img.tar.gz + $(MOBY) run gcp -skip-cleanup -machine n1-highcpu-4 test-ltp | tee test-ltp.log $(call check_test_log, test-ltp.log) .PHONY: ci ci-tag ci-pr diff --git a/docs/gcp.md b/docs/gcp.md index e16a0b4d5..c82dec8da 100644 --- a/docs/gcp.md +++ b/docs/gcp.md @@ -22,7 +22,7 @@ brew cask install google-cloud-sdk Or via source code: ```shell -wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-151.0.0-darwin-x86_64.tar.gz +curl -SsL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-151.0.0-darwin-x86_64.tar.gz tar xzvf google-cloud-sdk-151.0.0-darwin-x86_64.tar.gz ./google-cloud-sdk/install.sh ``` @@ -50,12 +50,16 @@ Make sure to download the credentials in JSON format and store them somewhere sa ## Build an image -Add a `gcp` output line to your yaml config, see the example in `examples/gcp.yml`. +Add a `gcp-img` output line to your yaml config, see the example in `examples/gcp.yml`. Then do `moby build myfile.yml` -This will create a local `myfile.img.tar.gz` compressed image file, upload it to the -specified bucket, and create a bootable image. +This will create a local `myfile.img.tar.gz` compressed image file. + +## Push image + +Do `moby push gcp -project myproject-1234 -bucket bucketname myfile.img.tar.gz` to upload it to the +specified bucket, and create a bootable image from the stored image. ## Create an instance and connect to it diff --git a/examples/gcp.yml b/examples/gcp.yml index 08ed0be3a..64f139531 100644 --- a/examples/gcp.yml +++ b/examples/gcp.yml @@ -65,8 +65,4 @@ trust: - linuxkit/kernel outputs: - format: kernel+initrd - - format: gcp - project: moby - bucket: mobytestjustin - family: moby-dev - replace: true + - format: gcp-img diff --git a/src/cmd/moby/config.go b/src/cmd/moby/config.go index 7e93a7816..d0d6897f0 100644 --- a/src/cmd/moby/config.go +++ b/src/cmd/moby/config.go @@ -35,13 +35,7 @@ type Moby struct { Contents string } Outputs []struct { - Format string - Project string - Bucket string - Family string - Keys string - Public bool - Replace bool + Format string } } diff --git a/src/cmd/moby/main.go b/src/cmd/moby/main.go index 47871f3eb..faf7b09b2 100644 --- a/src/cmd/moby/main.go +++ b/src/cmd/moby/main.go @@ -42,7 +42,8 @@ func main() { fmt.Printf("USAGE: %s [options] COMMAND\n\n", filepath.Base(os.Args[0])) fmt.Printf("Commands:\n") fmt.Printf(" build Build a Moby image from a YAML file\n") - fmt.Printf(" run Run a Moby image on a local hypervisor or remote cloud\n") + fmt.Printf(" push Push a VM image to a cloud or image store\n") + fmt.Printf(" run Run a VM image on a local hypervisor or remote cloud\n") fmt.Printf(" version Print version information\n") fmt.Printf(" help Print this message\n") fmt.Printf("\n") @@ -81,6 +82,8 @@ func main() { switch args[0] { case "build": build(args[1:]) + case "push": + push(args[1:]) case "run": run(args[1:]) case "version": diff --git a/src/cmd/moby/output.go b/src/cmd/moby/output.go index e5a0c8e12..e34e0e254 100644 --- a/src/cmd/moby/output.go +++ b/src/cmd/moby/output.go @@ -43,42 +43,6 @@ func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error { if err != nil { return fmt.Errorf("Error writing %s output: %v", o.Format, err) } - case "gcp-storage": - err := outputImg(gcp, base+".img.tar.gz", bzimage, initrd, m.Kernel.Cmdline) - if err != nil { - return fmt.Errorf("Error writing %s output: %v", o.Format, err) - } - if o.Bucket == "" { - return fmt.Errorf("No bucket specified for GCP output") - } - gClient, err := NewGCPClient(o.Keys, o.Project) - if err != nil { - return fmt.Errorf("Unable to connect to GCP") - } - err = gClient.UploadFile(base+".img.tar.gz", base+".img.tar.gz", o.Bucket, o.Public) - if err != nil { - return fmt.Errorf("Error copying to Google Storage: %v", err) - } - case "gcp": - err := outputImg(gcp, base+".img.tar.gz", bzimage, initrd, m.Kernel.Cmdline) - if err != nil { - return fmt.Errorf("Error writing %s output: %v", o.Format, err) - } - if o.Bucket == "" { - return fmt.Errorf("No bucket specified for GCP output") - } - gClient, err := NewGCPClient(o.Keys, o.Project) - if err != nil { - return fmt.Errorf("Unable to connect to GCP") - } - err = gClient.UploadFile(base+".img.tar.gz", base+".img.tar.gz", o.Bucket, o.Public) - if err != nil { - return fmt.Errorf("Error copying to Google Storage: %v", err) - } - err = gClient.CreateImage(base, "https://storage.googleapis.com/"+o.Bucket+"/"+base+".img.tar.gz", o.Family, o.Replace) - if err != nil { - return fmt.Errorf("Error creating Google Compute Image: %v", err) - } case "qcow", "qcow2": err := outputImg(qcow, base+".qcow2", bzimage, initrd, m.Kernel.Cmdline) if err != nil { diff --git a/src/cmd/moby/push.go b/src/cmd/moby/push.go new file mode 100644 index 000000000..e168ba3af --- /dev/null +++ b/src/cmd/moby/push.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "os" + + log "github.com/Sirupsen/logrus" +) + +func pushUsage() { + fmt.Printf("USAGE: %s push [backend] [options] [prefix]\n\n", os.Args[0]) + + fmt.Printf("'backend' specifies the push backend.\n") + fmt.Printf("Supported backends are\n") + fmt.Printf(" gcp\n") + fmt.Printf("\n") + fmt.Printf("'options' are the backend specific options.\n") + fmt.Printf("See 'moby push [backend] --help' for details.\n\n") + fmt.Printf("'prefix' specifies the path to the VM image.\n") +} + +func push(args []string) { + if len(args) < 1 { + runUsage() + os.Exit(1) + } + + switch args[0] { + case "help", "-h", "-help", "--help": + pushUsage() + os.Exit(0) + case "gcp": + pushGcp(args[1:]) + default: + log.Errorf("No 'push' backend specified.") + } +} diff --git a/src/cmd/moby/push_gcp.go b/src/cmd/moby/push_gcp.go new file mode 100644 index 000000000..68818aa57 --- /dev/null +++ b/src/cmd/moby/push_gcp.go @@ -0,0 +1,73 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + log "github.com/Sirupsen/logrus" +) + +// Process the run arguments and execute run +func pushGcp(args []string) { + gcpCmd := flag.NewFlagSet("gcp", flag.ExitOnError) + gcpCmd.Usage = func() { + fmt.Printf("USAGE: %s push gcp [options] [name]\n\n", os.Args[0]) + fmt.Printf("'name' specifies the full path of an image file which will be uploaded\n") + fmt.Printf("Options:\n\n") + gcpCmd.PrintDefaults() + } + keysFlag := gcpCmd.String("keys", "", "Path to Service Account JSON key file") + projectFlag := gcpCmd.String("project", "", "GCP Project Name") + bucketFlag := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required*") + publicFlag := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional*") + familyFlag := gcpCmd.String("family", "", "GCP Image Family. A group of images where the family name points to the most recent image. *Optional*") + nameFlag := gcpCmd.String("img-name", "", "Overrides the Name used to identify the file in Google Storage and Image. Defaults to [name]") + + if err := gcpCmd.Parse(args); err != nil { + log.Fatal("Unable to parse args") + } + + remArgs := gcpCmd.Args() + if len(remArgs) == 0 { + fmt.Printf("Please specify the prefix to the image to push\n") + gcpCmd.Usage() + os.Exit(1) + } + prefix := remArgs[0] + + keys := getStringValue(keysVar, *keysFlag, "") + project := getStringValue(projectVar, *projectFlag, "") + bucket := getStringValue(bucketVar, *bucketFlag, "") + public := getBoolValue(publicVar, *publicFlag) + family := getStringValue(familyVar, *familyFlag, "") + name := getStringValue(nameVar, *nameFlag, "") + + client, err := NewGCPClient(keys, project) + if err != nil { + log.Fatalf("Unable to connect to GCP") + } + + suffix := ".img.tar.gz" + src := prefix + if strings.HasSuffix(prefix, suffix) { + prefix = prefix[:len(prefix)-len(suffix)] + } else { + src = prefix + suffix + } + if name != "" { + prefix = name + } + if bucket == "" { + log.Fatalf("No bucket specified. Please provide one using the -bucket flag") + } + err = client.UploadFile(src, prefix+suffix, bucket, public) + if err != nil { + log.Fatalf("Error copying to Google Storage: %v", err) + } + err = client.CreateImage(prefix, "https://storage.googleapis.com/"+bucket+"/"+prefix+".img.tar.gz", family, true) + if err != nil { + log.Fatalf("Error creating Google Compute Image: %v", err) + } +} diff --git a/src/cmd/moby/run_gcp.go b/src/cmd/moby/run_gcp.go index b983b05b4..b29e8714e 100644 --- a/src/cmd/moby/run_gcp.go +++ b/src/cmd/moby/run_gcp.go @@ -4,7 +4,6 @@ import ( "flag" "fmt" "os" - "strings" log "github.com/Sirupsen/logrus" ) @@ -40,10 +39,6 @@ func runGcp(args []string) { machineFlag := gcpCmd.String("machine", defaultMachine, "GCP Machine Type") keysFlag := gcpCmd.String("keys", "", "Path to Service Account JSON key file") projectFlag := gcpCmd.String("project", "", "GCP Project Name") - bucketFlag := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required* when 'prefix' is a filename") - publicFlag := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional* when 'prefix' is a filename") - familyFlag := gcpCmd.String("family", "", "GCP Image Family. A group of images where the family name points to the most recent image. *Optional* when 'prefix' is a filename") - nameFlag := gcpCmd.String("img-name", "", "Overrides the Name used to identify the file in Google Storage, Image and Instance. Defaults to [name]") diskSizeFlag := gcpCmd.Int("disk-size", 0, "Size of system disk in GB") skipCleanup := gcpCmd.Bool("skip-cleanup", false, "Don't remove images or VMs") @@ -53,20 +48,16 @@ func runGcp(args []string) { remArgs := gcpCmd.Args() if len(remArgs) == 0 { - fmt.Printf("Please specify the prefix to the image to boot\n") + fmt.Printf("Please specify the name of the image to boot\n") gcpCmd.Usage() os.Exit(1) } - prefix := remArgs[0] + name := remArgs[0] zone := getStringValue(zoneVar, *zoneFlag, defaultZone) machine := getStringValue(machineVar, *machineFlag, defaultMachine) keys := getStringValue(keysVar, *keysFlag, "") project := getStringValue(projectVar, *projectFlag, "") - bucket := getStringValue(bucketVar, *bucketFlag, "") - public := getBoolValue(publicVar, *publicFlag) - family := getStringValue(familyVar, *familyFlag, "") - name := getStringValue(nameVar, *nameFlag, "") diskSize := getIntValue(diskSizeVar, *diskSizeFlag, defaultDiskSize) client, err := NewGCPClient(keys, project) @@ -74,33 +65,7 @@ func runGcp(args []string) { log.Fatalf("Unable to connect to GCP") } - suffix := ".img.tar.gz" - if strings.HasSuffix(prefix, suffix) { - src := prefix - if name != "" { - prefix = name - } else { - prefix = prefix[:len(prefix)-len(suffix)] - } - if bucket == "" { - log.Fatalf("No bucket specified. Please provide one using the -bucket flag") - } - err = client.UploadFile(src, prefix+suffix, bucket, public) - if err != nil { - log.Fatalf("Error copying to Google Storage: %v", err) - } - err = client.CreateImage(prefix, "https://storage.googleapis.com/"+bucket+"/"+prefix+".img.tar.gz", family, true) - if err != nil { - log.Fatalf("Error creating Google Compute Image: %v", err) - } - } - - // If no name was supplied, use the prefix - if name == "" { - name = prefix - } - - if err = client.CreateInstance(name, prefix, zone, machine, diskSize, true); err != nil { + if err = client.CreateInstance(name, name, zone, machine, diskSize, true); err != nil { log.Fatal(err) } diff --git a/src/cmd/moby/schema.go b/src/cmd/moby/schema.go index 1123fd4c8..379c480aa 100644 --- a/src/cmd/moby/schema.go +++ b/src/cmd/moby/schema.go @@ -31,13 +31,7 @@ var schema = string(` "type": "object", "additionalProperties": false, "properties": { - "format": {"type": "string"}, - "project": {"type": "string"}, - "bucket": {"type": "string"}, - "family": {"type": "string"}, - "keys": {"type": "string"}, - "public": {"type": "boolean"}, - "replace": {"type": "boolean"} + "format": {"type": "string"} } }, "outputs": {