From fddfd0b3ec5ac643884bd2ea226957a4e314ca6a Mon Sep 17 00:00:00 2001
From: Dave Tucker
Date: Thu, 6 Apr 2017 15:14:25 +0100
Subject: [PATCH 1/4] makefile: Allow a bin/moby to be provided
This is needed for CI to be able to run `moby run $platform` with a
known good version of moby. For example, the one generated when building
the `master`.
This protects CI from any nefarious changes to the `moby` tool and
allows `moby run` to be used safely to test PRs.
Signed-off-by: Dave Tucker
---
Makefile | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/Makefile b/Makefile
index a8979370a..674db1e96 100644
--- a/Makefile
+++ b/Makefile
@@ -7,6 +7,7 @@ GIT_COMMIT=$(shell git rev-list -1 HEAD)
GO_COMPILE=mobylinux/go-compile:a2ff853b00d687f845d0f67189fa645a567c006e@sha256:09fff8a5c022fc9ead35b2779209c043196b09193c6e61d98603d402c0971f03
+MOBY?=bin/moby
GOOS=$(shell uname -s | tr '[:upper:]' '[:lower:]')
GOARCH=amd64
ifneq ($(GOOS),linux)
@@ -27,7 +28,7 @@ bin/infrakit-instance-hyperkit: $(INFRAKIT_DEPS) | bin
tar cf - vendor -C src/cmd/infrakit-instance-hyperkit . | docker run --rm --net=none --log-driver=none -i $(CROSS) $(GO_COMPILE) --package github.com/docker/moby -o $@ | tar xf -
touch $@
-test-initrd.img: bin/moby test/test.yml
+test-initrd.img: $(MOBY) test/test.yml
bin/moby build test/test.yml
test-bzImage: test-initrd.img
@@ -46,9 +47,9 @@ define check_test_log
endef
.PHONY: hyperkit-test
-hyperkit-test: bin/moby test-initrd.img test-bzImage test-cmdline
+hyperkit-test: $(MOBY) test-initrd.img test-bzImage test-cmdline
rm -f disk.img
- script -q /dev/null bin/moby run test | tee test.log
+ script -q /dev/null $(MOBY) run test | tee test.log
$(call check_test_log, test.log)
.PHONY: test
From a4980c8bfb26c3347e42c1dc0d6c822733fbd684 Mon Sep 17 00:00:00 2001
From: Dave Tucker
Date: Thu, 6 Apr 2017 15:23:23 +0100
Subject: [PATCH 2/4] makefile: Add targets to test GCP
Signed-off-by: Dave Tucker
---
Makefile | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/Makefile b/Makefile
index 674db1e96..838a48e97 100644
--- a/Makefile
+++ b/Makefile
@@ -52,6 +52,11 @@ hyperkit-test: $(MOBY) test-initrd.img test-bzImage test-cmdline
script -q /dev/null $(MOBY) run test | tee test.log
$(call check_test_log, test.log)
+.PHONY: test-gcp
+test-gcp: $(MOBY) test.img.tar.gz
+ script -q /dev/null $(MOBY) run gcp test.img.tar.gz | tee test-gcp.log
+ $(call check_test_log, test-gcp.log)
+
.PHONY: test
test: test-initrd.img test-bzImage test-cmdline
tar cf - $^ | ./scripts/qemu.sh 2>&1 | tee test.log
From c50e0de0c134f6b82dff89cbe164d80f62ed5e47 Mon Sep 17 00:00:00 2001
From: Dave Tucker
Date: Fri, 7 Apr 2017 13:25:30 +0100
Subject: [PATCH 3/4] moby: Allow combination of env vars and flags for gcp
Signed-off-by: Dave Tucker
---
src/cmd/moby/run_gcp.go | 49 +++++++++++++++++++++++++-----------
src/cmd/moby/util.go | 55 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 89 insertions(+), 15 deletions(-)
create mode 100644 src/cmd/moby/util.go
diff --git a/src/cmd/moby/run_gcp.go b/src/cmd/moby/run_gcp.go
index 32cedf4dc..856944a00 100644
--- a/src/cmd/moby/run_gcp.go
+++ b/src/cmd/moby/run_gcp.go
@@ -9,6 +9,18 @@ import (
log "github.com/Sirupsen/logrus"
)
+const (
+ defaultZone = "europe-west1-d"
+ defaultMachine = "g1-small"
+ zoneVar = "MOBY_GCP_ZONE"
+ machineVar = "MOBY_GCP_MACHINE"
+ keysVar = "MOBY_GCP_KEYS"
+ projectVar = "MOBY_GCP_PROJECT"
+ bucketVar = "MOBY_GCP_BUCKET"
+ familyVar = "MOBY_GCP_FAMILY"
+ publicVar = "MOBY_GCP_PUBLIC"
+)
+
// Process the run arguments and execute run
func runGcp(args []string) {
gcpCmd := flag.NewFlagSet("gcp", flag.ExitOnError)
@@ -20,14 +32,13 @@ func runGcp(args []string) {
fmt.Printf("Options:\n\n")
gcpCmd.PrintDefaults()
}
- zone := gcpCmd.String("zone", "europe-west1-d", "GCP Zone")
- machine := gcpCmd.String("machine", "g1-small", "GCP Machine Type")
- keys := gcpCmd.String("keys", "", "Path to Service Account JSON key file")
- project := gcpCmd.String("project", "", "GCP Project Name")
- bucket := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required* when 'prefix' is a filename")
- public := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional* when 'prefix' is a filename")
- family := gcpCmd.String("family", "", "GCP Image Family. A group of images where the family name points to the most recent image. *Optional* when 'prefix' is a filename")
-
+ zoneFlag := gcpCmd.String("zone", defaultZone, "GCP Zone")
+ machineFlag := gcpCmd.String("machine", defaultMachine, "GCP Machine Type")
+ keysFlag := gcpCmd.String("keys", "", "Path to Service Account JSON key file")
+ projectFlag := gcpCmd.String("project", "", "GCP Project Name")
+ bucketFlag := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required* when 'prefix' is a filename")
+ publicFlag := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional* when 'prefix' is a filename")
+ familyFlag := gcpCmd.String("family", "", "GCP Image Family. A group of images where the family name points to the most recent image. *Optional* when 'prefix' is a filename")
gcpCmd.Parse(args)
remArgs := gcpCmd.Args()
if len(remArgs) == 0 {
@@ -37,7 +48,15 @@ func runGcp(args []string) {
}
prefix := remArgs[0]
- client, err := NewGCPClient(*keys, *project)
+ zone := getStringValue(zoneVar, *zoneFlag, defaultZone)
+ machine := getStringValue(machineVar, *machineFlag, defaultMachine)
+ keys := getStringValue(keysVar, *keysFlag, "")
+ project := getStringValue(projectVar, *projectFlag, "")
+ bucket := getStringValue(bucketVar, *bucketFlag, "")
+ public := getBoolValue(publicVar, *publicFlag)
+ family := getStringValue(familyVar, *familyFlag, "")
+
+ client, err := NewGCPClient(keys, project)
if err != nil {
log.Fatalf("Unable to connect to GCP")
}
@@ -46,28 +65,28 @@ func runGcp(args []string) {
if strings.HasSuffix(prefix, suffix) {
filename := prefix
prefix = prefix[:len(prefix)-len(suffix)]
- if *bucket == "" {
+ if bucket == "" {
log.Fatalf("No bucket specified. Please provide one using the -bucket flag")
}
- err = client.UploadFile(filename, *bucket, *public)
+ err = client.UploadFile(filename, bucket, public)
if err != nil {
log.Fatalf("Error copying to Google Storage: %v", err)
}
- err = client.CreateImage(prefix, "https://storage.googleapis.com/"+*bucket+"/"+prefix+".img.tar.gz", *family, true)
+ err = client.CreateImage(prefix, "https://storage.googleapis.com/"+bucket+"/"+prefix+".img.tar.gz", family, true)
if err != nil {
log.Fatalf("Error creating Google Compute Image: %v", err)
}
}
- if err = client.CreateInstance(prefix, *zone, *machine, true); err != nil {
+ if err = client.CreateInstance(prefix, zone, machine, true); err != nil {
log.Fatal(err)
}
- if err = client.ConnectToInstanceSerialPort(prefix, *zone); err != nil {
+ if err = client.ConnectToInstanceSerialPort(prefix, zone); err != nil {
log.Fatal(err)
}
- if err = client.DeleteInstance(prefix, *zone, true); err != nil {
+ if err = client.DeleteInstance(prefix, zone, true); err != nil {
log.Fatal(err)
}
}
diff --git a/src/cmd/moby/util.go b/src/cmd/moby/util.go
new file mode 100644
index 000000000..e43c3f493
--- /dev/null
+++ b/src/cmd/moby/util.go
@@ -0,0 +1,55 @@
+package main
+
+import "os"
+
+func getStringValue(envKey string, flagVal string, defaultVal string) string {
+ var res string
+
+ // If defined, take the env variable
+ if _, ok := os.LookupEnv(envKey); ok {
+ res = os.Getenv(envKey)
+ }
+
+ // If a flag is specified, this value takes precedence
+ // Ignore cases where the flag carries the default value
+ if flagVal != "" && flagVal != defaultVal {
+ res = flagVal
+ }
+
+ // if we still don't have a value, use the default
+ if res == "" {
+ res = defaultVal
+ }
+ return res
+}
+
+func getBoolValue(envKey string, flagVal bool) bool {
+ var res bool
+
+ // If defined, take the env variable
+ if _, ok := os.LookupEnv(envKey); ok {
+ switch os.Getenv(envKey) {
+ case "":
+ res = false
+ case "0":
+ res = false
+ case "false":
+ res = false
+ case "FALSE":
+ res = false
+ case "1":
+ res = true
+ default:
+ // catches "true", "TRUE" or anything else
+ res = true
+
+ }
+ }
+
+ // If a flag is specified, this value takes precedence
+ if res != flagVal {
+ res = flagVal
+ }
+
+ return res
+}
From e2da49c7e27610552def1b26d087a32fe29db4b4 Mon Sep 17 00:00:00 2001
From: Dave Tucker
Date: Fri, 7 Apr 2017 14:39:13 +0100
Subject: [PATCH 4/4] moby: Add -img-name flag to moby run gcp
This allows overriding the name used of the file in google storage,
image name or instance name. This will vary depending on how much `moby
run` is doing which is goverened by whether the positional argument
contains an `.img.tar.gz` or not.
For example:
`moby run gcp -img-name test-ea34d1 test` creates an instance called
`test-ea34d1` from the image `test`
`moby run gcp -img-name test-ea34d1` test.img.tar.gz` will upload the
file as `test-ea34d1.tar.gz`, create image `test-ea34d1` and create an
instance called `test-ea34d1`.
The use case for this is for CI to be able to spawn many concurrent test
machines and provide it's own name for them.
Signed-off-by: Dave Tucker
---
src/cmd/moby/gcp.go | 34 +++++++++++++++++-----------------
src/cmd/moby/output.go | 4 ++--
src/cmd/moby/run_gcp.go | 25 +++++++++++++++++++------
3 files changed, 38 insertions(+), 25 deletions(-)
diff --git a/src/cmd/moby/gcp.go b/src/cmd/moby/gcp.go
index 80d2f435f..d3e3b9ca9 100644
--- a/src/cmd/moby/gcp.go
+++ b/src/cmd/moby/gcp.go
@@ -99,15 +99,15 @@ func NewGCPClient(keys, projectName string) (*GCPClient, error) {
}
// UploadFile uploads a file to Google Storage
-func (g GCPClient) UploadFile(filename, bucketName string, public bool) error {
- log.Infof("Uploading file %s to Google Storage", filename)
- f, err := os.Open(filename)
+func (g GCPClient) UploadFile(src, dst, bucketName string, public bool) error {
+ log.Infof("Uploading file %s to Google Storage as %s", src, dst)
+ f, err := os.Open(src)
if err != nil {
return err
}
defer f.Close()
- objectCall := g.storage.Objects.Insert(bucketName, &storage.Object{Name: filename}).Media(f)
+ objectCall := g.storage.Objects.Insert(bucketName, &storage.Object{Name: dst}).Media(f)
if public {
objectCall.PredefinedAcl("publicRead")
@@ -118,24 +118,24 @@ func (g GCPClient) UploadFile(filename, bucketName string, public bool) error {
return err
}
log.Infof("Upload Complete!")
- fmt.Println("gs://" + bucketName + "/" + filename)
+ fmt.Println("gs://" + bucketName + "/" + dst)
return nil
}
// CreateImage creates a GCP image using the a source from Google Storage
-func (g GCPClient) CreateImage(filename, storageURL, family string, replace bool) error {
+func (g GCPClient) CreateImage(name, storageURL, family string, replace bool) error {
if replace {
- if err := g.DeleteImage(filename); err != nil {
+ if err := g.DeleteImage(name); err != nil {
return err
}
}
- log.Infof("Creating image: %s", filename)
+ log.Infof("Creating image: %s", name)
imgObj := &compute.Image{
RawDisk: &compute.ImageRawDisk{
Source: storageURL,
},
- Name: filename,
+ Name: name,
}
if family != "" {
@@ -150,14 +150,14 @@ func (g GCPClient) CreateImage(filename, storageURL, family string, replace bool
if err := g.pollOperationStatus(op.Name); err != nil {
return err
}
- log.Infof("Image %s created", filename)
+ log.Infof("Image %s created", name)
return nil
}
// DeleteImage deletes and image
-func (g GCPClient) DeleteImage(filename string) error {
+func (g GCPClient) DeleteImage(name string) error {
var notFound bool
- op, err := g.compute.Images.Delete(g.projectName, filename).Do()
+ op, err := g.compute.Images.Delete(g.projectName, name).Do()
if err != nil {
if err.(*googleapi.Error).Code != 404 {
return err
@@ -169,20 +169,20 @@ func (g GCPClient) DeleteImage(filename string) error {
if err := g.pollOperationStatus(op.Name); err != nil {
return err
}
- log.Infof("Image %s deleted", filename)
+ log.Infof("Image %s deleted", name)
}
return nil
}
// CreateInstance creates and starts an instance on GCP
-func (g GCPClient) CreateInstance(image, zone, machineType string, replace bool) error {
+func (g GCPClient) CreateInstance(name, image, zone, machineType string, replace bool) error {
if replace {
- if err := g.DeleteInstance(image, zone, true); err != nil {
+ if err := g.DeleteInstance(name, zone, true); err != nil {
return err
}
}
- log.Infof("Creating instance %s", image)
+ log.Infof("Creating instance %s from image %s", name, image)
enabled := new(string)
*enabled = "1"
@@ -195,7 +195,7 @@ func (g GCPClient) CreateInstance(image, zone, machineType string, replace bool)
instanceObj := &compute.Instance{
MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", zone, machineType),
- Name: image,
+ Name: name,
Disks: []*compute.AttachedDisk{
{
AutoDelete: true,
diff --git a/src/cmd/moby/output.go b/src/cmd/moby/output.go
index 5d7bf1d5a..a51d49444 100644
--- a/src/cmd/moby/output.go
+++ b/src/cmd/moby/output.go
@@ -55,7 +55,7 @@ func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error {
if err != nil {
return fmt.Errorf("Unable to connect to GCP")
}
- err = gClient.UploadFile(base+".img.tar.gz", o.Bucket, o.Public)
+ err = gClient.UploadFile(base+".img.tar.gz", base+".img.tar.gz", o.Bucket, o.Public)
if err != nil {
return fmt.Errorf("Error copying to Google Storage: %v", err)
}
@@ -71,7 +71,7 @@ func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error {
if err != nil {
return fmt.Errorf("Unable to connect to GCP")
}
- err = gClient.UploadFile(base+".img.tar.gz", o.Bucket, o.Public)
+ err = gClient.UploadFile(base+".img.tar.gz", base+".img.tar.gz", o.Bucket, o.Public)
if err != nil {
return fmt.Errorf("Error copying to Google Storage: %v", err)
}
diff --git a/src/cmd/moby/run_gcp.go b/src/cmd/moby/run_gcp.go
index 856944a00..9b88ebf95 100644
--- a/src/cmd/moby/run_gcp.go
+++ b/src/cmd/moby/run_gcp.go
@@ -19,6 +19,7 @@ const (
bucketVar = "MOBY_GCP_BUCKET"
familyVar = "MOBY_GCP_FAMILY"
publicVar = "MOBY_GCP_PUBLIC"
+ nameVar = "MOBY_GCP_IMAGE_NAME"
)
// Process the run arguments and execute run
@@ -39,7 +40,9 @@ func runGcp(args []string) {
bucketFlag := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required* when 'prefix' is a filename")
publicFlag := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional* when 'prefix' is a filename")
familyFlag := gcpCmd.String("family", "", "GCP Image Family. A group of images where the family name points to the most recent image. *Optional* when 'prefix' is a filename")
+ nameFlag := gcpCmd.String("img-name", "", "Overrides the Name used to identify the file in Google Storage, Image and Instance. Defaults to [name]")
gcpCmd.Parse(args)
+
remArgs := gcpCmd.Args()
if len(remArgs) == 0 {
fmt.Printf("Please specify the prefix to the image to boot\n")
@@ -55,6 +58,7 @@ func runGcp(args []string) {
bucket := getStringValue(bucketVar, *bucketFlag, "")
public := getBoolValue(publicVar, *publicFlag)
family := getStringValue(familyVar, *familyFlag, "")
+ name := getStringValue(nameVar, *nameFlag, "")
client, err := NewGCPClient(keys, project)
if err != nil {
@@ -63,12 +67,16 @@ func runGcp(args []string) {
suffix := ".img.tar.gz"
if strings.HasSuffix(prefix, suffix) {
- filename := prefix
- prefix = prefix[:len(prefix)-len(suffix)]
+ src := prefix
+ if name != "" {
+ prefix = name
+ } else {
+ prefix = prefix[:len(prefix)-len(suffix)]
+ }
if bucket == "" {
log.Fatalf("No bucket specified. Please provide one using the -bucket flag")
}
- err = client.UploadFile(filename, bucket, public)
+ err = client.UploadFile(src, prefix+suffix, bucket, public)
if err != nil {
log.Fatalf("Error copying to Google Storage: %v", err)
}
@@ -78,15 +86,20 @@ func runGcp(args []string) {
}
}
- if err = client.CreateInstance(prefix, zone, machine, true); err != nil {
+ // If no name was supplied, use the prefix
+ if name == "" {
+ name = prefix
+ }
+
+ if err = client.CreateInstance(name, prefix, zone, machine, true); err != nil {
log.Fatal(err)
}
- if err = client.ConnectToInstanceSerialPort(prefix, zone); err != nil {
+ if err = client.ConnectToInstanceSerialPort(name, zone); err != nil {
log.Fatal(err)
}
- if err = client.DeleteInstance(prefix, zone, true); err != nil {
+ if err = client.DeleteInstance(name, zone, true); err != nil {
log.Fatal(err)
}
}