From d5a8e23cdda0a54df71557989a336200b3ec72f8 Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Wed, 5 Apr 2017 01:08:54 +0100 Subject: [PATCH] build: Use older GCP API and support service account auth This commit uses the older GCP API as it supports both compute and storage. As a result, we can now use either Application Default Credentials that are generated using the `gcloud` tool or by supplying the service account credentials in JSON format Signed-off-by: Dave Tucker --- src/cmd/moby/config.go | 1 + src/cmd/moby/gcp.go | 190 ++++++++++++++++++++++++++++------------- src/cmd/moby/output.go | 14 ++- 3 files changed, 142 insertions(+), 63 deletions(-) diff --git a/src/cmd/moby/config.go b/src/cmd/moby/config.go index d5c17e3dc..5d314c641 100644 --- a/src/cmd/moby/config.go +++ b/src/cmd/moby/config.go @@ -33,6 +33,7 @@ type Moby struct { Project string Bucket string Family string + Keys string Public bool Replace bool } diff --git a/src/cmd/moby/gcp.go b/src/cmd/moby/gcp.go index 5390af538..77e9b3aeb 100644 --- a/src/cmd/moby/gcp.go +++ b/src/cmd/moby/gcp.go @@ -1,100 +1,170 @@ package main import ( - "errors" "fmt" - "io" + "io/ioutil" + "net/http" "os" - "os/exec" + "time" - "cloud.google.com/go/storage" + log "github.com/Sirupsen/logrus" "golang.org/x/net/context" + "golang.org/x/oauth2/google" + "google.golang.org/api/compute/v1" + "google.golang.org/api/googleapi" + "google.golang.org/api/storage/v1" ) -func uploadGS(filename, project, bucket string, public bool) error { - if project != "" { - err := os.Setenv("GOOGLE_CLOUD_PROJECT", project) +// GCPClient contains state required for communication with GCP +type GCPClient struct { + client *http.Client + compute *compute.Service + storage *storage.Service + projectName string + fileName string +} + +// NewGCPClient creates a new GCP client +func NewGCPClient(keys, projectName string) (*GCPClient, error) { + log.Debugf("Connecting to GCP") + ctx := context.Background() + var client *GCPClient + if keys != "" { + log.Debugf("Using Keys %s", keys) + f, err := os.Open(keys) if err != nil { - return err + return nil, err + } + + jsonKey, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + config, err := google.JWTConfigFromJSON(jsonKey, + storage.DevstorageReadWriteScope, + compute.ComputeScope, + ) + if err != nil { + return nil, err + } + + client = &GCPClient{ + client: config.Client(ctx), + projectName: projectName, + } + } else { + log.Debugf("Using Application Default crednetials") + gc, err := google.DefaultClient( + ctx, + storage.DevstorageReadWriteScope, + compute.ComputeScope, + ) + if err != nil { + return nil, err + } + client = &GCPClient{ + client: gc, + projectName: projectName, } } - if os.Getenv("GOOGLE_CLOUD_PROJECT") == "" { - return errors.New("GOOGLE_CLOUD_PROJECT environment variable must be set or project specified in config") - } - ctx := context.Background() - client, err := storage.NewClient(ctx) + var err error + client.compute, err = compute.New(client.client) if err != nil { - return err + return nil, err } + client.storage, err = storage.New(client.client) + if err != nil { + return nil, err + } + + return client, nil +} + +// UploadFile uploads a file to Google Storage +func (g GCPClient) UploadFile(filename, bucketName string, public bool) error { + log.Infof("Uploading file %s to Google Storage", filename) f, err := os.Open(filename) if err != nil { return err } defer f.Close() - obj := client.Bucket(bucket).Object(filename) - wc := obj.NewWriter(ctx) - _, err = io.Copy(wc, f) - if err != nil { - return err - } - err = wc.Close() - if err != nil { - return err - } + objectCall := g.storage.Objects.Insert(bucketName, &storage.Object{Name: filename}).Media(f) if public { - err = obj.ACL().Set(ctx, storage.AllUsers, storage.RoleReader) - if err != nil { - return err - } + objectCall.PredefinedAcl("publicRead") } - fmt.Println("gs://" + bucket + "/" + filename) - + _, err = objectCall.Do() + if err != nil { + return err + } + log.Infof("Upload Complete!") + fmt.Println("gs://" + bucketName + "/" + filename) return nil } -func imageGS(filename, project, storage, family string, replace bool) error { - if project != "" { - err := os.Setenv("GOOGLE_CLOUD_PROJECT", project) - if err != nil { - return err - } - } - if os.Getenv("GOOGLE_CLOUD_PROJECT") == "" { - return errors.New("GOOGLE_CLOUD_PROJECT environment variable must be set or project specified in config") - } - - // TODO do not shell out to gcloud tool, use the API - - gcloud, err := exec.LookPath("gcloud") - if err != nil { - return errors.New("Please install the gcloud binary") - } - +// CreateImage creates a GCE image using the a source from Google Storage +func (g GCPClient) CreateImage(filename, storageURL, family string, replace bool) error { if replace { - args := []string{"compute", "images", "delete", filename} - cmd := exec.Command(gcloud, args...) - // ignore failures; it may not exist - _ = cmd.Run() + var notFound bool + op, err := g.compute.Images.Delete(g.projectName, filename).Do() + if err != nil { + if err.(*googleapi.Error).Code != 404 { + return err + } + notFound = true + } + if !notFound { + log.Infof("Deleting existing image...") + if err := g.pollOperationStatus(op.Name); err != nil { + return err + } + log.Infof("Image %s deleted", filename) + } + } + + log.Infof("Creating image: %s", filename) + imgObj := &compute.Image{ + RawDisk: &compute.ImageRawDisk{ + Source: storageURL, + }, + Name: filename, } - args := []string{"compute", "images", "create", "--source-uri", storage} if family != "" { - args = append(args, "--family", family) + imgObj.Family = family } - args = append(args, filename) - cmd := exec.Command(gcloud, args...) - out, err := cmd.CombinedOutput() + op, err := g.compute.Images.Insert(g.projectName, imgObj).Do() if err != nil { - return fmt.Errorf("Image creation failed: %v - %s", err, string(out)) + return err } - fmt.Println(filename) - + if err := g.pollOperationStatus(op.Name); err != nil { + return err + } + log.Infof("Image %s created", filename) return nil } + +func (g *GCPClient) pollOperationStatus(operationName string) error { + for i := 0; i < timeout; i++ { + operation, err := g.compute.GlobalOperations.Get(g.projectName, operationName).Do() + if err != nil { + return fmt.Errorf("error fetching operation status: %v", err) + } + if operation.Error != nil { + return fmt.Errorf("error running operation: %v", operation.Error) + } + if operation.Status == "DONE" { + return nil + } + time.Sleep(pollingInterval) + } + return fmt.Errorf("timeout waiting for operation to finish") + +} diff --git a/src/cmd/moby/output.go b/src/cmd/moby/output.go index f69f5a8a4..171c58907 100644 --- a/src/cmd/moby/output.go +++ b/src/cmd/moby/output.go @@ -51,7 +51,11 @@ func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error { if o.Bucket == "" { return fmt.Errorf("No bucket specified for GCE output") } - err = uploadGS(base+".img.tar.gz", o.Project, o.Bucket, o.Public) + gClient, err := NewGCPClient(o.Keys, o.Project) + if err != nil { + return fmt.Errorf("Unable to connect to GCP") + } + err = gClient.UploadFile(base+".img.tar.gz", o.Bucket, o.Public) if err != nil { return fmt.Errorf("Error copying to Google Storage: %v", err) } @@ -63,11 +67,15 @@ func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error { if o.Bucket == "" { return fmt.Errorf("No bucket specified for GCE output") } - err = uploadGS(base+".img.tar.gz", o.Project, o.Bucket, o.Public) + gClient, err := NewGCPClient(o.Keys, o.Project) + if err != nil { + return fmt.Errorf("Unable to connect to GCP") + } + err = gClient.UploadFile(base+".img.tar.gz", o.Bucket, o.Public) if err != nil { return fmt.Errorf("Error copying to Google Storage: %v", err) } - err = imageGS(base, o.Project, "https://storage.googleapis.com/"+o.Bucket+"/"+base+".img.tar.gz", o.Family, o.Replace) + err = gClient.CreateImage(base, "https://storage.googleapis.com/"+o.Bucket+"/"+base+".img.tar.gz", o.Family, o.Replace) if err != nil { return fmt.Errorf("Error creating Google Compute Image: %v", err) }