build: Use older GCP API and support service account auth

This commit uses the older GCP API as it supports both compute and
storage. As a result, we can now use either Application Default
Credentials that are generated using the `gcloud` tool or by supplying the
service account credentials in JSON format

Signed-off-by: Dave Tucker <dt@docker.com>

src/cmd/moby/config.go

@@ -33,6 +33,7 @@ type Moby struct {
 		Project string
 		Bucket  string
 		Family  string
+		Keys    string
 		Public  bool
 		Replace bool
 	}

src/cmd/moby/gcp.go

@@ -1,100 +1,170 @@
 package main
 
 import (
-	"errors"
 	"fmt"
-	"io"
+	"io/ioutil"
+	"net/http"
 	"os"
-	"os/exec"
+	"time"
 
-	"cloud.google.com/go/storage"
+	log "github.com/Sirupsen/logrus"
 	"golang.org/x/net/context"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/compute/v1"
+	"google.golang.org/api/googleapi"
+	"google.golang.org/api/storage/v1"
 )
 
-func uploadGS(filename, project, bucket string, public bool) error {
+// GCPClient contains state required for communication with GCP
-	if project != "" {
+type GCPClient struct {
-		err := os.Setenv("GOOGLE_CLOUD_PROJECT", project)
+	client      *http.Client
-		if err != nil {
+	compute     *compute.Service
-			return err
+	storage     *storage.Service
-		}
+	projectName string
-	}
+	fileName    string
-	if os.Getenv("GOOGLE_CLOUD_PROJECT") == "" {
-		return errors.New("GOOGLE_CLOUD_PROJECT environment variable must be set or project specified in config")
 }
 
+// NewGCPClient creates a new GCP client
+func NewGCPClient(keys, projectName string) (*GCPClient, error) {
+	log.Debugf("Connecting to GCP")
 	ctx := context.Background()
-	client, err := storage.NewClient(ctx)
+	var client *GCPClient
+	if keys != "" {
+		log.Debugf("Using Keys %s", keys)
+		f, err := os.Open(keys)
 		if err != nil {
-		return err
+			return nil, err
 		}
 
+		jsonKey, err := ioutil.ReadAll(f)
+		if err != nil {
+			return nil, err
+		}
+
+		config, err := google.JWTConfigFromJSON(jsonKey,
+			storage.DevstorageReadWriteScope,
+			compute.ComputeScope,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		client = &GCPClient{
+			client:      config.Client(ctx),
+			projectName: projectName,
+		}
+	} else {
+		log.Debugf("Using Application Default crednetials")
+		gc, err := google.DefaultClient(
+			ctx,
+			storage.DevstorageReadWriteScope,
+			compute.ComputeScope,
+		)
+		if err != nil {
+			return nil, err
+		}
+		client = &GCPClient{
+			client:      gc,
+			projectName: projectName,
+		}
+	}
+
+	var err error
+	client.compute, err = compute.New(client.client)
+	if err != nil {
+		return nil, err
+	}
+
+	client.storage, err = storage.New(client.client)
+	if err != nil {
+		return nil, err
+	}
+
+	return client, nil
+}
+
+// UploadFile uploads a file to Google Storage
+func (g GCPClient) UploadFile(filename, bucketName string, public bool) error {
+	log.Infof("Uploading file %s to Google Storage", filename)
 	f, err := os.Open(filename)
 	if err != nil {
 		return err
 	}
 	defer f.Close()
 
-	obj := client.Bucket(bucket).Object(filename)
+	objectCall := g.storage.Objects.Insert(bucketName, &storage.Object{Name: filename}).Media(f)
-	wc := obj.NewWriter(ctx)
-	_, err = io.Copy(wc, f)
-	if err != nil {
-		return err
-	}
-	err = wc.Close()
-	if err != nil {
-		return err
-	}
 
 	if public {
-		err = obj.ACL().Set(ctx, storage.AllUsers, storage.RoleReader)
+		objectCall.PredefinedAcl("publicRead")
+	}
+
+	_, err = objectCall.Do()
 	if err != nil {
 		return err
 	}
-	}
+	log.Infof("Upload Complete!")
-
+	fmt.Println("gs://" + bucketName + "/" + filename)
-	fmt.Println("gs://" + bucket + "/" + filename)
-
 	return nil
 }
 
-func imageGS(filename, project, storage, family string, replace bool) error {
+// CreateImage creates a GCE image using the a source from Google Storage
-	if project != "" {
+func (g GCPClient) CreateImage(filename, storageURL, family string, replace bool) error {
-		err := os.Setenv("GOOGLE_CLOUD_PROJECT", project)
-		if err != nil {
-			return err
-		}
-	}
-	if os.Getenv("GOOGLE_CLOUD_PROJECT") == "" {
-		return errors.New("GOOGLE_CLOUD_PROJECT environment variable must be set or project specified in config")
-	}
-
-	// TODO do not shell out to gcloud tool, use the API
-
-	gcloud, err := exec.LookPath("gcloud")
-	if err != nil {
-		return errors.New("Please install the gcloud binary")
-	}
-
 	if replace {
-		args := []string{"compute", "images", "delete", filename}
+		var notFound bool
-		cmd := exec.Command(gcloud, args...)
+		op, err := g.compute.Images.Delete(g.projectName, filename).Do()
-		// ignore failures; it may not exist
-		_ = cmd.Run()
-	}
-
-	args := []string{"compute", "images", "create", "--source-uri", storage}
-	if family != "" {
-		args = append(args, "--family", family)
-	}
-	args = append(args, filename)
-	cmd := exec.Command(gcloud, args...)
-
-	out, err := cmd.CombinedOutput()
 		if err != nil {
-		return fmt.Errorf("Image creation failed: %v - %s", err, string(out))
+			if err.(*googleapi.Error).Code != 404 {
+				return err
+			}
+			notFound = true
+		}
+		if !notFound {
+			log.Infof("Deleting existing image...")
+			if err := g.pollOperationStatus(op.Name); err != nil {
+				return err
+			}
+			log.Infof("Image %s deleted", filename)
+		}
 	}
 
-	fmt.Println(filename)
+	log.Infof("Creating image: %s", filename)
+	imgObj := &compute.Image{
+		RawDisk: &compute.ImageRawDisk{
+			Source: storageURL,
+		},
+		Name: filename,
+	}
 
+	if family != "" {
+		imgObj.Family = family
+	}
+
+	op, err := g.compute.Images.Insert(g.projectName, imgObj).Do()
+	if err != nil {
+		return err
+	}
+
+	if err := g.pollOperationStatus(op.Name); err != nil {
+		return err
+	}
+	log.Infof("Image %s created", filename)
 	return nil
 }
+
+func (g *GCPClient) pollOperationStatus(operationName string) error {
+	for i := 0; i < timeout; i++ {
+		operation, err := g.compute.GlobalOperations.Get(g.projectName, operationName).Do()
+		if err != nil {
+			return fmt.Errorf("error fetching operation status: %v", err)
+		}
+		if operation.Error != nil {
+			return fmt.Errorf("error running operation: %v", operation.Error)
+		}
+		if operation.Status == "DONE" {
+			return nil
+		}
+		time.Sleep(pollingInterval)
+	}
+	return fmt.Errorf("timeout waiting for operation to finish")
+
+}

src/cmd/moby/output.go

@@ -51,7 +51,11 @@ func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error {
 			if o.Bucket == "" {
 				return fmt.Errorf("No bucket specified for GCE output")
 			}
-			err = uploadGS(base+".img.tar.gz", o.Project, o.Bucket, o.Public)
+			gClient, err := NewGCPClient(o.Keys, o.Project)
+			if err != nil {
+				return fmt.Errorf("Unable to connect to GCP")
+			}
+			err = gClient.UploadFile(base+".img.tar.gz", o.Bucket, o.Public)
 			if err != nil {
 				return fmt.Errorf("Error copying to Google Storage: %v", err)
 			}
@@ -63,11 +67,15 @@ func outputs(m *Moby, base string, bzimage []byte, initrd []byte) error {
 			if o.Bucket == "" {
 				return fmt.Errorf("No bucket specified for GCE output")
 			}
-			err = uploadGS(base+".img.tar.gz", o.Project, o.Bucket, o.Public)
+			gClient, err := NewGCPClient(o.Keys, o.Project)
+			if err != nil {
+				return fmt.Errorf("Unable to connect to GCP")
+			}
+			err = gClient.UploadFile(base+".img.tar.gz", o.Bucket, o.Public)
 			if err != nil {
 				return fmt.Errorf("Error copying to Google Storage: %v", err)
 			}
-			err = imageGS(base, o.Project, "https://storage.googleapis.com/"+o.Bucket+"/"+base+".img.tar.gz", o.Family, o.Replace)
+			err = gClient.CreateImage(base, "https://storage.googleapis.com/"+o.Bucket+"/"+base+".img.tar.gz", o.Family, o.Replace)
 			if err != nil {
 				return fmt.Errorf("Error creating Google Compute Image: %v", err)
 			}