run: Add gcp backend

This commit implements `moby run gcp` which allows for testing of moby
images on the Google Cloud Platform

This backend attaches (via SSH) to the serial console.
It generates instance-only SSH keys and adds the public key to the
image metadata. These are used by the `moby` tool only.

It will also automatically upload a file and creates an image if the prefix
given to `moby run` is a filename

Signed-off-by: Dave Tucker <dt@docker.com>

docs/gcp.md

@@ -4,6 +4,13 @@ This is a quick guide to run Moby on GCP.
 
 ## Setup
 
+You have two choices for authentication with Google Cloud
+
+1. You can use [Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials)
+2. You can use a Service Account
+
+### Application Default Credentials
+
 You need the [Google Cloud SDK](https://cloud.google.com/sdk/)
 installed.  Either install it from the URL or view `brew` (on a Mac):
 ```shell
@@ -21,9 +28,16 @@ The authentication will redirect to a browser with Google login.
 
 Also authenticate local applications with
 ```
-gcloud beta auth application-default login
+gcloud auth application-default login
 ```
 
+### Service Account
+
+You can use [this guide](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#createanewserviceaccount)
+to create a Service Account.
+
+Make sure to download the credentials in JSON format and store them somewhere safe.
+
 ## Build a moby image
 
 Add a `gcp` output line to your yaml config, see the example in `examples/gcp.yml`.
@@ -38,10 +52,6 @@ specified bucket, and create a bootable image.
 With the image created, we can now create an instance and connect to
 the serial port.
 
-```shell
+```
-gcloud compute instances create my-node \
+moby run gcp -project myproject-1234 myfile
-  --image="myfile" --metadata serial-port-enable=true \
-  --machine-type="g1-small" --boot-disk-size=200
-
-gcloud compute connect-to-serial-port my-node
 ```

src/cmd/moby/gcp.go

@@ -1,13 +1,18 @@
 package main
 
 import (
+	"crypto/rand"
+	"crypto/rsa"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"time"
 
 	log "github.com/Sirupsen/logrus"
+	"github.com/docker/docker/pkg/term"
+	"golang.org/x/crypto/ssh"
 	"golang.org/x/net/context"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/compute/v1"
@@ -15,6 +20,9 @@ import (
 	"google.golang.org/api/storage/v1"
 )
 
+const pollingInterval = 500 * time.Millisecond
+const timeout = 300
+
 // GCPClient contains state required for communication with GCP
 type GCPClient struct {
 	client      *http.Client
@@ -22,6 +30,7 @@ type GCPClient struct {
 	storage     *storage.Service
 	projectName string
 	fileName    string
+	privKey     *rsa.PrivateKey
 }
 
 // NewGCPClient creates a new GCP client
@@ -80,6 +89,12 @@ func NewGCPClient(keys, projectName string) (*GCPClient, error) {
 		return nil, err
 	}
 
+	log.Debugf("Generating SSH Keypair")
+	client.privKey, err = rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, err
+	}
+
 	return client, nil
 }
 
@@ -110,20 +125,8 @@ func (g GCPClient) UploadFile(filename, bucketName string, public bool) error {
 // CreateImage creates a GCE image using the a source from Google Storage
 func (g GCPClient) CreateImage(filename, storageURL, family string, replace bool) error {
 	if replace {
-		var notFound bool
+		if err := g.DeleteImage(filename); err != nil {
-		op, err := g.compute.Images.Delete(g.projectName, filename).Do()
+			return err
-		if err != nil {
-			if err.(*googleapi.Error).Code != 404 {
-				return err
-			}
-			notFound = true
-		}
-		if !notFound {
-			log.Infof("Deleting existing image...")
-			if err := g.pollOperationStatus(op.Name); err != nil {
-				return err
-			}
-			log.Infof("Image %s deleted", filename)
 		}
 	}
 
@@ -151,6 +154,260 @@ func (g GCPClient) CreateImage(filename, storageURL, family string, replace bool
 	return nil
 }
 
+// DeleteImage deletes and image
+func (g GCPClient) DeleteImage(filename string) error {
+	var notFound bool
+	op, err := g.compute.Images.Delete(g.projectName, filename).Do()
+	if err != nil {
+		if err.(*googleapi.Error).Code != 404 {
+			return err
+		}
+		notFound = true
+	}
+	if !notFound {
+		log.Infof("Deleting existing image...")
+		if err := g.pollOperationStatus(op.Name); err != nil {
+			return err
+		}
+		log.Infof("Image %s deleted", filename)
+	}
+	return nil
+}
+
+// CreateInstance creates and starts an instance on GCE
+func (g GCPClient) CreateInstance(image, zone, machineType string, replace bool) error {
+	if replace {
+		if err := g.DeleteInstance(image, zone, true); err != nil {
+			return err
+		}
+	}
+
+	log.Infof("Creating instance %s", image)
+	enabled := new(string)
+	*enabled = "1"
+
+	k, err := ssh.NewPublicKey(g.privKey.Public())
+	if err != nil {
+		return err
+	}
+	sshKey := new(string)
+	*sshKey = fmt.Sprintf("moby:%s moby", string(ssh.MarshalAuthorizedKey(k)))
+
+	instanceObj := &compute.Instance{
+		MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", zone, machineType),
+		Name:        image,
+		Disks: []*compute.AttachedDisk{
+			{
+				AutoDelete: true,
+				Boot:       true,
+				InitializeParams: &compute.AttachedDiskInitializeParams{
+					SourceImage: fmt.Sprintf("global/images/%s", image),
+				},
+			},
+		},
+		NetworkInterfaces: []*compute.NetworkInterface{
+			{
+				Network: "global/networks/default",
+			},
+		},
+		Metadata: &compute.Metadata{
+			Items: []*compute.MetadataItems{
+				{
+					Key:   "serial-port-enable",
+					Value: enabled,
+				},
+				{
+					Key:   "ssh-keys",
+					Value: sshKey,
+				},
+			},
+		},
+	}
+
+	// Don't wait for operation to complete!
+	// A headstart is needed as by the time we've polled for this event to be
+	// completed, the instance may have already terminated
+	_, err = g.compute.Instances.Insert(g.projectName, zone, instanceObj).Do()
+	if err != nil {
+		return err
+	}
+	log.Infof("Instance created")
+	return nil
+}
+
+// DeleteInstance removes an instance
+func (g GCPClient) DeleteInstance(instance, zone string, wait bool) error {
+	var notFound bool
+	op, err := g.compute.Instances.Delete(g.projectName, zone, instance).Do()
+	if err != nil {
+		if err.(*googleapi.Error).Code != 404 {
+			return err
+		}
+		notFound = true
+	}
+	if !notFound && wait {
+		log.Infof("Deleting existing instance...")
+		if err := g.pollZoneOperationStatus(op.Name, zone); err != nil {
+			return err
+		}
+		log.Infof("Instance %s deleted", instance)
+	}
+	return nil
+}
+
+// GetInstanceSerialOutput streams the serial output of an instance
+func (g GCPClient) GetInstanceSerialOutput(instance, zone string) error {
+	log.Infof("Getting serial port output for instance %s", instance)
+	var next int64
+	for {
+		res, err := g.compute.Instances.GetSerialPortOutput(g.projectName, zone, instance).Start(next).Do()
+		if err != nil {
+			if err.(*googleapi.Error).Code == 400 {
+				// Instance may not be ready yet...
+				time.Sleep(pollingInterval)
+				continue
+			}
+			if err.(*googleapi.Error).Code == 503 {
+				// Timeout received when the instance has terminated
+				break
+			}
+			return err
+		}
+		fmt.Printf(res.Contents)
+		next = res.Next
+		// When the instance has been stopped, Start and Next will both be 0
+		if res.Start > 0 && next == 0 {
+			break
+		}
+	}
+	return nil
+}
+
+// ConnectToInstanceSerialPort uses SSH to connect to the serial port of the instance
+func (g GCPClient) ConnectToInstanceSerialPort(instance, zone string) error {
+	log.Infof("Connecting to serial port of instance %s", instance)
+	gPubKeyURL := "https://cloud-certs.storage.googleapis.com/google-cloud-serialport-host-key.pub"
+	resp, err := http.Get(gPubKeyURL)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	gPubKey, _, _, _, err := ssh.ParseAuthorizedKey(body)
+	if err != nil {
+		return err
+	}
+
+	signer, err := ssh.NewSignerFromKey(g.privKey)
+	if err != nil {
+		return err
+	}
+	config := &ssh.ClientConfig{
+		User: fmt.Sprintf("%s.%s.%s.moby", g.projectName, zone, instance),
+		Auth: []ssh.AuthMethod{
+			ssh.PublicKeys(signer),
+		},
+		HostKeyCallback: ssh.FixedHostKey(gPubKey),
+		Timeout:         5 * time.Second,
+	}
+
+	var conn *ssh.Client
+	// Retry connection as VM may not be ready yet
+	for i := 0; i < timeout; i++ {
+		conn, err = ssh.Dial("tcp", "ssh-serialport.googleapis.com:9600", config)
+		if err != nil {
+			time.Sleep(pollingInterval)
+			continue
+		}
+		break
+	}
+	if conn == nil {
+		return fmt.Errorf(err.Error())
+	}
+	defer conn.Close()
+
+	session, err := conn.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	stdin, err := session.StdinPipe()
+	if err != nil {
+		return fmt.Errorf("Unable to setup stdin for session: %v", err)
+	}
+	go io.Copy(stdin, os.Stdin)
+
+	stdout, err := session.StdoutPipe()
+	if err != nil {
+		return fmt.Errorf("Unable to setup stdout for session: %v", err)
+	}
+	go io.Copy(os.Stdout, stdout)
+
+	stderr, err := session.StderrPipe()
+	if err != nil {
+		return fmt.Errorf("Unable to setup stderr for session: %v", err)
+	}
+	go io.Copy(os.Stderr, stderr)
+	/*
+		c := make(chan os.Signal, 1)
+		exit := make(chan bool, 1)
+		signal.Notify(c)
+		go func(exit <-chan bool, c <-chan os.Signal) {
+			select {
+			case <-exit:
+				return
+			case s := <-c:
+				switch s {
+				// CTRL+C
+				case os.Interrupt:
+					session.Signal(ssh.SIGINT)
+				// CTRL+\
+				case os.Kill:
+					session.Signal(ssh.SIGQUIT)
+				default:
+					log.Debugf("Received signal %s but not forwarding to ssh", s)
+				}
+			}
+		}(exit, c)
+	*/
+	var termWidth, termHeight int
+	fd := os.Stdin.Fd()
+
+	if term.IsTerminal(fd) {
+		oldState, err := term.MakeRaw(fd)
+		if err != nil {
+			return err
+		}
+
+		defer term.RestoreTerminal(fd, oldState)
+
+		winsize, err := term.GetWinsize(fd)
+		if err != nil {
+			termWidth = 80
+			termHeight = 24
+		} else {
+			termWidth = int(winsize.Width)
+			termHeight = int(winsize.Height)
+		}
+	}
+
+	session.RequestPty("xterm", termHeight, termWidth, ssh.TerminalModes{
+		ssh.ECHO: 1,
+	})
+	session.Shell()
+
+	err = session.Wait()
+	//exit <- true
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (g *GCPClient) pollOperationStatus(operationName string) error {
 	for i := 0; i < timeout; i++ {
 		operation, err := g.compute.GlobalOperations.Get(g.projectName, operationName).Do()
@@ -168,3 +425,19 @@ func (g *GCPClient) pollOperationStatus(operationName string) error {
 	return fmt.Errorf("timeout waiting for operation to finish")
 
 }
+func (g *GCPClient) pollZoneOperationStatus(operationName, zone string) error {
+	for i := 0; i < timeout; i++ {
+		operation, err := g.compute.ZoneOperations.Get(g.projectName, zone, operationName).Do()
+		if err != nil {
+			return fmt.Errorf("error fetching operation status: %v", err)
+		}
+		if operation.Error != nil {
+			return fmt.Errorf("error running operation: %v", operation.Error)
+		}
+		if operation.Status == "DONE" {
+			return nil
+		}
+		time.Sleep(pollingInterval)
+	}
+	return fmt.Errorf("timeout waiting for operation to finish")
+}

src/cmd/moby/run.go

@@ -37,6 +37,8 @@ func run(args []string) {
 		runHyperKit(args[1:])
 	case "vmware":
 		runVMware(args[1:])
+	case "gcp":
+		runGcp(args[1:])
 	default:
 		switch runtime.GOOS {
 		case "darwin":

src/cmd/moby/run_gcp.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+// Process the run arguments and execute run
+func runGcp(args []string) {
+	gcpCmd := flag.NewFlagSet("gcp", flag.ExitOnError)
+	gcpCmd.Usage = func() {
+		fmt.Printf("USAGE: %s run gcp [options] [name]\n\n", os.Args[0])
+		fmt.Printf("'name' specifies either the name of an already uploaded\n")
+		fmt.Printf("GCE image or the full path to a image file which will be\n")
+		fmt.Printf("uploaded before it is run.\n\n")
+		fmt.Printf("Options:\n\n")
+		gcpCmd.PrintDefaults()
+	}
+	zone := gcpCmd.String("zone", "europe-west1-d", "GCP Zone")
+	machine := gcpCmd.String("machine", "g1-small", "GCE Machine Type")
+	keys := gcpCmd.String("keys", "", "Path to Service Account JSON key file")
+	project := gcpCmd.String("project", "", "GCP Project Name")
+	bucket := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required* when 'prefix' is a filename")
+	public := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional* when 'prefix' is a filename")
+	family := gcpCmd.String("family", "", "GCE Image Family. A group of images where the family name points to the most recent image. *Optional* when 'prefix' is a filename")
+
+	gcpCmd.Parse(args)
+	remArgs := gcpCmd.Args()
+	if len(remArgs) == 0 {
+		fmt.Printf("Please specify the prefix to the image to boot\n")
+		gcpCmd.Usage()
+		os.Exit(1)
+	}
+	prefix := remArgs[0]
+
+	client, err := NewGCPClient(*keys, *project)
+	if err != nil {
+		log.Fatalf("Unable to connect to GCP")
+	}
+
+	suffix := ".img.tar.gz"
+	if strings.HasSuffix(prefix, suffix) {
+		filename := prefix
+		prefix = prefix[:len(prefix)-len(suffix)]
+		if *bucket == "" {
+			log.Fatalf("No bucket specified. Please provide one using the -bucket flag")
+		}
+		err = client.UploadFile(filename, *bucket, *public)
+		if err != nil {
+			log.Fatalf("Error copying to Google Storage: %v", err)
+		}
+		err = client.CreateImage(prefix, "https://storage.googleapis.com/"+*bucket+"/"+prefix+".img.tar.gz", *family, true)
+		if err != nil {
+			log.Fatalf("Error creating Google Compute Image: %v", err)
+		}
+	}
+
+	if err = client.CreateInstance(prefix, *zone, *machine, true); err != nil {
+		log.Fatal(err)
+	}
+
+	if err = client.ConnectToInstanceSerialPort(prefix, *zone); err != nil {
+		log.Fatal(err)
+	}
+
+	if err = client.DeleteInstance(prefix, *zone, true); err != nil {
+		log.Fatal(err)
+	}
+}

test/test.yml

@@ -30,3 +30,4 @@ outputs:
   - format: kernel+initrd
   - format: iso-bios
   - format: iso-efi
+  - format: gce-img