From db10280f5fe365c57af0c0dddf7fd820f83d9c68 Mon Sep 17 00:00:00 2001 From: Dave Tucker Date: Wed, 5 Apr 2017 01:12:38 +0100 Subject: [PATCH] run: Add gcp backend This commit implements `moby run gcp` which allows for testing of moby images on the Google Cloud Platform This backend attaches (via SSH) to the serial console. It generates instance-only SSH keys and adds the public key to the image metadata. These are used by the `moby` tool only. It will also automatically upload a file and creates an image if the prefix given to `moby run` is a filename Signed-off-by: Dave Tucker --- docs/gcp.md | 24 +++- src/cmd/moby/gcp.go | 301 ++++++++++++++++++++++++++++++++++++++-- src/cmd/moby/run.go | 2 + src/cmd/moby/run_gcp.go | 73 ++++++++++ test/test.yml | 1 + 5 files changed, 380 insertions(+), 21 deletions(-) create mode 100644 src/cmd/moby/run_gcp.go diff --git a/docs/gcp.md b/docs/gcp.md index f8734c26c..3079bf66e 100644 --- a/docs/gcp.md +++ b/docs/gcp.md @@ -4,6 +4,13 @@ This is a quick guide to run Moby on GCP. ## Setup +You have two choices for authentication with Google Cloud + +1. You can use [Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials) +2. You can use a Service Account + +### Application Default Credentials + You need the [Google Cloud SDK](https://cloud.google.com/sdk/) installed. Either install it from the URL or view `brew` (on a Mac): ```shell @@ -21,9 +28,16 @@ The authentication will redirect to a browser with Google login. Also authenticate local applications with ``` -gcloud beta auth application-default login +gcloud auth application-default login ``` +### Service Account + +You can use [this guide](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#createanewserviceaccount) +to create a Service Account. + +Make sure to download the credentials in JSON format and store them somewhere safe. + ## Build a moby image Add a `gcp` output line to your yaml config, see the example in `examples/gcp.yml`. @@ -38,10 +52,6 @@ specified bucket, and create a bootable image. With the image created, we can now create an instance and connect to the serial port. -```shell -gcloud compute instances create my-node \ - --image="myfile" --metadata serial-port-enable=true \ - --machine-type="g1-small" --boot-disk-size=200 - -gcloud compute connect-to-serial-port my-node +``` +moby run gcp -project myproject-1234 myfile ``` diff --git a/src/cmd/moby/gcp.go b/src/cmd/moby/gcp.go index 77e9b3aeb..aefc6698a 100644 --- a/src/cmd/moby/gcp.go +++ b/src/cmd/moby/gcp.go @@ -1,13 +1,18 @@ package main import ( + "crypto/rand" + "crypto/rsa" "fmt" + "io" "io/ioutil" "net/http" "os" "time" log "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/term" + "golang.org/x/crypto/ssh" "golang.org/x/net/context" "golang.org/x/oauth2/google" "google.golang.org/api/compute/v1" @@ -15,6 +20,9 @@ import ( "google.golang.org/api/storage/v1" ) +const pollingInterval = 500 * time.Millisecond +const timeout = 300 + // GCPClient contains state required for communication with GCP type GCPClient struct { client *http.Client @@ -22,6 +30,7 @@ type GCPClient struct { storage *storage.Service projectName string fileName string + privKey *rsa.PrivateKey } // NewGCPClient creates a new GCP client @@ -80,6 +89,12 @@ func NewGCPClient(keys, projectName string) (*GCPClient, error) { return nil, err } + log.Debugf("Generating SSH Keypair") + client.privKey, err = rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + return client, nil } @@ -110,20 +125,8 @@ func (g GCPClient) UploadFile(filename, bucketName string, public bool) error { // CreateImage creates a GCE image using the a source from Google Storage func (g GCPClient) CreateImage(filename, storageURL, family string, replace bool) error { if replace { - var notFound bool - op, err := g.compute.Images.Delete(g.projectName, filename).Do() - if err != nil { - if err.(*googleapi.Error).Code != 404 { - return err - } - notFound = true - } - if !notFound { - log.Infof("Deleting existing image...") - if err := g.pollOperationStatus(op.Name); err != nil { - return err - } - log.Infof("Image %s deleted", filename) + if err := g.DeleteImage(filename); err != nil { + return err } } @@ -151,6 +154,260 @@ func (g GCPClient) CreateImage(filename, storageURL, family string, replace bool return nil } +// DeleteImage deletes and image +func (g GCPClient) DeleteImage(filename string) error { + var notFound bool + op, err := g.compute.Images.Delete(g.projectName, filename).Do() + if err != nil { + if err.(*googleapi.Error).Code != 404 { + return err + } + notFound = true + } + if !notFound { + log.Infof("Deleting existing image...") + if err := g.pollOperationStatus(op.Name); err != nil { + return err + } + log.Infof("Image %s deleted", filename) + } + return nil +} + +// CreateInstance creates and starts an instance on GCE +func (g GCPClient) CreateInstance(image, zone, machineType string, replace bool) error { + if replace { + if err := g.DeleteInstance(image, zone, true); err != nil { + return err + } + } + + log.Infof("Creating instance %s", image) + enabled := new(string) + *enabled = "1" + + k, err := ssh.NewPublicKey(g.privKey.Public()) + if err != nil { + return err + } + sshKey := new(string) + *sshKey = fmt.Sprintf("moby:%s moby", string(ssh.MarshalAuthorizedKey(k))) + + instanceObj := &compute.Instance{ + MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", zone, machineType), + Name: image, + Disks: []*compute.AttachedDisk{ + { + AutoDelete: true, + Boot: true, + InitializeParams: &compute.AttachedDiskInitializeParams{ + SourceImage: fmt.Sprintf("global/images/%s", image), + }, + }, + }, + NetworkInterfaces: []*compute.NetworkInterface{ + { + Network: "global/networks/default", + }, + }, + Metadata: &compute.Metadata{ + Items: []*compute.MetadataItems{ + { + Key: "serial-port-enable", + Value: enabled, + }, + { + Key: "ssh-keys", + Value: sshKey, + }, + }, + }, + } + + // Don't wait for operation to complete! + // A headstart is needed as by the time we've polled for this event to be + // completed, the instance may have already terminated + _, err = g.compute.Instances.Insert(g.projectName, zone, instanceObj).Do() + if err != nil { + return err + } + log.Infof("Instance created") + return nil +} + +// DeleteInstance removes an instance +func (g GCPClient) DeleteInstance(instance, zone string, wait bool) error { + var notFound bool + op, err := g.compute.Instances.Delete(g.projectName, zone, instance).Do() + if err != nil { + if err.(*googleapi.Error).Code != 404 { + return err + } + notFound = true + } + if !notFound && wait { + log.Infof("Deleting existing instance...") + if err := g.pollZoneOperationStatus(op.Name, zone); err != nil { + return err + } + log.Infof("Instance %s deleted", instance) + } + return nil +} + +// GetInstanceSerialOutput streams the serial output of an instance +func (g GCPClient) GetInstanceSerialOutput(instance, zone string) error { + log.Infof("Getting serial port output for instance %s", instance) + var next int64 + for { + res, err := g.compute.Instances.GetSerialPortOutput(g.projectName, zone, instance).Start(next).Do() + if err != nil { + if err.(*googleapi.Error).Code == 400 { + // Instance may not be ready yet... + time.Sleep(pollingInterval) + continue + } + if err.(*googleapi.Error).Code == 503 { + // Timeout received when the instance has terminated + break + } + return err + } + fmt.Printf(res.Contents) + next = res.Next + // When the instance has been stopped, Start and Next will both be 0 + if res.Start > 0 && next == 0 { + break + } + } + return nil +} + +// ConnectToInstanceSerialPort uses SSH to connect to the serial port of the instance +func (g GCPClient) ConnectToInstanceSerialPort(instance, zone string) error { + log.Infof("Connecting to serial port of instance %s", instance) + gPubKeyURL := "https://cloud-certs.storage.googleapis.com/google-cloud-serialport-host-key.pub" + resp, err := http.Get(gPubKeyURL) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + gPubKey, _, _, _, err := ssh.ParseAuthorizedKey(body) + if err != nil { + return err + } + + signer, err := ssh.NewSignerFromKey(g.privKey) + if err != nil { + return err + } + config := &ssh.ClientConfig{ + User: fmt.Sprintf("%s.%s.%s.moby", g.projectName, zone, instance), + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + HostKeyCallback: ssh.FixedHostKey(gPubKey), + Timeout: 5 * time.Second, + } + + var conn *ssh.Client + // Retry connection as VM may not be ready yet + for i := 0; i < timeout; i++ { + conn, err = ssh.Dial("tcp", "ssh-serialport.googleapis.com:9600", config) + if err != nil { + time.Sleep(pollingInterval) + continue + } + break + } + if conn == nil { + return fmt.Errorf(err.Error()) + } + defer conn.Close() + + session, err := conn.NewSession() + if err != nil { + return err + } + defer session.Close() + + stdin, err := session.StdinPipe() + if err != nil { + return fmt.Errorf("Unable to setup stdin for session: %v", err) + } + go io.Copy(stdin, os.Stdin) + + stdout, err := session.StdoutPipe() + if err != nil { + return fmt.Errorf("Unable to setup stdout for session: %v", err) + } + go io.Copy(os.Stdout, stdout) + + stderr, err := session.StderrPipe() + if err != nil { + return fmt.Errorf("Unable to setup stderr for session: %v", err) + } + go io.Copy(os.Stderr, stderr) + /* + c := make(chan os.Signal, 1) + exit := make(chan bool, 1) + signal.Notify(c) + go func(exit <-chan bool, c <-chan os.Signal) { + select { + case <-exit: + return + case s := <-c: + switch s { + // CTRL+C + case os.Interrupt: + session.Signal(ssh.SIGINT) + // CTRL+\ + case os.Kill: + session.Signal(ssh.SIGQUIT) + default: + log.Debugf("Received signal %s but not forwarding to ssh", s) + } + } + }(exit, c) + */ + var termWidth, termHeight int + fd := os.Stdin.Fd() + + if term.IsTerminal(fd) { + oldState, err := term.MakeRaw(fd) + if err != nil { + return err + } + + defer term.RestoreTerminal(fd, oldState) + + winsize, err := term.GetWinsize(fd) + if err != nil { + termWidth = 80 + termHeight = 24 + } else { + termWidth = int(winsize.Width) + termHeight = int(winsize.Height) + } + } + + session.RequestPty("xterm", termHeight, termWidth, ssh.TerminalModes{ + ssh.ECHO: 1, + }) + session.Shell() + + err = session.Wait() + //exit <- true + if err != nil { + return err + } + return nil +} + func (g *GCPClient) pollOperationStatus(operationName string) error { for i := 0; i < timeout; i++ { operation, err := g.compute.GlobalOperations.Get(g.projectName, operationName).Do() @@ -168,3 +425,19 @@ func (g *GCPClient) pollOperationStatus(operationName string) error { return fmt.Errorf("timeout waiting for operation to finish") } +func (g *GCPClient) pollZoneOperationStatus(operationName, zone string) error { + for i := 0; i < timeout; i++ { + operation, err := g.compute.ZoneOperations.Get(g.projectName, zone, operationName).Do() + if err != nil { + return fmt.Errorf("error fetching operation status: %v", err) + } + if operation.Error != nil { + return fmt.Errorf("error running operation: %v", operation.Error) + } + if operation.Status == "DONE" { + return nil + } + time.Sleep(pollingInterval) + } + return fmt.Errorf("timeout waiting for operation to finish") +} diff --git a/src/cmd/moby/run.go b/src/cmd/moby/run.go index d5662b1fe..24c470c1f 100644 --- a/src/cmd/moby/run.go +++ b/src/cmd/moby/run.go @@ -37,6 +37,8 @@ func run(args []string) { runHyperKit(args[1:]) case "vmware": runVMware(args[1:]) + case "gcp": + runGcp(args[1:]) default: switch runtime.GOOS { case "darwin": diff --git a/src/cmd/moby/run_gcp.go b/src/cmd/moby/run_gcp.go new file mode 100644 index 000000000..4c743b992 --- /dev/null +++ b/src/cmd/moby/run_gcp.go @@ -0,0 +1,73 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + log "github.com/Sirupsen/logrus" +) + +// Process the run arguments and execute run +func runGcp(args []string) { + gcpCmd := flag.NewFlagSet("gcp", flag.ExitOnError) + gcpCmd.Usage = func() { + fmt.Printf("USAGE: %s run gcp [options] [name]\n\n", os.Args[0]) + fmt.Printf("'name' specifies either the name of an already uploaded\n") + fmt.Printf("GCE image or the full path to a image file which will be\n") + fmt.Printf("uploaded before it is run.\n\n") + fmt.Printf("Options:\n\n") + gcpCmd.PrintDefaults() + } + zone := gcpCmd.String("zone", "europe-west1-d", "GCP Zone") + machine := gcpCmd.String("machine", "g1-small", "GCE Machine Type") + keys := gcpCmd.String("keys", "", "Path to Service Account JSON key file") + project := gcpCmd.String("project", "", "GCP Project Name") + bucket := gcpCmd.String("bucket", "", "GS Bucket to upload to. *Required* when 'prefix' is a filename") + public := gcpCmd.Bool("public", false, "Select if file on GS should be public. *Optional* when 'prefix' is a filename") + family := gcpCmd.String("family", "", "GCE Image Family. A group of images where the family name points to the most recent image. *Optional* when 'prefix' is a filename") + + gcpCmd.Parse(args) + remArgs := gcpCmd.Args() + if len(remArgs) == 0 { + fmt.Printf("Please specify the prefix to the image to boot\n") + gcpCmd.Usage() + os.Exit(1) + } + prefix := remArgs[0] + + client, err := NewGCPClient(*keys, *project) + if err != nil { + log.Fatalf("Unable to connect to GCP") + } + + suffix := ".img.tar.gz" + if strings.HasSuffix(prefix, suffix) { + filename := prefix + prefix = prefix[:len(prefix)-len(suffix)] + if *bucket == "" { + log.Fatalf("No bucket specified. Please provide one using the -bucket flag") + } + err = client.UploadFile(filename, *bucket, *public) + if err != nil { + log.Fatalf("Error copying to Google Storage: %v", err) + } + err = client.CreateImage(prefix, "https://storage.googleapis.com/"+*bucket+"/"+prefix+".img.tar.gz", *family, true) + if err != nil { + log.Fatalf("Error creating Google Compute Image: %v", err) + } + } + + if err = client.CreateInstance(prefix, *zone, *machine, true); err != nil { + log.Fatal(err) + } + + if err = client.ConnectToInstanceSerialPort(prefix, *zone); err != nil { + log.Fatal(err) + } + + if err = client.DeleteInstance(prefix, *zone, true); err != nil { + log.Fatal(err) + } +} diff --git a/test/test.yml b/test/test.yml index f9f6db32c..ad47387e1 100644 --- a/test/test.yml +++ b/test/test.yml @@ -30,3 +30,4 @@ outputs: - format: kernel+initrd - format: iso-bios - format: iso-efi + - format: gce-img