Merge pull request #1964 from dave-tucker/aws

Add AWS Support to LinuxKit Push and Run

src/cmd/linuxkit/push.go

@@ -11,12 +11,14 @@ import (
 func pushUsage() {
 	invoked := filepath.Base(os.Args[0])
 	fmt.Printf("USAGE: %s push [backend] [options] [prefix]\n\n", invoked)
-
 	fmt.Printf("'backend' specifies the push backend.\n")
 	fmt.Printf("Supported backends are\n")
+	// Please keep these in alphabetical order
+	fmt.Printf("  aws\n")
+	fmt.Printf("  azure\n")
 	fmt.Printf("  gcp\n")
 	fmt.Printf("  vcenter\n")
-	fmt.Printf("  azure\n")
+	fmt.Printf("\n")
 	fmt.Printf("'options' are the backend specific options.\n")
 	fmt.Printf("See '%s push [backend] --help' for details.\n\n", invoked)
 	fmt.Printf("'prefix' specifies the path to the VM image.\n")
@@ -29,15 +31,18 @@ func push(args []string) {
 	}
 
 	switch args[0] {
+	// Please keep cases in alphabetical order
+	case "aws":
+		pushAWS(args[1:])
+	case "azure":
+		pushAzure(args[1:])
+	case "gcp":
+		pushGcp(args[1:])
 	case "help", "-h", "-help", "--help":
 		pushUsage()
 		os.Exit(0)
-	case "gcp":
-		pushGcp(args[1:])
 	case "vcenter":
 		pushVCenter(args[1:])
-	case "azure":
-		pushAzure(args[1:])
 	default:
 		log.Errorf("No 'push' backend specified.")
 	}

src/cmd/linuxkit/push_aws.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+const timeoutVar = "LINUXKIT_UPLOAD_TIMEOUT"
+
+func pushAWS(args []string) {
+	awsCmd := flag.NewFlagSet("aws", flag.ExitOnError)
+	invoked := filepath.Base(os.Args[0])
+	awsCmd.Usage = func() {
+		fmt.Printf("USAGE: %s push aws [options] [name]\n\n", invoked)
+		fmt.Printf("'name' specifies the full path of an image file which will be uploaded\n")
+		fmt.Printf("Options:\n\n")
+		awsCmd.PrintDefaults()
+	}
+	timeoutFlag := awsCmd.Int("timeout", 0, "Upload timeout in seconds")
+	bucketFlag := awsCmd.String("bucket", "", "S3 Bucket to upload to. *Required*")
+	nameFlag := awsCmd.String("img-name", "", "Overrides the Name used to identify the file in Amazon S3 and Image. Defaults to [name] with the file extension removed.")
+
+	if err := awsCmd.Parse(args); err != nil {
+		log.Fatal("Unable to parse args")
+	}
+
+	remArgs := awsCmd.Args()
+	if len(remArgs) == 0 {
+		fmt.Printf("Please specify the path to the image to push\n")
+		awsCmd.Usage()
+		os.Exit(1)
+	}
+	src := remArgs[0]
+
+	timeout := getIntValue(timeoutVar, *timeoutFlag, 600)
+	bucket := getStringValue(bucketVar, *bucketFlag, "")
+	name := getStringValue(nameVar, *nameFlag, "")
+
+	sess := session.Must(session.NewSession())
+	storage := s3.New(sess)
+
+	ctx, cancelFn := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
+	defer cancelFn()
+
+	if bucket == "" {
+		log.Fatalf("No bucket specified. Please provide one using the -bucket flag")
+	}
+
+	f, err := os.Open(src)
+	if err != nil {
+		log.Fatalf("Error opening file: %s", err)
+	}
+	defer f.Close()
+
+	if name == "" {
+		name = strings.TrimSuffix(name, filepath.Ext(src))
+	}
+
+	content, err := ioutil.ReadAll(f)
+	if err != nil {
+		log.Fatalf("error reading file: %s", err)
+	}
+
+	dst := name + filepath.Ext(src)
+	putParams := &s3.PutObjectInput{
+		Bucket:        aws.String(bucket),
+		Key:           aws.String(dst),
+		Body:          f,
+		ContentLength: aws.Int64(int64(len(content))),
+		ContentType:   aws.String("application/octet-stream"),
+	}
+	log.Debugf("PutObject:\n%v", putParams)
+
+	_, err = storage.PutObjectWithContext(ctx, putParams)
+	if err != nil {
+		log.Fatalf("Error uploading to S3: %s", err)
+	}
+
+	compute := ec2.New(sess)
+
+	importParams := &ec2.ImportSnapshotInput{
+		Description: aws.String(fmt.Sprintf("LinuxKit: %s", name)),
+		DiskContainer: &ec2.SnapshotDiskContainer{
+			Description: aws.String(fmt.Sprintf("LinuxKit: %s disk", name)),
+			Format:      aws.String("raw"),
+			UserBucket: &ec2.UserBucket{
+				S3Bucket: aws.String(bucket),
+				S3Key:    aws.String(dst),
+			},
+		},
+	}
+	log.Debugf("ImportSnapshot:\n%v", importParams)
+
+	resp, err := compute.ImportSnapshot(importParams)
+	if err != nil {
+		log.Fatalf("Error importing snapshot: %s", err)
+	}
+
+	var snapshotID *string
+	for {
+		describeParams := &ec2.DescribeImportSnapshotTasksInput{
+			ImportTaskIds: []*string{
+				resp.ImportTaskId,
+			},
+		}
+		log.Debugf("DescribeImportSnapshotTask:\n%v", describeParams)
+		status, err := compute.DescribeImportSnapshotTasks(describeParams)
+		if err != nil {
+			log.Fatalf("Error getting import snapshot status: %s", err)
+		}
+		if len(status.ImportSnapshotTasks) == 0 {
+			log.Fatalf("Unable to get import snapshot task status")
+		}
+		if *status.ImportSnapshotTasks[0].SnapshotTaskDetail.Status != "completed" {
+			progress := "0"
+			if status.ImportSnapshotTasks[0].SnapshotTaskDetail.Progress != nil {
+				progress = *status.ImportSnapshotTasks[0].SnapshotTaskDetail.Progress
+			}
+			log.Debugf("Task %s is %s%% complete. Waiting 60 seconds...\n", *resp.ImportTaskId, progress)
+			time.Sleep(60 * time.Second)
+			continue
+		}
+		snapshotID = status.ImportSnapshotTasks[0].SnapshotTaskDetail.SnapshotId
+		break
+	}
+
+	if snapshotID == nil {
+		log.Fatalf("SnapshotID unavailable after import completed")
+	}
+
+	regParams := &ec2.RegisterImageInput{
+		Name:         aws.String(name), // Required
+		Architecture: aws.String("x86_64"),
+		BlockDeviceMappings: []*ec2.BlockDeviceMapping{
+			{
+				DeviceName: aws.String("/dev/sda1"),
+				Ebs: &ec2.EbsBlockDevice{
+					DeleteOnTermination: aws.Bool(true),
+					SnapshotId:          snapshotID,
+					VolumeType:          aws.String("standard"),
+				},
+			},
+		},
+		Description:        aws.String(fmt.Sprintf("LinuxKit: %s image", name)),
+		RootDeviceName:     aws.String("/dev/sda1"),
+		VirtualizationType: aws.String("hvm"),
+	}
+	log.Debugf("RegisterImage:\n%v", regParams)
+	regResp, err := compute.RegisterImage(regParams)
+	if err != nil {
+		log.Fatalf("Error registering the image: %s", err)
+	}
+	log.Infof("Created AMI: %s", *regResp.ImageId)
+}

src/cmd/linuxkit/run.go

@@ -16,13 +16,15 @@ func runUsage() {
 	fmt.Printf("'backend' specifies the run backend.\n")
 	fmt.Printf("If not specified the platform specific default will be used\n")
 	fmt.Printf("Supported backends are (default platform in brackets):\n")
+	// Please keep these in alphabetical order
+	fmt.Printf("  aws\n")
 	fmt.Printf("  azure\n")
 	fmt.Printf("  gcp\n")
 	fmt.Printf("  hyperkit [macOS]\n")
+	fmt.Printf("  packet\n")
 	fmt.Printf("  qemu [linux]\n")
 	fmt.Printf("  vcenter\n")
 	fmt.Printf("  vmware\n")
-	fmt.Printf("  packet\n")
 	fmt.Printf("\n")
 	fmt.Printf("'options' are the backend specific options.\n")
 	fmt.Printf("See '%s run [backend] --help' for details.\n\n", invoked)
@@ -37,21 +39,24 @@ func run(args []string) {
 	}
 
 	switch args[0] {
+	// Please keep cases in alphabetical order
+	case "aws":
+		runAWS(args[1:])
+	case "azure":
+		runAzure(args[1:])
+	case "gcp":
+		runGcp(args[1:])
 	case "help", "-h", "-help", "--help":
 		runUsage()
 		os.Exit(0)
 	case "hyperkit":
 		runHyperKit(args[1:])
-	case "azure":
-		runAzure(args[1:])
-	case "vmware":
-		runVMware(args[1:])
-	case "gcp":
-		runGcp(args[1:])
-	case "qemu":
-		runQemu(args[1:])
 	case "packet":
 		runPacket(args[1:])
+	case "qemu":
+		runQemu(args[1:])
+	case "vmware":
+		runVMware(args[1:])
 	case "vcenter":
 		runVcenter(args[1:])
 	default:

src/cmd/linuxkit/run_aws.go

@@ -0,0 +1,186 @@
+package main
+
+import (
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+)
+
+const (
+	defaultAWSMachine  = "t2.micro"
+	defaultAWSDiskSize = 0
+	defaultAWSDiskType = "gp2"
+	defaultAWSZone     = "a"
+	// Environment variables. Some are non-standard
+	awsMachineVar  = "AWS_MACHINE"   // non-standard
+	awsDiskSizeVar = "AWS_DISK_SIZE" // non-standard
+	awsDiskTypeVar = "AWS_DISK_TYPE" // non-standard
+	awsZoneVar     = "AWS_ZONE"      // non-standard
+)
+
+// Process the run arguments and execute run
+func runAWS(args []string) {
+	flags := flag.NewFlagSet("aws", flag.ExitOnError)
+	invoked := filepath.Base(os.Args[0])
+	flags.Usage = func() {
+		fmt.Printf("USAGE: %s run aws [options] [name]\n\n", invoked)
+		fmt.Printf("'name' is the name of an AWS image that has already been\n")
+		fmt.Printf(" uploaded using 'linuxkit push'\n\n")
+		fmt.Printf("Options:\n\n")
+		flags.PrintDefaults()
+	}
+	machineFlag := flags.String("machine", defaultAWSMachine, "AWS Machine Type")
+	diskSizeFlag := flags.Int("disk-size", 0, "Size of system disk in GB")
+	diskTypeFlag := flags.String("disk-type", defaultAWSDiskType, "AWS Disk Type")
+	zoneFlag := flags.String("zone", defaultAWSZone, "AWS Availability Zone")
+
+	if err := flags.Parse(args); err != nil {
+		log.Fatal("Unable to parse args")
+	}
+
+	remArgs := flags.Args()
+	if len(remArgs) == 0 {
+		fmt.Printf("Please specify the name of the image to boot\n")
+		flags.Usage()
+		os.Exit(1)
+	}
+	name := remArgs[0]
+
+	machine := getStringValue(awsMachineVar, *machineFlag, defaultAWSMachine)
+	diskSize := getIntValue(awsDiskSizeVar, *diskSizeFlag, defaultAWSDiskSize)
+	diskType := getStringValue(awsDiskTypeVar, *diskTypeFlag, defaultAWSDiskType)
+	zone := os.Getenv("AWS_REGION") + getStringValue(awsZoneVar, *zoneFlag, defaultAWSZone)
+
+	sess := session.Must(session.NewSession())
+	compute := ec2.New(sess)
+
+	// 1. Find AMI
+	filter := &ec2.DescribeImagesInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("name"),
+				Values: []*string{aws.String(name)},
+			},
+		},
+	}
+	results, err := compute.DescribeImages(filter)
+	if err != nil {
+		log.Fatalf("Unable to describe images: %s", err)
+	}
+	if len(results.Images) < 1 {
+		log.Fatalf("Unable to find image with name %s", name)
+	}
+	if len(results.Images) > 1 {
+		log.Warnf("Found multiple images with the same name, using the first one")
+	}
+	imageID := results.Images[0].ImageId
+
+	// 2. Create Instance
+	params := &ec2.RunInstancesInput{
+		ImageId:      imageID,
+		InstanceType: aws.String(machine),
+		MinCount:     aws.Int64(1),
+		MaxCount:     aws.Int64(1),
+	}
+	runResult, err := compute.RunInstances(params)
+	if err != nil {
+		log.Fatalf("Unable to run instance: %s", err)
+
+	}
+	instanceID := runResult.Instances[0].InstanceId
+	log.Infof("Created instance %s", *instanceID)
+
+	instanceFilter := &ec2.DescribeInstancesInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("instance-id"),
+				Values: []*string{instanceID},
+			},
+		},
+	}
+
+	if err = compute.WaitUntilInstanceRunning(instanceFilter); err != nil {
+		log.Fatalf("Error waiting for instance to start: %s", err)
+	}
+	log.Infof("Instance %s is running", *instanceID)
+
+	if diskSize > 0 {
+		// 3. Create EBS Volume
+		diskParams := &ec2.CreateVolumeInput{
+			AvailabilityZone: aws.String(zone),
+			Size:             aws.Int64(int64(diskSize)),
+			VolumeType:       aws.String(diskType),
+		}
+		log.Debugf("CreateVolume:\n%v\n", diskParams)
+
+		volume, err := compute.CreateVolume(diskParams)
+		if err != nil {
+			log.Fatalf("Error creating volume: %s", err)
+		}
+
+		waitVol := &ec2.DescribeVolumesInput{
+			Filters: []*ec2.Filter{
+				{
+					Name:   aws.String("volume-id"),
+					Values: []*string{volume.VolumeId},
+				},
+			},
+		}
+
+		log.Infof("Waiting for volume %s to be available", *volume.VolumeId)
+
+		if err := compute.WaitUntilVolumeAvailable(waitVol); err != nil {
+			log.Fatalf("Error waiting for volume to be available: %s", err)
+		}
+
+		log.Infof("Attaching volume %s to instance %s", *volume.VolumeId, *instanceID)
+		volParams := &ec2.AttachVolumeInput{
+			Device:     aws.String("/dev/sda2"),
+			InstanceId: instanceID,
+			VolumeId:   volume.VolumeId,
+		}
+		_, err = compute.AttachVolume(volParams)
+		if err != nil {
+			log.Fatalf("Error attaching volume to instance: %s", err)
+		}
+	}
+
+	log.Warnf("AWS doesn't stream serial console output.\n Please use the AWS Management Console to obtain this output \n Console ouput will be displayed when the instance has been stopped.")
+	log.Warn("Waiting for instance to stop...")
+
+	if err = compute.WaitUntilInstanceStopped(instanceFilter); err != nil {
+		log.Fatalf("Error waiting for instance to stop: %s", err)
+	}
+
+	consoleParams := &ec2.GetConsoleOutputInput{
+		InstanceId: instanceID,
+	}
+	output, err := compute.GetConsoleOutput(consoleParams)
+	if err != nil {
+		log.Fatalf("Error getting output from instance %s: %s", *instanceID, err)
+	}
+
+	out, err := base64.StdEncoding.DecodeString(*output.Output)
+	if err != nil {
+		log.Fatalf("Error decoding output: %s", err)
+	}
+	fmt.Printf(string(out) + "\n")
+
+	log.Infof("Terminating instance %s", *instanceID)
+	terminateParams := &ec2.TerminateInstancesInput{
+		InstanceIds: []*string{instanceID},
+	}
+	if _, err := compute.TerminateInstances(terminateParams); err != nil {
+		log.Fatalf("Error terminating instance %s", *instanceID)
+	}
+	if err = compute.WaitUntilInstanceTerminated(instanceFilter); err != nil {
+		log.Fatalf("Error waiting for instance to terminate: %s", err)
+	}
+}