scaleway: Fix bugs and add option for changing image size

* Fix using ams1 as zone
* Allow specifying image size (+ calculate default from ISO size)
* Fix mangling logs when asking for ssh passphrase
* Some minor code and docs cleanups

Signed-off-by: Karol Woźniak <wozniakk@gmail.com>

docs/platform-scaleway.md

@@ -4,8 +4,11 @@ This is a quick guide to run LinuxKit on Scaleway (only VPS x86_64 for now)
 
 ## Setup
 
-You must create a Scaleway Secret Key (available ine the [Scaleway Console](https://console.scaleway.com/account/credentials)) first.
+You must create a Scaleway API Token (combination of Access and Secret Key), available at [Scaleway Console](https://console.scaleway.com/account/credentials), first.
-Then you can use it either with the `SCW_SECRET_KEY` environment variable or with the `-secret-key` flag of the `linuxkit push scaleway` and `linuxkit run scaleway` commands.
+Then you can use it either with the `SCW_ACCESS_KEY` and `SCW_SECRET_KEY` environment variables or the `-access-key` and `-secret-key` flags
+of the `linuxkit push scaleway` and `linuxkit run scaleway` commands.
+
+In addition, Organization ID value has to be set, either with the `SCW_DEFAULT_ORGANIZATION_ID` environment variable or the `-organization-id` command line flag.
 
 The environment variable `SCW_DEFAULT_ZONE` is used to set the zone (there is also the `-zone` flag)
 
@@ -25,7 +28,7 @@ $ linuxkit build -format iso-efi examples/scaleway.yml
 ## Push image
 
 You have to do `linuxkit push scaleway scaleway.iso` to upload it to your Scaleway images.
-By default the image name is the name of the ISO file without the extension. 
+By default the image name is the name of the ISO file without the extension.
 It can be overidden with the `-img-name` flag or the `SCW_IMAGE_NAME` environment variable.
 
 **Note 1:** If an image (and snapshot) of the same name exists, it will be replaced.

src/cmd/linuxkit/push_scaleway.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"math"
 	"os"
 	"path/filepath"
 	"strings"
@@ -10,6 +11,8 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+const defaultScalewayVolumeSize = 10 // GB
+
 func pushScaleway(args []string) {
 	flags := flag.NewFlagSet("scaleway", flag.ExitOnError)
 	invoked := filepath.Base(os.Args[0])
@@ -20,12 +23,14 @@ func pushScaleway(args []string) {
 		flags.PrintDefaults()
 	}
 	nameFlag := flags.String("img-name", "", "Overrides the name used to identify the image name in Scaleway's images. Defaults to the base of 'path' with the '.iso' suffix removed")
-	secretKeyFlag := flags.String("secret-key", "", "Secret Key to connet to Scaleway API")
+	accessKeyFlag := flags.String("access-key", "", "Access Key to connect to Scaleway API")
+	secretKeyFlag := flags.String("secret-key", "", "Secret Key to connect to Scaleway API")
 	sshKeyFlag := flags.String("ssh-key", os.Getenv("HOME")+"/.ssh/id_rsa", "SSH key file")
 	instanceIDFlag := flags.String("instance-id", "", "Instance ID of a running Scaleway instance, with a second volume.")
 	deviceNameFlag := flags.String("device-name", "/dev/vdb", "Device name on which the image will be copied")
+	volumeSizeFlag := flags.Int("volume-size", 0, "Size of the volume to use (in GB). Defaults to size of the ISO file rounded up to GB")
 	zoneFlag := flags.String("zone", defaultScalewayZone, "Select Scaleway zone")
-	projectIDFlag := flags.String("project-id", "", "Select Scaleway's project ID")
+	organizationIDFlag := flags.String("organization-id", "", "Select Scaleway's organization ID")
 	noCleanFlag := flags.Bool("no-clean", false, "Do not remove temporary instance and volumes")
 
 	if err := flags.Parse(args); err != nil {
@@ -41,12 +46,14 @@ func pushScaleway(args []string) {
 	path := remArgs[0]
 
 	name := getStringValue(scalewayNameVar, *nameFlag, "")
+	accessKey := getStringValue(accessKeyVar, *accessKeyFlag, "")
 	secretKey := getStringValue(secretKeyVar, *secretKeyFlag, "")
 	sshKeyFile := getStringValue(sshKeyVar, *sshKeyFlag, "")
 	instanceID := getStringValue(instanceIDVar, *instanceIDFlag, "")
 	deviceName := getStringValue(deviceNameVar, *deviceNameFlag, "")
+	volumeSize := getIntValue(volumeSizeVar, *volumeSizeFlag, 0)
 	zone := getStringValue(zoneVar, *zoneFlag, defaultScalewayZone)
-	projectID := getStringValue(projectIDVar, *projectIDFlag, "")
+	organizationID := getStringValue(organizationIDVar, *organizationIDFlag, "")
 
 	const suffix = ".iso"
 	if name == "" {
@@ -54,14 +61,25 @@ func pushScaleway(args []string) {
 		name = filepath.Base(name)
 	}
 
-	client, err := NewScalewayClient(secretKey, zone, projectID)
+	client, err := NewScalewayClient(accessKey, secretKey, zone, organizationID)
 	if err != nil {
 		log.Fatalf("Unable to connect to Scaleway: %v", err)
 	}
 
+	// if volume size not set, try to calculate it from file size
+	if volumeSize == 0 {
+		if fi, err := os.Stat(path); err == nil {
+			volumeSize = int(math.Ceil(float64(fi.Size()) / 1000000000)) // / 1 GB
+		} else {
+			// fallback to default
+			log.Warnf("Unable to calculate volume size, using default of %d GB: %v", defaultScalewayVolumeSize, err)
+			volumeSize = defaultScalewayVolumeSize
+		}
+	}
+
 	// if no instanceID is provided, we create the instance
 	if instanceID == "" {
-		instanceID, err = client.CreateInstance()
+		instanceID, err = client.CreateInstance(volumeSize)
 		if err != nil {
 			log.Fatalf("Error creating a Scaleway instance: %v", err)
 		}

src/cmd/linuxkit/run_scaleway.go

@@ -13,13 +13,15 @@ const (
 	defaultScalewayInstanceType = "DEV1-S"
 	defaultScalewayZone         = "par1"
 
-	scalewayNameVar = "SCW_IMAGE_NAME"   // non-standard
+	scalewayNameVar   = "SCW_IMAGE_NAME" // non-standard
-	secretKeyVar    = "SCW_SECRET_KEY"   // non-standard
+	accessKeyVar      = "SCW_ACCESS_KEY"
-	sshKeyVar       = "SCW_SSH_KEY_FILE" // non-standard
+	secretKeyVar      = "SCW_SECRET_KEY"
-	instanceIDVar   = "SCW_INSTANCE_ID"  // non-standard
+	sshKeyVar         = "SCW_SSH_KEY_FILE" // non-standard
-	deviceNameVar   = "SCW_DEVICE_NAME"  // non-standard
+	instanceIDVar     = "SCW_INSTANCE_ID"  // non-standard
-	scwZoneVar      = "SCW_DEFAULT_ZONE"
+	deviceNameVar     = "SCW_DEVICE_NAME"  // non-standard
-	projectIDVar    = "SCW_DEFAULT_PROJECT_ID"
+	volumeSizeVar     = "SCW_VOLUME_SIZE"  // non-standard
+	scwZoneVar        = "SCW_DEFAULT_ZONE"
+	organizationIDVar = "SCW_DEFAULT_ORGANIZATION_ID"
 
 	instanceTypeVar = "SCW_RUN_TYPE" // non-standard
 )
@@ -29,16 +31,17 @@ func runScaleway(args []string) {
 	invoked := filepath.Base(os.Args[0])
 	flags.Usage = func() {
 		fmt.Printf("USAGE: %s run scaleway [options] [name]\n\n", invoked)
-		fmt.Printf("'name' is the name of a Scaleway image that has alread \n")
+		fmt.Printf("'name' is the name of a Scaleway image that has already \n")
 		fmt.Printf("been uploaded using 'linuxkit push'\n\n")
 		fmt.Printf("Options:\n\n")
 		flags.PrintDefaults()
 	}
 	instanceTypeFlag := flags.String("instance-type", defaultScalewayInstanceType, "Scaleway instance type")
 	instanceNameFlag := flags.String("instance-name", "linuxkit", "Name of the create instance, default to the image name")
+	accessKeyFlag := flags.String("access-key", "", "Access Key to connect to Scaleway API")
 	secretKeyFlag := flags.String("secret-key", "", "Secret Key to connect to Scaleway API")
 	zoneFlag := flags.String("zone", defaultScalewayZone, "Select Scaleway zone")
-	projectIDFlag := flags.String("project-id", "", "Select Scaleway's project ID")
+	organizationIDFlag := flags.String("organization-id", "", "Select Scaleway's organization ID")
 	cleanFlag := flags.Bool("clean", false, "Remove instance")
 	noAttachFlag := flags.Bool("no-attach", false, "Don't attach to serial port, you will have to connect to instance manually")
 
@@ -56,11 +59,12 @@ func runScaleway(args []string) {
 
 	instanceType := getStringValue(instanceTypeVar, *instanceTypeFlag, defaultScalewayInstanceType)
 	instanceName := getStringValue("", *instanceNameFlag, name)
+	accessKey := getStringValue(accessKeyVar, *accessKeyFlag, "")
 	secretKey := getStringValue(secretKeyVar, *secretKeyFlag, "")
 	zone := getStringValue(scwZoneVar, *zoneFlag, defaultScalewayZone)
-	projectID := getStringValue(projectIDVar, *projectIDFlag, "")
+	organizationID := getStringValue(organizationIDVar, *organizationIDFlag, "")
 
-	client, err := NewScalewayClient(secretKey, zone, projectID)
+	client, err := NewScalewayClient(accessKey, secretKey, zone, organizationID)
 	if err != nil {
 		log.Fatalf("Unable to connect to Scaleway: %v", err)
 	}

src/cmd/linuxkit/scaleway.go

@@ -24,12 +24,12 @@ import (
 )
 
 var (
-	defaultScalewayCommercialType = "DEV1-S"
+	defaultScalewayCommercialType          = "DEV1-S"
-	defaultScalewayImageName      = "Ubuntu Bionic"
+	defaultScalewayImageName               = "Ubuntu Bionic"
-	defaultScalewayImageArch      = "x86_64"
+	defaultScalewayImageArch               = "x86_64"
-	defaultVolumeSize             = scw.GB * 10
+	scalewayDynamicIPRequired              = true
-	scalewayDynamicIPRequired     = true
+	scalewayBootType                       = instance.BootTypeLocal
-	scalewayBootType              = instance.BootTypeLocal
+	scalewayInstanceVolumeSize    scw.Size = 20
 )
 
 // ScalewayClient contains state required for communication with Scaleway as well as the instance
@@ -37,16 +37,18 @@ type ScalewayClient struct {
 	instanceAPI    *instance.API
 	marketplaceAPI *marketplace.API
 	fileName       string
-	zone           string
+	zone           scw.Zone
 	sshConfig      *ssh.ClientConfig
 	secretKey      string
 }
 
 // NewScalewayClient creates a new scaleway client
-func NewScalewayClient(secretKey, zone, organizationID string) (*ScalewayClient, error) {
+func NewScalewayClient(accessKey, secretKey, zone, organizationID string) (*ScalewayClient, error) {
-	var scwClient *scw.Client
 	log.Debugf("Connecting to Scaleway")
-	if secretKey == "" {
+
+	scwOptions := []scw.ClientOption{}
+
+	if accessKey == "" || secretKey == "" {
 		config, err := scw.LoadConfig()
 		if err != nil {
 			return nil, err
@@ -56,36 +58,35 @@ func NewScalewayClient(secretKey, zone, organizationID string) (*ScalewayClient,
 			return nil, err
 		}
 
-		scwClient, err = scw.NewClient(
+		scwOptions = append(scwOptions, scw.WithProfile(profile), scw.WithEnv())
-			scw.WithProfile(profile),
+		if *profile.DefaultZone != "" {
-			scw.WithEnv(),
+			zone = *profile.DefaultZone
-		)
-		if err != nil {
-			return nil, err
 		}
 	} else {
-		scwZone, err := scw.ParseZone(zone)
+		scwOptions = append(
-		if err != nil {
+			scwOptions,
-			return nil, err
+			scw.WithAuth(accessKey, secretKey),
-		}
-
-		scwClient, err = scw.NewClient(
-			scw.WithAuth("", secretKey),
-			scw.WithDefaultZone(scwZone),
 			scw.WithDefaultOrganizationID(organizationID),
 		)
-		if err != nil {
-			return nil, err
-		}
 	}
 
+	scwZone, err := scw.ParseZone(zone)
+	if err != nil {
+		return nil, err
+	}
+	scwOptions = append(scwOptions, scw.WithDefaultZone(scwZone))
+
+	scwClient, err := scw.NewClient(scwOptions...)
+	if err != nil {
+		return nil, err
+	}
 	instanceAPI := instance.NewAPI(scwClient)
 	marketplaceAPI := marketplace.NewAPI(scwClient)
 
 	client := &ScalewayClient{
 		instanceAPI:    instanceAPI,
 		marketplaceAPI: marketplaceAPI,
-		zone:           zone,
+		zone:           scwZone,
 		fileName:       "",
 		secretKey:      secretKey,
 	}
@@ -102,7 +103,7 @@ func (s *ScalewayClient) getImageID(imageName, commercialType, arch string) (str
 		if image.Name == imageName {
 			for _, version := range image.Versions {
 				for _, localImage := range version.LocalImages {
-					if localImage.Arch == arch {
+					if localImage.Arch == arch && localImage.Zone == s.zone {
 						for _, compatibleCommercialType := range localImage.CompatibleCommercialTypes {
 							if compatibleCommercialType == commercialType {
 								return localImage.ID, nil
@@ -117,17 +118,20 @@ func (s *ScalewayClient) getImageID(imageName, commercialType, arch string) (str
 }
 
 // CreateInstance create an instance with one additional volume
-func (s *ScalewayClient) CreateInstance() (string, error) {
+func (s *ScalewayClient) CreateInstance(volumeSize int) (string, error) {
-	// get the Ubuntu Xenial image id
+	// get the Ubuntu Bionic image id
 	imageID, err := s.getImageID(defaultScalewayImageName, defaultScalewayCommercialType, defaultScalewayImageArch)
 	if err != nil {
 		return "", err
 	}
 
+	scwVolumeSize := scw.Size(volumeSize)
+	builderVolumeSize := scwVolumeSize * scw.GB
+
 	createVolumeRequest := &instance.CreateVolumeRequest{
 		Name:       "linuxkit-builder-volume",
 		VolumeType: "l_ssd",
-		Size:       &defaultVolumeSize,
+		Size:       &builderVolumeSize,
 	}
 
 	log.Debugf("Creating volume on Scaleway")
@@ -136,11 +140,9 @@ func (s *ScalewayClient) CreateInstance() (string, error) {
 		return "", err
 	}
 
-	volumeMap := make(map[string]*instance.VolumeTemplate)
+	volumeMap := map[string]*instance.VolumeTemplate{
-	volumeTemplate := &instance.VolumeTemplate{
+		"0": {Size: (scalewayInstanceVolumeSize - scwVolumeSize) * scw.GB},
-		Size: defaultVolumeSize,
 	}
-	volumeMap["0"] = volumeTemplate
 
 	createServerRequest := &instance.CreateServerRequest{
 		Name:              "linuxkit-builder",
@@ -152,6 +154,7 @@ func (s *ScalewayClient) CreateInstance() (string, error) {
 		Volumes:           volumeMap,
 	}
 
+	log.Debug("Creating server on Scaleway")
 	serverResp, err := s.instanceAPI.CreateServer(createServerRequest)
 	if err != nil {
 		return "", err
@@ -268,7 +271,7 @@ func (s *ScalewayClient) BootInstanceAndWait(instanceID string) error {
 	}
 }
 
-// getSSHAuth is uses to get the ssh.Signer needed to connect via SSH
+// getSSHAuth gets the ssh.Signer needed to connect via SSH
 func getSSHAuth(sshKeyPath string) (ssh.Signer, error) {
 	f, err := os.Open(sshKeyPath)
 	if err != nil {
@@ -284,6 +287,8 @@ func getSSHAuth(sshKeyPath string) (ssh.Signer, error) {
 	if err != nil {
 		fmt.Print("Enter ssh key passphrase: ")
 		bytePassword, err := terminal.ReadPassword(int(syscall.Stdin))
+		// ReadPassword eats newline, put it back to avoid mangling logs
+		fmt.Println()
 		if err != nil {
 			return nil, err
 		}
@@ -581,9 +586,9 @@ func (s *ScalewayClient) BootInstance(instanceID string) error {
 func (s *ScalewayClient) ConnectSerialPort(instanceID string) error {
 	var gottyURL string
 	switch s.zone {
-	case "par1":
+	case scw.ZoneFrPar1:
 		gottyURL = "https://tty-par1.scaleway.com/v2/"
-	case "ams1":
+	case scw.ZoneNlAms1:
 		gottyURL = "https://tty-ams1.scaleway.com/"
 	default:
 		return errors.New("Instance have no region")