From d861987b79bd2a6f0e16660d387d50fbe8a0c76d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20Wo=C5=BAniak?= Date: Fri, 1 May 2020 01:15:24 +0200 Subject: [PATCH] scaleway: Fix bugs and add option for changing image size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix using ams1 as zone * Allow specifying image size (+ calculate default from ISO size) * Fix mangling logs when asking for ssh passphrase * Some minor code and docs cleanups Signed-off-by: Karol Woźniak --- docs/platform-scaleway.md | 9 ++-- src/cmd/linuxkit/push_scaleway.go | 28 +++++++++-- src/cmd/linuxkit/run_scaleway.go | 26 ++++++---- src/cmd/linuxkit/scaleway.go | 83 ++++++++++++++++--------------- 4 files changed, 88 insertions(+), 58 deletions(-) diff --git a/docs/platform-scaleway.md b/docs/platform-scaleway.md index 1c3a7d946..37ab701d5 100644 --- a/docs/platform-scaleway.md +++ b/docs/platform-scaleway.md @@ -4,8 +4,11 @@ This is a quick guide to run LinuxKit on Scaleway (only VPS x86_64 for now) ## Setup -You must create a Scaleway Secret Key (available ine the [Scaleway Console](https://console.scaleway.com/account/credentials)) first. -Then you can use it either with the `SCW_SECRET_KEY` environment variable or with the `-secret-key` flag of the `linuxkit push scaleway` and `linuxkit run scaleway` commands. +You must create a Scaleway API Token (combination of Access and Secret Key), available at [Scaleway Console](https://console.scaleway.com/account/credentials), first. +Then you can use it either with the `SCW_ACCESS_KEY` and `SCW_SECRET_KEY` environment variables or the `-access-key` and `-secret-key` flags +of the `linuxkit push scaleway` and `linuxkit run scaleway` commands. + +In addition, Organization ID value has to be set, either with the `SCW_DEFAULT_ORGANIZATION_ID` environment variable or the `-organization-id` command line flag. The environment variable `SCW_DEFAULT_ZONE` is used to set the zone (there is also the `-zone` flag) @@ -25,7 +28,7 @@ $ linuxkit build -format iso-efi examples/scaleway.yml ## Push image You have to do `linuxkit push scaleway scaleway.iso` to upload it to your Scaleway images. -By default the image name is the name of the ISO file without the extension. +By default the image name is the name of the ISO file without the extension. It can be overidden with the `-img-name` flag or the `SCW_IMAGE_NAME` environment variable. **Note 1:** If an image (and snapshot) of the same name exists, it will be replaced. diff --git a/src/cmd/linuxkit/push_scaleway.go b/src/cmd/linuxkit/push_scaleway.go index 4a9c11192..affcc35ed 100644 --- a/src/cmd/linuxkit/push_scaleway.go +++ b/src/cmd/linuxkit/push_scaleway.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "math" "os" "path/filepath" "strings" @@ -10,6 +11,8 @@ import ( log "github.com/sirupsen/logrus" ) +const defaultScalewayVolumeSize = 10 // GB + func pushScaleway(args []string) { flags := flag.NewFlagSet("scaleway", flag.ExitOnError) invoked := filepath.Base(os.Args[0]) @@ -20,12 +23,14 @@ func pushScaleway(args []string) { flags.PrintDefaults() } nameFlag := flags.String("img-name", "", "Overrides the name used to identify the image name in Scaleway's images. Defaults to the base of 'path' with the '.iso' suffix removed") - secretKeyFlag := flags.String("secret-key", "", "Secret Key to connet to Scaleway API") + accessKeyFlag := flags.String("access-key", "", "Access Key to connect to Scaleway API") + secretKeyFlag := flags.String("secret-key", "", "Secret Key to connect to Scaleway API") sshKeyFlag := flags.String("ssh-key", os.Getenv("HOME")+"/.ssh/id_rsa", "SSH key file") instanceIDFlag := flags.String("instance-id", "", "Instance ID of a running Scaleway instance, with a second volume.") deviceNameFlag := flags.String("device-name", "/dev/vdb", "Device name on which the image will be copied") + volumeSizeFlag := flags.Int("volume-size", 0, "Size of the volume to use (in GB). Defaults to size of the ISO file rounded up to GB") zoneFlag := flags.String("zone", defaultScalewayZone, "Select Scaleway zone") - projectIDFlag := flags.String("project-id", "", "Select Scaleway's project ID") + organizationIDFlag := flags.String("organization-id", "", "Select Scaleway's organization ID") noCleanFlag := flags.Bool("no-clean", false, "Do not remove temporary instance and volumes") if err := flags.Parse(args); err != nil { @@ -41,12 +46,14 @@ func pushScaleway(args []string) { path := remArgs[0] name := getStringValue(scalewayNameVar, *nameFlag, "") + accessKey := getStringValue(accessKeyVar, *accessKeyFlag, "") secretKey := getStringValue(secretKeyVar, *secretKeyFlag, "") sshKeyFile := getStringValue(sshKeyVar, *sshKeyFlag, "") instanceID := getStringValue(instanceIDVar, *instanceIDFlag, "") deviceName := getStringValue(deviceNameVar, *deviceNameFlag, "") + volumeSize := getIntValue(volumeSizeVar, *volumeSizeFlag, 0) zone := getStringValue(zoneVar, *zoneFlag, defaultScalewayZone) - projectID := getStringValue(projectIDVar, *projectIDFlag, "") + organizationID := getStringValue(organizationIDVar, *organizationIDFlag, "") const suffix = ".iso" if name == "" { @@ -54,14 +61,25 @@ func pushScaleway(args []string) { name = filepath.Base(name) } - client, err := NewScalewayClient(secretKey, zone, projectID) + client, err := NewScalewayClient(accessKey, secretKey, zone, organizationID) if err != nil { log.Fatalf("Unable to connect to Scaleway: %v", err) } + // if volume size not set, try to calculate it from file size + if volumeSize == 0 { + if fi, err := os.Stat(path); err == nil { + volumeSize = int(math.Ceil(float64(fi.Size()) / 1000000000)) // / 1 GB + } else { + // fallback to default + log.Warnf("Unable to calculate volume size, using default of %d GB: %v", defaultScalewayVolumeSize, err) + volumeSize = defaultScalewayVolumeSize + } + } + // if no instanceID is provided, we create the instance if instanceID == "" { - instanceID, err = client.CreateInstance() + instanceID, err = client.CreateInstance(volumeSize) if err != nil { log.Fatalf("Error creating a Scaleway instance: %v", err) } diff --git a/src/cmd/linuxkit/run_scaleway.go b/src/cmd/linuxkit/run_scaleway.go index 5c1a45b1e..d17f1a706 100644 --- a/src/cmd/linuxkit/run_scaleway.go +++ b/src/cmd/linuxkit/run_scaleway.go @@ -13,13 +13,15 @@ const ( defaultScalewayInstanceType = "DEV1-S" defaultScalewayZone = "par1" - scalewayNameVar = "SCW_IMAGE_NAME" // non-standard - secretKeyVar = "SCW_SECRET_KEY" // non-standard - sshKeyVar = "SCW_SSH_KEY_FILE" // non-standard - instanceIDVar = "SCW_INSTANCE_ID" // non-standard - deviceNameVar = "SCW_DEVICE_NAME" // non-standard - scwZoneVar = "SCW_DEFAULT_ZONE" - projectIDVar = "SCW_DEFAULT_PROJECT_ID" + scalewayNameVar = "SCW_IMAGE_NAME" // non-standard + accessKeyVar = "SCW_ACCESS_KEY" + secretKeyVar = "SCW_SECRET_KEY" + sshKeyVar = "SCW_SSH_KEY_FILE" // non-standard + instanceIDVar = "SCW_INSTANCE_ID" // non-standard + deviceNameVar = "SCW_DEVICE_NAME" // non-standard + volumeSizeVar = "SCW_VOLUME_SIZE" // non-standard + scwZoneVar = "SCW_DEFAULT_ZONE" + organizationIDVar = "SCW_DEFAULT_ORGANIZATION_ID" instanceTypeVar = "SCW_RUN_TYPE" // non-standard ) @@ -29,16 +31,17 @@ func runScaleway(args []string) { invoked := filepath.Base(os.Args[0]) flags.Usage = func() { fmt.Printf("USAGE: %s run scaleway [options] [name]\n\n", invoked) - fmt.Printf("'name' is the name of a Scaleway image that has alread \n") + fmt.Printf("'name' is the name of a Scaleway image that has already \n") fmt.Printf("been uploaded using 'linuxkit push'\n\n") fmt.Printf("Options:\n\n") flags.PrintDefaults() } instanceTypeFlag := flags.String("instance-type", defaultScalewayInstanceType, "Scaleway instance type") instanceNameFlag := flags.String("instance-name", "linuxkit", "Name of the create instance, default to the image name") + accessKeyFlag := flags.String("access-key", "", "Access Key to connect to Scaleway API") secretKeyFlag := flags.String("secret-key", "", "Secret Key to connect to Scaleway API") zoneFlag := flags.String("zone", defaultScalewayZone, "Select Scaleway zone") - projectIDFlag := flags.String("project-id", "", "Select Scaleway's project ID") + organizationIDFlag := flags.String("organization-id", "", "Select Scaleway's organization ID") cleanFlag := flags.Bool("clean", false, "Remove instance") noAttachFlag := flags.Bool("no-attach", false, "Don't attach to serial port, you will have to connect to instance manually") @@ -56,11 +59,12 @@ func runScaleway(args []string) { instanceType := getStringValue(instanceTypeVar, *instanceTypeFlag, defaultScalewayInstanceType) instanceName := getStringValue("", *instanceNameFlag, name) + accessKey := getStringValue(accessKeyVar, *accessKeyFlag, "") secretKey := getStringValue(secretKeyVar, *secretKeyFlag, "") zone := getStringValue(scwZoneVar, *zoneFlag, defaultScalewayZone) - projectID := getStringValue(projectIDVar, *projectIDFlag, "") + organizationID := getStringValue(organizationIDVar, *organizationIDFlag, "") - client, err := NewScalewayClient(secretKey, zone, projectID) + client, err := NewScalewayClient(accessKey, secretKey, zone, organizationID) if err != nil { log.Fatalf("Unable to connect to Scaleway: %v", err) } diff --git a/src/cmd/linuxkit/scaleway.go b/src/cmd/linuxkit/scaleway.go index d00b7a32b..277ef9704 100644 --- a/src/cmd/linuxkit/scaleway.go +++ b/src/cmd/linuxkit/scaleway.go @@ -24,12 +24,12 @@ import ( ) var ( - defaultScalewayCommercialType = "DEV1-S" - defaultScalewayImageName = "Ubuntu Bionic" - defaultScalewayImageArch = "x86_64" - defaultVolumeSize = scw.GB * 10 - scalewayDynamicIPRequired = true - scalewayBootType = instance.BootTypeLocal + defaultScalewayCommercialType = "DEV1-S" + defaultScalewayImageName = "Ubuntu Bionic" + defaultScalewayImageArch = "x86_64" + scalewayDynamicIPRequired = true + scalewayBootType = instance.BootTypeLocal + scalewayInstanceVolumeSize scw.Size = 20 ) // ScalewayClient contains state required for communication with Scaleway as well as the instance @@ -37,16 +37,18 @@ type ScalewayClient struct { instanceAPI *instance.API marketplaceAPI *marketplace.API fileName string - zone string + zone scw.Zone sshConfig *ssh.ClientConfig secretKey string } // NewScalewayClient creates a new scaleway client -func NewScalewayClient(secretKey, zone, organizationID string) (*ScalewayClient, error) { - var scwClient *scw.Client +func NewScalewayClient(accessKey, secretKey, zone, organizationID string) (*ScalewayClient, error) { log.Debugf("Connecting to Scaleway") - if secretKey == "" { + + scwOptions := []scw.ClientOption{} + + if accessKey == "" || secretKey == "" { config, err := scw.LoadConfig() if err != nil { return nil, err @@ -56,36 +58,35 @@ func NewScalewayClient(secretKey, zone, organizationID string) (*ScalewayClient, return nil, err } - scwClient, err = scw.NewClient( - scw.WithProfile(profile), - scw.WithEnv(), - ) - if err != nil { - return nil, err + scwOptions = append(scwOptions, scw.WithProfile(profile), scw.WithEnv()) + if *profile.DefaultZone != "" { + zone = *profile.DefaultZone } } else { - scwZone, err := scw.ParseZone(zone) - if err != nil { - return nil, err - } - - scwClient, err = scw.NewClient( - scw.WithAuth("", secretKey), - scw.WithDefaultZone(scwZone), + scwOptions = append( + scwOptions, + scw.WithAuth(accessKey, secretKey), scw.WithDefaultOrganizationID(organizationID), ) - if err != nil { - return nil, err - } } + scwZone, err := scw.ParseZone(zone) + if err != nil { + return nil, err + } + scwOptions = append(scwOptions, scw.WithDefaultZone(scwZone)) + + scwClient, err := scw.NewClient(scwOptions...) + if err != nil { + return nil, err + } instanceAPI := instance.NewAPI(scwClient) marketplaceAPI := marketplace.NewAPI(scwClient) client := &ScalewayClient{ instanceAPI: instanceAPI, marketplaceAPI: marketplaceAPI, - zone: zone, + zone: scwZone, fileName: "", secretKey: secretKey, } @@ -102,7 +103,7 @@ func (s *ScalewayClient) getImageID(imageName, commercialType, arch string) (str if image.Name == imageName { for _, version := range image.Versions { for _, localImage := range version.LocalImages { - if localImage.Arch == arch { + if localImage.Arch == arch && localImage.Zone == s.zone { for _, compatibleCommercialType := range localImage.CompatibleCommercialTypes { if compatibleCommercialType == commercialType { return localImage.ID, nil @@ -117,17 +118,20 @@ func (s *ScalewayClient) getImageID(imageName, commercialType, arch string) (str } // CreateInstance create an instance with one additional volume -func (s *ScalewayClient) CreateInstance() (string, error) { - // get the Ubuntu Xenial image id +func (s *ScalewayClient) CreateInstance(volumeSize int) (string, error) { + // get the Ubuntu Bionic image id imageID, err := s.getImageID(defaultScalewayImageName, defaultScalewayCommercialType, defaultScalewayImageArch) if err != nil { return "", err } + scwVolumeSize := scw.Size(volumeSize) + builderVolumeSize := scwVolumeSize * scw.GB + createVolumeRequest := &instance.CreateVolumeRequest{ Name: "linuxkit-builder-volume", VolumeType: "l_ssd", - Size: &defaultVolumeSize, + Size: &builderVolumeSize, } log.Debugf("Creating volume on Scaleway") @@ -136,11 +140,9 @@ func (s *ScalewayClient) CreateInstance() (string, error) { return "", err } - volumeMap := make(map[string]*instance.VolumeTemplate) - volumeTemplate := &instance.VolumeTemplate{ - Size: defaultVolumeSize, + volumeMap := map[string]*instance.VolumeTemplate{ + "0": {Size: (scalewayInstanceVolumeSize - scwVolumeSize) * scw.GB}, } - volumeMap["0"] = volumeTemplate createServerRequest := &instance.CreateServerRequest{ Name: "linuxkit-builder", @@ -152,6 +154,7 @@ func (s *ScalewayClient) CreateInstance() (string, error) { Volumes: volumeMap, } + log.Debug("Creating server on Scaleway") serverResp, err := s.instanceAPI.CreateServer(createServerRequest) if err != nil { return "", err @@ -268,7 +271,7 @@ func (s *ScalewayClient) BootInstanceAndWait(instanceID string) error { } } -// getSSHAuth is uses to get the ssh.Signer needed to connect via SSH +// getSSHAuth gets the ssh.Signer needed to connect via SSH func getSSHAuth(sshKeyPath string) (ssh.Signer, error) { f, err := os.Open(sshKeyPath) if err != nil { @@ -284,6 +287,8 @@ func getSSHAuth(sshKeyPath string) (ssh.Signer, error) { if err != nil { fmt.Print("Enter ssh key passphrase: ") bytePassword, err := terminal.ReadPassword(int(syscall.Stdin)) + // ReadPassword eats newline, put it back to avoid mangling logs + fmt.Println() if err != nil { return nil, err } @@ -581,9 +586,9 @@ func (s *ScalewayClient) BootInstance(instanceID string) error { func (s *ScalewayClient) ConnectSerialPort(instanceID string) error { var gottyURL string switch s.zone { - case "par1": + case scw.ZoneFrPar1: gottyURL = "https://tty-par1.scaleway.com/v2/" - case "ams1": + case scw.ZoneNlAms1: gottyURL = "https://tty-ams1.scaleway.com/" default: return errors.New("Instance have no region")