From 0821c9a4ea614316e0935b50c6fcfe4d7786bb02 Mon Sep 17 00:00:00 2001 From: Jason-ZW Date: Wed, 27 Jun 2018 12:08:32 +0800 Subject: [PATCH] Add multi docker command --- cmd/control/console_init.go | 22 ++ cmd/control/engine.go | 369 +++++++++++++++++++++++++++++++++ cmd/control/service/app/app.go | 1 + compose/project.go | 27 ++- compose/reload.go | 14 +- config/types.go | 23 +- os-config.tpl.yml | 1 + util/network/network.go | 28 +++ util/util.go | 18 ++ 9 files changed, 489 insertions(+), 14 deletions(-) diff --git a/cmd/control/console_init.go b/cmd/control/console_init.go index 616e1c89..d014fd31 100644 --- a/cmd/control/console_init.go +++ b/cmd/control/console_init.go @@ -13,6 +13,7 @@ import ( "github.com/codegangsta/cli" "github.com/rancher/os/cmd/cloudinitexecute" + "github.com/rancher/os/compose" "github.com/rancher/os/config" "github.com/rancher/os/config/cmdline" "github.com/rancher/os/log" @@ -90,6 +91,27 @@ func consoleInitFunc() error { log.Error(err) } + p, err := compose.GetProject(cfg, false, true) + if err != nil { + log.Error(err) + } + + // check the multi engine service & generate the multi engine script + for _, key := range p.ServiceConfigs.Keys() { + serviceConfig, ok := p.ServiceConfigs.Get(key) + if !ok { + log.Errorf("Failed to get service config from the project") + continue + } + if _, ok := serviceConfig.Labels[config.UserDockerLabel]; ok { + err = util.GenerateEngineScript(serviceConfig.Labels[config.UserDockerLabel]) + if err != nil { + log.Errorf("Failed to generate engine script: %v", err) + continue + } + } + } + for _, link := range []symlink{ {"/var/lib/rancher/engine/docker", "/usr/bin/docker"}, {"/var/lib/rancher/engine/docker-init", "/usr/bin/docker-init"}, diff --git a/cmd/control/engine.go b/cmd/control/engine.go index 486b938c..a40046a0 100644 --- a/cmd/control/engine.go +++ b/cmd/control/engine.go @@ -2,15 +2,27 @@ package control import ( "fmt" + "io/ioutil" + "net" + "os" + "os/user" + "path" "sort" + "strconv" "strings" "golang.org/x/net/context" + yaml "github.com/cloudfoundry-incubator/candiedyaml" "github.com/codegangsta/cli" "github.com/docker/docker/reference" + "github.com/docker/engine-api/types" + composeConfig "github.com/docker/libcompose/config" "github.com/docker/libcompose/project/options" + composeYaml "github.com/docker/libcompose/yaml" + "github.com/pkg/errors" "github.com/rancher/os/cmd/control/service" + "github.com/rancher/os/cmd/control/service/app" "github.com/rancher/os/compose" "github.com/rancher/os/config" "github.com/rancher/os/docker" @@ -19,6 +31,11 @@ import ( "github.com/rancher/os/util/network" ) +var ( + SupportedEngineVersions = []string{"docker:17.12.1-dind", "docker:18.03-dind", "docker:18.03.1-dind"} + SSHKeyPathDefault = "%s/.ssh/authorized_keys" +) + func engineSubcommands() []cli.Command { return []cli.Command{ { @@ -36,6 +53,64 @@ func engineSubcommands() []cli.Command { }, }, }, + { + Name: "create", + Usage: "create Docker engine without a reboot", + ArgsUsage: "", + Before: preFlightValidate, + Action: engineCreate, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "version, v", + Value: SupportedEngineVersions[0], + Usage: "set the version for the engine", + }, + cli.StringFlag{ + Name: "network", + Usage: "set the network for the engine", + }, + cli.StringFlag{ + Name: "fixed-ip", + Usage: "set the fix ip for the engine", + }, + cli.IntFlag{ + Name: "ssh-port", + Value: randomSSHPort(), + Usage: "set the ssh port for the engine", + }, + cli.StringFlag{ + Name: "authorized-keys", + Usage: "set the ssh authorized_keys path for the engine", + }, + }, + }, + { + Name: "rm", + Usage: "remove Docker engine without a reboot", + ArgsUsage: "", + Before: func(c *cli.Context) error { + if len(c.Args()) != 1 { + return errors.New("Must specify exactly one Docker engine to remove") + } + return nil + }, + Action: engineRemove, + Flags: []cli.Flag{ + cli.IntFlag{ + Name: "timeout,t", + Usage: "Specify a shutdown timeout in seconds.", + Value: 10, + }, + cli.BoolFlag{ + Name: "force,f", + Usage: "Allow deletion of all services", + }, + cli.BoolFlag{ + Name: "v", + Usage: "Remove volumes associated with containers", + }, + }, + }, { Name: "enable", Usage: "set Docker engine to be switched on next reboot", @@ -82,6 +157,96 @@ func engineSwitch(c *cli.Context) error { return nil } +func engineCreate(c *cli.Context) error { + name := c.Args()[0] + version := c.String("version") + sshPort := c.Int("ssh-port") + authorizedKeys := c.String("authorized-keys") + network := c.String("network") + fixedIP := c.String("fixed-ip") + + if authorizedKeys == "" { + authorizedKeys = authorizedKeysPath() + } + + // generate & create engine compose + err := generateEngineCompose(name, version, sshPort, authorizedKeys, network, fixedIP) + if err != nil { + return err + } + + // stage engine service + cfg := config.LoadConfig() + var enabledServices []string + if val, ok := cfg.Rancher.ServicesInclude[name]; !ok || !val { + cfg.Rancher.ServicesInclude[name] = true + enabledServices = append(enabledServices, name) + } + + if len(enabledServices) > 0 { + if err := compose.StageServices(cfg, enabledServices...); err != nil { + log.Fatal(err) + } + + if err := config.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil { + log.Fatal(err) + } + } + + // generate engine script + err = util.GenerateEngineScript(name) + if err != nil { + log.Fatal(err) + } + + return nil +} + +func engineRemove(c *cli.Context) error { + name := c.Args()[0] + cfg := config.LoadConfig() + p, err := compose.GetProject(cfg, true, false) + if err != nil { + log.Fatalf("Get project failed: %v", err) + } + + // 1. service stop + err = app.ProjectStop(p, c) + if err != nil { + log.Fatalf("Stop project service failed: %v", err) + } + + // 2. service delete + err = app.ProjectDelete(p, c) + if err != nil { + log.Fatalf("Delete project service failed: %v", err) + } + + // 3. service delete + changed := false + + if _, ok := cfg.Rancher.ServicesInclude[name]; !ok { + log.Fatalf("Failed to found enabled service %s", name) + } + + delete(cfg.Rancher.ServicesInclude, name) + changed = true + + if changed { + if err = config.Set("rancher.services_include", cfg.Rancher.ServicesInclude); err != nil { + log.Fatal(err) + } + } + + // 4. remove service from file + err = RemoveEngineFromCompose(name) + if err != nil { + log.Fatal(err) + } + + return nil +} + func engineEnable(c *cli.Context) error { if len(c.Args()) != 1 { log.Fatal("Must specify exactly one Docker engine to enable") @@ -166,3 +331,207 @@ func CurrentEngine() (engine string) { return } + +func preFlightValidate(c *cli.Context) error { + if len(c.Args()) != 1 { + return errors.New("Must specify one engine name") + } + name := c.Args()[0] + if name == "" { + return errors.New("Must specify one engine name") + } + + version := c.String("version") + if version == "" { + return errors.New("Must specify one engine version") + } + + port := c.Int("ssh-port") + if port == 0 { + return errors.New("Must specify one engine ssh port") + } + + network := c.String("network") + if network == "" { + return errors.New("Must specify network") + } + + userDefineNetwork, err := CheckUserDefineNetwork(network) + if err != nil { + return err + } + + fixedIP := c.String("fixed-ip") + if fixedIP == "" { + return errors.New("Must specify fix ip") + } + + err = CheckUserDefineIPv4Address(fixedIP, *userDefineNetwork) + if err != nil { + return err + } + + isVersionMatch := false + for _, v := range SupportedEngineVersions { + if v == version { + isVersionMatch = true + break + } + } + + if !isVersionMatch { + return errors.Errorf("Engine version not supported only %v are supported", SupportedEngineVersions) + } + + addr, err := net.ResolveTCPAddr("tcp", "localhost:"+strconv.Itoa(port)) + if err != nil { + return errors.Errorf("Failed to resolve tcp addr: %v", err) + } + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return errors.Errorf("Failed to listen tcp: %v", err) + } + defer l.Close() + + return nil +} + +func authorizedKeysPath() string { + home := "/home/rancher" + user, err := user.Current() + if err == nil { + home = user.HomeDir + } + + return fmt.Sprintf(SSHKeyPathDefault, home) +} + +func randomSSHPort() int { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + log.Errorf("Failed to resolve tcp addr: %v", err) + return 0 + } + + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return 0 + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port +} + +func generateEngineCompose(name, version string, sshPort int, authorizedKeys, network, fixedIP string) error { + if err := os.MkdirAll(path.Dir(config.MultiDockerConfFile), 0700); err != nil && !os.IsExist(err) { + log.Errorf("Failed to create directory for file %s: %v", config.MultiDockerConfFile, err) + return err + } + + composeConfigs := map[string]composeConfig.ServiceConfigV1{} + + if _, err := os.Stat(config.MultiDockerConfFile); err == nil { + // read from engine compose + bytes, err := ioutil.ReadFile(config.MultiDockerConfFile) + if err != nil { + return err + } + err = yaml.Unmarshal(bytes, &composeConfigs) + if err != nil { + return err + } + } + + if err := os.MkdirAll(config.MultiDockerDataDir+"/"+name, 0700); err != nil && !os.IsExist(err) { + log.Errorf("Failed to create directory for file %s: %v", config.MultiDockerDataDir+"/"+name, err) + return err + } + + composeConfigs[name] = composeConfig.ServiceConfigV1{ + Image: "${REGISTRY_DOMAIN}/" + version, + Restart: "always", + Privileged: true, + Net: network, + Ports: []string{strconv.Itoa(sshPort) + ":22"}, + Volumes: []string{ + "/lib/modules:/lib/modules", + config.MultiDockerDataDir + "/" + name + ":" + config.MultiDockerDataDir + "/" + name, + authorizedKeys + ":/root/.ssh/authorized_keys", + }, + VolumesFrom: []string{}, + Command: composeYaml.Command{ + "dockerd-entrypoint.sh", + "--storage-driver=overlay2", + "--data-root=" + config.MultiDockerDataDir + "/" + name, + "--host=unix://" + config.MultiDockerDataDir + "/" + name + "/docker-" + name + ".sock", + }, + Labels: composeYaml.SliceorMap{ + "io.rancher.os.scope": "system", + "io.rancher.os.after": "console", + config.UserDockerLabel: name, + config.UserDockerNetLabel: network, + config.UserDockerFIPLabel: fixedIP, + }, + } + + bytes, err := yaml.Marshal(composeConfigs) + if err != nil { + return err + } + + return ioutil.WriteFile(config.MultiDockerConfFile, bytes, 0640) +} + +func RemoveEngineFromCompose(name string) error { + composeConfigs := map[string]composeConfig.ServiceConfigV1{} + + if _, err := os.Stat(config.MultiDockerConfFile); err == nil { + // read from engine compose + bytes, err := ioutil.ReadFile(config.MultiDockerConfFile) + if err != nil { + return err + } + err = yaml.Unmarshal(bytes, &composeConfigs) + if err != nil { + return err + } + } + + delete(composeConfigs, name) + + bytes, err := yaml.Marshal(composeConfigs) + if err != nil { + return err + } + + return ioutil.WriteFile(config.MultiDockerConfFile, bytes, 0640) +} + +func CheckUserDefineNetwork(name string) (*types.NetworkResource, error) { + systemClient, err := docker.NewSystemClient() + if err != nil { + return nil, err + } + + networks, err := systemClient.NetworkList(context.Background(), types.NetworkListOptions{}) + if err != nil { + return nil, err + } + + for _, network := range networks { + if network.Name == name { + return &network, nil + } + } + + return nil, errors.Errorf("Failed to found the user define network: %s", name) +} + +func CheckUserDefineIPv4Address(ipv4 string, network types.NetworkResource) error { + for _, config := range network.IPAM.Config { + _, ipnet, _ := net.ParseCIDR(config.Subnet) + if ipnet.Contains(net.ParseIP(ipv4)) { + return nil + } + } + return errors.Errorf("IP %s is not in the specified cidr", ipv4) +} diff --git a/cmd/control/service/app/app.go b/cmd/control/service/app/app.go index dc2e8414..a213db21 100644 --- a/cmd/control/service/app/app.go +++ b/cmd/control/service/app/app.go @@ -85,6 +85,7 @@ func ProjectUp(p project.APIProject, c *cli.Context) error { if err != nil { return cli.NewExitError(err.Error(), 1) } + if c.Bool("foreground") { signalChan := make(chan os.Signal, 1) cleanupDone := make(chan bool) diff --git a/compose/project.go b/compose/project.go index 1c6319f0..6210cc32 100644 --- a/compose/project.go +++ b/compose/project.go @@ -2,6 +2,8 @@ package compose import ( "fmt" + "io/ioutil" + "os" "golang.org/x/net/context" @@ -224,8 +226,31 @@ func StageServices(cfg *config.CloudConfig, services ...string) error { return err } + // read engine services + composeConfigs := map[string]composeConfig.ServiceConfigV1{} + if _, err := os.Stat(config.MultiDockerConfFile); err == nil { + // read from engine compose + multiEngineBytes, err := ioutil.ReadFile(config.MultiDockerConfFile) + if err != nil { + return fmt.Errorf("Failed to read %s : %v", config.MultiDockerConfFile, err) + } + err = yaml.Unmarshal(multiEngineBytes, &composeConfigs) + if err != nil { + return fmt.Errorf("Failed to unmarshal %s : %v", config.MultiDockerConfFile, err) + } + } + for _, service := range services { - bytes, err := network.LoadServiceResource(service, true, cfg) + var bytes []byte + foundServiceConfig := map[string]composeConfig.ServiceConfigV1{} + + if _, ok := composeConfigs[service]; ok { + foundServiceConfig[service] = composeConfigs[service] + bytes, err = yaml.Marshal(foundServiceConfig) + } else { + bytes, err = network.LoadServiceResource(service, true, cfg) + } + if err != nil { return fmt.Errorf("Failed to load %s : %v", service, err) } diff --git a/compose/reload.go b/compose/reload.go index 7ad9c787..2d8253a1 100644 --- a/compose/reload.go +++ b/compose/reload.go @@ -13,10 +13,16 @@ import ( ) func LoadService(p *project.Project, cfg *config.CloudConfig, useNetwork bool, service string) error { - bytes, err := network.LoadServiceResource(service, useNetwork, cfg) - if err != nil { - log.Error(err) - return err + // First check the multi engine service file. + // If the name has been found in multi enging service file and matches, will not execute network.LoadServiceResource + // Otherwise will execute network.LoadServiceResource + bytes, err := network.LoadMultiEngineResource(service) + if err != nil || bytes == nil { + bytes, err = network.LoadServiceResource(service, useNetwork, cfg) + if err != nil { + log.Error(err) + return err + } } m := map[interface{}]interface{}{} diff --git a/config/types.go b/config/types.go index 6ab92c41..0d878803 100644 --- a/config/types.go +++ b/config/types.go @@ -27,15 +27,18 @@ const ( SystemDockerLog = "/var/log/system-docker.log" SystemDockerBin = "/usr/bin/system-dockerd" - HashLabel = "io.rancher.os.hash" - IDLabel = "io.rancher.os.id" - DetachLabel = "io.rancher.os.detach" - CreateOnlyLabel = "io.rancher.os.createonly" - ReloadConfigLabel = "io.rancher.os.reloadconfig" - ConsoleLabel = "io.rancher.os.console" - ScopeLabel = "io.rancher.os.scope" - RebuildLabel = "io.docker.compose.rebuild" - System = "system" + HashLabel = "io.rancher.os.hash" + IDLabel = "io.rancher.os.id" + DetachLabel = "io.rancher.os.detach" + CreateOnlyLabel = "io.rancher.os.createonly" + ReloadConfigLabel = "io.rancher.os.reloadconfig" + ConsoleLabel = "io.rancher.os.console" + ScopeLabel = "io.rancher.os.scope" + RebuildLabel = "io.docker.compose.rebuild" + UserDockerLabel = "io.rancher.user_docker.name" + UserDockerNetLabel = "io.rancher.user_docker.net" + UserDockerFIPLabel = "io.rancher.user_docker.fix_ip" + System = "system" OsConfigFile = "/usr/share/ros/os-config.yml" VarRancherDir = "/var/lib/rancher" @@ -47,6 +50,8 @@ const ( MetaDataFile = "/var/lib/rancher/conf/metadata" CloudConfigFile = "/var/lib/rancher/conf/cloud-config.yml" EtcResolvConfFile = "/etc/resolv.conf" + MultiDockerConfFile = "/var/lib/rancher/conf.d/m-user-docker.yml" + MultiDockerDataDir = "/var/lib/m-user-docker" ) var ( diff --git a/os-config.tpl.yml b/os-config.tpl.yml index 72525beb..c5f6f958 100644 --- a/os-config.tpl.yml +++ b/os-config.tpl.yml @@ -139,6 +139,7 @@ rancher: read_only: true volumes: - /var/lib/user-docker:/var/lib/docker + - /var/lib/m-user-docker:/var/lib/m-user-docker user-volumes: image: {{.OS_REPO}}/os-base:{{.VERSION}}{{.SUFFIX}} command: echo diff --git a/util/network/network.go b/util/network/network.go index 14553f8b..1a522c98 100644 --- a/util/network/network.go +++ b/util/network/network.go @@ -10,6 +10,7 @@ import ( yaml "github.com/cloudfoundry-incubator/candiedyaml" + composeConfig "github.com/docker/libcompose/config" "github.com/rancher/os/config" "github.com/rancher/os/log" ) @@ -165,3 +166,30 @@ func LoadServiceResource(name string, useNetwork bool, cfg *config.CloudConfig) return nil, err } + +func LoadMultiEngineResource(name string) ([]byte, error) { + composeConfigs := map[string]composeConfig.ServiceConfigV1{} + if _, err := os.Stat(config.MultiDockerConfFile); err == nil { + multiEngineBytes, err := ioutil.ReadFile(config.MultiDockerConfFile) + if err != nil { + return nil, err + } + err = yaml.Unmarshal(multiEngineBytes, &composeConfigs) + if err != nil { + return nil, err + } + } + + if _, ok := composeConfigs[name]; !ok { + return nil, errors.New("Failed to found " + name + " from " + config.MultiDockerConfFile + " will load from network") + } + + foundServiceConfig := map[string]composeConfig.ServiceConfigV1{} + foundServiceConfig[name] = composeConfigs[name] + bytes, err := yaml.Marshal(foundServiceConfig) + if err == nil { + return bytes, err + } + + return nil, err +} diff --git a/util/util.go b/util/util.go index 96cc34e7..e0fd538a 100644 --- a/util/util.go +++ b/util/util.go @@ -295,3 +295,21 @@ func RunCommandSequence(commandSequence []osYaml.StringandSlice) error { } return nil } + +func GenerateEngineScript(name string) error { + if _, err := os.Stat("/usr/bin/docker-" + name); err == nil { + err = os.Remove("/usr/bin/docker-" + name) + if err != nil { + return err + } + } + + bytes := []byte("/usr/bin/docker -H unix:///var/lib/m-user-docker/" + name + "/docker-" + name + ".sock $@") + + err := ioutil.WriteFile("/usr/bin/docker-"+name, bytes, 755) + if err != nil { + return err + } + + return nil +}