From f8b92797d0bfa57f2eb00668ad19bdae95027923 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Fri, 28 Aug 2015 21:16:54 -0700 Subject: [PATCH 1/3] Bump godeps for libcompose --- Godeps/Godeps.json | 12 +- .../libcompose/cli/logger/color_logger.go | 6 + .../docker/libcompose/cli/logger/colors.go | 8 +- .../docker/libcompose/docker/container.go | 67 +++-- .../docker/libcompose/docker/convert.go | 4 +- .../docker/libcompose/docker/service.go | 71 ++--- .../docker/libcompose/lookup/file.go | 5 + .../docker/libcompose/lookup/file_test.go | 65 +++++ .../docker/libcompose/lookup/simple_env.go | 8 +- .../libcompose/lookup/simple_env_test.go | 31 ++ .../docker/libcompose/project/types.go | 5 +- .../docker/libcompose/utils/util.go | 27 +- .../docker/libcompose/utils/util_test.go | 266 ++++++++++++++++++ 13 files changed, 474 insertions(+), 101 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/docker/libcompose/lookup/file_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libcompose/lookup/simple_env_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/libcompose/utils/util_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index e962fd34..464dd144 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -250,27 +250,27 @@ }, { "ImportPath": "github.com/docker/libcompose/cli/logger", - "Rev": "a8ac0a68c0552afe4ddb01c4ba3184e574ac416e" + "Rev": "a77147c9909a0ee7055c896826a01cb55352b4d9" }, { "ImportPath": "github.com/docker/libcompose/docker", - "Rev": "a8ac0a68c0552afe4ddb01c4ba3184e574ac416e" + "Rev": "a77147c9909a0ee7055c896826a01cb55352b4d9" }, { "ImportPath": "github.com/docker/libcompose/logger", - "Rev": "a8ac0a68c0552afe4ddb01c4ba3184e574ac416e" + "Rev": "a77147c9909a0ee7055c896826a01cb55352b4d9" }, { "ImportPath": "github.com/docker/libcompose/lookup", - "Rev": "a8ac0a68c0552afe4ddb01c4ba3184e574ac416e" + "Rev": "a77147c9909a0ee7055c896826a01cb55352b4d9" }, { "ImportPath": "github.com/docker/libcompose/project", - "Rev": "a8ac0a68c0552afe4ddb01c4ba3184e574ac416e" + "Rev": "a77147c9909a0ee7055c896826a01cb55352b4d9" }, { "ImportPath": "github.com/docker/libcompose/utils", - "Rev": "a8ac0a68c0552afe4ddb01c4ba3184e574ac416e" + "Rev": "a77147c9909a0ee7055c896826a01cb55352b4d9" }, { "ImportPath": "github.com/docker/libcontainer/netlink", diff --git a/Godeps/_workspace/src/github.com/docker/libcompose/cli/logger/color_logger.go b/Godeps/_workspace/src/github.com/docker/libcompose/cli/logger/color_logger.go index 04c5bcd6..51e59a60 100644 --- a/Godeps/_workspace/src/github.com/docker/libcompose/cli/logger/color_logger.go +++ b/Godeps/_workspace/src/github.com/docker/libcompose/cli/logger/color_logger.go @@ -9,23 +9,27 @@ import ( "golang.org/x/crypto/ssh/terminal" ) +// ColorLoggerFactory implements logger.Factory interface using ColorLogger. type ColorLoggerFactory struct { maxLength int tty bool } +// ColorLogger implements logger.Logger interface with color support. type ColorLogger struct { name string colorPrefix string factory *ColorLoggerFactory } +// NewColorLoggerFactory creates a new ColorLoggerFactory. func NewColorLoggerFactory() *ColorLoggerFactory { return &ColorLoggerFactory{ tty: terminal.IsTerminal(int(os.Stdout.Fd())), } } +// Create implements logger.Factory.Create. func (c *ColorLoggerFactory) Create(name string) logger.Logger { if c.maxLength < len(name) { c.maxLength = len(name) @@ -38,6 +42,7 @@ func (c *ColorLoggerFactory) Create(name string) logger.Logger { } } +// Out implements logger.Logger.Out. func (c *ColorLogger) Out(bytes []byte) { if len(bytes) == 0 { return @@ -47,6 +52,7 @@ func (c *ColorLogger) Out(bytes []byte) { fmt.Print(message) } +// Err implements logger.Logger.Err. func (c *ColorLogger) Err(bytes []byte) { if len(bytes) == 0 { return diff --git a/Godeps/_workspace/src/github.com/docker/libcompose/cli/logger/colors.go b/Godeps/_workspace/src/github.com/docker/libcompose/cli/logger/colors.go index 21aa46a2..4f9e62ad 100644 --- a/Godeps/_workspace/src/github.com/docker/libcompose/cli/logger/colors.go +++ b/Godeps/_workspace/src/github.com/docker/libcompose/cli/logger/colors.go @@ -3,12 +3,12 @@ package logger import "fmt" var ( - colorPrefix chan string = make(chan string) + colorPrefix = make(chan string) ) func generateColors() { i := 0 - color_order := []string{ + colorOrder := []string{ "36", // cyan "33", // yellow "32", // green @@ -24,8 +24,8 @@ func generateColors() { } for { - colorPrefix <- fmt.Sprintf("\033[%sm%%s |\033[0m", color_order[i]) - i = (i + 1) % len(color_order) + colorPrefix <- fmt.Sprintf("\033[%sm%%s |\033[0m", colorOrder[i]) + i = (i + 1) % len(colorOrder) } } diff --git a/Godeps/_workspace/src/github.com/docker/libcompose/docker/container.go b/Godeps/_workspace/src/github.com/docker/libcompose/docker/container.go index 0fce2abb..d112c5bd 100644 --- a/Godeps/_workspace/src/github.com/docker/libcompose/docker/container.go +++ b/Godeps/_workspace/src/github.com/docker/libcompose/docker/container.go @@ -55,10 +55,10 @@ func (c *Container) Info() (project.Info, error) { result := project.Info{} - result = append(result, project.InfoPart{"Name", name(container.Names)}) - result = append(result, project.InfoPart{"Command", container.Command}) - result = append(result, project.InfoPart{"State", container.Status}) - result = append(result, project.InfoPart{"Ports", portString(container.Ports)}) + result = append(result, project.InfoPart{Key: "Name", Value: name(container.Names)}) + result = append(result, project.InfoPart{Key: "Command", Value: container.Command}) + result = append(result, project.InfoPart{Key: "State", Value: container.Status}) + result = append(result, project.InfoPart{Key: "Ports", Value: portString(container.Ports)}) return result, nil } @@ -206,8 +206,9 @@ func (c *Container) Up(imageName string) error { if !info.State.Running { logrus.Debugf("Starting container: %s", container.Id) - err := c.client.StartContainer(container.Id, nil) - return err + if err := c.client.StartContainer(container.Id, nil); err != nil { + return err + } c.service.context.Project.Notify(project.CONTAINER_STARTED, c.service.Name(), map[string]string{ "name": c.Name(), @@ -218,29 +219,35 @@ func (c *Container) Up(imageName string) error { } func (c *Container) OutOfSync(imageName string) (bool, error) { - container, err := c.findExisting() - if err != nil || container == nil { + info, err := c.findInfo() + if err != nil || info == nil { return false, err } - info, err := c.client.InspectContainer(container.Id) - if err != nil { + if info.Config.Image != imageName { + logrus.Debugf("Images for %s do not match %s!=%s", c.name, info.Config.Image, imageName) + return true, nil + } + + if info.Config.Labels[HASH.Str()] != c.getHash() { + logrus.Debugf("Hashes for %s do not match %s!=%s", c.name, info.Config.Labels[HASH.Str()], c.getHash()) + return true, nil + } + + image, err := c.client.InspectImage(info.Config.Image) + if err != nil && (err.Error() == "Not found" || image == nil) { + logrus.Debugf("Image %s do not exist, do not know if it's out of sync", info.Config.Image) + return false, nil + } else if err != nil { return false, err } - return info.Config.Labels[HASH.Str()] != c.getHash(imageName), nil + logrus.Debugf("Checking existing image name vs id: %s == %s", image.Id, info.Image) + return image.Id != info.Image, err } -func (c *Container) getHash(imageName string) string { - serviceConfig := *c.service.Config() - imageInfo, err := c.client.InspectImage(imageName) - if imageInfo != nil && err == nil { - serviceConfig.Image = imageInfo.Id - } else { - serviceConfig.Image = imageName - } - - return project.GetServiceHash(c.service.Name(), serviceConfig) +func (c *Container) getHash() string { + return project.GetServiceHash(c.service.Name(), *c.service.Config()) } func (c *Container) createContainer(imageName, oldContainer string) (*dockerclient.Container, error) { @@ -258,7 +265,7 @@ func (c *Container) createContainer(imageName, oldContainer string) (*dockerclie config.Labels[NAME.Str()] = c.name config.Labels[SERVICE.Str()] = c.service.name config.Labels[PROJECT.Str()] = c.service.context.Project.Name - config.Labels[HASH.Str()] = c.getHash(imageName) + config.Labels[HASH.Str()] = c.getHash() err = c.populateAdditionalHostConfig(&config.HostConfig) if err != nil { @@ -344,7 +351,7 @@ func (c *Container) addLinks(links map[string]string, service project.Service, r func (c *Container) addIpc(config *dockerclient.HostConfig, service project.Service, containers []project.Container) (*dockerclient.HostConfig, error) { if len(containers) == 0 { - return nil, fmt.Errorf("Failed to find container for IPC %", c.service.Config().Ipc) + return nil, fmt.Errorf("Failed to find container for IPC %v", c.service.Config().Ipc) } id, err := containers[0].Id() @@ -358,7 +365,7 @@ func (c *Container) addIpc(config *dockerclient.HostConfig, service project.Serv func (c *Container) addNetNs(config *dockerclient.HostConfig, service project.Service, containers []project.Container) (*dockerclient.HostConfig, error) { if len(containers) == 0 { - return nil, fmt.Errorf("Failed to find container for networks ns %", c.service.Config().Net) + return nil, fmt.Errorf("Failed to find container for networks ns %v", c.service.Config().Net) } id, err := containers[0].Id() @@ -434,11 +441,13 @@ func (c *Container) Log() error { }, output) return err } - - return nil } func (c *Container) pull(image string) error { + return PullImage(c.client, c.service, image) +} + +func PullImage(client dockerclient.Client, service *Service, image string) error { taglessRemote, tag := parsers.ParseRepositoryTag(image) if tag == "" { image = utils.ImageReference(taglessRemote, tags.DEFAULTTAG) @@ -450,11 +459,11 @@ func (c *Container) pull(image string) error { } authConfig := cliconfig.AuthConfig{} - if c.service.context.ConfigFile != nil && repoInfo != nil && repoInfo.Index != nil { - authConfig = registry.ResolveAuthConfig(c.service.context.ConfigFile, repoInfo.Index) + if service.context.ConfigFile != nil && repoInfo != nil && repoInfo.Index != nil { + authConfig = registry.ResolveAuthConfig(service.context.ConfigFile, repoInfo.Index) } - err = c.client.PullImage(image, &dockerclient.AuthConfig{ + err = client.PullImage(image, &dockerclient.AuthConfig{ Username: authConfig.Username, Password: authConfig.Password, Email: authConfig.Email, diff --git a/Godeps/_workspace/src/github.com/docker/libcompose/docker/convert.go b/Godeps/_workspace/src/github.com/docker/libcompose/docker/convert.go index 937805ec..82aa04f2 100644 --- a/Godeps/_workspace/src/github.com/docker/libcompose/docker/convert.go +++ b/Godeps/_workspace/src/github.com/docker/libcompose/docker/convert.go @@ -36,13 +36,13 @@ func ConvertToApi(c *project.ServiceConfig) (*dockerclient.ContainerConfig, erro } var result dockerclient.ContainerConfig - err = utils.ConvertByJson(config, &result) + err = utils.ConvertByJSON(config, &result) if err != nil { logrus.Errorf("Failed to convert config to API structure: %v\n%#v", err, config) return nil, err } - err = utils.ConvertByJson(hostConfig, &result.HostConfig) + err = utils.ConvertByJSON(hostConfig, &result.HostConfig) if err != nil { logrus.Errorf("Failed to convert hostConfig to API structure: %v\n%#v", err, hostConfig) } diff --git a/Godeps/_workspace/src/github.com/docker/libcompose/docker/service.go b/Godeps/_workspace/src/github.com/docker/libcompose/docker/service.go index 38b73b6d..ca5f5241 100644 --- a/Godeps/_workspace/src/github.com/docker/libcompose/docker/service.go +++ b/Godeps/_workspace/src/github.com/docker/libcompose/docker/service.go @@ -10,7 +10,6 @@ type Service struct { name string serviceConfig *project.ServiceConfig context *Context - imageName string } func NewService(name string, serviceConfig *project.ServiceConfig, context *Context) *Service { @@ -34,7 +33,12 @@ func (s *Service) DependentServices() []project.ServiceRelationship { } func (s *Service) Create() error { - _, err := s.createOne() + imageName, err := s.build() + if err != nil { + return err + } + + _, err = s.createOne(imageName) return err } @@ -59,8 +63,8 @@ func (s *Service) collectContainers() ([]*Container, error) { return result, nil } -func (s *Service) createOne() (*Container, error) { - containers, err := s.constructContainers(true, 1) +func (s *Service) createOne(imageName string) (*Container, error) { + containers, err := s.constructContainers(imageName, 1) if err != nil { return nil, err } @@ -74,24 +78,14 @@ func (s *Service) Build() error { } func (s *Service) build() (string, error) { - if s.imageName != "" { - return s.imageName, nil - } - if s.context.Builder == nil { - s.imageName = s.Config().Image - } else { - var err error - s.imageName, err = s.context.Builder.Build(s.context.Project, s) - if err != nil { - return "", err - } + return s.Config().Image, nil } - return s.imageName, nil + return s.context.Builder.Build(s.context.Project, s) } -func (s *Service) constructContainers(create bool, count int) ([]*Container, error) { +func (s *Service) constructContainers(imageName string, count int) ([]*Container, error) { result, err := s.collectContainers() if err != nil { return nil, err @@ -107,20 +101,13 @@ func (s *Service) constructContainers(create bool, count int) ([]*Container, err c := NewContainer(client, containerName, s) - if create { - imageName, err := s.build() - if err != nil { - return nil, err - } - - dockerContainer, err := c.Create(imageName) - if err != nil { - return nil, err - } else { - logrus.Debugf("Created container %s: %v", dockerContainer.Id, dockerContainer.Names) - } + dockerContainer, err := c.Create(imageName) + if err != nil { + return nil, err } + logrus.Debugf("Created container %s: %v", dockerContainer.Id, dockerContainer.Names) + result = append(result, NewContainer(client, containerName, s)) } @@ -167,7 +154,7 @@ func (s *Service) up(imageName string, create bool) error { logrus.Debugf("Found %d existing containers for service %s", len(containers), s.name) if len(containers) == 0 && create { - c, err := s.createOne() + c, err := s.createOne(imageName) if err != nil { return err } @@ -176,7 +163,7 @@ func (s *Service) up(imageName string, create bool) error { return s.eachContainer(func(c *Container) error { if s.context.Rebuild && create { - if err := s.rebuildIfNeeded(c); err != nil { + if err := s.rebuildIfNeeded(imageName, c); err != nil { return err } } @@ -185,12 +172,7 @@ func (s *Service) up(imageName string, create bool) error { }) } -func (s *Service) rebuildIfNeeded(c *Container) error { - imageName, err := s.build() - if err != nil { - return err - } - +func (s *Service) rebuildIfNeeded(imageName string, c *Container) error { outOfSync, err := c.OutOfSync(imageName) if err != nil { return err @@ -208,7 +190,8 @@ func (s *Service) rebuildIfNeeded(c *Container) error { logrus.WithFields(logrus.Fields{ "origRebuildLabel": origRebuildLabel, "newRebuildLabel": newRebuildLabel, - "rebuildLabelChanged": rebuildLabelChanged}).Debug("Rebuild values") + "rebuildLabelChanged": rebuildLabelChanged, + "outOfSync": outOfSync}).Debug("Rebuild values") if origRebuildLabel == "always" || rebuildLabelChanged || origRebuildLabel != "false" && outOfSync { logrus.Infof("Rebuilding %s", name) @@ -292,23 +275,25 @@ func (s *Service) Scale(scale int) error { } if foundCount != scale { - _, err := s.constructContainers(true, scale) + imageName, err := s.build() if err != nil { return err } + if _, err = s.constructContainers(imageName, scale); err != nil { + return err + } } return s.up("", false) } func (s *Service) Pull() error { - containers, err := s.constructContainers(false, 1) - if err != nil { - return err + if s.Config().Image == "" { + return nil } - return containers[0].Pull() + return PullImage(s.context.ClientFactory.Create(s), s, s.Config().Image) } func (s *Service) Containers() ([]project.Container, error) { diff --git a/Godeps/_workspace/src/github.com/docker/libcompose/lookup/file.go b/Godeps/_workspace/src/github.com/docker/libcompose/lookup/file.go index 96cce945..88a585ab 100644 --- a/Godeps/_workspace/src/github.com/docker/libcompose/lookup/file.go +++ b/Godeps/_workspace/src/github.com/docker/libcompose/lookup/file.go @@ -8,9 +8,14 @@ import ( "github.com/Sirupsen/logrus" ) +// FileConfigLookup is a "bare" structure that implements the project.ConfigLookup interface type FileConfigLookup struct { } +// Lookup returns the content and the actual filename of the file that is "built" using the +// specified file and relativeTo string. file and relativeTo are supposed to be file path. +// If file starts with a slash ('/'), it tries to load it, otherwise it will build a +// filename using the folder part of relativeTo joined with file. func (f *FileConfigLookup) Lookup(file, relativeTo string) ([]byte, string, error) { if strings.HasPrefix(file, "/") { logrus.Debugf("Reading file %s", file) diff --git a/Godeps/_workspace/src/github.com/docker/libcompose/lookup/file_test.go b/Godeps/_workspace/src/github.com/docker/libcompose/lookup/file_test.go new file mode 100644 index 00000000..6814e6b5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libcompose/lookup/file_test.go @@ -0,0 +1,65 @@ +package lookup + +import ( + "io/ioutil" + "path/filepath" + "testing" +) + +type input struct { + file string + relativeTo string +} + +func TestLookupError(t *testing.T) { + invalids := map[input]string{ + input{"", ""}: "read .: is a directory", + input{"", "/tmp/"}: "read /tmp: is a directory", + input{"file", "/does/not/exists/"}: "open /does/not/exists/file: no such file or directory", + input{"file", "/does/not/something"}: "open /does/not/file: no such file or directory", + input{"file", "/does/not/exists/another"}: "open /does/not/exists/file: no such file or directory", + input{"/does/not/exists/file", "/tmp/"}: "open /does/not/exists/file: no such file or directory", + input{"does/not/exists/file", "/tmp/"}: "open /tmp/does/not/exists/file: no such file or directory", + } + + fileConfigLookup := FileConfigLookup{} + + for invalid, expectedError := range invalids { + _, _, err := fileConfigLookup.Lookup(invalid.file, invalid.relativeTo) + if err == nil || err.Error() != expectedError { + t.Fatalf("Expected error with '%s', got '%v'", expectedError, err) + } + } +} + +func TestLookupOK(t *testing.T) { + tmpFolder, err := ioutil.TempDir("", "lookup-tests") + if err != nil { + t.Fatal(err) + } + tmpFile1 := filepath.Join(tmpFolder, "file1") + tmpFile2 := filepath.Join(tmpFolder, "file2") + if err = ioutil.WriteFile(tmpFile1, []byte("content1"), 0755); err != nil { + t.Fatal(err) + } + if err = ioutil.WriteFile(tmpFile2, []byte("content2"), 0755); err != nil { + t.Fatal(err) + } + + fileConfigLookup := FileConfigLookup{} + + valids := map[input]string{ + input{"file1", tmpFolder + "/"}: "content1", + input{"file2", tmpFolder + "/"}: "content2", + input{tmpFile1, tmpFolder}: "content1", + input{tmpFile1, "/does/not/exists"}: "content1", + input{"file2", tmpFile1}: "content2", + } + + for valid, expectedContent := range valids { + out, _, err := fileConfigLookup.Lookup(valid.file, valid.relativeTo) + if err != nil || string(out) != expectedContent { + t.Fatalf("Expected %s to contains '%s', got %s, %v.", valid.file, expectedContent, out, err) + } + } +} diff --git a/Godeps/_workspace/src/github.com/docker/libcompose/lookup/simple_env.go b/Godeps/_workspace/src/github.com/docker/libcompose/lookup/simple_env.go index ab48ea57..7b8aacd9 100644 --- a/Godeps/_workspace/src/github.com/docker/libcompose/lookup/simple_env.go +++ b/Godeps/_workspace/src/github.com/docker/libcompose/lookup/simple_env.go @@ -7,14 +7,18 @@ import ( "github.com/docker/libcompose/project" ) +// OsEnvLookup is a "bare" structure that implements the project.EnvironmentLookup interface type OsEnvLookup struct { } +// Lookup creates a string slice of string containing a "docker-friendly" environment string +// in the form of 'key=value'. It gets environment values using os.Getenv. +// If the os environment variable does not exists, the slice is empty. serviceName and config +// are not used at all in this implementation. func (o *OsEnvLookup) Lookup(key, serviceName string, config *project.ServiceConfig) []string { ret := os.Getenv(key) if ret == "" { return []string{} - } else { - return []string{fmt.Sprintf("%s=%s", key, ret)} } + return []string{fmt.Sprintf("%s=%s", key, ret)} } diff --git a/Godeps/_workspace/src/github.com/docker/libcompose/lookup/simple_env_test.go b/Godeps/_workspace/src/github.com/docker/libcompose/lookup/simple_env_test.go new file mode 100644 index 00000000..4d3e4e1f --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libcompose/lookup/simple_env_test.go @@ -0,0 +1,31 @@ +package lookup + +import ( + "testing" + + "github.com/docker/libcompose/project" +) + +func TestOsEnvLookup(t *testing.T) { + // Putting bare minimun value for serviceName and config as there are + // not important on this test. + serviceName := "anything" + config := &project.ServiceConfig{} + + osEnvLookup := &OsEnvLookup{} + + envs := osEnvLookup.Lookup("PATH", serviceName, config) + if len(envs) != 1 { + t.Fatalf("Expected envs to contains one element, but was %v", envs) + } + + envs = osEnvLookup.Lookup("path", serviceName, config) + if len(envs) != 0 { + t.Fatalf("Expected envs to be empty, but was %v", envs) + } + + envs = osEnvLookup.Lookup("DOES_NOT_EXIST", serviceName, config) + if len(envs) != 0 { + t.Fatalf("Expected envs to be empty, but was %v", envs) + } +} diff --git a/Godeps/_workspace/src/github.com/docker/libcompose/project/types.go b/Godeps/_workspace/src/github.com/docker/libcompose/project/types.go index 24032be2..e8fb0645 100644 --- a/Godeps/_workspace/src/github.com/docker/libcompose/project/types.go +++ b/Godeps/_workspace/src/github.com/docker/libcompose/project/types.go @@ -158,10 +158,9 @@ type ServiceConfig struct { Build string `yaml:"build,omitempty"` CapAdd []string `yaml:"cap_add,omitempty"` CapDrop []string `yaml:"cap_drop,omitempty"` - CpuSet string `yaml:"cpu_set,omitempty"` + CpuSet string `yaml:"cpuset,omitempty"` CpuShares int64 `yaml:"cpu_shares,omitempty"` Command Command `yaml:"command"` // omitempty breaks serialization! - Detach string `yaml:"detach,omitempty"` Devices []string `yaml:"devices,omitempty"` Dns Stringorslice `yaml:"dns"` // omitempty breaks serialization! DnsSearch Stringorslice `yaml:"dns_search"` // omitempty breaks serialization! @@ -176,7 +175,7 @@ type ServiceConfig struct { Links MaporColonSlice `yaml:"links"` // omitempty breaks serialization! LogDriver string `yaml:"log_driver,omitempty"` MemLimit int64 `yaml:"mem_limit,omitempty"` - MemSwapLimit int64 `yaml:"mem_swap_limit,omitempty"` + MemSwapLimit int64 `yaml:"memswap_limit,omitempty"` Name string `yaml:"name,omitempty"` Net string `yaml:"net,omitempty"` Pid string `yaml:"pid,omitempty"` diff --git a/Godeps/_workspace/src/github.com/docker/libcompose/utils/util.go b/Godeps/_workspace/src/github.com/docker/libcompose/utils/util.go index 11cfce6c..bc28cbab 100644 --- a/Godeps/_workspace/src/github.com/docker/libcompose/utils/util.go +++ b/Godeps/_workspace/src/github.com/docker/libcompose/utils/util.go @@ -10,11 +10,14 @@ import ( "gopkg.in/yaml.v2" ) +// InParallel holds a pool and a waitgroup to execute tasks in parallel and to be able +// to wait for completion of all tasks. type InParallel struct { wg sync.WaitGroup pool sync.Pool } +// Add adds runs the specified task in parallel and add it to the waitGroup. func (i *InParallel) Add(task func() error) { i.wg.Add(1) @@ -27,17 +30,19 @@ func (i *InParallel) Add(task func() error) { }() } +// Wait waits for all tasks to complete and returns the latests error encountered if any. func (i *InParallel) Wait() error { i.wg.Wait() obj := i.pool.Get() if err, ok := obj.(error); ok { return err - } else { - return nil } + return nil } -func ConvertByJson(src, target interface{}) error { +// ConvertByJSON converts a struct (src) to another one (target) using json marshalling/unmarshalling. +// If the structure are not compatible, this will throw an error as the unmarshalling will fail. +func ConvertByJSON(src, target interface{}) error { newBytes, err := json.Marshal(src) if err != nil { return err @@ -50,6 +55,8 @@ func ConvertByJson(src, target interface{}) error { return err } +// Convert converts a struct (src) to another one (target) using yaml marshalling/unmarshalling. +// If the structure are not compatible, this will throw an error as the unmarshalling will fail. func Convert(src, target interface{}) error { newBytes, err := yaml.Marshal(src) if err != nil { @@ -63,27 +70,23 @@ func Convert(src, target interface{}) error { return err } -func ConvertToInterfaceMap(input map[string]string) map[string]interface{} { - result := map[string]interface{}{} - for k, v := range input { - result[k] = v - } - - return result -} - +// FilterString returns a json representation of the specified map +// that is used as filter for docker. func FilterString(data map[string][]string) string { // I can't imagine this would ever fail bytes, _ := json.Marshal(data) return string(bytes) } +// LabelFilter returns a label json representation of the specifed couple (key,value) +// that is used as filter for docker. func LabelFilter(key, value string) string { return FilterString(map[string][]string{ "label": {fmt.Sprintf("%s=%s", key, value)}, }) } +// Contains checks if the specified string (key) is present in the specified collection. func Contains(collection []string, key string) bool { for _, value := range collection { if value == key { diff --git a/Godeps/_workspace/src/github.com/docker/libcompose/utils/util_test.go b/Godeps/_workspace/src/github.com/docker/libcompose/utils/util_test.go new file mode 100644 index 00000000..71bfc3d7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/libcompose/utils/util_test.go @@ -0,0 +1,266 @@ +package utils + +import ( + "fmt" + "testing" +) + +type jsonfrom struct { + Element1 string `json:"element2"` + Element2 int `json:"element1"` +} +type jsonto struct { + Elt1 int `json:"element1"` + Elt2 string `json:"element2"` + Elt3 int +} + +func TestInParallel(t *testing.T) { + size := 5 + booleanMap := make(map[int]bool, size+1) + tasks := InParallel{} + for i := 0; i < size; i++ { + task := func(index int) func() error { + return func() error { + booleanMap[index] = true + return nil + } + }(i) + tasks.Add(task) + } + err := tasks.Wait() + if err != nil { + t.Fatal(err) + } + // Make sure every value is true + for _, value := range booleanMap { + if !value { + t.Fatalf("booleanMap expected to contain only true values, got at least one false") + } + } +} + +func TestInParallelError(t *testing.T) { + size := 5 + booleanMap := make(map[int]bool, size+1) + tasks := InParallel{} + for i := 0; i < size; i++ { + task := func(index int) func() error { + return func() error { + booleanMap[index] = true + if index%2 == 0 { + return fmt.Errorf("Error with %v", index) + } + return nil + } + }(i) + tasks.Add(task) + } + err := tasks.Wait() + if err == nil { + t.Fatalf("Expected an error on Wait, got nothing.") + } + for key, value := range booleanMap { + if key%2 != 0 && !value { + t.Fatalf("booleanMap expected to contain true values on odd number, got %v", booleanMap) + } + } +} + +func TestConvertByJSON(t *testing.T) { + valids := []struct { + src jsonfrom + expected jsonto + }{ + { + jsonfrom{Element2: 1}, + jsonto{1, "", 0}, + }, + { + jsonfrom{}, + jsonto{0, "", 0}, + }, + { + jsonfrom{"element1", 2}, + jsonto{2, "element1", 0}, + }, + } + for _, valid := range valids { + var target jsonto + err := ConvertByJSON(valid.src, &target) + if err != nil || target.Elt1 != valid.expected.Elt1 || target.Elt2 != valid.expected.Elt2 || target.Elt3 != 0 { + t.Fatalf("Expected %v from %v got %v, %v", valid.expected, valid.src, target, err) + } + } +} + +func TestConvertByJSONInvalid(t *testing.T) { + invalids := []interface{}{ + // Incompatible struct + struct { + Element1 int `json:"element2"` + Element2 string `json:"element1"` + }{1, "element1"}, + // Not marshable struct + struct { + Element1 func(int) int + }{ + func(i int) int { return 0 }, + }, + } + for _, invalid := range invalids { + var target jsonto + if err := ConvertByJSON(invalid, &target); err == nil { + t.Fatalf("Expected an error converting %v to %v, got nothing", invalid, target) + } + } +} + +type yamlfrom struct { + Element1 string `yaml:"element2"` + Element2 int `yaml:"element1"` +} +type yamlto struct { + Elt1 int `yaml:"element1"` + Elt2 string `yaml:"element2"` + Elt3 int +} + +func TestConvert(t *testing.T) { + valids := []struct { + src yamlfrom + expected yamlto + }{ + { + yamlfrom{Element2: 1}, + yamlto{1, "", 0}, + }, + { + yamlfrom{}, + yamlto{0, "", 0}, + }, + { + yamlfrom{"element1", 2}, + yamlto{2, "element1", 0}, + }, + } + for _, valid := range valids { + var target yamlto + err := Convert(valid.src, &target) + if err != nil || target.Elt1 != valid.expected.Elt1 || target.Elt2 != valid.expected.Elt2 || target.Elt3 != 0 { + t.Fatalf("Expected %v from %v got %v, %v", valid.expected, valid.src, target, err) + } + } +} + +func TestConvertInvalid(t *testing.T) { + invalids := []interface{}{ + // Incompatible struct + struct { + Element1 int `yaml:"element2"` + Element2 string `yaml:"element1"` + }{1, "element1"}, + // Not marshable struct + // This one panics :-| + // struct { + // Element1 func(int) int + // }{ + // func(i int) int { return 0 }, + // }, + } + for _, invalid := range invalids { + var target yamlto + if err := Convert(invalid, &target); err == nil { + t.Fatalf("Expected an error converting %v to %v, got nothing", invalid, target) + } + } +} + +func TestFilterString(t *testing.T) { + datas := []struct { + value map[string][]string + expected string + }{ + { + map[string][]string{}, + "{}", + }, + { + map[string][]string{ + "key": {}, + }, + `{"key":[]}`, + }, + { + map[string][]string{ + "key": {"value1", "value2"}, + }, + `{"key":["value1","value2"]}`, + }, + { + map[string][]string{ + "key1": {"value1", "value2"}, + "key2": {"value3", "value4"}, + }, + `{"key1":["value1","value2"],"key2":["value3","value4"]}`, + }, + } + for _, data := range datas { + actual := FilterString(data.value) + if actual != data.expected { + t.Fatalf("Expected '%v' for %v, got '%v'", data.expected, data.value, actual) + } + } +} + +func TestLabelFilter(t *testing.T) { + filters := []struct { + key string + value string + expected string + }{ + { + "key", "value", `{"label":["key=value"]}`, + }, { + "key", "", `{"label":["key="]}`, + }, { + "", "", `{"label":["="]}`, + }, + } + for _, filter := range filters { + actual := LabelFilter(filter.key, filter.value) + if actual != filter.expected { + t.Fatalf("Expected '%s for key=%s and value=%s, got %s", filter.expected, filter.key, filter.value, actual) + } + } +} + +func TestContains(t *testing.T) { + cases := []struct { + collection []string + key string + contains bool + }{ + { + []string{}, "", false, + }, + { + []string{""}, "", true, + }, + { + []string{"value1", "value2"}, "value3", false, + }, + { + []string{"value1", "value2"}, "value1", true, + }, + { + []string{"value1", "value2"}, "value2", true, + }, + } + for _, element := range cases { + actual := Contains(element.collection, element.key) + if actual != element.contains { + t.Fatalf("Expected contains to be %v for %v in %v, but was %v", element.contains, element.key, element.collection, actual) + } + } +} From 0f1b7741d7758d9e014c7a627f65e82c1adc42af Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Fri, 28 Aug 2015 21:13:01 -0700 Subject: [PATCH 2/3] Fix setting up state partition --- init/root.go | 59 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/init/root.go b/init/root.go index 86aad6fb..8815dd26 100644 --- a/init/root.go +++ b/init/root.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "path" + "strings" "syscall" log "github.com/Sirupsen/logrus" @@ -13,25 +14,67 @@ import ( "github.com/rancherio/os/config" ) -func prepareRoot(rootfs string) error { - usr := path.Join(rootfs, "usr") - if err := os.Remove(usr); err != nil && !os.IsNotExist(err) { - log.Errorf("Failed to delete %s, possibly invalid RancherOS state partition: %v", usr, err) - return err +func cleanupTarget(rootfs, targetUsr, usr, usrVer, tmpDir string) (bool, error) { + log.Debugf("Deleting %s", targetUsr) + if err := os.Remove(targetUsr); err != nil && !os.IsNotExist(err) { + log.Errorf("Failed to delete %s, possibly invalid RancherOS state partition: %v", targetUsr, err) + return false, err } - return nil + if err := dockerlaunch.CreateSymlink(usrVer, path.Join(rootfs, "usr")); err != nil { + return false, err + } + + log.Debugf("Deleting %s", tmpDir) + if err := os.RemoveAll(tmpDir); err != nil { + // Don't care if this fails + log.Errorf("Failed to cleanup temp directory %s: %v", tmpDir, err) + } + + if strings.HasSuffix(usrVer, "dev") { + log.Debugf("Deleteing old usr: %s", usr) + if err := os.RemoveAll(usr); err != nil { + // Don't care if this fails + log.Errorf("Failed to remove %s: %v", usr, err) + } + return true, nil + } + + if _, err := os.Stat(usrVer); os.IsNotExist(err) { + return false, nil + } + + return true, nil } func copyMoveRoot(rootfs string) error { usrVer := fmt.Sprintf("usr-%s", config.VERSION) usr := path.Join(rootfs, usrVer) + targetUsr := path.Join(rootfs, "usr") + tmpDir := path.Join(rootfs, "tmp") - if err := archive.CopyWithTar("/usr", usr); err != nil { + if cont, err := cleanupTarget(rootfs, targetUsr, usr, usrVer, tmpDir); !cont { return err } - if err := dockerlaunch.CreateSymlink(usrVer, path.Join(rootfs, "usr")); err != nil { + log.Debugf("Creating temp dir directory %s", tmpDir) + if err := os.MkdirAll(tmpDir, 0755); err != nil { + return err + } + + usrVerTmp, err := ioutil.TempDir(tmpDir, usrVer) + if err != nil { + return err + } + + log.Debugf("Copying to temp dir %s", usrVerTmp) + + if err := archive.CopyWithTar("/usr", usrVerTmp); err != nil { + return err + } + + log.Debugf("Renaming %s => %s", usrVerTmp, usr) + if err := os.Rename(usrVerTmp, usr); err != nil { return err } From 55c4d76e609e950686408ce73db3f1b3cb2d84e9 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Fri, 28 Aug 2015 21:13:21 -0700 Subject: [PATCH 3/3] Make sure services_include overrides services --- compose/project.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compose/project.go b/compose/project.go index c05faf3b..adaec8f4 100644 --- a/compose/project.go +++ b/compose/project.go @@ -1,6 +1,8 @@ package compose import ( + "fmt" + log "github.com/Sirupsen/logrus" "github.com/docker/libcompose/cli/logger" "github.com/docker/libcompose/docker" @@ -115,6 +117,8 @@ func newCoreServiceProject(cfg *config.CloudConfig) (*project.Project, error) { return err } + addServices(p, cfg, enabled, cfg.Rancher.Services) + for service, serviceEnabled := range cfg.Rancher.ServicesInclude { if enabled[service] != "" || !serviceEnabled { continue @@ -130,6 +134,7 @@ func newCoreServiceProject(cfg *config.CloudConfig) (*project.Project, error) { continue } + fmt.Println("Loading config: %s", string(bytes)) err = p.Load(bytes) if err != nil { log.Errorf("Failed to load %s : %v", service, err) @@ -139,8 +144,6 @@ func newCoreServiceProject(cfg *config.CloudConfig) (*project.Project, error) { enabled[service] = service } - addServices(p, cfg, enabled, cfg.Rancher.Services) - return nil }