diff --git a/cmd/skopeo/sync.go b/cmd/skopeo/sync.go index a93f1669..0665eec8 100644 --- a/cmd/skopeo/sync.go +++ b/cmd/skopeo/sync.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "regexp" "strings" "github.com/containers/image/v5/copy" @@ -50,7 +51,7 @@ type tlsVerifyConfig struct { // registrySyncConfig contains information about a single registry, read from // the source YAML file type registrySyncConfig struct { - Images map[string][]string // Images map images name to slices with the images' tags + Images map[string]interface{} // Images map images name to slices or regular expression with the images' tags Credentials types.DockerAuthConfig // Username and password used to authenticate with the registry TLSVerify tlsVerifyConfig `yaml:"tls-verify"` // TLS verification mode (enabled by default) CertDir string `yaml:"cert-dir"` // Path to the TLS certificates of the registry @@ -278,21 +279,82 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc serverCtx.DockerAuthConfig = &cfg.Credentials var sourceReferences []types.ImageReference - for _, tag := range tags { - source := fmt.Sprintf("%s:%s", repoName, tag) - imageRef, err := docker.ParseReference(source) + switch tags.(type) { + case []string, []interface{}, nil: + tagList := make([]string, 0) + if tagIns, ok := tags.([]interface{}); ok { + for _, tagValue := range tagIns { + switch tagValue.(type) { + case string, int, float64: + tagList = append(tagList, fmt.Sprintf("%v", tagValue)) + default: + logrus.WithFields(logrus.Fields{ + "repo": imageName, + "registry": registryName, + }).Error("Error processing repo, skipping") + logrus.Errorf("Elements can only be strings if they are of type array, wrong value (%v|%T)", tagValue, tagValue) + continue + } + } + } else { + // nil is equl full tags + if tags != nil { + tagList = tags.([]string) + } + } + + for _, tag := range tagList { + source := fmt.Sprintf("%s:%s", repoName, tag) + + imageRef, err := docker.ParseReference(source) + if err != nil { + logrus.WithFields(logrus.Fields{ + "tag": source, + }).Error("Error processing tag, skipping") + logrus.Errorf("Error getting image reference: %s", err) + continue + } + sourceReferences = append(sourceReferences, imageRef) + } + + if len(tagList) == 0 { + logrus.WithFields(logrus.Fields{ + "repo": imageName, + "registry": registryName, + }).Info("Querying registry for image tags") + + imageRef, err := docker.ParseReference(repoName) + if err != nil { + logrus.WithFields(logrus.Fields{ + "repo": imageName, + "registry": registryName, + }).Error("Error processing repo, skipping") + logrus.Error(err) + continue + } + + sourceReferences, err = imagesToCopyFromRepo(imageRef, repoName, serverCtx) + if err != nil { + logrus.WithFields(logrus.Fields{ + "repo": imageName, + "registry": registryName, + }).Error("Error processing repo, skipping") + logrus.Error(err) + continue + } + } + + case string: + tagReg, err := regexp.Compile(tags.(string)) if err != nil { logrus.WithFields(logrus.Fields{ - "tag": source, - }).Error("Error processing tag, skipping") - logrus.Errorf("Error getting image reference: %s", err) - continue + "repo": imageName, + "registry": registryName, + }).Error("Error processing repo, skipping") + logrus.Error(err) } - sourceReferences = append(sourceReferences, imageRef) - } - if len(tags) == 0 { logrus.WithFields(logrus.Fields{ "repo": imageName, "registry": registryName, @@ -308,7 +370,7 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc continue } - sourceReferences, err = imagesToCopyFromRepo(imageRef, repoName, serverCtx) + allSourceReferences, err := imagesToCopyFromRepo(imageRef, repoName, serverCtx) if err != nil { logrus.WithFields(logrus.Fields{ "repo": imageName, @@ -317,6 +379,25 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc logrus.Error(err) continue } + + logrus.WithFields(logrus.Fields{ + "repo": imageName, + "registry": registryName, + }).Infof("Start filtering using the regular expression: %v", tags.(string)) + for _, sReference := range allSourceReferences { + // get the tag names to match, [1] default is "latest" by .DockerReference().String() + if tagReg.Match([]byte(strings.Split(sReference.DockerReference().String(), ":")[1])) { + sourceReferences = append(sourceReferences, sReference) + } + } + + default: + logrus.WithFields(logrus.Fields{ + "repo": imageName, + "registry": registryName, + }).Error("Error processing repo, skipping") + logrus.Errorf("Tags's type only support []string or regular expression string, wrong type:(%v %T)", tags, tags) + continue } if len(sourceReferences) == 0 { diff --git a/docs/skopeo-sync.1.md b/docs/skopeo-sync.1.md index 9903a81c..5e99230b 100644 --- a/docs/skopeo-sync.1.md +++ b/docs/skopeo-sync.1.md @@ -86,6 +86,21 @@ Images are located at: /media/usb/busybox:latest ``` +### Synchronizing to a container registry from local +The Image's locate info: +``` +/media/usb/busybox:1-glibc/ +``` +Sync run +``` +$ skopeo sync --src dir --dest docker /media/usb/busybox\:1-glibc my-registry.local.lan/test/ +``` +Destination registry content: +``` +REPO TAGS +my-registry.local.lan/test/busybox 1-glibc +``` + ### Synchronizing to a local directory, scoped ``` $ skopeo sync --src docker --dest dir --scoped registry.example.com/busybox /media/usb @@ -128,6 +143,7 @@ registry.example.com: redis: - "1.0" - "2.0" + nginx: ^1\.13\.[12]-alpine-perl$ # String types are used for regular expressions, it will match `1.13.1-alpine-perl` and `1.13.2-alpine-perl` credentials: username: john password: this is a secret @@ -139,10 +155,14 @@ quay.io: coreos/etcd: - latest ``` - +If the yaml filename is `sync.yml`, sync run: +``` +skopeo sync --src yaml --dest docker sync.yml my-registry.local.lan/repo/ +``` This will copy the following images: - Repository `registry.example.com/busybox`: all images, as no tags are specified. - Repository `registry.example.com/redis`: images tagged "1.0" and "2.0". +- Repository `registry.example.com/nginx`: images tagged "1.13.1-alpine-perl" and "1.13.2-alpine-perl". - Repository `quay.io/coreos/etcd`: images tagged "latest". For the registry `registry.example.com`, the "john"/"this is a secret" credentials are used, with server TLS certificates located at `/home/john/certs`. diff --git a/integration/sync_test.go b/integration/sync_test.go index 18d27062..e7640526 100644 --- a/integration/sync_test.go +++ b/integration/sync_test.go @@ -255,6 +255,40 @@ docker.io: c.Assert(nManifests, check.Equals, len(tags)) } +func (s *SyncSuite) TestYamlRegex2Dir(c *check.C) { + tmpDir, err := ioutil.TempDir("", "skopeo-sync-test") + c.Assert(err, check.IsNil) + defer os.RemoveAll(tmpDir) + dir1 := path.Join(tmpDir, "dir1") + + yamlConfig := ` +docker.io: + images: + nginx: ^1\.13\.[12]-alpine-perl$ # regex string test +` + // the ↑ regex strings always matches only 2 images + var nTags = 2 + c.Assert(nTags, check.Not(check.Equals), 0) + + yamlFile := path.Join(tmpDir, "registries.yaml") + ioutil.WriteFile(yamlFile, []byte(yamlConfig), 0644) + assertSkopeoSucceeds(c, "", "sync", "--scoped", "--src", "yaml", "--dest", "dir", yamlFile, dir1) + + nManifests := 0 + err = filepath.Walk(dir1, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && info.Name() == "manifest.json" { + nManifests++ + return filepath.SkipDir + } + return nil + }) + c.Assert(err, check.IsNil) + c.Assert(nManifests, check.Equals, nTags) +} + func (s *SyncSuite) TestYaml2Dir(c *check.C) { tmpDir, err := ioutil.TempDir("", "skopeo-sync-test") c.Assert(err, check.IsNil) @@ -270,7 +304,6 @@ docker.io: alpine: - edge - 3.8 - opensuse/leap: - latest