Merge pull request #920 from zhangguanzhang/master

Add tags to support regular expressions without breaking the old ones in yaml conf

cmd/skopeo/sync.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/containers/image/v5/copy"
@@ -50,7 +51,7 @@ type tlsVerifyConfig struct {
 // registrySyncConfig contains information about a single registry, read from
 // the source YAML file
 type registrySyncConfig struct {
-	Images      map[string][]string    // Images map images name to slices with the images' tags
+	Images      map[string]interface{} // Images map images name to slices or regular expression with the images' tags
 	Credentials types.DockerAuthConfig // Username and password used to authenticate with the registry
 	TLSVerify   tlsVerifyConfig        `yaml:"tls-verify"` // TLS verification mode (enabled by default)
 	CertDir     string                 `yaml:"cert-dir"`   // Path to the TLS certificates of the registry
@@ -278,21 +279,82 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
 		serverCtx.DockerAuthConfig = &cfg.Credentials
 
 		var sourceReferences []types.ImageReference
-		for _, tag := range tags {
-			source := fmt.Sprintf("%s:%s", repoName, tag)
 
-			imageRef, err := docker.ParseReference(source)
+		switch tags.(type) {
+		case []string, []interface{}, nil:
+			tagList := make([]string, 0)
+			if tagIns, ok := tags.([]interface{}); ok {
+				for _, tagValue := range tagIns {
+					switch tagValue.(type) {
+					case string, int, float64:
+						tagList = append(tagList, fmt.Sprintf("%v", tagValue))
+					default:
+						logrus.WithFields(logrus.Fields{
+							"repo":     imageName,
+							"registry": registryName,
+						}).Error("Error processing repo, skipping")
+						logrus.Errorf("Elements can only be strings if they are of type array, wrong value (%v|%T)", tagValue, tagValue)
+						continue
+					}
+				}
+			} else {
+				// nil is equl full tags
+				if tags != nil {
+					tagList = tags.([]string)
+				}
+			}
+
+			for _, tag := range tagList {
+				source := fmt.Sprintf("%s:%s", repoName, tag)
+
+				imageRef, err := docker.ParseReference(source)
+				if err != nil {
+					logrus.WithFields(logrus.Fields{
+						"tag": source,
+					}).Error("Error processing tag, skipping")
+					logrus.Errorf("Error getting image reference: %s", err)
+					continue
+				}
+				sourceReferences = append(sourceReferences, imageRef)
+			}
+
+			if len(tagList) == 0 {
+				logrus.WithFields(logrus.Fields{
+					"repo":     imageName,
+					"registry": registryName,
+				}).Info("Querying registry for image tags")
+
+				imageRef, err := docker.ParseReference(repoName)
+				if err != nil {
+					logrus.WithFields(logrus.Fields{
+						"repo":     imageName,
+						"registry": registryName,
+					}).Error("Error processing repo, skipping")
+					logrus.Error(err)
+					continue
+				}
+
+				sourceReferences, err = imagesToCopyFromRepo(imageRef, repoName, serverCtx)
+				if err != nil {
+					logrus.WithFields(logrus.Fields{
+						"repo":     imageName,
+						"registry": registryName,
+					}).Error("Error processing repo, skipping")
+					logrus.Error(err)
+					continue
+				}
+			}
+
+		case string:
+			tagReg, err := regexp.Compile(tags.(string))
 			if err != nil {
 				logrus.WithFields(logrus.Fields{
-					"tag": source,
-				}).Error("Error processing tag, skipping")
-				logrus.Errorf("Error getting image reference: %s", err)
-				continue
+					"repo":     imageName,
+					"registry": registryName,
+				}).Error("Error processing repo, skipping")
+				logrus.Error(err)
 			}
-			sourceReferences = append(sourceReferences, imageRef)
-		}
 
-		if len(tags) == 0 {
 			logrus.WithFields(logrus.Fields{
 				"repo":     imageName,
 				"registry": registryName,
@@ -308,7 +370,7 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
 				continue
 			}
 
-			sourceReferences, err = imagesToCopyFromRepo(imageRef, repoName, serverCtx)
+			allSourceReferences, err := imagesToCopyFromRepo(imageRef, repoName, serverCtx)
 			if err != nil {
 				logrus.WithFields(logrus.Fields{
 					"repo":     imageName,
@@ -317,6 +379,25 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
 				logrus.Error(err)
 				continue
 			}
+
+			logrus.WithFields(logrus.Fields{
+				"repo":     imageName,
+				"registry": registryName,
+			}).Infof("Start filtering using the regular expression: %v", tags.(string))
+			for _, sReference := range allSourceReferences {
+				// get the tag names to match, [1] default is "latest" by .DockerReference().String()
+				if tagReg.Match([]byte(strings.Split(sReference.DockerReference().String(), ":")[1])) {
+					sourceReferences = append(sourceReferences, sReference)
+				}
+			}
+
+		default:
+			logrus.WithFields(logrus.Fields{
+				"repo":     imageName,
+				"registry": registryName,
+			}).Error("Error processing repo, skipping")
+			logrus.Errorf("Tags's type only support []string or regular expression string, wrong type:(%v %T)", tags, tags)
+			continue
 		}
 
 		if len(sourceReferences) == 0 {

docs/skopeo-sync.1.md

@@ -86,6 +86,21 @@ Images are located at:
 /media/usb/busybox:latest
 ```
 
+### Synchronizing to a container registry from local
+The Image's locate info:
+```
+/media/usb/busybox:1-glibc/
+```
+Sync run
+```
+$ skopeo sync --src dir --dest docker /media/usb/busybox\:1-glibc my-registry.local.lan/test/
+```
+Destination registry content:
+```
+REPO                                 TAGS
+my-registry.local.lan/test/busybox   1-glibc
+```
+
 ### Synchronizing to a local directory, scoped
 ```
 $ skopeo sync --src docker --dest dir --scoped registry.example.com/busybox /media/usb
@@ -128,6 +143,7 @@ registry.example.com:
         redis:
             - "1.0"
             - "2.0"
+        nginx: ^1\.13\.[12]-alpine-perl$ # String types are used for regular expressions, it will match `1.13.1-alpine-perl` and `1.13.2-alpine-perl`
     credentials:
         username: john
         password: this is a secret
@@ -139,10 +155,14 @@ quay.io:
         coreos/etcd:
             - latest
 ```
-
+If the yaml filename is `sync.yml`, sync run:
+```
+skopeo sync --src yaml --dest docker sync.yml my-registry.local.lan/repo/
+```
 This will copy the following images:
 - Repository `registry.example.com/busybox`: all images, as no tags are specified.
 - Repository `registry.example.com/redis`: images tagged "1.0" and "2.0".
+- Repository `registry.example.com/nginx`: images tagged "1.13.1-alpine-perl" and "1.13.2-alpine-perl".
 - Repository `quay.io/coreos/etcd`: images tagged "latest".
 
 For the registry `registry.example.com`, the "john"/"this is a secret" credentials are used, with server TLS certificates located at `/home/john/certs`.

integration/sync_test.go

@@ -255,6 +255,40 @@ docker.io:
 	c.Assert(nManifests, check.Equals, len(tags))
 }
 
+func (s *SyncSuite) TestYamlRegex2Dir(c *check.C) {
+	tmpDir, err := ioutil.TempDir("", "skopeo-sync-test")
+	c.Assert(err, check.IsNil)
+	defer os.RemoveAll(tmpDir)
+	dir1 := path.Join(tmpDir, "dir1")
+
+	yamlConfig := `
+docker.io:
+  images:
+    nginx: ^1\.13\.[12]-alpine-perl$  # regex string test
+`
+	// the       ↑    regex strings always matches only 2 images
+	var nTags = 2
+	c.Assert(nTags, check.Not(check.Equals), 0)
+
+	yamlFile := path.Join(tmpDir, "registries.yaml")
+	ioutil.WriteFile(yamlFile, []byte(yamlConfig), 0644)
+	assertSkopeoSucceeds(c, "", "sync", "--scoped", "--src", "yaml", "--dest", "dir", yamlFile, dir1)
+
+	nManifests := 0
+	err = filepath.Walk(dir1, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && info.Name() == "manifest.json" {
+			nManifests++
+			return filepath.SkipDir
+		}
+		return nil
+	})
+	c.Assert(err, check.IsNil)
+	c.Assert(nManifests, check.Equals, nTags)
+}
+
 func (s *SyncSuite) TestYaml2Dir(c *check.C) {
 	tmpDir, err := ioutil.TempDir("", "skopeo-sync-test")
 	c.Assert(err, check.IsNil)
@@ -270,7 +304,6 @@ docker.io:
     alpine:
       - edge
       - 3.8
-
     opensuse/leap:
       - latest