Use a separate field for the "sync images with tag matching regex" feature

Fields that magically change their behavior depending on type of the value
are too much hassle for no benefit.

For now, this just copies&pastes the full loop in imagesToCopyFromRegistry
to create another loop handling the new ImagesByTagRegex field. Simplifications
to reduce duplication will follow.

Signed-off-by: Miloslav Trmač <mitr@redhat.com>

cmd/skopeo/sync.go

@@ -51,10 +51,11 @@ type tlsVerifyConfig struct {
 // registrySyncConfig contains information about a single registry, read from
 // the source YAML file
 type registrySyncConfig struct {
-	Images      map[string]interface{} // Images map images name to slices or regular expression with the images' tags
+	Images           map[string][]string    // Images map images name to slices with the images' tags
-	Credentials types.DockerAuthConfig // Username and password used to authenticate with the registry
+	ImagesByTagRegex map[string]string      `yaml:"images-by-tag-regex"` // Images map images name to regular expression with the images' tags
-	TLSVerify   tlsVerifyConfig        `yaml:"tls-verify"` // TLS verification mode (enabled by default)
+	Credentials      types.DockerAuthConfig // Username and password used to authenticate with the registry
-	CertDir     string                 `yaml:"cert-dir"`   // Path to the TLS certificates of the registry
+	TLSVerify        tlsVerifyConfig        `yaml:"tls-verify"` // TLS verification mode (enabled by default)
+	CertDir          string                 `yaml:"cert-dir"`   // Path to the TLS certificates of the registry
 }
 
 // sourceConfig contains all registries information read from the source YAML file
@@ -280,81 +281,21 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
 
 		var sourceReferences []types.ImageReference
 
-		switch tags.(type) {
+		for _, tag := range tags {
-		case []string, []interface{}, nil:
+			source := fmt.Sprintf("%s:%s", repoName, tag)
-			tagList := make([]string, 0)
-			if tagIns, ok := tags.([]interface{}); ok {
-				for _, tagValue := range tagIns {
-					switch tagValue.(type) {
-					case string, int, float64:
-						tagList = append(tagList, fmt.Sprintf("%v", tagValue))
-					default:
-						logrus.WithFields(logrus.Fields{
-							"repo":     imageName,
-							"registry": registryName,
-						}).Error("Error processing repo, skipping")
-						logrus.Errorf("Elements can only be strings if they are of type array, wrong value (%v|%T)", tagValue, tagValue)
-						continue
-					}
-				}
-			} else {
-				// nil is equl full tags
-				if tags != nil {
-					tagList = tags.([]string)
-				}
-			}
 
-			for _, tag := range tagList {
+			imageRef, err := docker.ParseReference(source)
-				source := fmt.Sprintf("%s:%s", repoName, tag)
-
-				imageRef, err := docker.ParseReference(source)
-				if err != nil {
-					logrus.WithFields(logrus.Fields{
-						"tag": source,
-					}).Error("Error processing tag, skipping")
-					logrus.Errorf("Error getting image reference: %s", err)
-					continue
-				}
-				sourceReferences = append(sourceReferences, imageRef)
-			}
-
-			if len(tagList) == 0 {
-				logrus.WithFields(logrus.Fields{
-					"repo":     imageName,
-					"registry": registryName,
-				}).Info("Querying registry for image tags")
-
-				imageRef, err := docker.ParseReference(repoName)
-				if err != nil {
-					logrus.WithFields(logrus.Fields{
-						"repo":     imageName,
-						"registry": registryName,
-					}).Error("Error processing repo, skipping")
-					logrus.Error(err)
-					continue
-				}
-
-				sourceReferences, err = imagesToCopyFromRepo(imageRef, repoName, serverCtx)
-				if err != nil {
-					logrus.WithFields(logrus.Fields{
-						"repo":     imageName,
-						"registry": registryName,
-					}).Error("Error processing repo, skipping")
-					logrus.Error(err)
-					continue
-				}
-			}
-
-		case string:
-			tagReg, err := regexp.Compile(tags.(string))
 			if err != nil {
 				logrus.WithFields(logrus.Fields{
-					"repo":     imageName,
+					"tag": source,
-					"registry": registryName,
+				}).Error("Error processing tag, skipping")
-				}).Error("Error processing repo, skipping")
+				logrus.Errorf("Error getting image reference: %s", err)
-				logrus.Error(err)
+				continue
 			}
+			sourceReferences = append(sourceReferences, imageRef)
+		}
 
+		if len(tags) == 0 {
 			logrus.WithFields(logrus.Fields{
 				"repo":     imageName,
 				"registry": registryName,
@@ -370,7 +311,7 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
 				continue
 			}
 
-			allSourceReferences, err := imagesToCopyFromRepo(imageRef, repoName, serverCtx)
+			sourceReferences, err = imagesToCopyFromRepo(imageRef, repoName, serverCtx)
 			if err != nil {
 				logrus.WithFields(logrus.Fields{
 					"repo":     imageName,
@@ -379,27 +320,82 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
 				logrus.Error(err)
 				continue
 			}
+		}
 
+		if len(sourceReferences) == 0 {
 			logrus.WithFields(logrus.Fields{
 				"repo":     imageName,
 				"registry": registryName,
-			}).Infof("Start filtering using the regular expression: %v", tags.(string))
+			}).Warnf("No tags to sync found")
-			for _, sReference := range allSourceReferences {
+			continue
-				// get the tag names to match, [1] default is "latest" by .DockerReference().String()
+		}
-				if tagReg.Match([]byte(strings.Split(sReference.DockerReference().String(), ":")[1])) {
+		repoDescList = append(repoDescList, repoDescriptor{
-					sourceReferences = append(sourceReferences, sReference)
+			TaggedImages: sourceReferences,
-				}
+			Context:      serverCtx})
-			}
+	}
 
-		default:
+	for imageName, tagRegex := range cfg.ImagesByTagRegex {
+		repoName := fmt.Sprintf("//%s", path.Join(registryName, imageName))
+		logrus.WithFields(logrus.Fields{
+			"repo":     imageName,
+			"registry": registryName,
+		}).Info("Processing repo")
+
+		serverCtx := &sourceCtx
+		// override ctx with per-registryName options
+		serverCtx.DockerCertPath = cfg.CertDir
+		serverCtx.DockerDaemonCertPath = cfg.CertDir
+		serverCtx.DockerDaemonInsecureSkipTLSVerify = (cfg.TLSVerify.skip == types.OptionalBoolTrue)
+		serverCtx.DockerInsecureSkipTLSVerify = cfg.TLSVerify.skip
+		serverCtx.DockerAuthConfig = &cfg.Credentials
+
+		var sourceReferences []types.ImageReference
+
+		tagReg, err := regexp.Compile(tagRegex)
+		if err != nil {
 			logrus.WithFields(logrus.Fields{
 				"repo":     imageName,
 				"registry": registryName,
 			}).Error("Error processing repo, skipping")
-			logrus.Errorf("Tags's type only support []string or regular expression string, wrong type:(%v %T)", tags, tags)
+			logrus.Error(err)
+		}
+
+		logrus.WithFields(logrus.Fields{
+			"repo":     imageName,
+			"registry": registryName,
+		}).Info("Querying registry for image tags")
+
+		imageRef, err := docker.ParseReference(repoName)
+		if err != nil {
+			logrus.WithFields(logrus.Fields{
+				"repo":     imageName,
+				"registry": registryName,
+			}).Error("Error processing repo, skipping")
+			logrus.Error(err)
 			continue
 		}
 
+		allSourceReferences, err := imagesToCopyFromRepo(imageRef, repoName, serverCtx)
+		if err != nil {
+			logrus.WithFields(logrus.Fields{
+				"repo":     imageName,
+				"registry": registryName,
+			}).Error("Error processing repo, skipping")
+			logrus.Error(err)
+			continue
+		}
+
+		logrus.WithFields(logrus.Fields{
+			"repo":     imageName,
+			"registry": registryName,
+		}).Infof("Start filtering using the regular expression: %v", tagRegex)
+		for _, sReference := range allSourceReferences {
+			// get the tag names to match, [1] default is "latest" by .DockerReference().String()
+			if tagReg.Match([]byte(strings.Split(sReference.DockerReference().String(), ":")[1])) {
+				sourceReferences = append(sourceReferences, sReference)
+			}
+		}
+
 		if len(sourceReferences) == 0 {
 			logrus.WithFields(logrus.Fields{
 				"repo":     imageName,
@@ -482,7 +478,7 @@ func imagesToCopy(source string, transport string, sourceCtx *types.SystemContex
 			return descriptors, err
 		}
 		for registryName, registryConfig := range cfg {
-			if len(registryConfig.Images) == 0 {
+			if len(registryConfig.Images) == 0 && len(registryConfig.ImagesByTagRegex) == 0 {
 				logrus.WithFields(logrus.Fields{
 					"registry": registryName,
 				}).Warn("No images specified for registry")

docs/skopeo-sync.1.md

@@ -143,7 +143,8 @@ registry.example.com:
         redis:
             - "1.0"
             - "2.0"
-        nginx: ^1\.13\.[12]-alpine-perl$ # String types are used for regular expressions, it will match `1.13.1-alpine-perl` and `1.13.2-alpine-perl`
+    images-by-tag-regex:
+        nginx: ^1\.13\.[12]-alpine-perl$
     credentials:
         username: john
         password: this is a secret

integration/sync_test.go

@@ -263,7 +263,7 @@ func (s *SyncSuite) TestYamlRegex2Dir(c *check.C) {
 
 	yamlConfig := `
 docker.io:
-  images:
+  images-by-tag-regex:
     nginx: ^1\.13\.[12]-alpine-perl$  # regex string test
 `
 	// the       ↑    regex strings always matches only 2 images