Merge pull request #3988 from deitch/yaml-template

template in yaml file

docs/yaml.md

@@ -123,6 +123,9 @@ file:
     metadata: yaml
 ```
 
+Note that if you use templates in the yaml, the final resolved version will be included in the image,
+and not the original input template.
+
 Because a `tmpfs` is mounted onto `/var`, `/run`, and `/tmp` by default, the `tmpfs` mounts will shadow anything specified in `files` section for those directories.
 
 ## Image specification
@@ -293,3 +296,43 @@ binds:
  - /var:/var:rshared,rbind
 rootfsPropagation: shared
 ```
+
+## Templates
+
+The `yaml` file supports templates for the names of images. Anyplace an image is used in a file and begins
+with the character `@`, it indicates that it is not an actual name, but a template. The first word after
+the `@` indicates the type of template, and the rest of the line is the argument to the template. The
+templates currently supported are:
+
+* `@pkg:` - the argument is the path to a linuxkit package. For example, `@pkg:./pkg/init`.
+
+For `pkg`, linuxkit will resolve the path to the package, and then run the equivalent of `linuxkit pkg show-tag <dir>`.
+For example:
+
+```yaml
+init:
+  - "@pkg:../pkg/init"
+```
+
+Will cause linuxkit to resolve `../pkg/init` to a package, and then run `linuxkit pkg show-tag ../pkg/init`.
+
+The paths are relative to the directory of the yaml file.
+You can specify absolute paths, although it is not recommended, as that can make the yaml file less portable.
+
+The `@pkg:` templating is supported **only** when the yaml file is being read from a local filesystem. It does not
+support when using via stdin, e.g. `cat linuxkit.yml | linuxkit build -`, or URLs, e.g. `linuxkit build https://example.com/foo.yml`.
+
+The `@pkg:` template currently supports only default `linuxkit pkg` options, i.e. `build.yml` and `tag` options. There
+are no command-line options to override them.
+
+**Note:** The character `@` is reserved in yaml. To use it in the beginning of a string, you must put the entire string in
+quotes.
+
+If you use the template, the actual derived value, and not the initial template, is what will be stored in the final
+image when adding it via:
+
+```yaml
+files:
+  - path: etc/linuxkit.yml
+    metadata: yaml
+```

linuxkit-template.yml

@@ -0,0 +1,38 @@
+kernel:
+  image: linuxkit/kernel:6.6.13
+  cmdline: "console=tty0 console=ttyS0 console=ttyAMA0"
+init:
+  - "@pkg:./pkg/init"
+  - "@pkg:./pkg/runc"
+  - "@pkg:./pkg/containerd"
+  - "@pkg:./pkg/ca-certificates"
+onboot:
+  - name: sysctl
+    image: "@pkg:./pkg/sysctl"
+  - name: dhcpcd
+    image: "@pkg:./pkg/dhcpcd"
+    command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"]
+onshutdown:
+  - name: shutdown
+    image: busybox:latest
+    command: ["/bin/echo", "so long and thanks for all the fish"]
+services:
+  - name: getty
+    image: "@pkg:./pkg/getty"
+    env:
+     - INSECURE=true
+  - name: rngd
+    image: "@pkg:./pkg/rngd"
+  - name: nginx
+    image: nginx:1.19.5-alpine
+    capabilities:
+     - CAP_NET_BIND_SERVICE
+     - CAP_CHOWN
+     - CAP_SETUID
+     - CAP_SETGID
+     - CAP_DAC_OVERRIDE
+    binds:
+     - /etc/resolv.conf:/etc/resolv.conf
+files:
+  - path: etc/linuxkit-config
+    metadata: yaml

src/cmd/linuxkit/build.go

@@ -11,8 +11,10 @@ import (
 	"strings"
 
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/moby"
+	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -54,6 +56,7 @@ func buildCmd() *cobra.Command {
 		noSbom             bool
 		sbomOutputFilename string
 		sbomCurrentTime    bool
+		dryRun             bool
 	)
 	cmd := &cobra.Command{
 		Use:   "build",
@@ -141,7 +144,10 @@ The generated image can be in one of multiple formats which can be run on variou
 				log.Fatalf("Unable to parse disk size: %v", err)
 			}
 
-			var m moby.Moby
+			var (
+				m                  moby.Moby
+				templatesSupported bool
+			)
 			for _, arg := range args {
 				var config []byte
 				if conf := arg; conf == "-" {
@@ -168,9 +174,14 @@ The generated image can be in one of multiple formats which can be run on variou
 					if err != nil {
 						return fmt.Errorf("Cannot open config file: %v", err)
 					}
+					// templates are only supported for local files
+					templatesSupported = true
 				}
-
+				var pkgFinder spec.PackageResolver
-				c, err := moby.NewConfig(config)
+				if templatesSupported {
+					pkgFinder = createPackageResolver(filepath.Dir(arg))
+				}
+				c, err := moby.NewConfig(config, pkgFinder)
 				if err != nil {
 					return fmt.Errorf("Invalid config: %v", err)
 				}
@@ -180,6 +191,15 @@ The generated image can be in one of multiple formats which can be run on variou
 				}
 			}
 
+			if dryRun {
+				yml, err := yaml.Marshal(m)
+				if err != nil {
+					return fmt.Errorf("Error generating YAML: %v", err)
+				}
+				fmt.Println(string(yml))
+				return nil
+			}
+
 			var tf *os.File
 			var w io.Writer
 			if outfile != nil {
@@ -240,6 +260,7 @@ The generated image can be in one of multiple formats which can be run on variou
 	cmd.Flags().BoolVar(&noSbom, "no-sbom", false, "suppress consolidation of sboms on input container images to a single sbom and saving in the output filesystem")
 	cmd.Flags().BoolVar(&sbomCurrentTime, "sbom-current-time", false, "whether to use the current time as the build time in the sbom; this will make the build non-reproducible (default false)")
 	cmd.Flags().StringVar(&sbomOutputFilename, "sbom-output", defaultSbomFilename, "filename to save the output to in the root filesystem")
+	cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Do not actually build, just print the final yml file that would be used, including all merges and templates")
 
 	return cmd
 }

src/cmd/linuxkit/buildtemplate.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"fmt"
+	"path"
+	"strings"
+
+	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/pkglib"
+	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
+)
+
+const (
+	templateFlag = "@"
+	templatePkg  = "pkg:"
+)
+
+func createPackageResolver(baseDir string) spec.PackageResolver {
+	return func(pkgTmpl string) (tag string, err error) {
+		var pkgValue string
+		switch {
+		case len(pkgTmpl) == 0, pkgTmpl[0:1] != templateFlag:
+			pkgValue = pkgTmpl
+		case strings.HasPrefix(pkgTmpl, templateFlag+templatePkg):
+			pkgPath := strings.TrimPrefix(pkgTmpl, templateFlag+templatePkg)
+
+			var pkgs []pkglib.Pkg
+			pkgConfig := pkglib.PkglibConfig{
+				BuildYML:   defaultPkgBuildYML,
+				HashCommit: defaultPkgCommit,
+				Tag:        defaultPkgTag,
+			}
+			pkgs, err = pkglib.NewFromConfig(pkgConfig, path.Join(baseDir, pkgPath))
+			if err != nil {
+				return tag, err
+			}
+			if len(pkgs) == 0 {
+				return tag, fmt.Errorf("no packages found")
+			}
+			if len(pkgs) > 1 {
+				return tag, fmt.Errorf("multiple packages found")
+			}
+			pkgValue = pkgs[0].FullTag()
+		}
+		return pkgValue, nil
+	}
+}

src/cmd/linuxkit/const.go

@@ -0,0 +1,7 @@
+package main
+
+const (
+	defaultPkgBuildYML = "build.yml"
+	defaultPkgCommit   = "HEAD"
+	defaultPkgTag      = "{{.Hash}}"
+)

src/cmd/linuxkit/moby/config.go

@@ -8,13 +8,14 @@ import (
 	"strings"
 
 	"github.com/containerd/containerd/reference"
+	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec"
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/util"
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	log "github.com/sirupsen/logrus"
 	"github.com/syndtr/gocapability/capability"
 	"github.com/xeipuuv/gojsonschema"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 // Moby is the type of a Moby config file
@@ -239,7 +240,7 @@ func updateImages(m *Moby) {
 }
 
 // NewConfig parses a config file
-func NewConfig(config []byte) (Moby, error) {
+func NewConfig(config []byte, packageFinder spec.PackageResolver) (Moby, error) {
 	m := Moby{}
 
 	// Parse raw yaml
@@ -267,6 +268,12 @@ func NewConfig(config []byte) (Moby, error) {
 		return m, fmt.Errorf("invalid configuration file")
 	}
 
+	// process the template fields
+	config, err = processTemplates(config, packageFinder)
+	if err != nil {
+		return m, err
+	}
+
 	// Parse yaml
 	err = yaml.Unmarshal(config, &m)
 	if err != nil {
@@ -1071,3 +1078,33 @@ func deviceCgroup(device specs.LinuxDevice) specs.LinuxDeviceCgroup {
 		Access: "rwm", // read, write, mknod
 	}
 }
+
+// processTemplates given a raw config []byte and a package finder, process the templates to find the packages.
+// This eventually should expand to other types of templates. Since we only have @pkg: for now,
+// this will do to start.
+func processTemplates(b []byte, packageFinder spec.PackageResolver) ([]byte, error) {
+	if packageFinder == nil {
+		return b, nil
+	}
+	var node yaml.Node
+	if err := yaml.Unmarshal(b, &node); err != nil {
+		return nil, err
+	}
+	handleTemplate(&node, packageFinder)
+	return yaml.Marshal(&node)
+}
+
+func handleTemplate(node *yaml.Node, packageFinder spec.PackageResolver) {
+	switch node.Kind {
+	case yaml.SequenceNode, yaml.MappingNode, yaml.DocumentNode:
+		for i := 0; i < len(node.Content); i++ {
+			handleTemplate(node.Content[i], packageFinder)
+		}
+	case yaml.ScalarNode:
+		val := node.Value
+		if resolved, err := packageFinder(node.Value); err == nil {
+			val = resolved
+		}
+		node.Value = val
+	}
+}

src/cmd/linuxkit/moby/linuxkit.go

@@ -43,7 +43,7 @@ func ensureLinuxkitImage(name, cache string) error {
 
 	yaml := linuxkitYaml[name]
 
-	m, err := NewConfig([]byte(yaml))
+	m, err := NewConfig([]byte(yaml), nil)
 	if err != nil {
 		return err
 	}

src/cmd/linuxkit/pkg.go

@@ -85,10 +85,10 @@ func pkgCmd() *cobra.Command {
 	cmd.PersistentFlags().BoolVar(&argNetwork, "network", piBase.Network, "Allow network use during build")
 
 	cmd.PersistentFlags().StringVar(&argOrg, "org", piBase.Org, "Override the hub org")
-	cmd.PersistentFlags().StringVar(&buildYML, "build-yml", "build.yml", "Override the name of the yml file")
+	cmd.PersistentFlags().StringVar(&buildYML, "build-yml", defaultPkgBuildYML, "Override the name of the yml file")
 	cmd.PersistentFlags().StringVar(&hash, "hash", "", "Override the image hash (default is to query git for the package's tree-sh)")
-	cmd.PersistentFlags().StringVar(&tag, "tag", "{{.Hash}}", "Override the tag using fixed strings and/or text templates. Acceptable are .Hash for the hash")
+	cmd.PersistentFlags().StringVar(&tag, "tag", defaultPkgTag, "Override the tag using fixed strings and/or text templates. Acceptable are .Hash for the hash")
-	cmd.PersistentFlags().StringVar(&hashCommit, "hash-commit", "HEAD", "Override the git commit to use for the hash")
+	cmd.PersistentFlags().StringVar(&hashCommit, "hash-commit", defaultPkgCommit, "Override the git commit to use for the hash")
 	cmd.PersistentFlags().StringVar(&hashPath, "hash-path", "", "Override the directory to use for the image hash, must be a parent of the package dir (default is to use the package dir)")
 	cmd.PersistentFlags().BoolVar(&dirty, "force-dirty", false, "Force the pkg(s) to be considered dirty")
 	cmd.PersistentFlags().BoolVar(&devMode, "dev", false, "Force org and hash to $USER and \"dev\" respectively")

src/cmd/linuxkit/spec/packagefinder.go

@@ -0,0 +1,4 @@
+package spec
+
+// PackageResolver is an interface for resolving a template into a proper tagged package name
+type PackageResolver func(path string) (tag string, err error)

test/cases/000_build/010_reproducible/test.yml

@@ -39,8 +39,7 @@ services:
       - /etc/aaa:/etc/aaa
     # And some runtime settings
     runtime:
-      mkdir: ["/var/lib/docker"]
+      mkdir: ["/var/lib/docker","/var/lib/aaa"]
-      mkdir: ["/var/lib/aaa"]
 
 files:
   - path: etc/linuxkit-config