diff --git a/docs/yaml.md b/docs/yaml.md index bf4278a3f..c04ad17e0 100644 --- a/docs/yaml.md +++ b/docs/yaml.md @@ -123,6 +123,9 @@ file: metadata: yaml ``` +Note that if you use templates in the yaml, the final resolved version will be included in the image, +and not the original input template. + Because a `tmpfs` is mounted onto `/var`, `/run`, and `/tmp` by default, the `tmpfs` mounts will shadow anything specified in `files` section for those directories. ## Image specification @@ -293,3 +296,43 @@ binds: - /var:/var:rshared,rbind rootfsPropagation: shared ``` + +## Templates + +The `yaml` file supports templates for the names of images. Anyplace an image is used in a file and begins +with the character `@`, it indicates that it is not an actual name, but a template. The first word after +the `@` indicates the type of template, and the rest of the line is the argument to the template. The +templates currently supported are: + +* `@pkg:` - the argument is the path to a linuxkit package. For example, `@pkg:./pkg/init`. + +For `pkg`, linuxkit will resolve the path to the package, and then run the equivalent of `linuxkit pkg show-tag `. +For example: + +```yaml +init: + - "@pkg:../pkg/init" +``` + +Will cause linuxkit to resolve `../pkg/init` to a package, and then run `linuxkit pkg show-tag ../pkg/init`. + +The paths are relative to the directory of the yaml file. +You can specify absolute paths, although it is not recommended, as that can make the yaml file less portable. + +The `@pkg:` templating is supported **only** when the yaml file is being read from a local filesystem. It does not +support when using via stdin, e.g. `cat linuxkit.yml | linuxkit build -`, or URLs, e.g. `linuxkit build https://example.com/foo.yml`. + +The `@pkg:` template currently supports only default `linuxkit pkg` options, i.e. `build.yml` and `tag` options. There +are no command-line options to override them. + +**Note:** The character `@` is reserved in yaml. To use it in the beginning of a string, you must put the entire string in +quotes. + +If you use the template, the actual derived value, and not the initial template, is what will be stored in the final +image when adding it via: + +```yaml +files: + - path: etc/linuxkit.yml + metadata: yaml +``` diff --git a/linuxkit-template.yml b/linuxkit-template.yml new file mode 100644 index 000000000..e5a12152f --- /dev/null +++ b/linuxkit-template.yml @@ -0,0 +1,38 @@ +kernel: + image: linuxkit/kernel:6.6.13 + cmdline: "console=tty0 console=ttyS0 console=ttyAMA0" +init: + - "@pkg:./pkg/init" + - "@pkg:./pkg/runc" + - "@pkg:./pkg/containerd" + - "@pkg:./pkg/ca-certificates" +onboot: + - name: sysctl + image: "@pkg:./pkg/sysctl" + - name: dhcpcd + image: "@pkg:./pkg/dhcpcd" + command: ["/sbin/dhcpcd", "--nobackground", "-f", "/dhcpcd.conf", "-1"] +onshutdown: + - name: shutdown + image: busybox:latest + command: ["/bin/echo", "so long and thanks for all the fish"] +services: + - name: getty + image: "@pkg:./pkg/getty" + env: + - INSECURE=true + - name: rngd + image: "@pkg:./pkg/rngd" + - name: nginx + image: nginx:1.19.5-alpine + capabilities: + - CAP_NET_BIND_SERVICE + - CAP_CHOWN + - CAP_SETUID + - CAP_SETGID + - CAP_DAC_OVERRIDE + binds: + - /etc/resolv.conf:/etc/resolv.conf +files: + - path: etc/linuxkit-config + metadata: yaml diff --git a/src/cmd/linuxkit/build.go b/src/cmd/linuxkit/build.go index b434f98f5..95d106b3e 100644 --- a/src/cmd/linuxkit/build.go +++ b/src/cmd/linuxkit/build.go @@ -11,8 +11,10 @@ import ( "strings" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/moby" + "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "gopkg.in/yaml.v3" ) const ( @@ -54,6 +56,7 @@ func buildCmd() *cobra.Command { noSbom bool sbomOutputFilename string sbomCurrentTime bool + dryRun bool ) cmd := &cobra.Command{ Use: "build", @@ -141,7 +144,10 @@ The generated image can be in one of multiple formats which can be run on variou log.Fatalf("Unable to parse disk size: %v", err) } - var m moby.Moby + var ( + m moby.Moby + templatesSupported bool + ) for _, arg := range args { var config []byte if conf := arg; conf == "-" { @@ -168,9 +174,14 @@ The generated image can be in one of multiple formats which can be run on variou if err != nil { return fmt.Errorf("Cannot open config file: %v", err) } + // templates are only supported for local files + templatesSupported = true } - - c, err := moby.NewConfig(config) + var pkgFinder spec.PackageResolver + if templatesSupported { + pkgFinder = createPackageResolver(filepath.Dir(arg)) + } + c, err := moby.NewConfig(config, pkgFinder) if err != nil { return fmt.Errorf("Invalid config: %v", err) } @@ -180,6 +191,15 @@ The generated image can be in one of multiple formats which can be run on variou } } + if dryRun { + yml, err := yaml.Marshal(m) + if err != nil { + return fmt.Errorf("Error generating YAML: %v", err) + } + fmt.Println(string(yml)) + return nil + } + var tf *os.File var w io.Writer if outfile != nil { @@ -240,6 +260,7 @@ The generated image can be in one of multiple formats which can be run on variou cmd.Flags().BoolVar(&noSbom, "no-sbom", false, "suppress consolidation of sboms on input container images to a single sbom and saving in the output filesystem") cmd.Flags().BoolVar(&sbomCurrentTime, "sbom-current-time", false, "whether to use the current time as the build time in the sbom; this will make the build non-reproducible (default false)") cmd.Flags().StringVar(&sbomOutputFilename, "sbom-output", defaultSbomFilename, "filename to save the output to in the root filesystem") + cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Do not actually build, just print the final yml file that would be used, including all merges and templates") return cmd } diff --git a/src/cmd/linuxkit/buildtemplate.go b/src/cmd/linuxkit/buildtemplate.go new file mode 100644 index 000000000..d534b4a4d --- /dev/null +++ b/src/cmd/linuxkit/buildtemplate.go @@ -0,0 +1,46 @@ +package main + +import ( + "fmt" + "path" + "strings" + + "github.com/linuxkit/linuxkit/src/cmd/linuxkit/pkglib" + "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec" +) + +const ( + templateFlag = "@" + templatePkg = "pkg:" +) + +func createPackageResolver(baseDir string) spec.PackageResolver { + return func(pkgTmpl string) (tag string, err error) { + var pkgValue string + switch { + case len(pkgTmpl) == 0, pkgTmpl[0:1] != templateFlag: + pkgValue = pkgTmpl + case strings.HasPrefix(pkgTmpl, templateFlag+templatePkg): + pkgPath := strings.TrimPrefix(pkgTmpl, templateFlag+templatePkg) + + var pkgs []pkglib.Pkg + pkgConfig := pkglib.PkglibConfig{ + BuildYML: defaultPkgBuildYML, + HashCommit: defaultPkgCommit, + Tag: defaultPkgTag, + } + pkgs, err = pkglib.NewFromConfig(pkgConfig, path.Join(baseDir, pkgPath)) + if err != nil { + return tag, err + } + if len(pkgs) == 0 { + return tag, fmt.Errorf("no packages found") + } + if len(pkgs) > 1 { + return tag, fmt.Errorf("multiple packages found") + } + pkgValue = pkgs[0].FullTag() + } + return pkgValue, nil + } +} diff --git a/src/cmd/linuxkit/const.go b/src/cmd/linuxkit/const.go new file mode 100644 index 000000000..93161cf02 --- /dev/null +++ b/src/cmd/linuxkit/const.go @@ -0,0 +1,7 @@ +package main + +const ( + defaultPkgBuildYML = "build.yml" + defaultPkgCommit = "HEAD" + defaultPkgTag = "{{.Hash}}" +) diff --git a/src/cmd/linuxkit/moby/config.go b/src/cmd/linuxkit/moby/config.go index ed467eb32..ae5cb0268 100644 --- a/src/cmd/linuxkit/moby/config.go +++ b/src/cmd/linuxkit/moby/config.go @@ -8,13 +8,14 @@ import ( "strings" "github.com/containerd/containerd/reference" + "github.com/linuxkit/linuxkit/src/cmd/linuxkit/spec" "github.com/linuxkit/linuxkit/src/cmd/linuxkit/util" imagespec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-spec/specs-go" log "github.com/sirupsen/logrus" "github.com/syndtr/gocapability/capability" "github.com/xeipuuv/gojsonschema" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // Moby is the type of a Moby config file @@ -239,7 +240,7 @@ func updateImages(m *Moby) { } // NewConfig parses a config file -func NewConfig(config []byte) (Moby, error) { +func NewConfig(config []byte, packageFinder spec.PackageResolver) (Moby, error) { m := Moby{} // Parse raw yaml @@ -267,6 +268,12 @@ func NewConfig(config []byte) (Moby, error) { return m, fmt.Errorf("invalid configuration file") } + // process the template fields + config, err = processTemplates(config, packageFinder) + if err != nil { + return m, err + } + // Parse yaml err = yaml.Unmarshal(config, &m) if err != nil { @@ -1071,3 +1078,33 @@ func deviceCgroup(device specs.LinuxDevice) specs.LinuxDeviceCgroup { Access: "rwm", // read, write, mknod } } + +// processTemplates given a raw config []byte and a package finder, process the templates to find the packages. +// This eventually should expand to other types of templates. Since we only have @pkg: for now, +// this will do to start. +func processTemplates(b []byte, packageFinder spec.PackageResolver) ([]byte, error) { + if packageFinder == nil { + return b, nil + } + var node yaml.Node + if err := yaml.Unmarshal(b, &node); err != nil { + return nil, err + } + handleTemplate(&node, packageFinder) + return yaml.Marshal(&node) +} + +func handleTemplate(node *yaml.Node, packageFinder spec.PackageResolver) { + switch node.Kind { + case yaml.SequenceNode, yaml.MappingNode, yaml.DocumentNode: + for i := 0; i < len(node.Content); i++ { + handleTemplate(node.Content[i], packageFinder) + } + case yaml.ScalarNode: + val := node.Value + if resolved, err := packageFinder(node.Value); err == nil { + val = resolved + } + node.Value = val + } +} diff --git a/src/cmd/linuxkit/moby/linuxkit.go b/src/cmd/linuxkit/moby/linuxkit.go index dc674743c..91fe42bf3 100644 --- a/src/cmd/linuxkit/moby/linuxkit.go +++ b/src/cmd/linuxkit/moby/linuxkit.go @@ -43,7 +43,7 @@ func ensureLinuxkitImage(name, cache string) error { yaml := linuxkitYaml[name] - m, err := NewConfig([]byte(yaml)) + m, err := NewConfig([]byte(yaml), nil) if err != nil { return err } diff --git a/src/cmd/linuxkit/pkg.go b/src/cmd/linuxkit/pkg.go index e16a1bc5f..1a0e6ad61 100644 --- a/src/cmd/linuxkit/pkg.go +++ b/src/cmd/linuxkit/pkg.go @@ -85,10 +85,10 @@ func pkgCmd() *cobra.Command { cmd.PersistentFlags().BoolVar(&argNetwork, "network", piBase.Network, "Allow network use during build") cmd.PersistentFlags().StringVar(&argOrg, "org", piBase.Org, "Override the hub org") - cmd.PersistentFlags().StringVar(&buildYML, "build-yml", "build.yml", "Override the name of the yml file") + cmd.PersistentFlags().StringVar(&buildYML, "build-yml", defaultPkgBuildYML, "Override the name of the yml file") cmd.PersistentFlags().StringVar(&hash, "hash", "", "Override the image hash (default is to query git for the package's tree-sh)") - cmd.PersistentFlags().StringVar(&tag, "tag", "{{.Hash}}", "Override the tag using fixed strings and/or text templates. Acceptable are .Hash for the hash") - cmd.PersistentFlags().StringVar(&hashCommit, "hash-commit", "HEAD", "Override the git commit to use for the hash") + cmd.PersistentFlags().StringVar(&tag, "tag", defaultPkgTag, "Override the tag using fixed strings and/or text templates. Acceptable are .Hash for the hash") + cmd.PersistentFlags().StringVar(&hashCommit, "hash-commit", defaultPkgCommit, "Override the git commit to use for the hash") cmd.PersistentFlags().StringVar(&hashPath, "hash-path", "", "Override the directory to use for the image hash, must be a parent of the package dir (default is to use the package dir)") cmd.PersistentFlags().BoolVar(&dirty, "force-dirty", false, "Force the pkg(s) to be considered dirty") cmd.PersistentFlags().BoolVar(&devMode, "dev", false, "Force org and hash to $USER and \"dev\" respectively") diff --git a/src/cmd/linuxkit/spec/packagefinder.go b/src/cmd/linuxkit/spec/packagefinder.go new file mode 100644 index 000000000..0bad821f7 --- /dev/null +++ b/src/cmd/linuxkit/spec/packagefinder.go @@ -0,0 +1,4 @@ +package spec + +// PackageResolver is an interface for resolving a template into a proper tagged package name +type PackageResolver func(path string) (tag string, err error) diff --git a/test/cases/000_build/010_reproducible/test.yml b/test/cases/000_build/010_reproducible/test.yml index 4f20ca45f..d3bd18f58 100644 --- a/test/cases/000_build/010_reproducible/test.yml +++ b/test/cases/000_build/010_reproducible/test.yml @@ -39,8 +39,7 @@ services: - /etc/aaa:/etc/aaa # And some runtime settings runtime: - mkdir: ["/var/lib/docker"] - mkdir: ["/var/lib/aaa"] + mkdir: ["/var/lib/docker","/var/lib/aaa"] files: - path: etc/linuxkit-config