Add ability to interpolate during build

Now build takes a --values argument, which is a yaml file that can be
used to interpolate the specs that are going to be compiled.

cmd/build.go

@@ -49,6 +49,7 @@ var buildCmd = &cobra.Command{
 		viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
 		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
 		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("values", cmd.Flags().Lookup("values"))
 
 		viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
 		viper.BindPFlag("push", cmd.Flags().Lookup("push"))
@@ -75,6 +76,8 @@ var buildCmd = &cobra.Command{
 		databaseType := viper.GetString("database")
 		compressionType := viper.GetString("compression")
 		imageRepository := viper.GetString("image-repository")
+		values := viper.GetString("values")
+
 		push := viper.GetBool("push")
 		pull := viper.GetBool("pull")
 		keepImages := viper.GetBool("keep-images")
@@ -157,7 +160,7 @@ var buildCmd = &cobra.Command{
 		opts.KeepImageExport = keepExportedImages
 		opts.SkipIfMetadataExists = skip
 		opts.PackageTargetOnly = onlyTarget
-
+		opts.BuildValuesFile = values
 		var solverOpts solver.Options
 		if concurrent {
 			solverOpts = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
@@ -291,6 +294,7 @@ func init() {
 	buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
 	buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
 	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
+	buildCmd.Flags().String("values", "", "Build values file to interpolate with each package")
 
 	buildCmd.Flags().String("destination", path, "Destination folder")
 	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")

pkg/compiler/compiler.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	bus "github.com/mudler/luet/pkg/bus"
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
@@ -743,13 +744,25 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 
 		raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
 
-		dat, err := helpers.RenderHelm(string(dataBuild), raw)
+		d := map[string]interface{}{}
+		if len(cs.Options.BuildValuesFile) > 0 {
+			defBuild, err := ioutil.ReadFile(cs.Options.BuildValuesFile)
+			if err != nil {
+				return nil, errors.Wrap(err, "rendering file "+val)
+			}
+			err = yaml.Unmarshal(defBuild, &d)
+			if err != nil {
+				return nil, errors.Wrap(err, "rendering file "+val)
+			}
+		}
+
+		dat, err := helpers.RenderHelm(string(dataBuild), raw, d)
 		if err != nil {
 			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
 		}
 		dataresult = []byte(dat)
 	} else {
-		out, err := helpers.RenderFiles(pack.Rel(BuildFile), val)
+		out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, cs.Options.BuildValuesFile)
 		if err != nil {
 			return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
 		}

pkg/compiler/interface.go

@@ -56,6 +56,7 @@ type CompilerOptions struct {
 	NoDeps               bool
 	SolverOptions        config.LuetSolverOptions
 	SkipIfMetadataExists bool
+	BuildValuesFile      string
 
 	PackageTargetOnly bool
 }

pkg/helpers/helm.go

@@ -11,7 +11,7 @@ import (
 )
 
 // RenderHelm renders the template string with helm
-func RenderHelm(template string, values map[string]interface{}) (string, error) {
+func RenderHelm(template string, values, d map[string]interface{}) (string, error) {
 	c := &chart.Chart{
 		Metadata: &chart.Metadata{
 			Name:    "",
@@ -23,7 +23,7 @@ func RenderHelm(template string, values map[string]interface{}) (string, error)
 		Values: map[string]interface{}{"Values": values},
 	}
 
-	v, err := chartutil.CoalesceValues(c, map[string]interface{}{})
+	v, err := chartutil.CoalesceValues(c, map[string]interface{}{"Values": d})
 	if err != nil {
 		return "", errors.Wrap(err, "while rendering template")
 	}
@@ -37,7 +37,7 @@ func RenderHelm(template string, values map[string]interface{}) (string, error)
 
 type templatedata map[string]interface{}
 
-func RenderFiles(toTemplate, valuesFile string) (string, error) {
+func RenderFiles(toTemplate, valuesFile string, defaultFile string) (string, error) {
 	raw, err := ioutil.ReadFile(toTemplate)
 	if err != nil {
 		return "", errors.Wrap(err, "reading file "+toTemplate)
@@ -46,14 +46,26 @@ func RenderFiles(toTemplate, valuesFile string) (string, error) {
 	if !Exists(valuesFile) {
 		return "", errors.Wrap(err, "file not existing "+valuesFile)
 	}
-	def, err := ioutil.ReadFile(valuesFile)
+	val, err := ioutil.ReadFile(valuesFile)
 	if err != nil {
 		return "", errors.Wrap(err, "reading file "+valuesFile)
 	}
 
 	var values templatedata
-	if err = yaml.Unmarshal(def, &values); err != nil {
+	d := templatedata{}
+	if len(defaultFile) > 0 {
+		def, err := ioutil.ReadFile(defaultFile)
+		if err != nil {
+			return "", errors.Wrap(err, "reading file "+valuesFile)
+		}
+		if err = yaml.Unmarshal(def, &d); err != nil {
+			return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
+		}
+	}
+
+	if err = yaml.Unmarshal(val, &values); err != nil {
 		return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
 	}
-	return RenderHelm(string(raw), values)
+
+	return RenderHelm(string(raw), values, d)
 }

pkg/helpers/helm_test.go

@@ -16,17 +16,132 @@
 package helpers_test
 
 import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
 	. "github.com/mudler/luet/pkg/helpers"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
+func writeFile(path string, content string) {
+	err := ioutil.WriteFile(path, []byte(content), 0644)
+	Expect(err).ToNot(HaveOccurred())
+}
+
 var _ = Describe("Helpers", func() {
 	Context("RenderHelm", func() {
 		It("Renders templates", func() {
-			out, err := RenderHelm("{{.Values.Test}}", map[string]interface{}{"Test": "foo"})
+			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"})
 			Expect(err).ToNot(HaveOccurred())
-			Expect(out).To(Equal("foo"))
+			Expect(out).To(Equal("foobar"))
+		})
+		It("Renders templates with overrides", func() {
+			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foobar"))
+		})
+
+		It("Renders templates", func() {
+			out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foobar"))
+		})
+
+		It("Render files default overrides", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foo: "bar"
+`)
+			writeFile(d, `
+foo: "baz"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("baz"))
+
+		})
+
+		It("Render files from values", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foo: "bar"
+`)
+			writeFile(d, `
+faa: "baz"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("bar"))
+
+		})
+
+		It("Render files from values if no default", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foo: "bar"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, "")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal("bar"))
+		})
+
+		It("doesn't interpolate if no one provides the values", func() {
+			testDir, err := ioutil.TempDir(os.TempDir(), "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(testDir)
+
+			toTemplate := filepath.Join(testDir, "totemplate.yaml")
+			values := filepath.Join(testDir, "values.yaml")
+			d := filepath.Join(testDir, "default.yaml")
+
+			writeFile(toTemplate, `{{.Values.foo}}`)
+			writeFile(values, `
+foao: "bar"
+`)
+			writeFile(d, `
+faa: "baz"
+`)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			res, err := RenderFiles(toTemplate, values, d)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(Equal(""))
+
 		})
 	})
 })

pkg/installer/system.go

@@ -24,7 +24,7 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package, force bool) error {
 	executedFinalizer := map[string]bool{}
 	for _, p := range packs {
 		if helpers.Exists(p.Rel(tree.FinalizerFile)) {
-			out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile))
+			out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile), "")
 			if err != nil && !force {
 				return errors.Wrap(err, "reading file "+p.Rel(tree.FinalizerFile))
 			}

pkg/tree/compiler_recipe.go

@@ -92,7 +92,7 @@ func (r *CompilerRecipe) Load(path string) error {
 			compileDefPath := pack.Rel(CompilerDefinitionFile)
 			if helpers.Exists(compileDefPath) {
 
-				dat, err := helpers.RenderFiles(compileDefPath, currentpath)
+				dat, err := helpers.RenderFiles(compileDefPath, currentpath, "")
 				if err != nil {
 					return errors.Wrap(err,
 						"Error templating file "+CompilerDefinitionFile+" from "+
@@ -137,7 +137,7 @@ func (r *CompilerRecipe) Load(path string) error {
 					if err != nil {
 						return errors.Wrap(err, "Error reading file "+currentpath)
 					}
-					dat, err := helpers.RenderHelm(string(buildyaml), raw)
+					dat, err := helpers.RenderHelm(string(buildyaml), raw, map[string]interface{}{})
 					if err != nil {
 						return errors.Wrap(err,
 							"Error templating file "+CompilerDefinitionFile+" from "+

tests/fixtures/build_values/build.yaml

@@ -0,0 +1,7 @@
+image: quay.io/mocaccino/extra
+steps:
+- touch /{{.Values.name}}
+- touch /build-extra-{{.Values.foo}}
+- touch /{{.Values.name}}-{{.Values.bb}}
+
+unpack: true

tests/fixtures/build_values/collection.yaml

@@ -0,0 +1,13 @@
+packages:
+- name: "a"
+  category: "distro"
+  version: "0.1"
+  foo: "baz"
+- name: "b"
+  category: "distro"
+  version: "0.3"
+  foo: "f"
+- name: "c"
+  category: "distro"
+  version: "0.3"
+  foo: "bar"

tests/fixtures/build_values/finalize.yaml

@@ -0,0 +1,2 @@
+install:
+- touch /finalize-{{.Values.name}}

tests/fixtures/build_values/pack/build.yaml

@@ -0,0 +1,4 @@
+image: quay.io/mocaccino/extra
+steps:
+- touch /{{.Values.name}}
+- touch /{{.Values.name}}-{{.Values.bb}}

tests/fixtures/build_values/pack/definition.yaml

@@ -0,0 +1,3 @@
+name: foo
+category: test
+version: "1.1"

tests/integration/22_build_values.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    cat <<EOF > $tmpdir/default.yaml
+bb: "ttt"
+EOF
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/build_values" --values $tmpdir/default.yaml --destination $tmpdir/testbuild --compression gzip --all 
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package B' "[ -e '$tmpdir/testbuild/b-distro-0.3.package.tar.gz' ]"
+    assertTrue 'create package A' "[ -e '$tmpdir/testbuild/a-distro-0.1.package.tar.gz' ]"
+    assertTrue 'create package C' "[ -e '$tmpdir/testbuild/c-distro-0.3.package.tar.gz' ]"
+    assertTrue 'create package foo' "[ -e '$tmpdir/testbuild/foo-test-1.1.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/build_values" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install -y --config $tmpdir/luet.yaml distro/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/a' ]"
+    # Build time can interpolate on fields which aren't package properties.
+    assertTrue 'extra field on A' "[ -e '$tmpdir/testrootfs/build-extra-baz' ]"
+    assertTrue 'package installed A interpolated with values' "[ -e '$tmpdir/testrootfs/a-ttt' ]"
+    # Finalizers can interpolate only on package field. No extra fields are allowed at this time.
+    assertTrue 'finalizer executed on A' "[ -e '$tmpdir/testrootfs/finalize-a' ]"
+
+    installed=$(luet --config $tmpdir/luet.yaml search --installed .)
+    searchst=$?
+    assertEquals 'search exists successfully' "$searchst" "0"
+
+    assertContains 'contains distro/a-0.1' "$installed" 'distro/a-0.1'
+
+    luet uninstall -y --config $tmpdir/luet.yaml distro/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    # We do the same check for the others
+    luet install -y --config $tmpdir/luet.yaml distro/b
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/b' ]"
+    assertTrue 'package installed B interpolated with values' "[ -e '$tmpdir/testrootfs/b-ttt' ]"
+    assertTrue 'extra field on B' "[ -e '$tmpdir/testrootfs/build-extra-f' ]"
+    assertTrue 'finalizer executed on B' "[ -e '$tmpdir/testrootfs/finalize-b' ]"
+
+    installed=$(luet --config $tmpdir/luet.yaml search --installed .)
+    searchst=$?
+    assertEquals 'search exists successfully' "$searchst" "0"
+
+    assertContains 'contains distro/b-0.3' "$installed" 'distro/b-0.3'
+
+    luet uninstall -y --config $tmpdir/luet.yaml distro/b
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    luet install -y --config $tmpdir/luet.yaml distro/c
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    assertTrue 'package installed C' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'extra field on C' "[ -e '$tmpdir/testrootfs/build-extra-bar' ]"
+    assertTrue 'package installed C interpolated with values' "[ -e '$tmpdir/testrootfs/c-ttt' ]"
+    assertTrue 'finalizer executed on C' "[ -e '$tmpdir/testrootfs/finalize-c' ]"
+
+    installed=$(luet --config $tmpdir/luet.yaml search --installed .)
+    searchst=$?
+    assertEquals 'search exists successfully' "$searchst" "0"
+
+    assertContains 'contains distro/c-0.3' "$installed" 'distro/c-0.3'
+
+    luet uninstall -y --config $tmpdir/luet.yaml distro/c
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    luet install -y --config $tmpdir/luet.yaml test/foo
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+    assertTrue 'package installed foo' "[ -e '$tmpdir/testrootfs/foo' ]"
+    assertTrue 'package installed foo interpolated with values' "[ -e '$tmpdir/testrootfs/foo-ttt' ]"
+}
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+