diff --git a/cmd/build.go b/cmd/build.go index d9d3c265..f696ef6c 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -49,6 +49,7 @@ var buildCmd = &cobra.Command{ viper.BindPFlag("compression", cmd.Flags().Lookup("compression")) viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps")) viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps")) + viper.BindPFlag("values", cmd.Flags().Lookup("values")) viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository")) viper.BindPFlag("push", cmd.Flags().Lookup("push")) @@ -75,6 +76,8 @@ var buildCmd = &cobra.Command{ databaseType := viper.GetString("database") compressionType := viper.GetString("compression") imageRepository := viper.GetString("image-repository") + values := viper.GetString("values") + push := viper.GetBool("push") pull := viper.GetBool("pull") keepImages := viper.GetBool("keep-images") @@ -157,7 +160,7 @@ var buildCmd = &cobra.Command{ opts.KeepImageExport = keepExportedImages opts.SkipIfMetadataExists = skip opts.PackageTargetOnly = onlyTarget - + opts.BuildValuesFile = values var solverOpts solver.Options if concurrent { solverOpts = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency} @@ -291,6 +294,7 @@ func init() { buildCmd.Flags().Bool("revdeps", false, "Build with revdeps") buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree") buildCmd.Flags().Bool("full", false, "Build all packages (optimized)") + buildCmd.Flags().String("values", "", "Build values file to interpolate with each package") buildCmd.Flags().String("destination", path, "Destination folder") buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip") diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 6d252b8f..6514d881 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -27,6 +27,7 @@ import ( "time" bus "github.com/mudler/luet/pkg/bus" + yaml "gopkg.in/yaml.v2" "github.com/mudler/luet/pkg/helpers" . "github.com/mudler/luet/pkg/logger" @@ -743,13 +744,25 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) { raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion()) - dat, err := helpers.RenderHelm(string(dataBuild), raw) + d := map[string]interface{}{} + if len(cs.Options.BuildValuesFile) > 0 { + defBuild, err := ioutil.ReadFile(cs.Options.BuildValuesFile) + if err != nil { + return nil, errors.Wrap(err, "rendering file "+val) + } + err = yaml.Unmarshal(defBuild, &d) + if err != nil { + return nil, errors.Wrap(err, "rendering file "+val) + } + } + + dat, err := helpers.RenderHelm(string(dataBuild), raw, d) if err != nil { return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile)) } dataresult = []byte(dat) } else { - out, err := helpers.RenderFiles(pack.Rel(BuildFile), val) + out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, cs.Options.BuildValuesFile) if err != nil { return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile)) } diff --git a/pkg/compiler/interface.go b/pkg/compiler/interface.go index e80a21ae..8d9e339d 100644 --- a/pkg/compiler/interface.go +++ b/pkg/compiler/interface.go @@ -56,6 +56,7 @@ type CompilerOptions struct { NoDeps bool SolverOptions config.LuetSolverOptions SkipIfMetadataExists bool + BuildValuesFile string PackageTargetOnly bool } diff --git a/pkg/helpers/helm.go b/pkg/helpers/helm.go index 2f038fbc..a3f21cb5 100644 --- a/pkg/helpers/helm.go +++ b/pkg/helpers/helm.go @@ -11,7 +11,7 @@ import ( ) // RenderHelm renders the template string with helm -func RenderHelm(template string, values map[string]interface{}) (string, error) { +func RenderHelm(template string, values, d map[string]interface{}) (string, error) { c := &chart.Chart{ Metadata: &chart.Metadata{ Name: "", @@ -23,7 +23,7 @@ func RenderHelm(template string, values map[string]interface{}) (string, error) Values: map[string]interface{}{"Values": values}, } - v, err := chartutil.CoalesceValues(c, map[string]interface{}{}) + v, err := chartutil.CoalesceValues(c, map[string]interface{}{"Values": d}) if err != nil { return "", errors.Wrap(err, "while rendering template") } @@ -37,7 +37,7 @@ func RenderHelm(template string, values map[string]interface{}) (string, error) type templatedata map[string]interface{} -func RenderFiles(toTemplate, valuesFile string) (string, error) { +func RenderFiles(toTemplate, valuesFile string, defaultFile string) (string, error) { raw, err := ioutil.ReadFile(toTemplate) if err != nil { return "", errors.Wrap(err, "reading file "+toTemplate) @@ -46,14 +46,26 @@ func RenderFiles(toTemplate, valuesFile string) (string, error) { if !Exists(valuesFile) { return "", errors.Wrap(err, "file not existing "+valuesFile) } - def, err := ioutil.ReadFile(valuesFile) + val, err := ioutil.ReadFile(valuesFile) if err != nil { return "", errors.Wrap(err, "reading file "+valuesFile) } var values templatedata - if err = yaml.Unmarshal(def, &values); err != nil { + d := templatedata{} + if len(defaultFile) > 0 { + def, err := ioutil.ReadFile(defaultFile) + if err != nil { + return "", errors.Wrap(err, "reading file "+valuesFile) + } + if err = yaml.Unmarshal(def, &d); err != nil { + return "", errors.Wrap(err, "unmarshalling file "+toTemplate) + } + } + + if err = yaml.Unmarshal(val, &values); err != nil { return "", errors.Wrap(err, "unmarshalling file "+toTemplate) } - return RenderHelm(string(raw), values) + + return RenderHelm(string(raw), values, d) } diff --git a/pkg/helpers/helm_test.go b/pkg/helpers/helm_test.go index 2c9e3739..9d0c62e5 100644 --- a/pkg/helpers/helm_test.go +++ b/pkg/helpers/helm_test.go @@ -16,17 +16,132 @@ package helpers_test import ( + "io/ioutil" + "os" + "path/filepath" + . "github.com/mudler/luet/pkg/helpers" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) +func writeFile(path string, content string) { + err := ioutil.WriteFile(path, []byte(content), 0644) + Expect(err).ToNot(HaveOccurred()) +} + var _ = Describe("Helpers", func() { Context("RenderHelm", func() { It("Renders templates", func() { - out, err := RenderHelm("{{.Values.Test}}", map[string]interface{}{"Test": "foo"}) + out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"}) Expect(err).ToNot(HaveOccurred()) - Expect(out).To(Equal("foo")) + Expect(out).To(Equal("foobar")) + }) + It("Renders templates with overrides", func() { + out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"}) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(Equal("foobar")) + }) + + It("Renders templates", func() { + out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{}) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(Equal("foobar")) + }) + + It("Render files default overrides", func() { + testDir, err := ioutil.TempDir(os.TempDir(), "test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(testDir) + + toTemplate := filepath.Join(testDir, "totemplate.yaml") + values := filepath.Join(testDir, "values.yaml") + d := filepath.Join(testDir, "default.yaml") + + writeFile(toTemplate, `{{.Values.foo}}`) + writeFile(values, ` +foo: "bar" +`) + writeFile(d, ` +foo: "baz" +`) + + Expect(err).ToNot(HaveOccurred()) + + res, err := RenderFiles(toTemplate, values, d) + Expect(err).ToNot(HaveOccurred()) + Expect(res).To(Equal("baz")) + + }) + + It("Render files from values", func() { + testDir, err := ioutil.TempDir(os.TempDir(), "test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(testDir) + + toTemplate := filepath.Join(testDir, "totemplate.yaml") + values := filepath.Join(testDir, "values.yaml") + d := filepath.Join(testDir, "default.yaml") + + writeFile(toTemplate, `{{.Values.foo}}`) + writeFile(values, ` +foo: "bar" +`) + writeFile(d, ` +faa: "baz" +`) + + Expect(err).ToNot(HaveOccurred()) + + res, err := RenderFiles(toTemplate, values, d) + Expect(err).ToNot(HaveOccurred()) + Expect(res).To(Equal("bar")) + + }) + + It("Render files from values if no default", func() { + testDir, err := ioutil.TempDir(os.TempDir(), "test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(testDir) + + toTemplate := filepath.Join(testDir, "totemplate.yaml") + values := filepath.Join(testDir, "values.yaml") + + writeFile(toTemplate, `{{.Values.foo}}`) + writeFile(values, ` +foo: "bar" +`) + + Expect(err).ToNot(HaveOccurred()) + + res, err := RenderFiles(toTemplate, values, "") + Expect(err).ToNot(HaveOccurred()) + Expect(res).To(Equal("bar")) + }) + + It("doesn't interpolate if no one provides the values", func() { + testDir, err := ioutil.TempDir(os.TempDir(), "test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(testDir) + + toTemplate := filepath.Join(testDir, "totemplate.yaml") + values := filepath.Join(testDir, "values.yaml") + d := filepath.Join(testDir, "default.yaml") + + writeFile(toTemplate, `{{.Values.foo}}`) + writeFile(values, ` +foao: "bar" +`) + writeFile(d, ` +faa: "baz" +`) + + Expect(err).ToNot(HaveOccurred()) + + res, err := RenderFiles(toTemplate, values, d) + Expect(err).ToNot(HaveOccurred()) + Expect(res).To(Equal("")) + }) }) }) diff --git a/pkg/installer/system.go b/pkg/installer/system.go index eb4e91da..a7cd4044 100644 --- a/pkg/installer/system.go +++ b/pkg/installer/system.go @@ -24,7 +24,7 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package, force bool) error { executedFinalizer := map[string]bool{} for _, p := range packs { if helpers.Exists(p.Rel(tree.FinalizerFile)) { - out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile)) + out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile), "") if err != nil && !force { return errors.Wrap(err, "reading file "+p.Rel(tree.FinalizerFile)) } diff --git a/pkg/tree/compiler_recipe.go b/pkg/tree/compiler_recipe.go index 6d1b378c..a7f57147 100644 --- a/pkg/tree/compiler_recipe.go +++ b/pkg/tree/compiler_recipe.go @@ -92,7 +92,7 @@ func (r *CompilerRecipe) Load(path string) error { compileDefPath := pack.Rel(CompilerDefinitionFile) if helpers.Exists(compileDefPath) { - dat, err := helpers.RenderFiles(compileDefPath, currentpath) + dat, err := helpers.RenderFiles(compileDefPath, currentpath, "") if err != nil { return errors.Wrap(err, "Error templating file "+CompilerDefinitionFile+" from "+ @@ -137,7 +137,7 @@ func (r *CompilerRecipe) Load(path string) error { if err != nil { return errors.Wrap(err, "Error reading file "+currentpath) } - dat, err := helpers.RenderHelm(string(buildyaml), raw) + dat, err := helpers.RenderHelm(string(buildyaml), raw, map[string]interface{}{}) if err != nil { return errors.Wrap(err, "Error templating file "+CompilerDefinitionFile+" from "+ diff --git a/tests/fixtures/build_values/build.yaml b/tests/fixtures/build_values/build.yaml new file mode 100644 index 00000000..b6d549e0 --- /dev/null +++ b/tests/fixtures/build_values/build.yaml @@ -0,0 +1,7 @@ +image: quay.io/mocaccino/extra +steps: +- touch /{{.Values.name}} +- touch /build-extra-{{.Values.foo}} +- touch /{{.Values.name}}-{{.Values.bb}} + +unpack: true diff --git a/tests/fixtures/build_values/collection.yaml b/tests/fixtures/build_values/collection.yaml new file mode 100644 index 00000000..26861d89 --- /dev/null +++ b/tests/fixtures/build_values/collection.yaml @@ -0,0 +1,13 @@ +packages: +- name: "a" + category: "distro" + version: "0.1" + foo: "baz" +- name: "b" + category: "distro" + version: "0.3" + foo: "f" +- name: "c" + category: "distro" + version: "0.3" + foo: "bar" \ No newline at end of file diff --git a/tests/fixtures/build_values/finalize.yaml b/tests/fixtures/build_values/finalize.yaml new file mode 100644 index 00000000..a057217f --- /dev/null +++ b/tests/fixtures/build_values/finalize.yaml @@ -0,0 +1,2 @@ +install: +- touch /finalize-{{.Values.name}} \ No newline at end of file diff --git a/tests/fixtures/build_values/pack/build.yaml b/tests/fixtures/build_values/pack/build.yaml new file mode 100644 index 00000000..f95b274f --- /dev/null +++ b/tests/fixtures/build_values/pack/build.yaml @@ -0,0 +1,4 @@ +image: quay.io/mocaccino/extra +steps: +- touch /{{.Values.name}} +- touch /{{.Values.name}}-{{.Values.bb}} diff --git a/tests/fixtures/build_values/pack/definition.yaml b/tests/fixtures/build_values/pack/definition.yaml new file mode 100644 index 00000000..79d17775 --- /dev/null +++ b/tests/fixtures/build_values/pack/definition.yaml @@ -0,0 +1,3 @@ +name: foo +category: test +version: "1.1" \ No newline at end of file diff --git a/tests/integration/22_build_values.sh b/tests/integration/22_build_values.sh new file mode 100755 index 00000000..89f55967 --- /dev/null +++ b/tests/integration/22_build_values.sh @@ -0,0 +1,133 @@ +#!/bin/bash + +export LUET_NOLOCK=true +oneTimeSetUp() { +export tmpdir="$(mktemp -d)" +} + +oneTimeTearDown() { + rm -rf "$tmpdir" +} + +testBuild() { + cat < $tmpdir/default.yaml +bb: "ttt" +EOF + mkdir $tmpdir/testbuild + luet build --tree "$ROOT_DIR/tests/fixtures/build_values" --values $tmpdir/default.yaml --destination $tmpdir/testbuild --compression gzip --all + buildst=$? + assertEquals 'builds successfully' "$buildst" "0" + assertTrue 'create package B' "[ -e '$tmpdir/testbuild/b-distro-0.3.package.tar.gz' ]" + assertTrue 'create package A' "[ -e '$tmpdir/testbuild/a-distro-0.1.package.tar.gz' ]" + assertTrue 'create package C' "[ -e '$tmpdir/testbuild/c-distro-0.3.package.tar.gz' ]" + assertTrue 'create package foo' "[ -e '$tmpdir/testbuild/foo-test-1.1.package.tar.gz' ]" +} + +testRepo() { + assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]" + luet create-repo --tree "$ROOT_DIR/tests/fixtures/build_values" \ + --output $tmpdir/testbuild \ + --packages $tmpdir/testbuild \ + --name "test" \ + --descr "Test Repo" \ + --urls $tmpdir/testrootfs \ + --type disk + + createst=$? + assertEquals 'create repo successfully' "$createst" "0" + assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]" +} + +testConfig() { + mkdir $tmpdir/testrootfs + cat < $tmpdir/luet.yaml +general: + debug: true +system: + rootfs: $tmpdir/testrootfs + database_path: "/" + database_engine: "boltdb" +config_from_host: true +repositories: + - name: "main" + type: "disk" + enable: true + urls: + - "$tmpdir/testbuild" +EOF + luet config --config $tmpdir/luet.yaml + res=$? + assertEquals 'config test successfully' "$res" "0" +} + +testInstall() { + luet install -y --config $tmpdir/luet.yaml distro/a + installst=$? + assertEquals 'install test successfully' "$installst" "0" + + assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/a' ]" + # Build time can interpolate on fields which aren't package properties. + assertTrue 'extra field on A' "[ -e '$tmpdir/testrootfs/build-extra-baz' ]" + assertTrue 'package installed A interpolated with values' "[ -e '$tmpdir/testrootfs/a-ttt' ]" + # Finalizers can interpolate only on package field. No extra fields are allowed at this time. + assertTrue 'finalizer executed on A' "[ -e '$tmpdir/testrootfs/finalize-a' ]" + + installed=$(luet --config $tmpdir/luet.yaml search --installed .) + searchst=$? + assertEquals 'search exists successfully' "$searchst" "0" + + assertContains 'contains distro/a-0.1' "$installed" 'distro/a-0.1' + + luet uninstall -y --config $tmpdir/luet.yaml distro/a + installst=$? + assertEquals 'install test successfully' "$installst" "0" + + # We do the same check for the others + luet install -y --config $tmpdir/luet.yaml distro/b + installst=$? + assertEquals 'install test successfully' "$installst" "0" + + assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/b' ]" + assertTrue 'package installed B interpolated with values' "[ -e '$tmpdir/testrootfs/b-ttt' ]" + assertTrue 'extra field on B' "[ -e '$tmpdir/testrootfs/build-extra-f' ]" + assertTrue 'finalizer executed on B' "[ -e '$tmpdir/testrootfs/finalize-b' ]" + + installed=$(luet --config $tmpdir/luet.yaml search --installed .) + searchst=$? + assertEquals 'search exists successfully' "$searchst" "0" + + assertContains 'contains distro/b-0.3' "$installed" 'distro/b-0.3' + + luet uninstall -y --config $tmpdir/luet.yaml distro/b + installst=$? + assertEquals 'install test successfully' "$installst" "0" + + luet install -y --config $tmpdir/luet.yaml distro/c + installst=$? + assertEquals 'install test successfully' "$installst" "0" + + assertTrue 'package installed C' "[ -e '$tmpdir/testrootfs/c' ]" + assertTrue 'extra field on C' "[ -e '$tmpdir/testrootfs/build-extra-bar' ]" + assertTrue 'package installed C interpolated with values' "[ -e '$tmpdir/testrootfs/c-ttt' ]" + assertTrue 'finalizer executed on C' "[ -e '$tmpdir/testrootfs/finalize-c' ]" + + installed=$(luet --config $tmpdir/luet.yaml search --installed .) + searchst=$? + assertEquals 'search exists successfully' "$searchst" "0" + + assertContains 'contains distro/c-0.3' "$installed" 'distro/c-0.3' + + luet uninstall -y --config $tmpdir/luet.yaml distro/c + installst=$? + assertEquals 'install test successfully' "$installst" "0" + + luet install -y --config $tmpdir/luet.yaml test/foo + installst=$? + assertEquals 'install test successfully' "$installst" "0" + + assertTrue 'package installed foo' "[ -e '$tmpdir/testrootfs/foo' ]" + assertTrue 'package installed foo interpolated with values' "[ -e '$tmpdir/testrootfs/foo-ttt' ]" +} +# Load shUnit2. +. "$ROOT_DIR/tests/integration/shunit2"/shunit2 +