From 33b1c638158d83e4b3b6edaca3ae826d1e8551a6 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 4 Aug 2021 16:16:54 +0200 Subject: [PATCH] Allow to have shared templates across packages This changeset allows to have shared templates in a static folder "templates" present in each luet tree. If the directory is present, it gets scanned and templated accordingly on top of each package. This allows to use such folder to store custom blocks to share between packages. This is still experimental and subject to change, this is just a first pass version to provide the feature. It needs to be refined still as it would be more elegant to use the helm engine properly and map our structure to the engine instead of adapting it roughly. Fixes #224 --- cmd/build.go | 12 +++ pkg/compiler/compiler.go | 20 +++- .../types/options/compiler_options.go | 10 ++ pkg/config/config.go | 15 ++- pkg/helpers/file/file.go | 12 +++ pkg/helpers/helm.go | 96 ++++++++++++++++--- pkg/helpers/helm_test.go | 20 ++-- pkg/installer/system.go | 2 +- pkg/tree/compiler_recipe.go | 11 ++- .../shared_templates/templates/writefile.yaml | 3 + .../fixtures/shared_templates/test/build.yaml | 6 ++ .../shared_templates/test/collection.yaml | 7 ++ tests/integration/34_templates.sh | 69 +++++++++++++ 13 files changed, 243 insertions(+), 40 deletions(-) create mode 100644 tests/fixtures/shared_templates/templates/writefile.yaml create mode 100644 tests/fixtures/shared_templates/test/build.yaml create mode 100644 tests/fixtures/shared_templates/test/collection.yaml create mode 100755 tests/integration/34_templates.sh diff --git a/cmd/build.go b/cmd/build.go index e07190dc..e1d66413 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -160,6 +160,17 @@ Build packages specifying multiple definition trees: opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency} } + templateFolders := []string{} + if !fromRepo { + for _, t := range treePaths { + templateFolders = append(templateFolders, filepath.Join(t, "templates")) + } + } else { + for _, s := range installer.SystemRepositories(LuetCfg) { + templateFolders = append(templateFolders, filepath.Join(s.TreePath, "templates")) + } + } + luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), options.NoDeps(nodeps), options.WithBackendType(backendType), @@ -168,6 +179,7 @@ Build packages specifying multiple definition trees: options.WithPullRepositories(pullRepo), options.WithPushRepository(imageRepository), options.Rebuild(rebuild), + options.WithTemplateFolder(templateFolders), options.WithSolverOptions(*opts), options.Wait(wait), options.OnlyTarget(onlyTarget), diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 01a3e75b..9b4a7f02 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -42,6 +42,7 @@ import ( "github.com/mudler/luet/pkg/solver" "github.com/pkg/errors" "gopkg.in/yaml.v2" + "helm.sh/helm/v3/pkg/chart" ) const BuildFile = "build.yaml" @@ -750,6 +751,7 @@ func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationS p.BuildOptions.PullImageRepository = append(p.BuildOptions.PullImageRepository, cs.Options.PullImageRepository...) Debug("Inheriting pull repository from PullImageRepository buildoptions", p.BuildOptions.PullImageRepository) } + Debug(p.GetPackage().HumanReadableString(), "Build options after inherit", p.BuildOptions) } @@ -1128,6 +1130,14 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF type templatedata map[string]interface{} func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.Package, dst templatedata) ([]byte, error) { + // Grab shared templates first + var chartFiles []*chart.File + if len(cs.Options.TemplatesFolder) != 0 { + c, err := helpers.ChartFiles(cs.Options.TemplatesFolder) + if err == nil { + chartFiles = c + } + } var dataresult []byte val := pack.Rel(DefinitionFile) @@ -1165,7 +1175,7 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg. return nil, errors.Wrap(err, "merging values maps") } - dat, err := helpers.RenderHelm(string(dataBuild), td, dst) + dat, err := helpers.RenderHelm(append(chartFiles, helpers.ChartFileB(dataBuild)...), td, dst) if err != nil { return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile)) } @@ -1190,7 +1200,13 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg. bv = append([]string{f}, bv...) } } - out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, bv...) + + raw, err := ioutil.ReadFile(pack.Rel(BuildFile)) + if err != nil { + return nil, err + } + + out, err := helpers.RenderFiles(append(chartFiles, helpers.ChartFileB(raw)...), val, bv...) if err != nil { return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile)) } diff --git a/pkg/compiler/types/options/compiler_options.go b/pkg/compiler/types/options/compiler_options.go index f93ea85e..5c90b36f 100644 --- a/pkg/compiler/types/options/compiler_options.go +++ b/pkg/compiler/types/options/compiler_options.go @@ -43,6 +43,9 @@ type Compiler struct { BackendArgs []string BackendType string + + // TemplatesFolder. should default to tree/templates + TemplatesFolder []string } func NewDefaultCompiler() *Compiler { @@ -87,6 +90,13 @@ func WithBackendType(r string) func(cfg *Compiler) error { } } +func WithTemplateFolder(r []string) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.TemplatesFolder = r + return nil + } +} + func WithBuildValues(r []string) func(cfg *Compiler) error { return func(cfg *Compiler) error { cfg.BuildValuesFile = r diff --git a/pkg/config/config.go b/pkg/config/config.go index 58a2d47e..b594310b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -27,6 +27,8 @@ import ( "strings" "time" + fileHelper "github.com/mudler/luet/pkg/helpers/file" + pkg "github.com/mudler/luet/pkg/package" solver "github.com/mudler/luet/pkg/solver" @@ -108,15 +110,12 @@ type LuetSystemConfig struct { } func (s *LuetSystemConfig) SetRootFS(path string) error { - pathToSet := path - if !filepath.IsAbs(path) { - abs, err := filepath.Abs(path) - if err != nil { - return err - } - pathToSet = abs + p, err := fileHelper.Rel2Abs(path) + if err != nil { + return err } - s.Rootfs = pathToSet + + s.Rootfs = p return nil } diff --git a/pkg/helpers/file/file.go b/pkg/helpers/file/file.go index 5c3af522..de951d25 100644 --- a/pkg/helpers/file/file.go +++ b/pkg/helpers/file/file.go @@ -287,3 +287,15 @@ func CopyDir(src string, dst string) (err error) { Sync: true, OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }}) } + +func Rel2Abs(s string) (string, error) { + pathToSet := s + if !filepath.IsAbs(s) { + abs, err := filepath.Abs(s) + if err != nil { + return "", err + } + pathToSet = abs + } + return pathToSet, nil +} diff --git a/pkg/helpers/helm.go b/pkg/helpers/helm.go index bb345613..29ea4964 100644 --- a/pkg/helpers/helm.go +++ b/pkg/helpers/helm.go @@ -2,6 +2,8 @@ package helpers import ( "io/ioutil" + "path/filepath" + "strings" fileHelper "github.com/mudler/luet/pkg/helpers/file" @@ -13,17 +15,86 @@ import ( "helm.sh/helm/v3/pkg/engine" ) +// ChartFileB is an helper that takes a slice of bytes and construct a chart.File slice from it +func ChartFileB(s []byte) []*chart.File { + return []*chart.File{ + {Name: "templates", Data: s}, + } +} + +// ChartFileS is an helper that takes a string and construct a chart.File slice from it +func ChartFileS(s string) []*chart.File { + return []*chart.File{ + {Name: "templates", Data: []byte(s)}, + } +} + +// ChartFile reads all the given files and returns a slice of []*chart.File +// containing the raw content and the file name for each file +func ChartFile(s ...string) []*chart.File { + files := []*chart.File{} + for _, c := range s { + raw, err := ioutil.ReadFile(c) + if err != nil { + return files + } + files = append(files, &chart.File{Name: c, Data: raw}) + } + + return files +} + +// ChartFiles reads a list of paths and reads all yaml file inside. It returns a +// slice of pointers of chart.File(s) with the raw content of the yaml +func ChartFiles(path []string) ([]*chart.File, error) { + var chartFiles []*chart.File + for _, t := range path { + rel, err := fileHelper.Rel2Abs(t) + if err != nil { + return nil, err + } + + if !fileHelper.Exists(rel) { + continue + } + files, err := fileHelper.ListDir(rel) + if err != nil { + return nil, err + } + + for _, f := range files { + if strings.ToLower(filepath.Ext(f)) == ".yaml" { + raw, err := ioutil.ReadFile(f) + if err != nil { + return nil, err + } + chartFiles = append(chartFiles, &chart.File{Name: f, Data: raw}) + } + } + } + return chartFiles, nil +} + // RenderHelm renders the template string with helm -func RenderHelm(template string, values, d map[string]interface{}) (string, error) { +func RenderHelm(files []*chart.File, values, d map[string]interface{}) (string, error) { + + // We slurp all the files into one here. This is not elegant, but still works. + // As a reminder, the files passed here have on the head the templates in the 'templates/' folder + // of each luet tree, and it have at the bottom the package buildpsec to be templated. + // TODO: Replace by correctly populating the files so that the helm render engine templates it + // correctly + toTemplate := "" + for _, f := range files { + toTemplate += string(f.Data) + } + c := &chart.Chart{ Metadata: &chart.Metadata{ Name: "", Version: "", }, - Templates: []*chart.File{ - {Name: "templates", Data: []byte(template)}, - }, - Values: map[string]interface{}{"Values": values}, + Templates: ChartFileS(toTemplate), + Values: map[string]interface{}{"Values": values}, } v, err := chartutil.CoalesceValues(c, map[string]interface{}{"Values": d}) @@ -71,23 +142,18 @@ func reverse(s []string) []string { return s } -func RenderFiles(toTemplate, valuesFile string, defaultFile ...string) (string, error) { - raw, err := ioutil.ReadFile(toTemplate) - if err != nil { - return "", errors.Wrap(err, "reading file "+toTemplate) - } - +func RenderFiles(files []*chart.File, valuesFile string, defaultFile ...string) (string, error) { if !fileHelper.Exists(valuesFile) { - return "", errors.Wrap(err, "file not existing "+valuesFile) + return "", errors.New("file does not exist: " + valuesFile) } val, err := ioutil.ReadFile(valuesFile) if err != nil { - return "", errors.Wrap(err, "reading file "+valuesFile) + return "", errors.Wrap(err, "reading file: "+valuesFile) } var values templatedata if err = yaml.Unmarshal(val, &values); err != nil { - return "", errors.Wrap(err, "unmarshalling file "+toTemplate) + return "", errors.Wrap(err, "unmarshalling values") } dst, err := UnMarshalValues(defaultFile) @@ -95,5 +161,5 @@ func RenderFiles(toTemplate, valuesFile string, defaultFile ...string) (string, return "", errors.Wrap(err, "unmarshalling values") } - return RenderHelm(string(raw), values, dst) + return RenderHelm(files, values, dst) } diff --git a/pkg/helpers/helm_test.go b/pkg/helpers/helm_test.go index f45150ea..bcf15ad3 100644 --- a/pkg/helpers/helm_test.go +++ b/pkg/helpers/helm_test.go @@ -30,21 +30,21 @@ func writeFile(path string, content string) { Expect(err).ToNot(HaveOccurred()) } -var _ = Describe("Helpers", func() { +var _ = Describe("Helm", func() { Context("RenderHelm", func() { It("Renders templates", func() { - out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"}) + out, err := RenderHelm(ChartFileS("{{.Values.Test}}{{.Values.Bar}}"), map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"}) Expect(err).ToNot(HaveOccurred()) Expect(out).To(Equal("foobar")) }) It("Renders templates with overrides", func() { - out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"}) + out, err := RenderHelm(ChartFileS("{{.Values.Test}}{{.Values.Bar}}"), map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"}) Expect(err).ToNot(HaveOccurred()) Expect(out).To(Equal("foobar")) }) It("Renders templates", func() { - out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{}) + out, err := RenderHelm(ChartFileS("{{.Values.Test}}{{.Values.Bar}}"), map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{}) Expect(err).ToNot(HaveOccurred()) Expect(out).To(Equal("foobar")) }) @@ -68,7 +68,7 @@ foo: "baz" Expect(err).ToNot(HaveOccurred()) - res, err := RenderFiles(toTemplate, values, d) + res, err := RenderFiles(ChartFile(toTemplate), values, d) Expect(err).ToNot(HaveOccurred()) Expect(res).To(Equal("baz")) @@ -93,7 +93,7 @@ faa: "baz" Expect(err).ToNot(HaveOccurred()) - res, err := RenderFiles(toTemplate, values, d) + res, err := RenderFiles(ChartFile(toTemplate), values, d) Expect(err).ToNot(HaveOccurred()) Expect(res).To(Equal("bar")) @@ -114,7 +114,7 @@ foo: "bar" Expect(err).ToNot(HaveOccurred()) - res, err := RenderFiles(toTemplate, values) + res, err := RenderFiles(ChartFile(toTemplate), values) Expect(err).ToNot(HaveOccurred()) Expect(res).To(Equal("bar")) }) @@ -145,11 +145,11 @@ bar: "nei" Expect(err).ToNot(HaveOccurred()) - res, err := RenderFiles(toTemplate, values, d2, d) + res, err := RenderFiles(ChartFile(toTemplate), values, d2, d) Expect(err).ToNot(HaveOccurred()) Expect(res).To(Equal("bazneif")) - res, err = RenderFiles(toTemplate, values, d, d2) + res, err = RenderFiles(ChartFile(toTemplate), values, d, d2) Expect(err).ToNot(HaveOccurred()) Expect(res).To(Equal("doneif")) }) @@ -173,7 +173,7 @@ faa: "baz" Expect(err).ToNot(HaveOccurred()) - res, err := RenderFiles(toTemplate, values, d) + res, err := RenderFiles(ChartFile(toTemplate), values, d) Expect(err).ToNot(HaveOccurred()) Expect(res).To(Equal("")) diff --git a/pkg/installer/system.go b/pkg/installer/system.go index 2dbc65ea..bd0604bc 100644 --- a/pkg/installer/system.go +++ b/pkg/installer/system.go @@ -29,7 +29,7 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package) error { executedFinalizer := map[string]bool{} for _, p := range packs { if fileHelper.Exists(p.Rel(tree.FinalizerFile)) { - out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile)) + out, err := helpers.RenderFiles(helpers.ChartFile(p.Rel(tree.FinalizerFile)), p.Rel(tree.DefinitionFile)) if err != nil { Warning("Failed rendering finalizer for ", p.HumanReadableString(), err.Error()) errs = multierror.Append(errs, err) diff --git a/pkg/tree/compiler_recipe.go b/pkg/tree/compiler_recipe.go index d95207fb..22ee1931 100644 --- a/pkg/tree/compiler_recipe.go +++ b/pkg/tree/compiler_recipe.go @@ -76,6 +76,10 @@ func (r *CompilerRecipe) Load(path string) error { //if err != nil { // return err //} + c, err := helpers.ChartFiles([]string{filepath.Join(path, "templates")}) + if err != nil { + return err + } //r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name())) // TODO: Handle cleaning after? Cleanup implemented in GetPackageSet().Clean() @@ -104,8 +108,7 @@ func (r *CompilerRecipe) Load(path string) error { // Instead of rdeps, have a different tree for build deps. compileDefPath := pack.Rel(CompilerDefinitionFile) if fileHelper.Exists(compileDefPath) { - - dat, err := helpers.RenderFiles(compileDefPath, currentpath) + dat, err := helpers.RenderFiles(append(c, helpers.ChartFile(compileDefPath)...), currentpath) if err != nil { return errors.Wrap(err, "Error templating file "+CompilerDefinitionFile+" from "+ @@ -157,7 +160,7 @@ func (r *CompilerRecipe) Load(path string) error { if err != nil { return errors.Wrap(err, "Error reading file "+currentpath) } - dat, err := helpers.RenderHelm(string(buildyaml), raw, map[string]interface{}{}) + dat, err := helpers.RenderHelm(append(c, helpers.ChartFileB(buildyaml)...), raw, map[string]interface{}{}) if err != nil { return errors.Wrap(err, "Error templating file "+CompilerDefinitionFile+" from "+ @@ -183,7 +186,7 @@ func (r *CompilerRecipe) Load(path string) error { return nil } - err := filepath.Walk(path, ff) + err = filepath.Walk(path, ff) if err != nil { return err } diff --git a/tests/fixtures/shared_templates/templates/writefile.yaml b/tests/fixtures/shared_templates/templates/writefile.yaml new file mode 100644 index 00000000..0b421606 --- /dev/null +++ b/tests/fixtures/shared_templates/templates/writefile.yaml @@ -0,0 +1,3 @@ +{{ define "writefile" }} +- echo conflict > /foo/{{.}} +{{end}} \ No newline at end of file diff --git a/tests/fixtures/shared_templates/test/build.yaml b/tests/fixtures/shared_templates/test/build.yaml new file mode 100644 index 00000000..0102537c --- /dev/null +++ b/tests/fixtures/shared_templates/test/build.yaml @@ -0,0 +1,6 @@ +image: "alpine" +prelude: + - mkdir /foo +steps: +{{ template "writefile" .Values.name }} +package_dir: /foo \ No newline at end of file diff --git a/tests/fixtures/shared_templates/test/collection.yaml b/tests/fixtures/shared_templates/test/collection.yaml new file mode 100644 index 00000000..fd590581 --- /dev/null +++ b/tests/fixtures/shared_templates/test/collection.yaml @@ -0,0 +1,7 @@ +packages: +- category: "test" + name: "foo" + version: "1.0" +- category: "test" + name: "bar" + version: "1.0" diff --git a/tests/integration/34_templates.sh b/tests/integration/34_templates.sh new file mode 100755 index 00000000..cd3f1b97 --- /dev/null +++ b/tests/integration/34_templates.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +export LUET_NOLOCK=true + +oneTimeSetUp() { +export tmpdir="$(mktemp -d)" +} + +oneTimeTearDown() { + rm -rf "$tmpdir" +} + +testBuild() { + mkdir $tmpdir/testbuild + luet build --tree "$ROOT_DIR/tests/fixtures/shared_templates" --destination $tmpdir/testbuild --compression gzip --all + buildst=$? + assertEquals 'builds successfully' "$buildst" "0" + assertTrue 'create packages' "[ -e '$tmpdir/testbuild/foo-test-1.0.package.tar.gz' ]" + assertTrue 'create packages' "[ -e '$tmpdir/testbuild/bar-test-1.0.package.tar.gz' ]" +} + +testRepo() { + assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]" + luet create-repo --tree "$ROOT_DIR/tests/fixtures/shared_templates" \ + --output $tmpdir/testbuild \ + --packages $tmpdir/testbuild \ + --name "test" \ + --descr "Test Repo" \ + --urls $tmpdir/testrootfs \ + --type disk > /dev/null + + createst=$? + assertEquals 'create repo successfully' "$createst" "0" + assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]" +} + +testConfig() { + mkdir $tmpdir/testrootfs + cat < $tmpdir/luet.yaml +general: + debug: true +system: + rootfs: $tmpdir/testrootfs + database_path: "/" + database_engine: "boltdb" +config_from_host: true +repositories: + - name: "main" + type: "disk" + enable: true + urls: + - "$tmpdir/testbuild" +EOF + luet config --config $tmpdir/luet.yaml + res=$? + assertEquals 'config test successfully' "$res" "0" +} + +testInstall() { + luet install -y --config $tmpdir/luet.yaml test/foo test/bar + installst=$? + assertEquals 'install test failed' "$installst" "0" + assertTrue 'package bar installed' "[ -e '$tmpdir/testrootfs/bar' ]" + assertTrue 'package foo installed' "[ -e '$tmpdir/testrootfs/foo' ]" +} + +# Load shUnit2. +. "$ROOT_DIR/tests/integration/shunit2"/shunit2 +