Tag 0.8.6

* Added command line option --skip-config-protect

* Added config option config_protect_skip

cmd/root.go

@@ -38,7 +38,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.8.5"
+	LuetCLIVersion = "0.8.6"
 	LuetEnvPrefix  = "LUET"
 )
 
@@ -152,6 +152,8 @@ func init() {
 	pflags.Bool("no-spinner", false, "Disable spinner.")
 	pflags.Bool("color", config.LuetCfg.GetLogging().Color, "Enable/Disable color.")
 	pflags.Bool("emoji", config.LuetCfg.GetLogging().EnableEmoji, "Enable/Disable emoji.")
+	pflags.Bool("skip-config-protect", config.LuetCfg.ConfigProtectSkip,
+		"Disable config protect analysis.")
 	pflags.StringP("logfile", "l", config.LuetCfg.GetLogging().Path,
 		"Logfile path. Empty value disable log to file.")
 
@@ -175,6 +177,7 @@ func init() {
 	config.LuetCfg.Viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
 	// Currently I maintain this only from cli.
 	config.LuetCfg.Viper.BindPFlag("no_spinner", pflags.Lookup("no-spinner"))
+	config.LuetCfg.Viper.BindPFlag("config_protect_skip", pflags.Lookup("skip-config-protect"))
 
 	// Extensions must be binary with the "luet-" prefix to be able to be shown in the help.
 	// we also accept extensions in the relative path where luet is being started, "extensions/"

contrib/config/config.protect.d/01_etc.yml.example

@@ -0,0 +1,3 @@
+name: "etc_conf"
+dirs:
+  - "/etc/"

contrib/config/luet.yaml

@@ -88,6 +88,11 @@
 # config_protect_confdir:
 #   - /etc/luet/config.protect.d
 #
+# Permit to ignore rules defined on
+# config protect confdir and packages
+# annotation.
+# config_protect_skip: false
+#
 # System repositories
 # ---------------------------------------------
 # In alternative to define repositories files

go.mod

@@ -4,7 +4,7 @@ go 1.12
 
 require (
 	github.com/DataDog/zstd v1.4.4 // indirect
-	github.com/Sabayon/pkgs-checker v0.6.2-0.20200404093625-076438c31739
+	github.com/Sabayon/pkgs-checker v0.6.3-0.20200912135508-97c41780e9b6
 	github.com/asdine/storm v0.0.0-20190418133842-e0f77eada154
 	github.com/briandowns/spinner v1.7.0
 	github.com/cavaliercoder/grab v2.0.0+incompatible
@@ -31,9 +31,7 @@ require (
 	github.com/pelletier/go-toml v1.6.0 // indirect
 	github.com/philopon/go-toposort v0.0.0-20170620085441-9be86dbd762f
 	github.com/pkg/errors v0.9.1
-	github.com/spf13/afero v1.2.2 // indirect
 	github.com/spf13/cobra v1.0.0
-	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.6.3
 	github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
 	go.etcd.io/bbolt v1.3.4
@@ -42,9 +40,9 @@ require (
 	go.uber.org/zap v1.13.0
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
 	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
-	golang.org/x/tools v0.0.0-20200102200121-6de373a2766c // indirect
-	gopkg.in/yaml.v2 v2.2.7
+	gopkg.in/yaml.v2 v2.2.8
 	gotest.tools/v3 v3.0.2 // indirect
+	helm.sh/helm/v3 v3.3.4
 	mvdan.cc/sh/v3 v3.0.0-beta1
 )
 

pkg/compiler/artifact.go

@@ -331,7 +331,9 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 func (a *PackageArtifact) GetProtectFiles() []string {
 	ans := []string{}
 
-	if LuetCfg.GetConfigProtectConfFiles() != nil && len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
+	if !LuetCfg.ConfigProtectSkip &&
+		LuetCfg.GetConfigProtectConfFiles() != nil &&
+		len(LuetCfg.GetConfigProtectConfFiles()) > 0 {
 
 		for _, file := range a.Files {
 			for _, conf := range LuetCfg.GetConfigProtectConfFiles() {
@@ -514,7 +516,7 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 
 		_, err := os.Lstat(job.Dst)
 		if err != nil {
-			fmt.Println("Copying ", job.Src)
+			Debug("Copying ", job.Src)
 			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}

pkg/compiler/compiler.go

@@ -17,9 +17,11 @@ package compiler
 
 import (
 	"fmt"
+	"github.com/ghodss/yaml"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+
 	"regexp"
 	"strings"
 	"sync"
@@ -34,6 +36,7 @@ import (
 )
 
 const BuildFile = "build.yaml"
+const DefinitionFile = "definition.yaml"
 
 type LuetCompiler struct {
 	*tree.CompilerRecipe
@@ -641,6 +644,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
 	}
 }
 
+type templatedata map[string]interface{}
+
 func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 
 	pack, err := cs.Database.FindPackageCandidate(p)
@@ -652,12 +657,28 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
 	if !helpers.Exists(buildFile) {
 		return nil, errors.New("No build file present for " + p.GetFingerPrint())
 	}
-
-	dat, err := ioutil.ReadFile(buildFile)
+	defFile := pack.Rel(DefinitionFile)
+	if !helpers.Exists(defFile) {
+		return nil, errors.New("No build file present for " + p.GetFingerPrint())
+	}
+	def, err := ioutil.ReadFile(defFile)
 	if err != nil {
 		return nil, err
 	}
-	return NewLuetCompilationSpec(dat, pack)
+
+	build, err := ioutil.ReadFile(buildFile)
+	if err != nil {
+		return nil, err
+	}
+	var values templatedata
+	if err = yaml.Unmarshal(def, &values); err != nil {
+		return nil, err
+	}
+	out, err := helpers.RenderHelm(string(build), values)
+	if err != nil {
+		return nil, err
+	}
+	return NewLuetCompilationSpec([]byte(out), pack)
 }
 
 func (cs *LuetCompiler) GetBackend() CompilerBackend {

pkg/compiler/compiler_test.go

@@ -108,6 +108,27 @@ var _ = Describe("Compiler", func() {
 		})
 	})
 
+	Context("Templated packages",func(){
+		It("Renders", func() {
+			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+			tmpdir, err := ioutil.TempDir("", "package")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(tmpdir) // clean up
+
+			err = generalRecipe.Load("../../tests/fixtures/templates")
+			Expect(err).ToNot(HaveOccurred())
+			compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
+
+			Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
+			pkg ,err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
+			Expect(err).ToNot(HaveOccurred())
+
+			spec, err := compiler.FromPackage(pkg)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(spec.GetImage()).To(Equal("b:bar"))
+		})
+	})
+
 	Context("Reconstruct image tree", func() {
 		It("Compiles it", func() {
 			generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))

pkg/config/config.go

@@ -202,6 +202,7 @@ type LuetConfig struct {
 
 	RepositoriesConfDir  []string         `mapstructure:"repos_confdir"`
 	ConfigProtectConfDir []string         `mapstructure:"config_protect_confdir"`
+	ConfigProtectSkip    bool             `mapstructure:"config_protect_skip"`
 	CacheRepositories    []LuetRepository `mapstructure:"repetitors"`
 	SystemRepositories   []LuetRepository `mapstructure:"repositories"`
 
@@ -248,6 +249,7 @@ func GenDefault(viper *v.Viper) {
 
 	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
 	viper.SetDefault("config_protect_confdir", []string{"/etc/luet/config.protect.d"})
+	viper.SetDefault("config_protect_skip", false)
 	viper.SetDefault("cache_repositories", []string{})
 	viper.SetDefault("system_repositories", []string{})
 

pkg/helpers/helm.go

@@ -0,0 +1,34 @@
+package helpers
+
+import (
+
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/engine"
+	"github.com/pkg/errors"
+)
+
+// RenderHelm renders the template string with helm
+func RenderHelm(template string, values map[string]interface{}) (string,error) {
+	c := &chart.Chart{
+		Metadata: &chart.Metadata{
+			Name:    "",
+			Version: "",
+		},
+		Templates: []*chart.File{
+			{Name: "templates", Data: []byte(template)},
+		},
+		Values:    map[string]interface{}{"Values":values},
+	}
+
+	v, err := chartutil.CoalesceValues(c, map[string]interface{}{})
+	if err != nil {
+		return "",errors.Wrap(err,"while rendering template")
+	}
+	out, err := engine.Render(c, v)
+	if err != nil {
+		return "",errors.Wrap(err,"while rendering template")
+	}
+
+	return out["templates"],nil
+}

pkg/helpers/helm_test.go

@@ -0,0 +1,32 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package helpers_test
+
+import (
+	. "github.com/mudler/luet/pkg/helpers"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Helpers", func() {
+	Context("RenderHelm", func() {
+		It("Renders templates", func() {
+			out, err := RenderHelm("{{.Values.Test}}",map[string]interface{}{"Test":"foo"})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).To(Equal("foo"))
+		})
+	})
+})

pkg/tree/installer_recipe.go

@@ -21,6 +21,7 @@
 package tree
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -70,10 +71,12 @@ func (r *InstallerRecipe) Save(path string) error {
 
 func (r *InstallerRecipe) Load(path string) error {
 
-	// tmpfile, err := ioutil.TempFile("", "luet")
-	// if err != nil {
-	// 	return err
-	// }
+	if !helpers.Exists(path) {
+		return errors.New(fmt.Sprintf(
+			"Path %s doesn't exit.", path,
+		))
+	}
+
 	r.SourcePath = append(r.SourcePath, path)
 
 	//r.Tree().SetPackageSet(pkg.NewBoltDatabase(tmpfile.Name()))

pkg/tree/recipes.go

@@ -21,10 +21,12 @@
 package tree
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
+	helpers "github.com/mudler/luet/pkg/helpers"
 	pkg "github.com/mudler/luet/pkg/package"
 	spectooling "github.com/mudler/luet/pkg/spectooling"
 	"github.com/pkg/errors"
@@ -74,6 +76,12 @@ func (r *Recipe) Load(path string) error {
 	// if err != nil {
 	// 	return err
 	// }
+	if !helpers.Exists(path) {
+		return errors.New(fmt.Sprintf(
+			"Path %s doesn't exit.", path,
+		))
+	}
+
 	r.SourcePath = append(r.SourcePath, path)
 
 	if r.Database == nil {

tests/fixtures/templates/build.yaml

@@ -0,0 +1 @@
+image: "{{.Values.name}}:{{.Values.test.foo}}"

tests/fixtures/templates/definition.yaml

@@ -0,0 +1,5 @@
+category: "test"
+name: "b"
+version: "1.0"
+test:
+  foo: "bar"

tests/integration/17_config_protect_skip.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/config_protect" --destination $tmpdir/testbuild --compression gzip test/a
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/a-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/config_protect" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+
+    mkdir $tmpdir/config.protect.d
+
+    cat <<EOF > $tmpdir/config.protect.d/conf1.yml
+name: "protect1"
+dirs:
+- /etc/
+EOF
+
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_protect_skip: true
+config_protect_confdir:
+    - $tmpdir/config.protect.d
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+
+
+testInstall() {
+
+    # Simulate previous installation
+    mkdir $tmpdir/testrootfs/etc/a -p
+    echo "fakeconf" > $tmpdir/testrootfs/etc/a/conf
+
+    luet install --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+
+    # Simulate config protect
+    assertTrue 'package A installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'config protect created' "[ ! -e '$tmpdir/testrootfs/etc/a/._cfg0001_conf' ]"
+}
+
+
+testUnInstall() {
+    luet uninstall --full --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+}
+
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/a-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

vendor/github.com/Sabayon/pkgs-checker/pkg/gentoo/pkg.go

@@ -60,16 +60,16 @@ const (
 )
 
 type GentooPackage struct {
-	Name          string `json:"name",omitempty"`
-	Category      string `json:"category",omitempty"`
-	Version       string `json:"version",omitempty"`
-	VersionSuffix string `json:"version_suffix",omitempty"`
-	VersionBuild  string `json:"version_build",omitempty"`
-	Slot          string `json:"slot",omitempty"`
+	Name          string `json:"name,omitempty"`
+	Category      string `json:"category,omitempty"`
+	Version       string `json:"version,omitempty"`
+	VersionSuffix string `json:"version_suffix,omitempty"`
+	VersionBuild  string `json:"version_build,omitempty"`
+	Slot          string `json:"slot,omitempty"`
 	Condition     PackageCond
-	Repository    string   `json:"repository",omitempty"`
-	UseFlags      []string `json:"use_flags",omitempty"`
-	License       string   `json:"license",omitempty"`
+	Repository    string   `json:"repository,omitempty"`
+	UseFlags      []string `json:"use_flags,omitempty"`
+	License       string   `json:"license,omitempty"`
 }
 
 func (p *GentooPackage) String() string {
@@ -358,15 +358,33 @@ func ParsePackageStr(pkg string) (*GentooPackage, error) {
 		"_beta",
 	)
 
+	words := strings.Split(pkg, "/")
+	if len(words) != 2 {
+		return nil, errors.New(fmt.Sprintf("Invalid package string %s", pkg))
+	}
+	ans.Category = words[0]
+	pkgname := words[1]
+
+	// Validate category
+
+	regexPkg := regexp.MustCompile(
+		fmt.Sprintf("%s$", RegexCatString),
+	)
+
+	matches := regexPkg.FindAllString(ans.Category, -1)
+	if len(matches) > 1 {
+		return nil, errors.New(fmt.Sprintf("Invalid category %s", ans.Category))
+	}
+
 	hasBuild, _ := regexp.MatchString(
-		fmt.Sprintf("(%s[/]%s%s([[:]{1,2}[0-9a-zA-Z]*]*)*[+])",
-			RegexCatString, RegexPkgNameString, regexVerString),
-		pkg,
+		fmt.Sprintf("(%s%s([[:]{1,2}[0-9a-zA-Z]*]*)*[+])",
+			RegexPkgNameString, regexVerString),
+		pkgname,
 	)
 
 	if hasBuild {
 		// Check if build number is present
-		buildIdx := strings.LastIndex(pkg, "+")
+		buildIdx := strings.LastIndex(pkgname, "+")
 		if buildIdx > 0 {
 			// <pre-release> ::= <dot-separated pre-release identifiers>
 			//
@@ -389,18 +407,11 @@ func ParsePackageStr(pkg string) (*GentooPackage, error) {
 			//      | <non-digit> <identifier characters>
 			//      | <identifier characters> <non-digit>
 			//      | <identifier characters> <non-digit> <identifier characters>
-			ans.VersionBuild = pkg[buildIdx+1:]
-			pkg = pkg[0:buildIdx]
+			ans.VersionBuild = pkgname[buildIdx+1:]
+			pkgname = pkgname[0:buildIdx]
 		}
 	}
 
-	words := strings.Split(pkg, "/")
-	if len(words) != 2 {
-		return nil, errors.New(fmt.Sprintf("Invalid package string %s", pkg))
-	}
-	ans.Category = words[0]
-	pkgname := words[1]
-
 	// Check if has repository
 	if strings.Contains(pkgname, "::") {
 		words = strings.Split(pkgname, "::")
@@ -415,11 +426,11 @@ func ParsePackageStr(pkg string) (*GentooPackage, error) {
 		pkgname = words[0]
 	}
 
-	regexPkg := regexp.MustCompile(
+	regexPkg = regexp.MustCompile(
 		fmt.Sprintf("%s$", regexVerString),
 	)
 
-	matches := regexPkg.FindAllString(pkgname, -1)
+	matches = regexPkg.FindAllString(pkgname, -1)
 
 	// NOTE: Now suffix comples like _alpha_rc1 are not supported.
 

vendor/modules.txt

@@ -25,7 +25,7 @@ github.com/Microsoft/hcsshim/internal/timeout
 github.com/Microsoft/hcsshim/internal/vmcompute
 github.com/Microsoft/hcsshim/internal/wclayer
 github.com/Microsoft/hcsshim/osversion
-# github.com/Sabayon/pkgs-checker v0.6.2-0.20200404093625-076438c31739
+# github.com/Sabayon/pkgs-checker v0.6.3-0.20200912135508-97c41780e9b6
 github.com/Sabayon/pkgs-checker/pkg/gentoo
 # github.com/apex/log v1.1.1
 github.com/apex/log