Tag 0.14.6 patchfix

Add checksum check to config protect files
Add test file

Fixes #214

cmd/root.go

@@ -40,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.14.3"
+	LuetCLIVersion = "0.14.6"
 	LuetEnvPrefix  = "LUET"
 )
 

go.mod

@@ -36,7 +36,7 @@ require (
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
 	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
-	github.com/mudler/go-pluggable v0.0.0-20210510180427-ba09243a8c65
+	github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
 	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3

go.sum

@@ -758,8 +758,8 @@ github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d h1:fKh+rvw
 github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d/go.mod h1:puRUWSwyecW2V355tKncwPVPRAjQBduPsFjG0mrV/Nw=
 github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87 h1:mGz7T8KvmHH0gLWPI5tQne8xl2cO3T8wrrb6Aa16Jxo=
 github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87/go.mod h1:1w4zI1LYXDeiUXqedPcrT5eQJnmKR6dbg5iJMgSIP/Y=
-github.com/mudler/go-pluggable v0.0.0-20210510180427-ba09243a8c65 h1:Xmfr2g3QU/Ci1mvLswuOhrzXnJ5OXGqAk/skUD1aOsY=
-github.com/mudler/go-pluggable v0.0.0-20210510180427-ba09243a8c65/go.mod h1:WmKcT8ONmhDQIqQ+HxU+tkGWjzBEyY/KFO8LTGCu4AI=
+github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af h1:jixIxEgLSqu24eMiyzfCI+roa5IaOUhF546ePSFyHeY=
+github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af/go.mod h1:WmKcT8ONmhDQIqQ+HxU+tkGWjzBEyY/KFO8LTGCu4AI=
 github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290 h1:426hFyXMpXeqIeGJn2cGAW9ogvM2Jf+Jv23gtVPvBLM=
 github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290/go.mod h1:uP5BBgFxq2wNWo7n1vnY5SSbgL0WDshVJrOO12tZ/lA=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=

pkg/compiler/types/artifact/artifact.go

@@ -19,6 +19,8 @@ import (
 	"archive/tar"
 	"bufio"
 	"bytes"
+	"crypto/sha1"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -316,7 +318,6 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	default:
 		return helpers.Tar(src, a.getCompressedName())
 	}
-	return errors.New("Compression type must be supplied")
 }
 
 func (a *PackageArtifact) getCompressedName() string {
@@ -339,6 +340,28 @@ func (a *PackageArtifact) GetUncompressedName() string {
 	return a.Path
 }
 
+func hashContent(bv []byte) string {
+	hasher := sha1.New()
+	hasher.Write(bv)
+	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
+	return sha
+}
+
+func hashFileContent(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	h := sha1.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+
+	return base64.URLEncoding.EncodeToString(h.Sum(nil)), nil
+}
+
 func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
 	// If the destination path already exists I rename target file name with postfix.
 	var destPath string
@@ -350,6 +373,7 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 			return nil, nil, err
 		}
 	}
+	tarHash := hashContent(buffer.Bytes())
 
 	// If file is not present on archive but is defined on mods
 	// I receive the callback. Prevent nil exception.
@@ -362,8 +386,21 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 			return header, buffer.Bytes(), nil
 		}
 
+		existingHash := ""
+		f, err := os.Lstat(destPath)
+		if err == nil {
+			Debug("File exists already, computing hash for", destPath)
+			hash, herr := hashFileContent(destPath)
+			if herr == nil {
+				existingHash = hash
+			}
+		}
+
+		Debug("Existing file hash: ", existingHash, "Tar file hashsum: ", tarHash)
+		// We want to protect file only if the hash of the files are differing OR the file size are
+		differs := (existingHash != "" && existingHash != tarHash) || (err != nil && f != nil && header.Size != f.Size())
 		// Check if exists
-		if helpers.Exists(destPath) {
+		if helpers.Exists(destPath) && differs {
 			for i := 1; i < 1000; i++ {
 				name := filepath.Join(filepath.Join(filepath.Dir(path),
 					fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))

pkg/helpers/archive.go

@@ -90,8 +90,7 @@ func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modi
 	}
 
 	if sameOwner {
-		// PRE: i have root privileged.
-
+		// we do have root permissions, so we can extract keeping the same permissions.
 		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
 
 		opts := &archive.TarOptions{

tests/integration/12_config_protect_samefile.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testrootfs/testbuild -p
+    luet build --tree "$ROOT_DIR/tests/fixtures/config_protect" \
+      --destination $tmpdir/testrootfs/testbuild --compression gzip test/a
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package' "[ -e '$tmpdir/testrootfs/testbuild/a-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/config_protect" \
+    --output $tmpdir/testrootfs/testbuild \
+    --packages $tmpdir/testrootfs/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testrootfs/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+
+    mkdir $tmpdir/testrootfs/etc/luet/config.protect.d -p
+
+    cat <<EOF > $tmpdir/testrootfs/etc/luet/config.protect.d/conf1.yml
+name: "protect1"
+dirs:
+- /etc/
+EOF
+
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_protect_confdir:
+    - /etc/luet/config.protect.d
+config_from_host: false
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+
+
+testInstall() {
+
+    # Simulate previous installation
+    mkdir $tmpdir/testrootfs/etc/a -p
+    echo config > $tmpdir/testrootfs/etc/a/conf
+
+    luet install -y --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+
+    # Simulate config protect
+    assertTrue 'package A installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'config protect not created, file is the same' "[ ! -e '$tmpdir/testrootfs/etc/a/._cfg0001_conf' ]"
+    assertEquals 'config protect content' "$(cat $tmpdir/testrootfs/etc/a/conf)" "config"
+}
+
+
+testUnInstall() {
+    luet uninstall -y --full --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'config protect maintains the protected files' "[ -e '$tmpdir/testrootfs/etc/a/conf' ]"
+}
+
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/a-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

vendor/github.com/mudler/go-pluggable/events.go

@@ -29,6 +29,7 @@ type EventType string
 type Event struct {
 	Name EventType `json:"name"`
 	Data string    `json:"data"`
+	File string    `json:"file"` // If Data >> 10K write content to file instead
 }
 
 // EventResponse describes the event response structure
@@ -45,6 +46,12 @@ func (e Event) JSON() (string, error) {
 	return string(dat), err
 }
 
+// Copy returns a copy of Event
+func (e Event) Copy() *Event {
+	copy := &e
+	return copy
+}
+
 func (e Event) ResponseEventName(s string) EventType {
 	return EventType(fmt.Sprintf("%s-%s", e.Name, s))
 }

vendor/github.com/mudler/go-pluggable/plugin.go

@@ -16,7 +16,9 @@
 package pluggable
 
 import (
+	"bytes"
 	"encoding/json"
+	"io/ioutil"
 	"os"
 	"os/exec"
 
@@ -29,19 +31,42 @@ type Plugin struct {
 	Executable string
 }
 
+// A safe threshold to avoid unpleasant exec buffer fill for argv too big. Seems 128K is the limit on Linux.
+const maxMessageSize = 1 << 13
+
 // Run runs the Event on the plugin, and returns an EventResponse
 func (p Plugin) Run(e Event) (EventResponse, error) {
 	r := EventResponse{}
-	k, err := e.JSON()
+
+	eventToprocess := &e
+
+	if len(e.Data) > maxMessageSize {
+		copy := e.Copy()
+		copy.Data = ""
+		f, err := ioutil.TempFile(os.TempDir(), "pluggable")
+		if err != nil {
+			return r, errors.Wrap(err, "while creating temporary file")
+		}
+		if err := ioutil.WriteFile(f.Name(), []byte(e.Data), os.ModePerm); err != nil {
+			return r, errors.Wrap(err, "while writing to temporary file")
+		}
+		copy.File = f.Name()
+		eventToprocess = copy
+		defer os.RemoveAll(f.Name())
+	}
+
+	k, err := eventToprocess.JSON()
 	if err != nil {
 		return r, errors.Wrap(err, "while marshalling event")
 	}
 	cmd := exec.Command(p.Executable, string(e.Name), k)
 	cmd.Env = os.Environ()
+	var b bytes.Buffer
+	cmd.Stderr = &b
 	out, err := cmd.Output()
 	if err != nil {
-		r.Error = err.Error()
-		return r, errors.Wrap(err, "while executing plugin")
+		r.Error = "error while executing plugin: " + err.Error() + string(b.String())
+		return r, errors.Wrap(err, "while executing plugin: "+string(b.String()))
 	}
 
 	if err := json.Unmarshal(out, &r); err != nil {

vendor/modules.txt

@@ -475,7 +475,7 @@ github.com/mudler/cobra-extensions
 # github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
 ## explicit
 github.com/mudler/docker-companion/api
-# github.com/mudler/go-pluggable v0.0.0-20210510180427-ba09243a8c65
+# github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
 ## explicit
 github.com/mudler/go-pluggable
 # github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290