Tag 0.14.6 patchfix

Add checksum check to config protect files
Add test file

Fixes #214

cmd/root.go

@@ -40,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.14.1"
+	LuetCLIVersion = "0.14.6"
 	LuetEnvPrefix  = "LUET"
 )
 

go.mod

@@ -16,7 +16,6 @@ require (
 	github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible
 	github.com/docker/go-units v0.4.0
 	github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
-	github.com/fsouza/go-dockerclient v1.6.4
 	github.com/genuinetools/img v0.5.11
 	github.com/ghodss/yaml v1.0.0
 	github.com/google/go-containerregistry v0.2.1
@@ -37,7 +36,7 @@ require (
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d
 	github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
-	github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82
+	github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
 	github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3

go.sum

@@ -188,8 +188,6 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
 github.com/containerd/cgroups v0.0.0-20200217135630-d732e370d46d h1:UKAt78F1OvM4ceTn1VvXuYuatXohsFU1eSI2IBtTw9g=
@@ -760,8 +758,8 @@ github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d h1:fKh+rvw
 github.com/mudler/cobra-extensions v0.0.0-20200612154940-31a47105fe3d/go.mod h1:puRUWSwyecW2V355tKncwPVPRAjQBduPsFjG0mrV/Nw=
 github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87 h1:mGz7T8KvmHH0gLWPI5tQne8xl2cO3T8wrrb6Aa16Jxo=
 github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87/go.mod h1:1w4zI1LYXDeiUXqedPcrT5eQJnmKR6dbg5iJMgSIP/Y=
-github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82 h1:Hkefw2tzoKATVUTFsCtDlUnY180+OE851qGbq45ATxk=
-github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82/go.mod h1:4P/ULate+2QxoAQtojaRjyO5VGMhV0KLnSdAS8nuBbo=
+github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af h1:jixIxEgLSqu24eMiyzfCI+roa5IaOUhF546ePSFyHeY=
+github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af/go.mod h1:WmKcT8ONmhDQIqQ+HxU+tkGWjzBEyY/KFO8LTGCu4AI=
 github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290 h1:426hFyXMpXeqIeGJn2cGAW9ogvM2Jf+Jv23gtVPvBLM=
 github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290/go.mod h1:uP5BBgFxq2wNWo7n1vnY5SSbgL0WDshVJrOO12tZ/lA=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=

pkg/compiler/backend/simpledocker.go

@@ -25,7 +25,6 @@ import (
 
 	bus "github.com/mudler/luet/pkg/bus"
 
-	docker "github.com/fsouza/go-dockerclient"
 	capi "github.com/mudler/docker-companion/api"
 
 	"github.com/mudler/luet/pkg/helpers"
@@ -56,24 +55,6 @@ func (*SimpleDocker) BuildImage(opts Options) error {
 
 	Info(":whale: Building image " + name + " done")
 
-	if os.Getenv("DOCKER_SQUASH") == "true" {
-		Info(":whale: Squashing image " + name)
-		var client *docker.Client
-
-		Spinner(22)
-		defer SpinnerStop()
-		client, err = docker.NewClientFromEnv()
-		if err != nil {
-			return errors.Wrap(err, "could not connect to the Docker daemon")
-		}
-		err = capi.Squash(client, name, name)
-		if err != nil {
-			return errors.Wrap(err, "Failed squashing image")
-		}
-
-		Info(":whale: Squashing image " + name + " done")
-	}
-
 	bus.Manager.Publish(bus.EventImagePostBuild, opts)
 
 	return nil

pkg/compiler/types/artifact/artifact.go

@@ -19,6 +19,8 @@ import (
 	"archive/tar"
 	"bufio"
 	"bytes"
+	"crypto/sha1"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -27,7 +29,6 @@ import (
 	"path/filepath"
 	"regexp"
 
-	system "github.com/docker/docker/pkg/system"
 	zstd "github.com/klauspost/compress/zstd"
 	gzip "github.com/klauspost/pgzip"
 
@@ -317,7 +318,6 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
 	default:
 		return helpers.Tar(src, a.getCompressedName())
 	}
-	return errors.New("Compression type must be supplied")
 }
 
 func (a *PackageArtifact) getCompressedName() string {
@@ -340,6 +340,28 @@ func (a *PackageArtifact) GetUncompressedName() string {
 	return a.Path
 }
 
+func hashContent(bv []byte) string {
+	hasher := sha1.New()
+	hasher.Write(bv)
+	sha := base64.URLEncoding.EncodeToString(hasher.Sum(nil))
+	return sha
+}
+
+func hashFileContent(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	h := sha1.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+
+	return base64.URLEncoding.EncodeToString(h.Sum(nil)), nil
+}
+
 func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
 	// If the destination path already exists I rename target file name with postfix.
 	var destPath string
@@ -351,6 +373,7 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 			return nil, nil, err
 		}
 	}
+	tarHash := hashContent(buffer.Bytes())
 
 	// If file is not present on archive but is defined on mods
 	// I receive the callback. Prevent nil exception.
@@ -363,8 +386,21 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
 			return header, buffer.Bytes(), nil
 		}
 
+		existingHash := ""
+		f, err := os.Lstat(destPath)
+		if err == nil {
+			Debug("File exists already, computing hash for", destPath)
+			hash, herr := hashFileContent(destPath)
+			if herr == nil {
+				existingHash = hash
+			}
+		}
+
+		Debug("Existing file hash: ", existingHash, "Tar file hashsum: ", tarHash)
+		// We want to protect file only if the hash of the files are differing OR the file size are
+		differs := (existingHash != "" && existingHash != tarHash) || (err != nil && f != nil && header.Size != f.Size())
 		// Check if exists
-		if helpers.Exists(destPath) {
+		if helpers.Exists(destPath) && differs {
 			for i := 1; i < 1000; i++ {
 				name := filepath.Join(filepath.Join(filepath.Dir(path),
 					fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
@@ -584,47 +620,16 @@ type CopyJob struct {
 	Artifact string
 }
 
-func copyXattr(srcPath, dstPath, attr string) error {
-	data, err := system.Lgetxattr(srcPath, attr)
-	if err != nil {
-		return err
-	}
-	if data != nil {
-		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func doCopyXattrs(srcPath, dstPath string) error {
-	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
-		return err
-	}
-
-	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
-}
-
 func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 	defer wg.Done()
 
 	for job := range s {
-		//Info("#"+strconv.Itoa(i), "copying", job.Src, "to", job.Dst)
-		// if dir, err := helpers.IsDirectory(job.Src); err == nil && dir {
-		// 	err = helpers.CopyDir(job.Src, job.Dst)
-		// 	if err != nil {
-		// 		Warning("Error copying dir", job, err)
-		// 	}
-		// 	continue
-		// }
-
 		_, err := os.Lstat(job.Dst)
 		if err != nil {
 			Debug("Copying ", job.Src)
-			if err := helpers.CopyFile(job.Src, job.Dst); err != nil {
+			if err := helpers.DeepCopyFile(job.Src, job.Dst); err != nil {
 				Warning("Error copying", job, err)
 			}
-			doCopyXattrs(job.Src, job.Dst)
 		}
 	}
 }

pkg/helpers/archive.go

@@ -90,8 +90,7 @@ func UntarProtect(src, dst string, sameOwner bool, protectedFiles []string, modi
 	}
 
 	if sameOwner {
-		// PRE: i have root privileged.
-
+		// we do have root permissions, so we can extract keeping the same permissions.
 		replacerArchive := archive.ReplaceFileTarWrapper(in, mods)
 
 		opts := &archive.TarOptions{

pkg/helpers/file.go

@@ -27,6 +27,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/docker/docker/pkg/system"
 	"github.com/google/renameio"
 	copy "github.com/otiai10/copy"
 	"github.com/pkg/errors"
@@ -167,9 +168,27 @@ func Read(file string) (string, error) {
 	return string(dat), nil
 }
 
+func EnsureDirPerm(src, dst string) {
+	if info, err := os.Lstat(filepath.Dir(src)); err == nil {
+		if _, err := os.Lstat(filepath.Dir(dst)); os.IsNotExist(err) {
+			err := os.MkdirAll(filepath.Dir(dst), info.Mode().Perm())
+			if err != nil {
+				fmt.Println("warning: failed creating", filepath.Dir(dst), err.Error())
+			}
+			if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+				if err := os.Lchown(filepath.Dir(dst), int(stat.Uid), int(stat.Gid)); err != nil {
+					fmt.Println("warning: failed chowning", filepath.Dir(dst), err.Error())
+				}
+			}
+		}
+	} else {
+		EnsureDir(dst)
+	}
+}
+
 func EnsureDir(fileName string) error {
 	dirName := filepath.Dir(fileName)
-	if _, serr := os.Stat(dirName); serr != nil {
+	if _, serr := os.Stat(dirName); os.IsNotExist(serr) {
 		merr := os.MkdirAll(dirName, os.ModePerm) // FIXME: It should preserve permissions from src to dst instead
 		if merr != nil {
 			return merr
@@ -178,12 +197,39 @@ func EnsureDir(fileName string) error {
 	return nil
 }
 
-// CopyFile copies the contents of the file named src to the file named
+func CopyFile(src, dst string) (err error) {
+	return copy.Copy(src, dst, copy.Options{
+		Sync:      true,
+		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
+}
+
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
+
+// DeepCopyFile copies the contents of the file named src to the file named
 // by dst. The file will be created if it does not already exist. If the
 // destination file exists, all it's contents will be replaced by the contents
 // of the source file. The file mode will be copied from the source and
 // the copied data is synced/flushed to stable storage.
-func CopyFile(src, dst string) (err error) {
+func DeepCopyFile(src, dst string) (err error) {
 	// Workaround for https://github.com/otiai10/copy/issues/47
 	fi, err := os.Lstat(src)
 	if err != nil {
@@ -193,7 +239,7 @@ func CopyFile(src, dst string) (err error) {
 	fm := fi.Mode()
 	switch {
 	case fm&os.ModeNamedPipe != 0:
-		EnsureDir(dst)
+		EnsureDirPerm(src, dst)
 		if err := syscall.Mkfifo(dst, uint32(fi.Mode())); err != nil {
 			return errors.Wrap(err, "failed creating pipe")
 		}
@@ -205,6 +251,9 @@ func CopyFile(src, dst string) (err error) {
 		return nil
 	}
 
+	//filepath.Dir(src)
+	EnsureDirPerm(src, dst)
+
 	err = copy.Copy(src, dst, copy.Options{
 		Sync:      true,
 		OnSymlink: func(string) copy.SymlinkAction { return copy.Shallow }})
@@ -216,7 +265,8 @@ func CopyFile(src, dst string) (err error) {
 			fmt.Println("warning: failed chowning", dst, err.Error())
 		}
 	}
-	return err
+
+	return doCopyXattrs(src, dst)
 }
 
 func IsDirectory(path string) (bool, error) {

tests/fixtures/perms/pkgA/0.1/build.yaml

@@ -0,0 +1,11 @@
+image: "alpine"
+unpack: true
+includes:
+  - /foo
+  - /foo/bar
+  - /foo/bar/.keep
+steps:
+- mkdir -p /foo/bar
+- touch /foo/bar/.keep
+- chown 100:100 /foo/bar
+- chown 101:101 /foo/bar/.keep

tests/fixtures/perms/pkgA/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "perms"
+version: "0.1"

tests/fixtures/perms/pkgB/0.1/build.yaml

@@ -0,0 +1,7 @@
+image: "alpine"
+
+steps:
+- mkdir -p /foo/baz
+- touch /foo/baz/.keep
+- chown 100:100 /foo/baz
+- chown 101:101 /foo/baz/.keep

tests/fixtures/perms/pkgB/0.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "perms2"
+version: "0.1"

tests/integration/12_config_protect_samefile.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testrootfs/testbuild -p
+    luet build --tree "$ROOT_DIR/tests/fixtures/config_protect" \
+      --destination $tmpdir/testrootfs/testbuild --compression gzip test/a
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package' "[ -e '$tmpdir/testrootfs/testbuild/a-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/config_protect" \
+    --output $tmpdir/testrootfs/testbuild \
+    --packages $tmpdir/testrootfs/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testrootfs/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+
+    mkdir $tmpdir/testrootfs/etc/luet/config.protect.d -p
+
+    cat <<EOF > $tmpdir/testrootfs/etc/luet/config.protect.d/conf1.yml
+name: "protect1"
+dirs:
+- /etc/
+EOF
+
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_protect_confdir:
+    - /etc/luet/config.protect.d
+config_from_host: false
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+
+
+testInstall() {
+
+    # Simulate previous installation
+    mkdir $tmpdir/testrootfs/etc/a -p
+    echo config > $tmpdir/testrootfs/etc/a/conf
+
+    luet install -y --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+
+
+    # Simulate config protect
+    assertTrue 'package A installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'config protect not created, file is the same' "[ ! -e '$tmpdir/testrootfs/etc/a/._cfg0001_conf' ]"
+    assertEquals 'config protect content' "$(cat $tmpdir/testrootfs/etc/a/conf)" "config"
+}
+
+
+testUnInstall() {
+    luet uninstall -y --full --config $tmpdir/luet.yaml test/a
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'config protect maintains the protected files' "[ -e '$tmpdir/testrootfs/etc/a/conf' ]"
+}
+
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/a-test-1.0.package.tar.gz' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

tests/integration/16_perms.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testbuild
+    luet build -d --tree "$ROOT_DIR/tests/fixtures/perms" --same-owner=true --destination $tmpdir/testbuild --compression gzip --full
+    buildst=$?
+    assertTrue 'create package perms 0.1' "[ -e '$tmpdir/testbuild/perms-test-0.1.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/perms" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    $ROOT_DIR/tests/integration/bin/luet install -y --config $tmpdir/luet.yaml test/perms@0.1 test/perms2@0.1
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+   
+    assertTrue 'package installed perms baz' "[ -d '$tmpdir/testrootfs/foo/baz' ]"
+    assertTrue 'package installed perms bar' "[ -d '$tmpdir/testrootfs/foo/bar' ]"
+
+    assertContains 'perms1' "$(stat -c %u:%g $tmpdir/testrootfs/foo/baz)" "100:100"
+    assertContains 'perms2' "$(stat -c %u:%g $tmpdir/testrootfs/foo/bar)" "100:100"
+    assertContains 'perms11' "$(stat -c %u:%g $tmpdir/testrootfs/foo/baz/.keep)" "101:101"
+    assertContains 'perms22' "$(stat -c %u:%g $tmpdir/testrootfs/foo/bar/.keep)" "101:101"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+

vendor/github.com/codegangsta/inject/.gitignore

@@ -1,2 +0,0 @@
-inject
-inject.test

vendor/github.com/codegangsta/inject/LICENSE

@@ -1,20 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2013 Jeremy Saenz
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/codegangsta/inject/README.md

@@ -1,92 +0,0 @@
-# inject
---
-    import "github.com/codegangsta/inject"
-
-Package inject provides utilities for mapping and injecting dependencies in
-various ways.
-
-Language Translations:
-* [简体中文](translations/README_zh_cn.md)
-
-## Usage
-
-#### func  InterfaceOf
-
-```go
-func InterfaceOf(value interface{}) reflect.Type
-```
-InterfaceOf dereferences a pointer to an Interface type. It panics if value is
-not an pointer to an interface.
-
-#### type Applicator
-
-```go
-type Applicator interface {
-	// Maps dependencies in the Type map to each field in the struct
-	// that is tagged with 'inject'. Returns an error if the injection
-	// fails.
-	Apply(interface{}) error
-}
-```
-
-Applicator represents an interface for mapping dependencies to a struct.
-
-#### type Injector
-
-```go
-type Injector interface {
-	Applicator
-	Invoker
-	TypeMapper
-	// SetParent sets the parent of the injector. If the injector cannot find a
-	// dependency in its Type map it will check its parent before returning an
-	// error.
-	SetParent(Injector)
-}
-```
-
-Injector represents an interface for mapping and injecting dependencies into
-structs and function arguments.
-
-#### func  New
-
-```go
-func New() Injector
-```
-New returns a new Injector.
-
-#### type Invoker
-
-```go
-type Invoker interface {
-	// Invoke attempts to call the interface{} provided as a function,
-	// providing dependencies for function arguments based on Type. Returns
-	// a slice of reflect.Value representing the returned values of the function.
-	// Returns an error if the injection fails.
-	Invoke(interface{}) ([]reflect.Value, error)
-}
-```
-
-Invoker represents an interface for calling functions via reflection.
-
-#### type TypeMapper
-
-```go
-type TypeMapper interface {
-	// Maps the interface{} value based on its immediate type from reflect.TypeOf.
-	Map(interface{}) TypeMapper
-	// Maps the interface{} value based on the pointer of an Interface provided.
-	// This is really only useful for mapping a value as an interface, as interfaces
-	// cannot at this time be referenced directly without a pointer.
-	MapTo(interface{}, interface{}) TypeMapper
-	// Provides a possibility to directly insert a mapping based on type and value.
-	// This makes it possible to directly map type arguments not possible to instantiate
-	// with reflect like unidirectional channels.
-	Set(reflect.Type, reflect.Value) TypeMapper
-	// Returns the Value that is mapped to the current type. Returns a zeroed Value if
-	// the Type has not been mapped.
-	Get(reflect.Type) reflect.Value
-}
-```
-
-TypeMapper represents an interface for mapping interface{} values based on type.

vendor/github.com/codegangsta/inject/inject.go

@@ -1,187 +0,0 @@
-// Package inject provides utilities for mapping and injecting dependencies in various ways.
-package inject
-
-import (
-	"fmt"
-	"reflect"
-)
-
-// Injector represents an interface for mapping and injecting dependencies into structs
-// and function arguments.
-type Injector interface {
-	Applicator
-	Invoker
-	TypeMapper
-	// SetParent sets the parent of the injector. If the injector cannot find a
-	// dependency in its Type map it will check its parent before returning an
-	// error.
-	SetParent(Injector)
-}
-
-// Applicator represents an interface for mapping dependencies to a struct.
-type Applicator interface {
-	// Maps dependencies in the Type map to each field in the struct
-	// that is tagged with 'inject'. Returns an error if the injection
-	// fails.
-	Apply(interface{}) error
-}
-
-// Invoker represents an interface for calling functions via reflection.
-type Invoker interface {
-	// Invoke attempts to call the interface{} provided as a function,
-	// providing dependencies for function arguments based on Type. Returns
-	// a slice of reflect.Value representing the returned values of the function.
-	// Returns an error if the injection fails.
-	Invoke(interface{}) ([]reflect.Value, error)
-}
-
-// TypeMapper represents an interface for mapping interface{} values based on type.
-type TypeMapper interface {
-	// Maps the interface{} value based on its immediate type from reflect.TypeOf.
-	Map(interface{}) TypeMapper
-	// Maps the interface{} value based on the pointer of an Interface provided.
-	// This is really only useful for mapping a value as an interface, as interfaces
-	// cannot at this time be referenced directly without a pointer.
-	MapTo(interface{}, interface{}) TypeMapper
-	// Provides a possibility to directly insert a mapping based on type and value.
-	// This makes it possible to directly map type arguments not possible to instantiate
-	// with reflect like unidirectional channels.
-	Set(reflect.Type, reflect.Value) TypeMapper
-	// Returns the Value that is mapped to the current type. Returns a zeroed Value if
-	// the Type has not been mapped.
-	Get(reflect.Type) reflect.Value
-}
-
-type injector struct {
-	values map[reflect.Type]reflect.Value
-	parent Injector
-}
-
-// InterfaceOf dereferences a pointer to an Interface type.
-// It panics if value is not an pointer to an interface.
-func InterfaceOf(value interface{}) reflect.Type {
-	t := reflect.TypeOf(value)
-
-	for t.Kind() == reflect.Ptr {
-		t = t.Elem()
-	}
-
-	if t.Kind() != reflect.Interface {
-		panic("Called inject.InterfaceOf with a value that is not a pointer to an interface. (*MyInterface)(nil)")
-	}
-
-	return t
-}
-
-// New returns a new Injector.
-func New() Injector {
-	return &injector{
-		values: make(map[reflect.Type]reflect.Value),
-	}
-}
-
-// Invoke attempts to call the interface{} provided as a function,
-// providing dependencies for function arguments based on Type.
-// Returns a slice of reflect.Value representing the returned values of the function.
-// Returns an error if the injection fails.
-// It panics if f is not a function
-func (inj *injector) Invoke(f interface{}) ([]reflect.Value, error) {
-	t := reflect.TypeOf(f)
-
-	var in = make([]reflect.Value, t.NumIn()) //Panic if t is not kind of Func
-	for i := 0; i < t.NumIn(); i++ {
-		argType := t.In(i)
-		val := inj.Get(argType)
-		if !val.IsValid() {
-			return nil, fmt.Errorf("Value not found for type %v", argType)
-		}
-
-		in[i] = val
-	}
-
-	return reflect.ValueOf(f).Call(in), nil
-}
-
-// Maps dependencies in the Type map to each field in the struct
-// that is tagged with 'inject'.
-// Returns an error if the injection fails.
-func (inj *injector) Apply(val interface{}) error {
-	v := reflect.ValueOf(val)
-
-	for v.Kind() == reflect.Ptr {
-		v = v.Elem()
-	}
-
-	if v.Kind() != reflect.Struct {
-		return nil // Should not panic here ?
-	}
-
-	t := v.Type()
-
-	for i := 0; i < v.NumField(); i++ {
-		f := v.Field(i)
-		structField := t.Field(i)
-		if f.CanSet() && (structField.Tag == "inject" || structField.Tag.Get("inject") != "") {
-			ft := f.Type()
-			v := inj.Get(ft)
-			if !v.IsValid() {
-				return fmt.Errorf("Value not found for type %v", ft)
-			}
-
-			f.Set(v)
-		}
-
-	}
-
-	return nil
-}
-
-// Maps the concrete value of val to its dynamic type using reflect.TypeOf,
-// It returns the TypeMapper registered in.
-func (i *injector) Map(val interface{}) TypeMapper {
-	i.values[reflect.TypeOf(val)] = reflect.ValueOf(val)
-	return i
-}
-
-func (i *injector) MapTo(val interface{}, ifacePtr interface{}) TypeMapper {
-	i.values[InterfaceOf(ifacePtr)] = reflect.ValueOf(val)
-	return i
-}
-
-// Maps the given reflect.Type to the given reflect.Value and returns
-// the Typemapper the mapping has been registered in.
-func (i *injector) Set(typ reflect.Type, val reflect.Value) TypeMapper {
-	i.values[typ] = val
-	return i
-}
-
-func (i *injector) Get(t reflect.Type) reflect.Value {
-	val := i.values[t]
-
-	if val.IsValid() {
-		return val
-	}
-
-	// no concrete types found, try to find implementors
-	// if t is an interface
-	if t.Kind() == reflect.Interface {
-		for k, v := range i.values {
-			if k.Implements(t) {
-				val = v
-				break
-			}
-		}
-	}
-
-	// Still no type found, try to look it up on the parent
-	if !val.IsValid() && i.parent != nil {
-		val = i.parent.Get(t)
-	}
-
-	return val
-
-}
-
-func (i *injector) SetParent(parent Injector) {
-	i.parent = parent
-}

vendor/github.com/codegangsta/inject/update_readme.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-go get github.com/robertkrimen/godocdown/godocdown
-godocdown > README.md

vendor/github.com/mudler/go-pluggable/README.md

@@ -1,6 +1,7 @@
-# go-pluggable
+# :bento: go-pluggable
+[![PkgGoDev](https://pkg.go.dev/badge/github.com/mudler/go-pluggable)](https://pkg.go.dev/github.com/mudler/go-pluggable) [![Go Report Card](https://goreportcard.com/badge/github.com/mudler/go-pluggable)](https://goreportcard.com/report/github.com/mudler/go-pluggable) [![Test](https://github.com/mudler/go-pluggable/workflows/Test/badge.svg)](https://github.com/mudler/go-pluggable/actions?query=workflow%3ATest)
 
-light Bus-event driven plugin library for Golang.
+:bento: *go-pluggable* is a light Bus-event driven plugin library for Golang.
 
 `go-pluggable` implements the event/sub pattern to extend your Golang project with external binary plugins that can be written in any language.
 
@@ -24,8 +25,24 @@ func main() {
     m.Autoload("test", temp)
     m.Register()
 
-    // ...
-    m.Publish(myEv,  map[string]string{"foo": "bar"}) // test-foo, will receive our data as json payload
+    // Optionally process plugin results response
+    // The plugins has to return as output a json in stdout in the format { 'state': "somestate", data: "some data", error: "some error" }
+    // e.g. with jq:  
+    // jq --arg key0   'state' \
+    // --arg value0 '' \
+    // --arg key1   'data' \
+    // --arg value1 "" \
+    // --arg key2   'error' \
+    // --arg value2 '' \
+    // '. | .[$key0]=$value0 | .[$key1]=$value1 | .[$key2]=$value2' \
+    // <<<'{}'
+    m.Response(myEv, func(p *pluggable.Plugin, r *pluggable.EventResponse) { ... }) 
+
+    // Emit events, they are encoded and passed as JSON payloads to the plugins.
+    // In our case, test-foo will receive the map as JSON
+    m.Publish(myEv,  map[string]string{"foo": "bar"})
+
+
 }
 
-```
+```

vendor/github.com/mudler/go-pluggable/bus.go

@@ -1,73 +0,0 @@
-// Copyright © 2020 Ettore Di Giacinto <mudler@mocaccino.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package pluggable
-
-import (
-	"sync"
-
-	"github.com/chuckpreslar/emission"
-	"github.com/codegangsta/inject"
-)
-
-// Bus represent the bus event system
-type Bus struct {
-	inject.Injector
-	emission.Emitter
-	sync.Mutex
-}
-
-// NewBus returns a new Bus instance
-func NewBus() *Bus {
-	return &Bus{
-		Injector: inject.New(),
-		Emitter:  *emission.NewEmitter(),
-	}
-}
-
-// Listen Binds a callback to an event, mapping the arguments on a global level
-func (a *Bus) Listen(event EventType, listener interface{}) *Bus {
-	a.Lock()
-	defer a.Unlock()
-	a.On(string(event), func() { a.Invoke(listener) })
-	return a
-}
-
-// Publish publishes an event, it does accept only the event as argument, since
-// the callback will have access to the service mapped by the injector
-func (a *Bus) Publish(e *Event) *Bus {
-	a.Lock()
-	defer a.Unlock()
-	a.Map(e)
-	a.Emit(string(e.Name))
-	return a
-}
-
-// OnlyOnce Binds a callback to an event, mapping the arguments on a global level
-// It is fired only once.
-func (a *Bus) OnlyOnce(event EventType, listener interface{}) *Bus {
-	a.Lock()
-	defer a.Unlock()
-	a.Once(string(event), func() { a.Invoke(listener) })
-	return a
-}
-
-func (a *Bus) propagateEvent(p Plugin) func(e *Event) {
-	return func(e *Event) {
-		resp, _ := p.Run(*e)
-		a.Map(&resp)
-		a.Emit(p.Name)
-	}
-}

vendor/github.com/mudler/go-pluggable/events.go

@@ -15,7 +15,10 @@
 
 package pluggable
 
-import "encoding/json"
+import (
+	"encoding/json"
+	"fmt"
+)
 
 // EventType describes an event type
 type EventType string
@@ -26,6 +29,7 @@ type EventType string
 type Event struct {
 	Name EventType `json:"name"`
 	Data string    `json:"data"`
+	File string    `json:"file"` // If Data >> 10K write content to file instead
 }
 
 // EventResponse describes the event response structure
@@ -42,6 +46,16 @@ func (e Event) JSON() (string, error) {
 	return string(dat), err
 }
 
+// Copy returns a copy of Event
+func (e Event) Copy() *Event {
+	copy := &e
+	return copy
+}
+
+func (e Event) ResponseEventName(s string) EventType {
+	return EventType(fmt.Sprintf("%s-%s", e.Name, s))
+}
+
 // Unmarshal decodes the json payload in the given parameteer
 func (r EventResponse) Unmarshal(i interface{}) error {
 	return json.Unmarshal([]byte(r.Data), i)
@@ -52,7 +66,7 @@ func (r EventResponse) Errored() bool {
 	return len(r.Error) != 0
 }
 
-// NewEvent retuns a new event which can be used for publishing
+// NewEvent returns a new event which can be used for publishing
 // the obj gets automatically serialized in json.
 func NewEvent(name EventType, obj interface{}) (*Event, error) {
 	dat, err := json.Marshal(obj)

vendor/github.com/mudler/go-pluggable/go.mod

@@ -4,7 +4,6 @@ go 1.14
 
 require (
 	github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9
-	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3
 	github.com/pkg/errors v0.9.1

vendor/github.com/mudler/go-pluggable/go.sum

@@ -1,7 +1,5 @@
 github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9 h1:xz6Nv3zcwO2Lila35hcb0QloCQsc38Al13RNEzWRpX4=
 github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9/go.mod h1:2wSM9zJkl1UQEFZgSd68NfCgRz1VL1jzy/RjCg+ULrs=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
-github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=

vendor/github.com/mudler/go-pluggable/manager.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/chuckpreslar/emission"
 	"github.com/pkg/errors"
 )
 
@@ -29,14 +30,14 @@ import (
 type Manager struct {
 	Plugins []Plugin
 	Events  []EventType
-	Bus     *Bus
+	Bus     *emission.Emitter
 }
 
 // NewManager returns a manager instance with a new bus and
 func NewManager(events []EventType) *Manager {
 	return &Manager{
 		Events: events,
-		Bus:    NewBus(),
+		Bus:    emission.NewEmitter(),
 	}
 }
 
@@ -47,19 +48,41 @@ func (m *Manager) Register() *Manager {
 }
 
 // Publish is a wrapper around NewEvent and the Manager internal Bus publishing system
+// It accepts optionally a list of functions that are called with the plugin result (only once)
 func (m *Manager) Publish(event EventType, obj interface{}) (*Manager, error) {
 	ev, err := NewEvent(event, obj)
 	if err == nil && ev != nil {
-		m.Bus.Publish(ev)
+		m.Bus.Emit(string(ev.Name), ev)
 	}
 	return m, err
 }
 
+// Response binds a set of listeners to an event type. The listeners are called for each result from
+// every plugin when Publish is called.
+func (m *Manager) Response(event EventType, listener ...func(p *Plugin, r *EventResponse)) *Manager {
+	ev, _ := NewEvent(event, nil)
+	for _, l := range listener {
+		m.Bus.On(string(ev.ResponseEventName("results")), l)
+	}
+	return m
+}
+
+func (m *Manager) propagateEvent(p Plugin) func(e *Event) {
+	return func(e *Event) {
+		resp, err := p.Run(*e)
+		r := &resp
+		if err != nil && !resp.Errored() {
+			resp.Error = err.Error()
+		}
+		m.Bus.Emit(string(e.ResponseEventName("results")), &p, r)
+	}
+}
+
 // Subscribe subscribes the plugin to the events in the given bus
-func (m *Manager) Subscribe(b *Bus) *Manager {
+func (m *Manager) Subscribe(b *emission.Emitter) *Manager {
 	for _, p := range m.Plugins {
 		for _, e := range m.Events {
-			b.Listen(e, b.propagateEvent(p))
+			b.On(string(e), m.propagateEvent(p))
 		}
 	}
 	return m
@@ -74,13 +97,6 @@ func relativeToCwd(p string) (string, error) {
 	return filepath.Join(cwd, p), nil
 }
 
-// ListenAll Binds a callback to all plugins event
-func (m *Manager) ListenAll(event EventType, listener interface{}) {
-	for _, p := range m.Plugins {
-		m.Bus.Listen(EventType(p.Name), listener)
-	}
-}
-
 // Autoload automatically loads plugins binaries prefixed by 'prefix' in the current path
 // optionally takes a list of paths to look also into
 func (m *Manager) Autoload(prefix string, extensionpath ...string) *Manager {

vendor/github.com/mudler/go-pluggable/plugin.go

@@ -16,7 +16,9 @@
 package pluggable
 
 import (
+	"bytes"
 	"encoding/json"
+	"io/ioutil"
 	"os"
 	"os/exec"
 
@@ -29,19 +31,42 @@ type Plugin struct {
 	Executable string
 }
 
+// A safe threshold to avoid unpleasant exec buffer fill for argv too big. Seems 128K is the limit on Linux.
+const maxMessageSize = 1 << 13
+
 // Run runs the Event on the plugin, and returns an EventResponse
 func (p Plugin) Run(e Event) (EventResponse, error) {
 	r := EventResponse{}
-	k, err := e.JSON()
+
+	eventToprocess := &e
+
+	if len(e.Data) > maxMessageSize {
+		copy := e.Copy()
+		copy.Data = ""
+		f, err := ioutil.TempFile(os.TempDir(), "pluggable")
+		if err != nil {
+			return r, errors.Wrap(err, "while creating temporary file")
+		}
+		if err := ioutil.WriteFile(f.Name(), []byte(e.Data), os.ModePerm); err != nil {
+			return r, errors.Wrap(err, "while writing to temporary file")
+		}
+		copy.File = f.Name()
+		eventToprocess = copy
+		defer os.RemoveAll(f.Name())
+	}
+
+	k, err := eventToprocess.JSON()
 	if err != nil {
 		return r, errors.Wrap(err, "while marshalling event")
 	}
 	cmd := exec.Command(p.Executable, string(e.Name), k)
 	cmd.Env = os.Environ()
+	var b bytes.Buffer
+	cmd.Stderr = &b
 	out, err := cmd.Output()
 	if err != nil {
-		r.Error = err.Error()
-		return r, errors.Wrap(err, "while executing plugin")
+		r.Error = "error while executing plugin: " + err.Error() + string(b.String())
+		return r, errors.Wrap(err, "while executing plugin: "+string(b.String()))
 	}
 
 	if err := json.Unmarshal(out, &r); err != nil {

vendor/modules.txt

@@ -62,8 +62,6 @@ github.com/cavaliercoder/grab/bps
 github.com/cespare/xxhash/v2
 # github.com/chuckpreslar/emission v0.0.0-20170206194824-a7ddd980baf9
 github.com/chuckpreslar/emission
-# github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0
-github.com/codegangsta/inject
 # github.com/containerd/cgroups v0.0.0-20200217135630-d732e370d46d
 github.com/containerd/cgroups/stats/v1
 # github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc => github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55
@@ -209,7 +207,6 @@ github.com/fatih/color
 # github.com/fsnotify/fsnotify v1.4.9
 github.com/fsnotify/fsnotify
 # github.com/fsouza/go-dockerclient v1.6.4
-## explicit
 github.com/fsouza/go-dockerclient
 # github.com/genuinetools/img v0.5.11
 ## explicit
@@ -478,7 +475,7 @@ github.com/mudler/cobra-extensions
 # github.com/mudler/docker-companion v0.4.6-0.20200418093252-41846f112d87
 ## explicit
 github.com/mudler/docker-companion/api
-# github.com/mudler/go-pluggable v0.0.0-20201113184918-d36448fc8f82
+# github.com/mudler/go-pluggable v0.0.0-20210513155700-54c6443073af
 ## explicit
 github.com/mudler/go-pluggable
 # github.com/mudler/topsort v0.0.0-20201103161459-db5c7901c290