Merge pull request #103 from mudler/tmpdir-cleanup

Tmpdir cleanup

cmd/root.go

@@ -61,6 +61,20 @@ var RootCmd = &cobra.Command{
 		if err != nil {
 			Fatal("failed to load configuration:", err.Error())
 		}
+		// Initialize tmpdir prefix. TODO: Move this with LoadConfig
+		// directly on sub command to ensure the creation only when it's
+		// needed.
+		err = config.LuetCfg.GetSystem().InitTmpDir()
+		if err != nil {
+			Fatal("failed on init tmp basedir:", err.Error())
+		}
+	},
+	PersistentPostRun: func(cmd *cobra.Command, args []string) {
+		// Cleanup all tmp directories used by luet
+		err := config.LuetCfg.GetSystem().CleanupTmpDir()
+		if err != nil {
+			Warning("failed on cleanup tmpdir:", err.Error())
+		}
 	},
 	SilenceErrors: true,
 }

contrib/config/luet.yaml

@@ -56,6 +56,10 @@
 #   The path is append to rootfs option path.
 #   database_path: "/var/cache/luet"
 #
+#   Define the tmpdir base directory where luet store temporary files.
+#   Default $TMPDIR/tmpluet
+#   tmpdir_base: "/tmp/tmpluet"
+#
 # ---------------------------------------------
 # Repositories configurations directories.
 # ---------------------------------------------

pkg/compiler/artifact.go

@@ -410,14 +410,14 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
 // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
 func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, t CompressionImplementation) (Artifact, error) {
 
-	archive, err := ioutil.TempDir(os.TempDir(), "archive")
+	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
 		return nil, errors.Wrap(err, "Error met while creating tempdir for archive")
 	}
 	defer os.RemoveAll(archive) // clean up
 
 	if strings.HasSuffix(src, ".tar") {
-		rootfs, err := ioutil.TempDir(os.TempDir(), "rootfs")
+		rootfs, err := LuetCfg.GetSystem().TempDir("rootfs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 		}

pkg/compiler/backend/simpledocker.go

@@ -237,7 +237,7 @@ func (*SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPer
 // ]
 // Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
 func (*SimpleDocker) Changes(fromImage, toImage string) ([]compiler.ArtifactLayer, error) {
-	tmpdiffs, err := ioutil.TempDir(os.TempDir(), "tmpdiffs")
+	tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("tmpdiffs")
 	if err != nil {
 		return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 	}

pkg/config/config.go

@@ -27,7 +27,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/mudler/luet/pkg/helpers"
 	solver "github.com/mudler/luet/pkg/solver"
+
 	v "github.com/spf13/viper"
 )
 
@@ -80,6 +82,7 @@ type LuetSystemConfig struct {
 	DatabasePath   string `yaml:"database_path" mapstructure:"database_path"`
 	Rootfs         string `yaml:"rootfs" mapstructure:"rootfs"`
 	PkgsCachePath  string `yaml:"pkgs_cache_path" mapstructure:"pkgs_cache_path"`
+	TmpDirBase     string `yaml:"tmpdir_base" mapstructure:"tmpdir_base"`
 }
 
 func (sc LuetSystemConfig) GetRepoDatabaseDirPath(name string) string {
@@ -223,6 +226,7 @@ func GenDefault(viper *v.Viper) {
 	viper.SetDefault("system.database_engine", "boltdb")
 	viper.SetDefault("system.database_path", "/var/cache/luet")
 	viper.SetDefault("system.rootfs", "/")
+	viper.SetDefault("system.tmpdir_base", filepath.Join(os.TempDir(), "tmpluet"))
 	viper.SetDefault("system.pkgs_cache_path", "packages")
 
 	viper.SetDefault("repos_confdir", []string{"/etc/luet/repos.conf.d"})
@@ -327,8 +331,37 @@ system:
   database_engine: %s
   database_path: %s
   pkgs_cache_path: %s
+  tmpdir_base: %s
   rootfs: %s`,
-		c.DatabaseEngine, c.DatabasePath, c.PkgsCachePath, c.Rootfs)
+		c.DatabaseEngine, c.DatabasePath, c.PkgsCachePath,
+		c.TmpDirBase, c.Rootfs)
 
 	return ans
 }
+
+func (c *LuetSystemConfig) InitTmpDir() error {
+	if !helpers.Exists(c.TmpDirBase) {
+		return os.MkdirAll(c.TmpDirBase, os.ModePerm)
+	}
+	return nil
+}
+
+func (c *LuetSystemConfig) CleanupTmpDir() error {
+	return os.RemoveAll(c.TmpDirBase)
+}
+
+func (c *LuetSystemConfig) TempDir(pattern string) (string, error) {
+	err := c.InitTmpDir()
+	if err != nil {
+		return "", err
+	}
+	return ioutil.TempDir(c.TmpDirBase, pattern)
+}
+
+func (c *LuetSystemConfig) TempFile(pattern string) (*os.File, error) {
+	err := c.InitTmpDir()
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.TempFile(c.TmpDirBase, pattern)
+}

pkg/config/config_suite_test.go

@@ -0,0 +1,33 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config_test
+
+import (
+	"testing"
+
+	. "github.com/mudler/luet/cmd"
+	config "github.com/mudler/luet/pkg/config"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSolver(t *testing.T) {
+	RegisterFailHandler(Fail)
+	LoadConfig(config.LuetCfg)
+	RunSpecs(t, "Config Suite")
+}

pkg/config/config_test.go

@@ -0,0 +1,59 @@
+// Copyright © 2019-2020 Ettore Di Giacinto <mudler@gentoo.org>
+//                       Daniele Rondina <geaaru@sabayonlinux.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package config_test
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	config "github.com/mudler/luet/pkg/config"
+	"github.com/mudler/luet/pkg/helpers"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Config", func() {
+
+	Context("Simple temporary directory creation", func() {
+
+		It("Create Temporary directory", func() {
+			// PRE: tmpdir_base contains default value.
+
+			tmpDir, err := config.LuetCfg.GetSystem().TempDir("test1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(strings.HasPrefix(tmpDir, filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
+			Expect(helpers.Exists(tmpDir)).To(BeTrue())
+
+			defer os.RemoveAll(tmpDir)
+		})
+
+		It("Create Temporary file", func() {
+			// PRE: tmpdir_base contains default value.
+
+			tmpFile, err := config.LuetCfg.GetSystem().TempFile("testfile1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(strings.HasPrefix(tmpFile.Name(), filepath.Join(os.TempDir(), "tmpluet"))).To(BeTrue())
+			Expect(helpers.Exists(tmpFile.Name())).To(BeTrue())
+
+			defer os.Remove(tmpFile.Name())
+		})
+
+	})
+
+})

pkg/installer/client/http.go

@@ -17,7 +17,6 @@ package client
 
 import (
 	"fmt"
-	"io/ioutil"
 	"math"
 	"net/url"
 	"os"
@@ -79,7 +78,7 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti
 		Info("Use artifact", artifactName, "from cache.")
 	} else {
 
-		temp, err = ioutil.TempDir(os.TempDir(), "tree")
+		temp, err = config.LuetCfg.GetSystem().TempDir("tree")
 		if err != nil {
 			return nil, err
 		}
@@ -139,7 +138,7 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 
 	ok := false
 
-	temp, err = ioutil.TempDir(os.TempDir(), "tree")
+	temp, err = config.LuetCfg.GetSystem().TempDir("tree")
 	if err != nil {
 		return "", err
 	}
@@ -148,7 +147,7 @@ func (c *HttpClient) DownloadFile(name string) (string, error) {
 
 	for _, uri := range c.RepoData.Urls {
 
-		file, err = ioutil.TempFile(os.TempDir(), "HttpClient")
+		file, err = config.LuetCfg.GetSystem().TempFile("HttpClient")
 		if err != nil {
 			continue
 		}

pkg/installer/client/local.go

@@ -16,7 +16,6 @@
 package client
 
 import (
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -76,7 +75,7 @@ func (c *LocalClient) DownloadFile(name string) (string, error) {
 	ok := false
 	for _, uri := range c.RepoData.Urls {
 		Info("Downloading file", name, "from", uri)
-		file, err = ioutil.TempFile(os.TempDir(), "localclient")
+		file, err = config.LuetCfg.GetSystem().TempFile("localclient")
 		if err != nil {
 			continue
 		}

pkg/installer/repository.go

@@ -418,7 +418,7 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 	))
 
 	// Create tree and repository file
-	archive, err := ioutil.TempDir(os.TempDir(), "archive")
+	archive, err := config.LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for archive")
 	}
@@ -454,7 +454,7 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 	meta, serialized := r.Serialize()
 
 	// Create metadata file and repository file
-	metaTmpDir, err := ioutil.TempDir(os.TempDir(), "metadata")
+	metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
 	defer os.RemoveAll(metaTmpDir) // clean up
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for metadata")
@@ -561,15 +561,14 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 		}
 
 	} else {
-		treefs, err = ioutil.TempDir(os.TempDir(), "treefs")
+		treefs, err = config.LuetCfg.GetSystem().TempDir("treefs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met while creating tempdir for rootfs")
 		}
-		// If we always remove them, later on, no other structure can access
+		// Note: If we always remove them, later on, no other structure can access
 		// to the tree for e.g. to retrieve finalizers
-		//defer os.RemoveAll(treefs)
 
-		metafs, err = ioutil.TempDir(os.TempDir(), "metafs")
+		metafs, err = config.LuetCfg.GetSystem().TempDir("metafs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Error met whilte creating tempdir for metafs")
 		}

tests/integration/10_tmpdir_cleanup.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+export LUET_SYSTEM__TMPDIR_BASE=${TMPDIR:-/tmp}/luet_integration10
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/buildableseed" --destination $tmpdir/testbuild --compression gzip test/c > /dev/null
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/buildableseed" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     cached: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testRepoUpdate() {
+    luet repo update --config $tmpdir/luet.yaml
+    res=$?
+
+    assertEquals 'repo update successfully' "$res" "0"
+    assertTrue 'repo cached correctly' "[ -e '$tmpdir/testrootfs/repos/main' ]"
+    assertTrue 'tmpdir cleanup' "[ ! -e '${TMPDIR:/tmp}/luet_integration10' ]"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+