wip/gc

CONTRIBUTING.md

@@ -17,7 +17,7 @@ Join us in [slack](https://luet.slack.com/join/shared_invite/enQtOTQxMjcyNDQ0MDU
 ## All Code Changes Happen Through Pull Requests
 Pull requests are the best way to propose changes to the codebase. We actively welcome your pull requests:
 
-1. Fork the repo you want to contribute to and create your branch from `develop`.
+1. Fork the repo you want to contribute to and create your branch from `master`.
 2. If you've added code that should be tested, add tests.
 3. If you've changed APIs, update the [documentation](https://github.com/Luet-lab/docs).
 4. Ensure the test suite passes.

cmd/search.go

@@ -371,6 +371,10 @@ Search can also return results in the terminal in different ways: as terminal ou
 		default:
 			if tableMode {
 				t.Render()
+			} else if util.DefaultContext.Config.General.Quiet {
+				for _, tt := range results.Packages {
+					fmt.Printf("%s/%s-%s\n", tt.Category, tt.Name, tt.Version)
+				}
 			} else {
 				l.Render()
 			}

cmd/util/cli.go

@@ -156,11 +156,15 @@ func DisplayVersionBanner(c *types.Context, banner func(), version func() string
 		}
 	}
 	if display {
-		banner()
-		pterm.DefaultCenter.Print(version())
-		for _, l := range license {
-			pterm.DefaultCenter.Print(l)
-
+		if c.Config.General.Quiet {
+			pterm.Info.Printf("Luet %s\n", version())
+			pterm.Info.Println(strings.Join(license, "\n"))
+		} else {
+			banner()
+			pterm.DefaultCenter.Print(version())
+			for _, l := range license {
+				pterm.DefaultCenter.Print(l)
+			}
 		}
 	}
 }

cmd/util/config.go

@@ -120,6 +120,7 @@ func setDefaults(viper *viper.Viper) {
 
 	viper.SetDefault("general.concurrency", runtime.NumCPU())
 	viper.SetDefault("general.debug", false)
+	viper.SetDefault("general.quiet", false)
 	viper.SetDefault("general.show_build_output", false)
 	viper.SetDefault("general.fatal_warnings", false)
 	viper.SetDefault("general.http_timeout", 360)
@@ -159,7 +160,8 @@ func InitViper(ctx *types.Context, RootCmd *cobra.Command) {
 	cobra.OnInitialize(initConfig)
 	pflags := RootCmd.PersistentFlags()
 	pflags.StringVar(&cfgFile, "config", "", "config file (default is $HOME/.luet.yaml)")
-	pflags.BoolP("debug", "d", false, "verbose output")
+	pflags.BoolP("debug", "d", false, "debug output")
+	pflags.BoolP("quiet", "q", false, "quiet output")
 	pflags.Bool("fatal", false, "Enables Warnings to exit")
 	pflags.Bool("enable-logfile", false, "Enable log to file")
 	pflags.Bool("no-spinner", false, "Disable spinner.")
@@ -188,6 +190,7 @@ func InitViper(ctx *types.Context, RootCmd *cobra.Command) {
 
 	viper.BindPFlag("general.concurrency", pflags.Lookup("concurrency"))
 	viper.BindPFlag("general.debug", pflags.Lookup("debug"))
+	viper.BindPFlag("general.quiet", pflags.Lookup("quiet"))
 	viper.BindPFlag("general.fatal_warnings", pflags.Lookup("fatal"))
 	viper.BindPFlag("general.same_owner", pflags.Lookup("same-owner"))
 	viper.BindPFlag("plugin", pflags.Lookup("plugin"))

pkg/api/core/types/config.go

@@ -61,6 +61,7 @@ type LuetGeneralConfig struct {
 	ShowBuildOutput bool `yaml:"show_build_output,omitempty" mapstructure:"show_build_output"`
 	FatalWarns      bool `yaml:"fatal_warnings,omitempty" mapstructure:"fatal_warnings"`
 	HTTPTimeout     int  `yaml:"http_timeout,omitempty" mapstructure:"http_timeout"`
+	Quiet           bool `yaml:"quiet" mapstructure:"quiet"`
 }
 
 type LuetSolverOptions struct {
@@ -430,43 +431,3 @@ func loadConfigProtectConFile(filename string, data []byte) (*config.ConfigProte
 func (c *LuetLoggingConfig) SetLogLevel(s LogLevel) {
 	c.Level = s
 }
-
-func (c *LuetSystemConfig) InitTmpDir() error {
-	if !filepath.IsAbs(c.TmpDirBase) {
-		abs, err := fileHelper.Rel2Abs(c.TmpDirBase)
-		if err != nil {
-			return errors.Wrap(err, "while converting relative path to absolute path")
-		}
-		c.TmpDirBase = abs
-	}
-
-	if _, err := os.Stat(c.TmpDirBase); err != nil {
-		if os.IsNotExist(err) {
-			err = os.MkdirAll(c.TmpDirBase, os.ModePerm)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func (c *LuetSystemConfig) CleanupTmpDir() error {
-	return os.RemoveAll(c.TmpDirBase)
-}
-
-func (c *LuetSystemConfig) TempDir(pattern string) (string, error) {
-	err := c.InitTmpDir()
-	if err != nil {
-		return "", err
-	}
-	return ioutil.TempDir(c.TmpDirBase, pattern)
-}
-
-func (c *LuetSystemConfig) TempFile(pattern string) (*os.File, error) {
-	err := c.InitTmpDir()
-	if err != nil {
-		return nil, err
-	}
-	return ioutil.TempFile(c.TmpDirBase, pattern)
-}

pkg/api/core/types/context.go

@@ -45,9 +45,11 @@ const (
 
 type Context struct {
 	context.Context
+	gc         *GarbageCollector
 	Config     *LuetConfig
 	IsTerminal bool
 	NoSpinner  bool
+	name       string
 
 	s           *pterm.SpinnerPrinter
 	spinnerLock *sync.Mutex
@@ -56,22 +58,40 @@ type Context struct {
 }
 
 func NewContext() *Context {
+	tmp := filepath.Join(os.TempDir(), "tmpluet")
+	gc, _ := NewGC(tmp)
 	return &Context{
 		spinnerLock: &sync.Mutex{},
 		IsTerminal:  terminal.IsTerminal(os.Stdout),
+		gc:          gc,
 		Config: &LuetConfig{
 			ConfigFromHost: true,
 			Logging:        LuetLoggingConfig{},
 			General:        LuetGeneralConfig{},
 			System: LuetSystemConfig{
 				DatabasePath: filepath.Join("var", "db", "packages"),
-				TmpDirBase:   filepath.Join(os.TempDir(), "tmpluet")},
+				TmpDirBase:   tmp},
 			Solver: LuetSolverOptions{},
 		},
 		s: pterm.DefaultSpinner.WithShowTimer(false).WithRemoveWhenDone(true),
 	}
 }
 
+func (c *Context) WithGC(dir string) error {
+	gc, err := NewGC(dir)
+	if err != nil {
+		return err
+	}
+	c.gc = gc
+	return nil
+}
+
+func (c *Context) WithName(name string) *Context {
+	newc := c.Copy()
+	newc.name = name
+	return newc
+}
+
 func (c *Context) Copy() *Context {
 
 	configCopy := *c.Config
@@ -112,6 +132,11 @@ func (c *Context) Init() (err error) {
 		c.NoColor()
 	}
 
+	if c.Config.General.Quiet {
+		c.NoColor()
+		pterm.DisableStyling()
+	}
+
 	c.Debug("Colors", c.Config.GetLogging().Color)
 	c.Debug("Logging level", c.Config.GetLogging().Level)
 	c.Debug("Debug mode", c.Config.GetGeneral().Debug)
@@ -306,6 +331,10 @@ func (c *Context) Msg(level LogLevel, ln bool, msg ...interface{}) {
 		levelMsg = re.ReplaceAllString(levelMsg, "")
 	}
 
+	if c.name != "" {
+		levelMsg = fmt.Sprintf("[%s] %s", c.name, levelMsg)
+	}
+
 	if c.z != nil {
 		c.log2File(level, message)
 	}

pkg/api/core/types/garbagecollector.go

@@ -0,0 +1,85 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package types
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/go-multierror"
+	fileHelper "github.com/mudler/luet/pkg/helpers/file"
+	"github.com/pkg/errors"
+)
+
+// GarbageCollector keeps track of directory assigned and cleans them up
+type GarbageCollector struct {
+	dir          string
+	createdDirs  []string
+	createdFiles []string
+}
+
+// NewGC returns a new GC instance on dir
+func NewGC(s string) (*GarbageCollector, error) {
+	if !filepath.IsAbs(s) {
+		abs, err := fileHelper.Rel2Abs(s)
+		if err != nil {
+			return nil, errors.Wrap(err, "while converting relative path to absolute path")
+		}
+		s = abs
+	}
+	if _, err := os.Stat(s); err != nil {
+		if os.IsNotExist(err) {
+			err = os.MkdirAll(s, os.ModePerm)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	return &GarbageCollector{dir: s}, nil
+}
+
+func (gc *GarbageCollector) Directory(pattern string) (string, error) {
+	dir, err := ioutil.TempDir(gc.dir, pattern)
+	if err != nil {
+		return "", err
+	}
+	gc.createdDirs = append(gc.createdDirs, dir)
+	return dir, err
+}
+
+func (gc *GarbageCollector) Remove() error {
+	return os.RemoveAll(gc.dir)
+}
+
+func (gc *GarbageCollector) Clean() (err error) {
+	for _, d := range append(gc.createdDirs, gc.createdFiles...) {
+		if fileHelper.Exists(d) {
+			multierror.Append(err, os.RemoveAll(d))
+		}
+	}
+
+	return
+}
+
+func (gc *GarbageCollector) File(pattern string) (*os.File, error) {
+	f, err := ioutil.TempFile(gc.dir, pattern)
+	if err != nil {
+		return nil, err
+	}
+	gc.createdFiles = append(gc.createdFiles, f.Name())
+	return f, err
+}

pkg/compiler/compiler.go

@@ -273,11 +273,11 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compi
 
 func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec, builderOpts, runnerOpts backend.Options) (*artifact.PackageArtifact, error) {
 
-	rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
+	rootfs, err := cs.Options.Context.Config.System.TempDir("rootfs")
 	if err != nil {
 		return nil, errors.Wrap(err, "Could not create tempdir")
 	}
-	defer os.RemoveAll(rootfs) // clean up
+	defer os.RemoveAll(rootfs)
 
 	pkgTag := ":package: " + p.GetPackage().HumanReadableString()
 	if cs.Options.PullFirst {
@@ -458,11 +458,11 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
 	if p.EmptyPackage() {
 		fakePackage := p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")
 
-		rootfs, err = ioutil.TempDir(p.GetOutputPath(), "rootfs")
+		rootfs, err = cs.Options.Context.Config.System.TempDir("rootfs")
 		if err != nil {
 			return nil, errors.Wrap(err, "Could not create tempdir")
 		}
-		defer os.RemoveAll(rootfs) // clean up
+		defer os.RemoveAll(rootfs)
 
 		a := artifact.NewPackageArtifact(fakePackage)
 		a.CompressionType = cs.Options.CompressionType
@@ -939,11 +939,11 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
 	}
 
 	// otherwise, generate it and push it aside
-	joinDir, err := ioutil.TempDir(p.GetOutputPath(), "join")
+	joinDir, err := cs.Options.Context.Config.System.TempDir("join")
 	if err != nil {
 		return errors.Wrap(err, "could not create tempdir for joining images")
 	}
-	defer os.RemoveAll(joinDir) // clean up
+	defer os.RemoveAll(joinDir)
 
 	for _, p := range fromPackages {
 		cs.Options.Context.Info(joinTag, ":arrow_right_hook:", p.HumanReadableString(), ":leaves:")
@@ -978,11 +978,11 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
 		}
 	}
 
-	artifactDir, err := ioutil.TempDir(p.GetOutputPath(), "artifact")
+	artifactDir, err := cs.Options.Context.Config.System.TempDir("join")
 	if err != nil {
 		return errors.Wrap(err, "could not create tempdir for final artifact")
 	}
-	defer os.RemoveAll(joinDir) // clean up
+	defer os.RemoveAll(artifactDir)
 
 	cs.Options.Context.Info(joinTag, ":droplet: generating artifact for source image of", p.GetPackage().HumanReadableString())
 
@@ -1092,7 +1092,10 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
 	}
 	for _, p := range dbCopy.World() {
 		copy := p.Clone()
-		spec, _ := cs.FromPackage(p)
+		spec, err := cs.FromPackage(p)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed getting compile spec for package "+p.HumanReadableString())
+		}
 		if spec.RequiresFinalImages {
 			copy.Requires([]*pkg.DefaultPackage{})
 		}
@@ -1332,11 +1335,12 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.
 	} else {
 		bv := cs.Options.BuildValuesFile
 		if len(vals) > 0 {
-			valuesdir, err := ioutil.TempDir("", "genvalues")
+			valuesdir, err := cs.Options.Context.Config.System.TempDir("genvalues")
 			if err != nil {
 				return nil, errors.Wrap(err, "Could not create tempdir")
 			}
-			defer os.RemoveAll(valuesdir) // clean up
+			defer os.RemoveAll(valuesdir)
+
 			for _, b := range vals {
 				out, err := yaml.Marshal(b)
 				if err != nil {