🆕 Tag 0.20.13

It is safer, and plays better with buildx

cmd/oscheck.go

@@ -0,0 +1,140 @@
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mudler/luet/pkg/api/core/types"
+	installer "github.com/mudler/luet/pkg/installer"
+	pkg "github.com/mudler/luet/pkg/package"
+	"github.com/mudler/luet/pkg/solver"
+
+	"github.com/mudler/luet/cmd/util"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var osCheckCmd = &cobra.Command{
+	Use:   "oscheck",
+	Short: "Checks packages integrity",
+	Long: `List packages that are installed in the system which files are missing in the system.
+
+	$ luet oscheck
+	
+To reinstall packages in the list:
+	
+	$ luet oscheck --reinstall
+`,
+	Aliases: []string{"i"},
+	PreRun: func(cmd *cobra.Command, args []string) {
+		util.BindSystemFlags(cmd)
+		util.BindSolverFlags(cmd)
+		viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
+		viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
+		viper.BindPFlag("force", cmd.Flags().Lookup("force"))
+		viper.BindPFlag("yes", cmd.Flags().Lookup("yes"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+
+		force := viper.GetBool("force")
+		onlydeps := viper.GetBool("onlydeps")
+		yes := viper.GetBool("yes")
+
+		downloadOnly, _ := cmd.Flags().GetBool("download-only")
+		util.SetSystemConfig(util.DefaultContext)
+
+		system := &installer.System{Database: util.DefaultContext.Config.GetSystemDB(), Target: util.DefaultContext.Config.GetSystem().Rootfs}
+		packs := system.OSCheck()
+		if !util.DefaultContext.Config.General.Quiet {
+			if len(packs) == 0 {
+				util.DefaultContext.Success("All good!")
+				os.Exit(0)
+			} else {
+				util.DefaultContext.Info("Following packages are missing files or are incomplete:")
+				for _, p := range packs {
+					util.DefaultContext.Info(p.HumanReadableString())
+				}
+			}
+		} else {
+			var s []string
+			for _, p := range packs {
+				s = append(s, p.HumanReadableString())
+			}
+			fmt.Println(strings.Join(s, " "))
+		}
+
+		reinstall, _ := cmd.Flags().GetBool("reinstall")
+		if reinstall {
+
+			// Strip version for reinstall
+			toInstall := pkg.Packages{}
+			for _, p := range packs {
+				new := p.Clone()
+				new.SetVersion(">=0")
+				toInstall = append(toInstall, new)
+			}
+
+			util.SetSolverConfig(util.DefaultContext)
+
+			util.DefaultContext.Config.GetSolverOptions().Implementation = solver.SingleCoreSimple
+
+			util.DefaultContext.Debug("Solver", util.DefaultContext.Config.GetSolverOptions().CompactString())
+
+			// Load config protect configs
+			util.DefaultContext.Config.LoadConfigProtect(util.DefaultContext)
+
+			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
+				Concurrency:                 util.DefaultContext.Config.GetGeneral().Concurrency,
+				SolverOptions:               *util.DefaultContext.Config.GetSolverOptions(),
+				NoDeps:                      true,
+				Force:                       force,
+				OnlyDeps:                    onlydeps,
+				PreserveSystemEssentialData: true,
+				Ask:                         !yes,
+				DownloadOnly:                downloadOnly,
+				Context:                     util.DefaultContext,
+				PackageRepositories:         util.DefaultContext.Config.SystemRepositories,
+			})
+
+			err := inst.Swap(packs, toInstall, system)
+			if err != nil {
+				util.DefaultContext.Fatal("Error: " + err.Error())
+			}
+		}
+	},
+}
+
+func init() {
+	osCheckCmd.Flags().String("system-dbpath", "", "System db path")
+	osCheckCmd.Flags().String("system-target", "", "System rootpath")
+	osCheckCmd.Flags().String("system-engine", "", "System DB engine")
+	osCheckCmd.Flags().Bool("reinstall", false, "reinstall")
+
+	osCheckCmd.Flags().String("solver-type", "", "Solver strategy ( Defaults none, available: "+types.AvailableResolvers+" )")
+	osCheckCmd.Flags().Float32("solver-rate", 0.7, "Solver learning rate")
+	osCheckCmd.Flags().Float32("solver-discount", 1.0, "Solver discount rate")
+	osCheckCmd.Flags().Int("solver-attempts", 9000, "Solver maximum attempts")
+	osCheckCmd.Flags().Bool("onlydeps", false, "Consider **only** package dependencies")
+	osCheckCmd.Flags().Bool("force", false, "Skip errors and keep going (potentially harmful)")
+	osCheckCmd.Flags().Bool("solver-concurrent", false, "Use concurrent solver (experimental)")
+	osCheckCmd.Flags().BoolP("yes", "y", false, "Don't ask questions")
+	osCheckCmd.Flags().Bool("download-only", false, "Download only")
+
+	RootCmd.AddCommand(osCheckCmd)
+}

cmd/root.go

@@ -30,7 +30,7 @@ var cfgFile string
 var Verbose bool
 
 const (
-	LuetCLIVersion = "0.20.12"
+	LuetCLIVersion = "0.20.13"
 	LuetEnvPrefix  = "LUET"
 )
 

pkg/api/core/types/config.go

@@ -431,3 +431,43 @@ func loadConfigProtectConFile(filename string, data []byte) (*config.ConfigProte
 func (c *LuetLoggingConfig) SetLogLevel(s LogLevel) {
 	c.Level = s
 }
+
+func (c *LuetSystemConfig) InitTmpDir() error {
+	if !filepath.IsAbs(c.TmpDirBase) {
+		abs, err := fileHelper.Rel2Abs(c.TmpDirBase)
+		if err != nil {
+			return errors.Wrap(err, "while converting relative path to absolute path")
+		}
+		c.TmpDirBase = abs
+	}
+
+	if _, err := os.Stat(c.TmpDirBase); err != nil {
+		if os.IsNotExist(err) {
+			err = os.MkdirAll(c.TmpDirBase, os.ModePerm)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (c *LuetSystemConfig) CleanupTmpDir() error {
+	return os.RemoveAll(c.TmpDirBase)
+}
+
+func (c *LuetSystemConfig) TempDir(pattern string) (string, error) {
+	err := c.InitTmpDir()
+	if err != nil {
+		return "", err
+	}
+	return ioutil.TempDir(c.TmpDirBase, pattern)
+}
+
+func (c *LuetSystemConfig) TempFile(pattern string) (*os.File, error) {
+	err := c.InitTmpDir()
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.TempFile(c.TmpDirBase, pattern)
+}

pkg/api/core/types/context.go

@@ -45,7 +45,6 @@ const (
 
 type Context struct {
 	context.Context
-	gc         *GarbageCollector
 	Config     *LuetConfig
 	IsTerminal bool
 	NoSpinner  bool
@@ -58,34 +57,22 @@ type Context struct {
 }
 
 func NewContext() *Context {
-	tmp := filepath.Join(os.TempDir(), "tmpluet")
-	gc, _ := NewGC(tmp)
 	return &Context{
 		spinnerLock: &sync.Mutex{},
 		IsTerminal:  terminal.IsTerminal(os.Stdout),
-		gc:          gc,
 		Config: &LuetConfig{
 			ConfigFromHost: true,
 			Logging:        LuetLoggingConfig{},
 			General:        LuetGeneralConfig{},
 			System: LuetSystemConfig{
 				DatabasePath: filepath.Join("var", "db", "packages"),
-				TmpDirBase:   tmp},
+				TmpDirBase:   filepath.Join(os.TempDir(), "tmpluet")},
 			Solver: LuetSolverOptions{},
 		},
 		s: pterm.DefaultSpinner.WithShowTimer(false).WithRemoveWhenDone(true),
 	}
 }
 
-func (c *Context) WithGC(dir string) error {
-	gc, err := NewGC(dir)
-	if err != nil {
-		return err
-	}
-	c.gc = gc
-	return nil
-}
-
 func (c *Context) WithName(name string) *Context {
 	newc := c.Copy()
 	newc.name = name

pkg/api/core/types/garbagecollector.go

@@ -1,85 +0,0 @@
-// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, see <http://www.gnu.org/licenses/>.
-
-package types
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	"github.com/hashicorp/go-multierror"
-	fileHelper "github.com/mudler/luet/pkg/helpers/file"
-	"github.com/pkg/errors"
-)
-
-// GarbageCollector keeps track of directory assigned and cleans them up
-type GarbageCollector struct {
-	dir          string
-	createdDirs  []string
-	createdFiles []string
-}
-
-// NewGC returns a new GC instance on dir
-func NewGC(s string) (*GarbageCollector, error) {
-	if !filepath.IsAbs(s) {
-		abs, err := fileHelper.Rel2Abs(s)
-		if err != nil {
-			return nil, errors.Wrap(err, "while converting relative path to absolute path")
-		}
-		s = abs
-	}
-	if _, err := os.Stat(s); err != nil {
-		if os.IsNotExist(err) {
-			err = os.MkdirAll(s, os.ModePerm)
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-	return &GarbageCollector{dir: s}, nil
-}
-
-func (gc *GarbageCollector) Directory(pattern string) (string, error) {
-	dir, err := ioutil.TempDir(gc.dir, pattern)
-	if err != nil {
-		return "", err
-	}
-	gc.createdDirs = append(gc.createdDirs, dir)
-	return dir, err
-}
-
-func (gc *GarbageCollector) Remove() error {
-	return os.RemoveAll(gc.dir)
-}
-
-func (gc *GarbageCollector) Clean() (err error) {
-	for _, d := range append(gc.createdDirs, gc.createdFiles...) {
-		if fileHelper.Exists(d) {
-			multierror.Append(err, os.RemoveAll(d))
-		}
-	}
-
-	return
-}
-
-func (gc *GarbageCollector) File(pattern string) (*os.File, error) {
-	f, err := ioutil.TempFile(gc.dir, pattern)
-	if err != nil {
-		return nil, err
-	}
-	gc.createdFiles = append(gc.createdFiles, f.Name())
-	return f, err
-}

pkg/compiler/compiler.go

@@ -371,7 +371,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	}
 
 	// First we create the builder image
-	if err := p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile")); err != nil {
+	if err := p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().ImageID()+"-builder.dockerfile")); err != nil {
 		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
 	}
 
@@ -386,21 +386,21 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
 	// steps in prelude are == 0 those are equivalent.
 
 	// Then we write the step image, which uses the builder one
-	if err := p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile")); err != nil {
+	if err := p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().ImageID()+".dockerfile")); err != nil {
 		return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
 	}
 
 	builderOpts = backend.Options{
 		ImageName:      buildertaggedImage,
 		SourcePath:     buildDir,
-		DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile",
+		DockerFileName: p.GetPackage().ImageID() + "-builder.dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
 		BackendArgs:    cs.Options.BackendArgs,
 	}
 	runnerOpts = backend.Options{
 		ImageName:      packageImage,
 		SourcePath:     buildDir,
-		DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile",
+		DockerFileName: p.GetPackage().ImageID() + ".dockerfile",
 		Destination:    p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
 		BackendArgs:    cs.Options.BackendArgs,
 	}

pkg/installer/system.go

@@ -1,6 +1,8 @@
 package installer
 
 import (
+	"os"
+	"path/filepath"
 	"sync"
 
 	"github.com/hashicorp/go-multierror"
@@ -22,6 +24,19 @@ func (s *System) World() (pkg.Packages, error) {
 	return s.Database.World(), nil
 }
 
+func (s *System) OSCheck() (notFound pkg.Packages) {
+	s.buildFileIndex()
+	s.Lock()
+	defer s.Unlock()
+	for f, p := range s.fileIndex {
+		if _, err := os.Lstat(filepath.Join(s.Target, f)); err != nil {
+			notFound = append(notFound, p)
+		}
+	}
+	notFound = notFound.Unique()
+	return
+}
+
 func (s *System) ExecuteFinalizers(ctx *types.Context, packs []pkg.Package) error {
 	var errs error
 	executedFinalizer := map[string]bool{}
@@ -56,6 +71,7 @@ func (s *System) ExecuteFinalizers(ctx *types.Context, packs []pkg.Package) erro
 }
 
 func (s *System) buildFileIndex() {
+	// XXX: Replace with cache
 	s.Lock()
 	defer s.Unlock()
 	// Check if cache is empty or if it got modified

pkg/installer/system_test.go

@@ -19,6 +19,10 @@ import (
 
 	//	. "github.com/mudler/luet/pkg/installer"
 
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
 	. "github.com/mudler/luet/pkg/installer"
 	pkg "github.com/mudler/luet/pkg/package"
 
@@ -66,5 +70,18 @@ var _ = Describe("System", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(p).To(Equal(b))
 		})
+
+		It("detect missing files", func() {
+			dir, err := ioutil.TempDir("", "test")
+			Expect(err).ToNot(HaveOccurred())
+			defer os.RemoveAll(dir)
+			s.Target = dir
+			notfound := s.OSCheck()
+			Expect(len(notfound)).To(Equal(2))
+			ioutil.WriteFile(filepath.Join(dir, "f"), []byte{}, os.ModePerm)
+			ioutil.WriteFile(filepath.Join(dir, "foo"), []byte{}, os.ModePerm)
+			notfound = s.OSCheck()
+			Expect(len(notfound)).To(Equal(1))
+		})
 	})
 })