🆕 Tag 0.21.2

Fixes #274
Fixes #212

cmd/create-repo.go

@@ -100,6 +100,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		helpers.CheckErr(err)
 		force := viper.GetBool("force-push")
 		imagePush := viper.GetBool("push-images")
+		snapshotID, _ := cmd.Flags().GetString("snapshot-id")
 
 		opts := []installer.RepositoryOption{
 			installer.WithSource(viper.GetString("packages")),
@@ -163,7 +164,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		if metaName != "" {
 			metaFile.SetFileName(metaName)
 		}
-
+		repo.SetSnapshotID(snapshotID)
 		repo.SetRepositoryFile(installer.REPOFILE_TREE_KEY, treeFile)
 		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
 
@@ -197,6 +198,7 @@ func init() {
 	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
 	createrepoCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
+	createrepoCmd.Flags().String("snapshot-id", "", "Unique ID to use when creating repository snapshots")
 
 	RootCmd.AddCommand(createrepoCmd)
 }

cmd/repo/update.go

@@ -1,5 +1,6 @@
 // Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
 //                  Daniele Rondina <geaaru@sabayonlinux.org>
+// Copyright © 2021 Ettore Di Giacinto <mudler@mocaccino.org>
 //
 // This program is free software; you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -24,7 +25,7 @@ import (
 )
 
 func NewRepoUpdateCommand() *cobra.Command {
-	var ans = &cobra.Command{
+	var repoUpdate = &cobra.Command{
 		Use:   "update [repo1] [repo2] [OPTIONS]",
 		Short: "Update a specific cached repository or all cached repositories.",
 		Example: `
@@ -72,8 +73,8 @@ $> luet repo update repo1 repo2
 		},
 	}
 
-	ans.Flags().BoolP("ignore-errors", "i", false, "Ignore errors on sync repositories.")
+	repoUpdate.Flags().BoolP("ignore-errors", "i", false, "Ignore errors on sync repositories.")
-	ans.Flags().BoolP("force", "f", false, "Force resync.")
+	repoUpdate.Flags().BoolP("force", "f", true, "Force resync.")
 
-	return ans
+	return repoUpdate
 }

cmd/root.go

@@ -30,7 +30,7 @@ var cfgFile string
 var Verbose bool
 
 const (
-	LuetCLIVersion = "0.21.0"
+	LuetCLIVersion = "0.21.2"
 	LuetEnvPrefix  = "LUET"
 )
 

pkg/installer/installer.go

@@ -29,7 +29,6 @@ import (
 	"github.com/mudler/luet/pkg/api/core/bus"
 	"github.com/mudler/luet/pkg/api/core/types"
 	artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
-	"github.com/mudler/luet/pkg/helpers"
 	fileHelper "github.com/mudler/luet/pkg/helpers/file"
 	"github.com/mudler/luet/pkg/helpers/match"
 	pkg "github.com/mudler/luet/pkg/package"
@@ -279,6 +278,20 @@ func (l *LuetInstaller) swap(o Option, syncedRepos Repositories, toRemove pkg.Pa
 		return errors.Wrap(err, "failed computing installer options")
 	}
 
+	l.Options.Context.Info("Computed operations")
+
+	for _, o := range ops {
+		toUninstall := ""
+		toInstall := ""
+		for _, u := range o.Uninstall {
+			toUninstall += fmt.Sprintf(" %s", u.Package.HumanReadableString())
+		}
+		for _, u := range o.Install {
+			toInstall += fmt.Sprintf(" %s", u.Package.HumanReadableString())
+		}
+		l.Options.Context.Info(fmt.Sprintf("%s -> %s", toUninstall, toInstall))
+	}
+
 	err = l.runOps(ops, s)
 	if err != nil {
 		return errors.Wrap(err, "failed running installer options")
@@ -402,16 +415,19 @@ func (l *LuetInstaller) getOpsWithOptions(
 	l.Options.Context.Debug("Computing installation order")
 	resOps := []installerOp{}
 
-	insertPackage := func(install pkg.Package, uninstall ...pkg.Package) {
+	insertPackage := func(install []pkg.Package, uninstall ...pkg.Package) {
+		if len(install) == 0 && len(uninstall) == 0 {
+			return
+		}
 		uOpts := []operation{}
 		for _, u := range uninstall {
 			uOpts = append(uOpts, operation{Package: u, Option: uninstallOpt})
 		}
-		resOps = append(resOps, installerOp{
+		iOpts := []installOperation{}
-			Uninstall: uOpts,
+		for _, u := range install {
-			Install: []installOperation{{
+			iOpts = append(iOpts, installOperation{
 				operation: operation{
-					Package: install,
+					Package: u,
 					Option:  installOpt,
 				},
 				Matches:     installMatch,
@@ -419,11 +435,46 @@ func (l *LuetInstaller) getOpsWithOptions(
 				Reposiories: syncedRepos,
 				Assertions:  solution,
 				Database:    allRepos,
-			}},
+			})
+		}
+		resOps = append(resOps, installerOp{
+			Uninstall: uOpts,
+			Install:   iOpts,
 		})
 	}
 
 	removals := make(map[string]interface{})
+	additions := make(map[string]interface{})
+
+	remove := func(p pkg.Package) {
+		removals[p.GetPackageName()] = nil
+	}
+
+	add := func(p pkg.Package) {
+		additions[p.GetPackageName()] = nil
+	}
+
+	added := func(p pkg.Package) bool {
+		_, exists := additions[p.GetPackageName()]
+		return exists
+	}
+
+	removed := func(p pkg.Package) bool {
+		_, exists := removals[p.GetPackageName()]
+		return exists
+	}
+
+	findToBeInstalled := func(p pkg.Package) pkg.Package {
+		for _, m := range installMatch {
+			if m.Package.GetPackageName() == p.GetPackageName() {
+				return m.Package
+			}
+		}
+		return nil
+	}
+
+	fileIndex := s.FileIndex()
+
 	for _, match := range installMatch {
 		a, err := l.getPackage(match, l.Options.Context)
 		if err != nil && !l.Options.Force {
@@ -434,39 +485,65 @@ func (l *LuetInstaller) getOpsWithOptions(
 			return nil, errors.Wrapf(err, "Could not get filelist for %s", a.CompileSpec.Package.HumanReadableString())
 		}
 
-		var foundPackages []pkg.Package
+		l.Options.Context.Debug(match.Package.HumanReadableString(), "files", len(files))
+
+		foundPackages := make(map[string]pkg.Package)
+	FILES:
 		for _, f := range files {
-			if exists, p, _ := s.ExistsPackageFile(f); exists {
+			if p, exists := fileIndex[f]; exists {
+				if _, exists := foundPackages[p.HumanReadableString()]; exists {
+					continue FILES
+				}
+
 				_, err := toUninstall.Find(p.GetPackageName())
 				if err == nil {
+					l.Options.Context.Debug(p.HumanReadableString(), "files", f, "matches")
+
 					// Packages that is being installed have a file that
 					// is going to be removed by another package
-					foundPackages = append(foundPackages, p)
+					foundPackages[p.HumanReadableString()] = p
 				}
 			}
 		}
-
+		l.Options.Context.Debug(match.Package.HumanReadableString(), "found", len(foundPackages), "packages")
-		foundPackages = pkg.Packages(foundPackages).Unique()
 		if len(foundPackages) > 0 {
 			if pack, err := toUninstall.Find(match.Package.GetPackageName()); err == nil {
-				foundPackages = append(foundPackages, pack)
+				foundPackages[pack.HumanReadableString()] = pack
 			}
 
+			toInstall := []pkg.Package{}
+			if !added(match.Package) {
+				toInstall = append(toInstall, match.Package)
+				add(match.Package)
+			}
 			toRemove := []pkg.Package{}
 			for _, p := range foundPackages {
-				if _, ok := removals[p.GetPackageName()]; !ok {
+				if !removed(p) {
 					toRemove = append(toRemove, p)
-					removals[p.GetPackageName()] = nil
+					if pp := findToBeInstalled(p); pp != nil && !added(pp) {
+						toInstall = append(toInstall, pp)
+						add(pp)
+					}
+					remove(p)
 				}
 			}
-			insertPackage(match.Package, toRemove...)
+			// toInstall needs to have:
+			// equivalent of what was found in foundPackages AND
+			// was not already added to installation
+			insertPackage(toInstall, toRemove...)
 		} else if pack, err := toUninstall.Find(match.Package.GetPackageName()); err == nil {
-			if _, ok := removals[pack.GetPackageName()]; !ok {
+			if !removed(pack) {
-				insertPackage(match.Package, pack)
+				toInstall := []pkg.Package{}
-				removals[pack.GetPackageName()] = nil
+				if !added(match.Package) {
+					toInstall = append(toInstall, match.Package)
+					add(match.Package)
+				}
+				insertPackage(toInstall, pack)
+				remove(pack)
 			}
-		} else {
+		} else if !added(match.Package) {
-			insertPackage(match.Package)
+			insertPackage([]pkg.Package{match.Package})
+			add(match.Package)
 		}
 	}
 
@@ -868,8 +945,10 @@ func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, c
 	l.Options.Context.Info("Checking for file conflicts..")
 	defer s.Clean() // Release memory
 
-	filesToInstall := []string{}
+	filesToInstall := map[string]interface{}{}
 	for _, m := range toInstall {
+		l.Options.Context.Debug("Checking file conflicts for", m.Package.HumanReadableString())
+
 		a, err := l.getPackage(m, l.Options.Context)
 		if err != nil && !l.Options.Force {
 			return errors.Wrap(err, "Failed downloading package")
@@ -880,7 +959,7 @@ func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, c
 		}
 
 		for _, f := range files {
-			if helpers.Contains(filesToInstall, f) {
+			if _, ok := filesToInstall[f]; ok {
 				return fmt.Errorf(
 					"file conflict between packages to be installed",
 				)
@@ -899,9 +978,10 @@ func (l *LuetInstaller) checkFileconflicts(toInstall map[string]ArtifactMatch, c
 					)
 				}
 			}
+			filesToInstall[f] = nil
 		}
-		filesToInstall = append(filesToInstall, files...)
 	}
+	l.Options.Context.Info("Done checking for file conflicts..")
 
 	return nil
 }

pkg/installer/repository.go

@@ -74,7 +74,7 @@ type LuetSystemRepository struct {
 	PushImages      bool                          `json:"-"`
 	ForcePush       bool                          `json:"-"`
 
-	imagePrefix string
+	imagePrefix, snapshotID string
 }
 
 type LuetSystemRepositoryMetadata struct {
@@ -379,7 +379,7 @@ func (r *LuetSystemRepository) SetPriority(n int) {
 }
 
 func (r *LuetSystemRepository) initialize(ctx *types.Context, src string) error {
-	generator, err := r.getGenerator(ctx)
+	generator, err := r.getGenerator(ctx, r.snapshotID)
 	if err != nil {
 		return errors.Wrap(err, "while constructing repository generator")
 	}
@@ -430,6 +430,11 @@ func (r *LuetSystemRepository) SetType(p string) {
 	r.LuetRepository.Type = p
 }
 
+// Sets snapshot ID
+func (r *LuetSystemRepository) SetSnapshotID(i string) {
+	r.snapshotID = i
+}
+
 func (r *LuetSystemRepository) GetVerify() bool {
 	return r.LuetRepository.Verify
 }
@@ -705,11 +710,15 @@ type RepositoryGenerator interface {
 	Initialize(string, pkg.PackageDatabase) ([]*artifact.PackageArtifact, error)
 }
 
-func (r *LuetSystemRepository) getGenerator(ctx *types.Context) (RepositoryGenerator, error) {
+func (r *LuetSystemRepository) getGenerator(ctx *types.Context, snapshotID string) (RepositoryGenerator, error) {
+	if snapshotID == "" {
+		snapshotID = time.Now().Format("20060102150405")
+	}
+
 	var rg RepositoryGenerator
 	switch r.GetType() {
 	case DiskRepositoryType, HttpRepositoryType:
-		rg = &localRepositoryGenerator{context: ctx}
+		rg = &localRepositoryGenerator{context: ctx, snapshotID: snapshotID}
 	case DockerRepositoryType:
 		rg = &dockerRepositoryGenerator{
 			b:           r.Backend,
@@ -717,6 +726,7 @@ func (r *LuetSystemRepository) getGenerator(ctx *types.Context) (RepositoryGener
 			imagePush:   r.PushImages,
 			force:       r.ForcePush,
 			context:     ctx,
+			snapshotID:  snapshotID,
 		}
 	default:
 		return nil, errors.New("invalid repository type")
@@ -726,7 +736,7 @@ func (r *LuetSystemRepository) getGenerator(ctx *types.Context) (RepositoryGener
 
 // Write writes the repository metadata to the supplied destination
 func (r *LuetSystemRepository) Write(ctx *types.Context, dst string, resetRevision, force bool) error {
-	rg, err := r.getGenerator(ctx)
+	rg, err := r.getGenerator(ctx, r.snapshotID)
 	if err != nil {
 		return err
 	}
@@ -846,7 +856,20 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 	var repoUpdated bool = false
 	var treefs, metafs string
 
+	repobasedir := ctx.Config.GetSystem().GetRepoDatabaseDirPath(r.GetName())
+
+	toTimeSync := false
+	dat, err := ioutil.ReadFile(filepath.Join(repobasedir, "SYNCTIME"))
+	if err == nil {
+		parsed, _ := time.Parse(time.RFC3339, string(dat))
+		if time.Now().After(parsed.Add(24 * time.Hour)) {
+			toTimeSync = true
+			ctx.Debug(r.Name, "is old, refresh is suggested")
+		}
+	}
+
 	ctx.Debug("Sync of the repository", r.Name, "in progress...")
+
 	c := r.Client(ctx)
 	if c == nil {
 		return nil, errors.New("no client could be generated from repository")
@@ -854,20 +877,29 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 
 	repositoryReferenceID := r.referenceID()
 
-	// Retrieve remote repository.yaml for retrieve revision and date
+	var downloadedRepoMeta *LuetSystemRepository
-	file, err := c.DownloadFile(repositoryReferenceID)
+	var file string
-	if err != nil {
+	repoFile := filepath.Join(repobasedir, repositoryReferenceID)
-		return nil, errors.Wrap(err, "while downloading "+repositoryReferenceID)
-	}
 
-	repobasedir := ctx.Config.GetSystem().GetRepoDatabaseDirPath(r.GetName())
+	_, repoExistsErr := os.Stat(repoFile)
-	downloadedRepoMeta, err := r.ReadSpecFile(file)
+	if toTimeSync || force || os.IsNotExist(repoExistsErr) {
-	if err != nil {
+		// Retrieve remote repository.yaml for retrieve revision and date
-		return nil, err
+		file, err = c.DownloadFile(repositoryReferenceID)
+		if err != nil {
+			return nil, errors.Wrap(err, "while downloading "+repositoryReferenceID)
+		}
+		downloadedRepoMeta, err = r.ReadSpecFile(file)
+		if err != nil {
+			return nil, err
+		}
+		defer os.RemoveAll(file)
+	} else {
+		downloadedRepoMeta, err = r.ReadSpecFile(repoFile)
+		if err != nil {
+			return nil, err
+		}
+		repoUpdated = true
 	}
-	// Remove temporary file that contains repository.yaml
-	// Example: /tmp/HttpClient236052003
-	defer os.RemoveAll(file)
 
 	if r.Cached {
 		if !force {
@@ -958,8 +990,8 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 				downloadedRepoMeta.GetRevision(),
 				time.Unix(tsec, 0).String()))
 
-	} else {
+		now := time.Now().Format(time.RFC3339)
-		ctx.Info("Repository", downloadedRepoMeta.GetName(), "is already up to date.")
+		ioutil.WriteFile(filepath.Join(repobasedir, "SYNCTIME"), []byte(now), os.ModePerm)
 	}
 
 	meta, err := NewLuetSystemRepositoryMetadata(
@@ -983,11 +1015,14 @@ func (r *LuetSystemRepository) Sync(ctx *types.Context, force bool) (*LuetSystem
 	// e.g. locally we can override the type (disk), or priority
 	// while remotely it could be advertized differently
 	r.fill(downloadedRepoMeta)
-	ctx.Info(
+
-		fmt.Sprintf(":information_source: Repository: %s Priority: %d Type: %s",
+	if !repoUpdated {
-			downloadedRepoMeta.GetName(),
+		ctx.Info(
-			downloadedRepoMeta.GetPriority(),
+			fmt.Sprintf(":information_source: Repository: %s Priority: %d Type: %s",
-			downloadedRepoMeta.GetType()))
+				downloadedRepoMeta.GetName(),
+				downloadedRepoMeta.GetPriority(),
+				downloadedRepoMeta.GetType()))
+	}
 	return downloadedRepoMeta, nil
 }
 

pkg/installer/repository_docker.go

@@ -37,10 +37,10 @@ import (
 )
 
 type dockerRepositoryGenerator struct {
-	b                compiler.CompilerBackend
+	b                       compiler.CompilerBackend
-	imagePrefix      string
+	imagePrefix, snapshotID string
-	imagePush, force bool
+	imagePush, force        bool
-	context          *types.Context
+	context                 *types.Context
 }
 
 func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
@@ -270,8 +270,7 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi
 	// Create a named snapshot and push it.
 	// It edits the metadata pointing at the repository files associated with the snapshot
 	// And copies the new files
-	id := time.Now().Format("20060102150405")
+	artifacts, snapshotRepoFile, err := r.Snapshot(d.snapshotID, repoTemp)
-	artifacts, snapshotRepoFile, err := r.Snapshot(id, repoTemp)
 	if err != nil {
 		return errors.Wrap(err, "while creating snapshot")
 	}

pkg/installer/repository_local.go

@@ -32,7 +32,10 @@ import (
 	"github.com/pkg/errors"
 )
 
-type localRepositoryGenerator struct{ context *types.Context }
+type localRepositoryGenerator struct {
+	context    *types.Context
+	snapshotID string
+}
 
 func (l *localRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) {
 	return buildPackageIndex(l.context, path, db)
@@ -124,7 +127,7 @@ func (g *localRepositoryGenerator) Generate(r *LuetSystemRepository, dst string,
 	// Create named snapshot.
 	// It edits the metadata pointing at the repository files associated with the snapshot
 	// And copies the new files
-	if _, _, err := r.Snapshot(time.Now().Format("20060102150405"), dst); err != nil {
+	if _, _, err := r.Snapshot(g.snapshotID, dst); err != nil {
 		return errors.Wrap(err, "while creating snapshot")
 	}
 

pkg/installer/system.go

@@ -90,13 +90,11 @@ func (s *System) buildFileIndex() {
 	if len(s.Database.GetPackages()) != len(s.fileIndexPackages) {
 		s.fileIndexPackages = make(map[string]pkg.Package)
 		for _, p := range s.Database.World() {
-			if _, ok := s.fileIndexPackages[p.GetPackageName()]; !ok {
+			files, _ := s.Database.GetPackageFiles(p)
-				files, _ := s.Database.GetPackageFiles(p)
+			for _, f := range files {
-				for _, f := range files {
+				s.fileIndex[f] = p
-					s.fileIndex[f] = p
-				}
-				s.fileIndexPackages[p.GetPackageName()] = p
 			}
+			s.fileIndexPackages[p.GetPackageName()] = p
 		}
 	}
 }
@@ -107,6 +105,13 @@ func (s *System) Clean() {
 	s.fileIndex = nil
 }
 
+func (s *System) FileIndex() map[string]pkg.Package {
+	s.buildFileIndex()
+	s.Lock()
+	defer s.Unlock()
+	return s.fileIndex
+}
+
 func (s *System) ExistsPackageFile(file string) (bool, pkg.Package, error) {
 	s.buildFileIndex()
 	s.Lock()