Add package buildtimestamp and luet upgrade --sync

Annotate the package build time when compiling, and use that from the
client to force upgrade of packages that changed the artifact, but
didn't changed any version.

The client can trigger this behavior with `luet upgrade --sync`

cmd/upgrade.go

@@ -62,6 +62,7 @@ var upgradeCmd = &cobra.Command{
 		full, _ := cmd.Flags().GetBool("full")
 		universe, _ := cmd.Flags().GetBool("universe")
 		clean, _ := cmd.Flags().GetBool("clean")
+		sync, _ := cmd.Flags().GetBool("sync")
 
 		LuetCfg.GetSolverOptions().Type = stype
 		LuetCfg.GetSolverOptions().LearnRate = float32(rate)
@@ -78,6 +79,7 @@ var upgradeCmd = &cobra.Command{
 			NoDeps:                     nodeps,
 			SolverUpgrade:              universe,
 			RemoveUnavailableOnUpgrade: clean,
+			UpgradeNewRevisions:        sync,
 		})
 		inst.Repositories(repos)
 		_, err := inst.SyncRepositories(false)
@@ -115,6 +117,7 @@ func init() {
 	upgradeCmd.Flags().Bool("full", true, "Attempts to remove as much packages as possible which aren't required (slow)")
 	upgradeCmd.Flags().Bool("universe", false, "Use ONLY the SAT solver to compute upgrades (experimental)")
 	upgradeCmd.Flags().Bool("clean", false, "Try to drop removed packages (experimental, only when --universe is enabled)")
+	upgradeCmd.Flags().Bool("sync", false, "Upgrade packages with new revisions (experimental)")
 
 	RootCmd.AddCommand(upgradeCmd)
 }

pkg/compiler/compiler.go

@@ -23,6 +23,7 @@ import (
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/mudler/luet/pkg/helpers"
 	. "github.com/mudler/luet/pkg/logger"
@@ -455,6 +456,8 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
 
 	artifact.SetFiles(filelist)
 
+	artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
+
 	err = artifact.WriteYaml(p.GetOutputPath())
 	if err != nil {
 		return artifact, errors.Wrap(err, "Failed while writing metadata file")

pkg/installer/installer.go

@@ -36,15 +36,15 @@ import (
 )
 
 type LuetInstallerOptions struct {
-	SolverOptions                             config.LuetSolverOptions
+	SolverOptions                                                  config.LuetSolverOptions
-	Concurrency                               int
+	Concurrency                                                    int
-	NoDeps                                    bool
+	NoDeps                                                         bool
-	OnlyDeps                                  bool
+	OnlyDeps                                                       bool
-	Force                                     bool
+	Force                                                          bool
-	PreserveSystemEssentialData               bool
+	PreserveSystemEssentialData                                    bool
-	FullUninstall, FullCleanUninstall         bool
+	FullUninstall, FullCleanUninstall                              bool
-	CheckConflicts                            bool
+	CheckConflicts                                                 bool
-	SolverUpgrade, RemoveUnavailableOnUpgrade bool
+	SolverUpgrade, RemoveUnavailableOnUpgrade, UpgradeNewRevisions bool
 }
 
 type LuetInstaller struct {
@@ -90,12 +90,18 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 		}
 	}
 
-	Info("Marked for uninstall")
+	if len(uninstall) > 0 {
+		Info("Packages marked for uninstall:")
+	}
+
 	for _, p := range uninstall {
 		Info(fmt.Sprintf("- %s", p.HumanReadableString()))
 	}
 
-	Info("Marked for upgrade")
+	if len(solution) > 0 {
+		Info("Packages marked for upgrade:")
+	}
+
 	toInstall := pkg.Packages{}
 	for _, assertion := range solution {
 		// Be sure to filter from solutions packages already installed in the system
@@ -105,6 +111,37 @@ func (l *LuetInstaller) Upgrade(s *System) error {
 		}
 	}
 
+	if l.Options.UpgradeNewRevisions {
+		Info("Checking packages with new revisions available")
+		for _, p := range s.Database.World() {
+			matches := syncedRepos.PackageMatches(pkg.Packages{p})
+			if len(matches) == 0 {
+				// Package missing. the user should run luet upgrade --universe
+				Info("Installed packages seems to be missing from remote repositories.")
+				Info("It is suggested to run 'luet upgrade --universe'")
+				continue
+			}
+			for _, artefact := range matches[0].Repo.GetIndex() {
+				if artefact.GetCompileSpec().GetPackage() == nil {
+					return errors.New("Package in compilespec empty")
+
+				}
+				if artefact.GetCompileSpec().GetPackage().Matches(p) && artefact.GetCompileSpec().GetPackage().GetBuildTimestamp() != p.GetBuildTimestamp() {
+					toInstall = append(toInstall, matches[0].Package).Unique()
+					uninstall = append(uninstall, p).Unique()
+					Info(
+						fmt.Sprintf("- %s ( %s vs %s ) repo: %s (date: %s)",
+							p.HumanReadableString(),
+							artefact.GetCompileSpec().GetPackage().GetBuildTimestamp(),
+							p.GetBuildTimestamp(),
+							matches[0].Repo.GetName(),
+							matches[0].Repo.GetLastUpdate(),
+						))
+				}
+			}
+		}
+	}
+
 	return l.swap(syncedRepos, uninstall, toInstall, s)
 }
 
@@ -349,6 +386,7 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
 
 			}
 			if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) {
+				currentPack.SetBuildTimestamp(artefact.GetCompileSpec().GetPackage().GetBuildTimestamp())
 				// Filter out already installed
 				if _, err := s.Database.FindPackage(currentPack); err != nil {
 					toInstall[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo}

pkg/package/package.go

@@ -107,6 +107,9 @@ type Package interface {
 	HumanReadableString() string
 	HashFingerprint(string) string
 
+	SetBuildTimestamp(s string)
+	GetBuildTimestamp() string
+
 	Clone() Package
 }
 
@@ -170,9 +173,10 @@ type DefaultPackage struct {
 	// Path is set only internally when tree is loaded from disk
 	Path string `json:"path,omitempty"`
 
-	Description string   `json:"description,omitempty"`
+	Description    string   `json:"description,omitempty"`
-	Uri         []string `json:"uri,omitempty"`
+	Uri            []string `json:"uri,omitempty"`
-	License     string   `json:"license,omitempty"`
+	License        string   `json:"license,omitempty"`
+	BuildTimestamp string   `json:"buildtimestamp,omitempty"`
 
 	Labels map[string]string `json:"labels,omitempty"` // Affects YAML field names too.
 }
@@ -207,7 +211,7 @@ func (p *DefaultPackage) GetFingerPrint() string {
 
 func (p *DefaultPackage) HashFingerprint(salt string) string {
 	h := md5.New()
-	io.WriteString(h, fmt.Sprintf("%s-%s",p.GetFingerPrint(),salt))
+	io.WriteString(h, fmt.Sprintf("%s-%s", p.GetFingerPrint(), salt))
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
@@ -229,6 +233,16 @@ func (p *DefaultPackage) GetPackageName() string {
 	return fmt.Sprintf("%s-%s", p.Name, p.Category)
 }
 
+// GetBuildTimestamp returns the package build timestamp
+func (p *DefaultPackage) GetBuildTimestamp() string {
+	return p.BuildTimestamp
+}
+
+// SetBuildTimestamp sets the package Build timestamp
+func (p *DefaultPackage) SetBuildTimestamp(s string) {
+	p.BuildTimestamp = s
+}
+
 // GetPath returns the path where the definition file was found
 func (p *DefaultPackage) GetPath() string {
 	return p.Path

tests/integration/14_upgrade_revision.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_old_repo" --destination $tmpdir/testbuild --compression gzip --full --clean=true
+    buildst=$?
+    assertTrue 'create package B 1.0' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+
+    mkdir $tmpdir/testbuild_revision
+    luet build --tree "$ROOT_DIR/tests/fixtures/upgrade_old_repo_revision" --destination $tmpdir/testbuild_revision --compression gzip --full --clean=true
+    buildst=$?
+    assertTrue 'create package B 1.0' "[ -e '$tmpdir/testbuild_revision/b-test-1.0.package.tar.gz' ]"
+    assertEquals 'builds successfully' "$buildst" "0"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/upgrade_old_repo" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild_revision/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/upgrade_old_repo_revision" \
+    --output $tmpdir/testbuild_revision \
+    --packages $tmpdir/testbuild_revision \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type http
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild_revision/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testUpgrade() {
+    luet install --config $tmpdir/luet.yaml test/b-1.0
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/test5' ]"
+
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild_revision"
+EOF
+
+    luet cleanup --config $tmpdir/luet.yaml
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+
+    luet upgrade --sync --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'upgrade test successfully' "$installst" "0"
+    assertTrue 'package uninstalled B' "[ ! -e '$tmpdir/testrootfs/test5' ]"
+    assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/newc' ]"
+
+    content=$(luet upgrade --sync --config $tmpdir/luet.yaml)
+    installst=$?
+    assertNotContains 'didn not upgrade' "$content" "Uninstalling"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+