Support compression and checksum for trees

Fixes #33 #34 #35

cmd/create-repo.go

@@ -17,6 +17,7 @@ package cmd
 import (
 	"os"
 
+	"github.com/mudler/luet/pkg/compiler"
 	installer "github.com/mudler/luet/pkg/installer"
 
 	. "github.com/mudler/luet/pkg/logger"
@@ -38,6 +39,8 @@ var createrepoCmd = &cobra.Command{
 		viper.BindPFlag("descr", cmd.Flags().Lookup("descr"))
 		viper.BindPFlag("urls", cmd.Flags().Lookup("urls"))
 		viper.BindPFlag("type", cmd.Flags().Lookup("type"))
+		viper.BindPFlag("tree-compression", cmd.Flags().Lookup("tree-compression"))
+		viper.BindPFlag("tree-path", cmd.Flags().Lookup("tree-path"))
 		viper.BindPFlag("reset-revision", cmd.Flags().Lookup("reset-revision"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
@@ -50,11 +53,21 @@ var createrepoCmd = &cobra.Command{
 		urls := viper.GetStringSlice("urls")
 		t := viper.GetString("type")
 		reset := viper.GetBool("reset-revision")
+		treetype := viper.GetString("tree-compression")
+		treepath := viper.GetString("tree-path")
 
 		repo, err := installer.GenerateRepository(name, descr, t, urls, 1, packages, tree, pkg.NewInMemoryDatabase(false))
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
+		if treetype != "" {
+			repo.SetTreeCompressionType(compiler.CompressionImplementation(treetype))
+		}
+
+		if treepath != "" {
+			repo.SetTreePath(treepath)
+		}
+
 		err = repo.Write(dst, reset)
 		if err != nil {
 			Fatal("Error: " + err.Error())
@@ -76,5 +89,8 @@ func init() {
 	createrepoCmd.Flags().String("type", "disk", "Repository type (disk)")
 	createrepoCmd.Flags().Bool("reset-revision", false, "Reset repository revision.")
 
+	createrepoCmd.Flags().String("tree-compression", "none", "Compression alg: none, gzip")
+	createrepoCmd.Flags().String("tree-path", installer.TREE_TARBALL, "Repository tree filename")
+
 	RootCmd.AddCommand(createrepoCmd)
 }

pkg/compiler/artifact.go

@@ -112,6 +112,12 @@ func (a *PackageArtifact) SetCompressionType(t CompressionImplementation) {
 	a.CompressionType = t
 }
 
+func (a *PackageArtifact) GetChecksums() Checksums {
+	return a.Checksums
+}
+func (a *PackageArtifact) SetChecksums(c Checksums) {
+	a.Checksums = c
+}
 func (a *PackageArtifact) Hash() error {
 	return a.Checksums.Generate(a)
 }

pkg/compiler/interface.go

@@ -90,6 +90,9 @@ type Artifact interface {
 	FileList() ([]string, error)
 	Hash() error
 	Verify() error
+
+	GetChecksums() Checksums
+	SetChecksums(c Checksums)
 }
 
 type ArtifactNode struct {

pkg/installer/interface.go

@@ -58,4 +58,9 @@ type Repository interface {
 	GetLastUpdate() string
 	SetLastUpdate(string)
 	Client() Client
+
+	GetTreeChecksums() compiler.Checksums
+	GetTreeCompressionType() compiler.CompressionImplementation
+	SetTreeCompressionType(c compiler.CompressionImplementation)
+	SetTreeChecksums(c compiler.Checksums)
 }

pkg/installer/repository.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"sort"
@@ -49,6 +50,9 @@ type LuetSystemRepository struct {
 
 	Index               compiler.ArtifactIndex             `json:"index"`
 	Tree                tree.Builder                       `json:"-"`
+	TreePath            string                             `json:"treepath"`
+	TreeCompressionType compiler.CompressionImplementation `json:"treecompressiontype"`
+	TreeChecksums       compiler.Checksums                 `json:"treechecksums"`
 }
 
 type LuetSystemRepositorySerialized struct {
@@ -60,6 +64,9 @@ type LuetSystemRepositorySerialized struct {
 	Type                string                             `json:"type"`
 	Revision            int                                `json:"revision,omitempty"`
 	LastUpdate          string                             `json:"last_update,omitempty"`
+	TreePath            string                             `json:"treepath"`
+	TreeCompressionType compiler.CompressionImplementation `json:"treecompressiontype"`
+	TreeChecksums       compiler.Checksums                 `json:"treechecksums"`
 }
 
 func GenerateRepository(name, descr, t string, urls []string, priority int, src, treeDir string, db pkg.PackageDatabase) (Repository, error) {
@@ -109,6 +116,9 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos
 			true,
 			false,
 		),
+		TreeCompressionType: p.TreeCompressionType,
+		TreeChecksums:       p.TreeChecksums,
+		TreePath:            p.TreePath,
 	}
 	if p.Revision > 0 {
 		r.Revision = p.Revision
@@ -163,6 +173,23 @@ func (r *LuetSystemRepository) GetName() string {
 func (r *LuetSystemRepository) GetDescription() string {
 	return r.LuetRepository.Description
 }
+
+func (r *LuetSystemRepository) GetTreeCompressionType() compiler.CompressionImplementation {
+	return r.TreeCompressionType
+}
+
+func (r *LuetSystemRepository) GetTreeChecksums() compiler.Checksums {
+	return r.TreeChecksums
+}
+
+func (r *LuetSystemRepository) SetTreeCompressionType(c compiler.CompressionImplementation) {
+	r.TreeCompressionType = c
+}
+
+func (r *LuetSystemRepository) SetTreeChecksums(c compiler.Checksums) {
+	r.TreeChecksums = c
+}
+
 func (r *LuetSystemRepository) GetType() string {
 	return r.LuetRepository.Type
 }
@@ -250,20 +277,11 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 	}
 	r.Revision++
 
-	data, err := yaml.Marshal(r)
-	if err != nil {
-		return err
-	}
-
 	Info(fmt.Sprintf(
 		"For repository %s creating revision %d and last update %s...",
 		r.Name, r.Revision, r.LastUpdate,
 	))
 
-	err = ioutil.WriteFile(repospec, data, os.ModePerm)
-	if err != nil {
-		return err
-	}
 	archive, err := ioutil.TempDir(os.TempDir(), "archive")
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating tempdir for archive")
@@ -273,10 +291,34 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 	if err != nil {
 		return errors.Wrap(err, "Error met while saving the tree")
 	}
-	err = helpers.Tar(archive, filepath.Join(dst, TREE_TARBALL))
+	tpath := r.GetTreePath()
+	if tpath == "" {
+		tpath = TREE_TARBALL
+	}
+
+	a := compiler.NewPackageArtifact(filepath.Join(dst, tpath))
+	a.SetCompressionType(r.TreeCompressionType)
+	err = a.Compress(archive, 1)
 	if err != nil {
 		return errors.Wrap(err, "Error met while creating package archive")
 	}
+
+	r.TreePath = path.Base(a.GetPath())
+	err = a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for tree")
+	}
+	r.TreeChecksums = a.GetChecksums()
+
+	data, err := yaml.Marshal(r)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(repospec, data, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -339,11 +381,25 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	}
 
 	if !repoUpdated {
-		archivetree, err := c.DownloadFile(TREE_TARBALL)
+		tpath := repo.GetTreePath()
-		if err != nil {
+		if tpath == "" {
-			return nil, errors.Wrap(err, "While downloading "+TREE_TARBALL)
+			tpath = TREE_TARBALL
+		}
+		a := compiler.NewPackageArtifact(tpath)
+
+		artifact, err := c.DownloadArtifact(a)
+		if err != nil {
+			return nil, errors.Wrap(err, "While downloading "+tpath)
+		}
+		defer os.Remove(artifact.GetPath())
+
+		artifact.SetChecksums(repo.GetTreeChecksums())
+		artifact.SetCompressionType(repo.GetTreeCompressionType())
+
+		err = artifact.Verify()
+		if err != nil {
+			return nil, errors.Wrap(err, "Tree integrity check failure")
 		}
-		defer os.RemoveAll(archivetree) // clean up
 
 		Debug("Tree tarball for the repository " + r.GetName() + " downloaded correctly.")
 
@@ -356,10 +412,11 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 			// Remove previous tree
 			os.RemoveAll(treefs)
 		}
-		Debug("Untar tree of the repository " + r.Name + "...")
+		Debug("Decompress tree of the repository " + r.Name + "...")
-		err = helpers.Untar(archivetree, treefs, true)
+
+		err = artifact.Unpack(treefs, true)
 		if err != nil {
-			return nil, errors.Wrap(err, "Error met while unpacking rootfs")
+			return nil, errors.Wrap(err, "Error met while unpacking tree")
 		}
 
 		tsec, _ := strconv.ParseInt(repo.GetLastUpdate(), 10, 64)