Generate repository metadata and packages for docker repository type

Drop image-repository on create-repo. In case of a docker repository, --output is the image reference to use.
Also restore default output build dir.

See also: #169

cmd/build.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 
 	"github.com/ghodss/yaml"
 	helpers "github.com/mudler/luet/cmd/helpers"
@@ -116,14 +117,9 @@ Build packages specifying multiple definition trees:
 		}
 		pretend, _ := cmd.Flags().GetBool("pretend")
 		compilerSpecs := compiler.NewLuetCompilationspecs()
-		var compilerBackend compiler.CompilerBackend
 		var db pkg.PackageDatabase
-		switch backendType {
+
-		case "img":
+		compilerBackend := backend.NewBackend(backendType)
-			compilerBackend = backend.NewSimpleImgBackend()
-		case "docker":
-			compilerBackend = backend.NewSimpleDockerBackend()
-		}
 
 		switch databaseType {
 		case "memory":
@@ -310,7 +306,7 @@ func init() {
 	buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
 	buildCmd.Flags().String("values", "", "Build values file to interpolate with each package")
 
-	buildCmd.Flags().String("destination", path, "Destination folder")
+	buildCmd.Flags().String("destination", filepath.Join(path, "build"), "Destination folder")
 	buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip, zstd")
 	buildCmd.Flags().String("image-repository", "luet/cache", "Default base image string for generated image")
 	buildCmd.Flags().Bool("push", false, "Push images to a hub")

cmd/create-repo.go

@@ -16,8 +16,10 @@ package cmd
 
 import (
 	"os"
+	"path/filepath"
 
 	"github.com/mudler/luet/pkg/compiler"
+	"github.com/mudler/luet/pkg/compiler/backend"
 	. "github.com/mudler/luet/pkg/config"
 	installer "github.com/mudler/luet/pkg/installer"
 	. "github.com/mudler/luet/pkg/logger"
@@ -54,6 +56,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		viper.BindPFlag("packages", cmd.Flags().Lookup("packages"))
 		viper.BindPFlag("tree", cmd.Flags().Lookup("tree"))
 		viper.BindPFlag("output", cmd.Flags().Lookup("output"))
+		viper.BindPFlag("backend", cmd.Flags().Lookup("backend"))
 		viper.BindPFlag("name", cmd.Flags().Lookup("name"))
 		viper.BindPFlag("descr", cmd.Flags().Lookup("descr"))
 		viper.BindPFlag("urls", cmd.Flags().Lookup("urls"))
@@ -82,9 +85,11 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		metatype := viper.GetString("meta-compression")
 		metaName := viper.GetString("meta-filename")
 		source_repo := viper.GetString("repo")
+		backendType := viper.GetString("backend")
 
 		treeFile := installer.NewDefaultTreeRepositoryFile()
 		metaFile := installer.NewDefaultMetaRepositoryFile()
+		compilerBackend := backend.NewBackend(backendType)
 
 		if source_repo != "" {
 			// Search for system repository
@@ -107,11 +112,11 @@ Create a repository from the metadata description defined in the luet.yaml confi
 				lrepo.Priority,
 				packages,
 				treePaths,
-				pkg.NewInMemoryDatabase(false))
+				pkg.NewInMemoryDatabase(false), compilerBackend, dst)
 
 		} else {
 			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
-				treePaths, pkg.NewInMemoryDatabase(false))
+				treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst)
 		}
 
 		if err != nil {
@@ -137,7 +142,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		repo.SetRepositoryFile(installer.REPOFILE_TREE_KEY, treeFile)
 		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
 
-		err = repo.Write(dst, reset)
+		err = repo.Write(dst, reset, true)
 		if err != nil {
 			Fatal("Error: " + err.Error())
 		}
@@ -149,15 +154,16 @@ func init() {
 	if err != nil {
 		Fatal(err)
 	}
-	createrepoCmd.Flags().String("packages", path, "Packages folder (output from build)")
+	createrepoCmd.Flags().String("packages", filepath.Join(path, "build"), "Packages folder (output from build)")
 	createrepoCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the source trees to use.")
-	createrepoCmd.Flags().String("output", path, "Destination folder")
+	createrepoCmd.Flags().String("output", filepath.Join(path, "build"), "Destination for generated archives. With 'docker' repository type, it should be an image reference (e.g 'foo/bar')")
 	createrepoCmd.Flags().String("name", "luet", "Repository name")
 	createrepoCmd.Flags().String("descr", "luet", "Repository description")
 	createrepoCmd.Flags().StringSlice("urls", []string{}, "Repository URLs")
 	createrepoCmd.Flags().String("type", "disk", "Repository type (disk)")
 	createrepoCmd.Flags().Bool("reset-revision", false, "Reset repository revision.")
 	createrepoCmd.Flags().String("repo", "", "Use repository defined in configuration.")
+	createrepoCmd.Flags().String("backend", "docker", "backend used (docker,img)")
 
 	createrepoCmd.Flags().String("tree-compression", "gzip", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")

pkg/compiler/backend/common.go

@@ -17,9 +17,27 @@ package backend
 
 import (
 	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/mudler/luet/pkg/compiler"
+)
+
+const (
+	ImgBackend    = "img"
+	DockerBackend = "docker"
 )
 
 func imageAvailable(image string) bool {
 	_, err := crane.Digest(image)
 	return err == nil
 }
+
+func NewBackend(s string) compiler.CompilerBackend {
+	var compilerBackend compiler.CompilerBackend
+
+	switch s {
+	case ImgBackend:
+		compilerBackend = NewSimpleImgBackend()
+	case DockerBackend:
+		compilerBackend = NewSimpleDockerBackend()
+	}
+	return compilerBackend
+}

pkg/installer/interface.go

@@ -51,7 +51,7 @@ type Repository interface {
 	SetIndex(i compiler.ArtifactIndex)
 	GetTree() tree.Builder
 	SetTree(tree.Builder)
-	Write(path string, resetRevision bool) error
+	Write(path string, resetRevision, force bool) error
 	Sync(bool) (Repository, error)
 	GetTreePath() string
 	SetTreePath(string)
@@ -72,4 +72,6 @@ type Repository interface {
 	SetRepositoryFile(string, LuetRepositoryFile)
 	SetName(p string)
 	Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized)
+	GetBackend() compiler.CompilerBackend
+	SetBackend(b compiler.CompilerBackend)
 }

pkg/installer/repository.go

@@ -46,6 +46,10 @@ const (
 
 	REPOFILE_TREE_KEY = "tree"
 	REPOFILE_META_KEY = "meta"
+
+	DiskRepositoryType   = "disk"
+	HttpRepositoryType   = "http"
+	DockerRepositoryType = "docker"
 )
 
 type LuetRepositoryFile struct {
@@ -60,6 +64,7 @@ type LuetSystemRepository struct {
 	Index           compiler.ArtifactIndex        `json:"index"`
 	Tree            tree.Builder                  `json:"-"`
 	RepositoryFiles map[string]LuetRepositoryFile `json:"repo_files"`
+	Backend         compiler.CompilerBackend      `json:"-"`
 }
 
 type LuetSystemRepositorySerialized struct {
@@ -177,7 +182,9 @@ func (f *LuetRepositoryFile) GetChecksums() compiler.Checksums {
 	return f.Checksums
 }
 
-func GenerateRepository(name, descr, t string, urls []string, priority int, src string, treesDir []string, db pkg.PackageDatabase) (Repository, error) {
+func GenerateRepository(name, descr, t string, urls []string,
+	priority int, src string, treesDir []string, db pkg.PackageDatabase,
+	b compiler.CompilerBackend, imagePrefix string) (Repository, error) {
 
 	tr := tree.NewInstallerRecipe(db)
 
@@ -188,14 +195,27 @@ func GenerateRepository(name, descr, t string, urls []string, priority int, src
 		}
 	}
 
-	art, err := buildPackageIndex(src, tr.GetDatabase())
+	var art []compiler.Artifact
+	var err error
+	switch t {
+	case DiskRepositoryType, HttpRepositoryType:
+		art, err = buildPackageIndex(src, tr.GetDatabase())
 		if err != nil {
 			return nil, err
 		}
 
-	return NewLuetSystemRepository(
+	case DockerRepositoryType:
+		art, err = generatePackageImages(b, imagePrefix, src, tr.GetDatabase())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	repo := NewLuetSystemRepository(
 		config.NewLuetRepository(name, t, descr, urls, priority, true, false),
-		art, tr), nil
+		art, tr)
+	repo.SetBackend(b)
+	return repo, nil
 }
 
 func NewSystemRepository(repo config.LuetRepository) Repository {
@@ -244,6 +264,53 @@ func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repos
 	return r, err
 }
 
+func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) {
+
+	var art []compiler.Artifact
+	var ff = func(currentpath string, info os.FileInfo, err error) error {
+
+		if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
+			return nil // Skip with no errors
+		}
+
+		dat, err := ioutil.ReadFile(currentpath)
+		if err != nil {
+			return errors.Wrap(err, "Error reading file "+currentpath)
+		}
+
+		artifact, err := compiler.NewPackageArtifactFromYaml(dat)
+		if err != nil {
+			return errors.Wrap(err, "Error reading yaml "+currentpath)
+		}
+
+		// We want to include packages that are ONLY referenced in the tree.
+		// the ones which aren't should be deleted. (TODO: by another cli command?)
+		if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
+			Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+				artifact.GetCompileSpec().GetPackage().HumanReadableString()))
+			return nil
+		}
+
+		Info("Generating final image", imagePrefix+artifact.GetCompileSpec().GetPackage().GetFingerPrint(),
+			"for package ", artifact.GetCompileSpec().GetPackage().HumanReadableString())
+		if opts, err := artifact.GenerateFinalImage(imagePrefix+artifact.GetCompileSpec().GetPackage().GetFingerPrint(), b, true); err != nil {
+			return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
+		}
+		// TODO: Push image
+
+		art = append(art, artifact)
+
+		return nil
+	}
+
+	err := filepath.Walk(path, ff)
+	if err != nil {
+		return nil, err
+
+	}
+	return art, nil
+}
+
 func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) {
 
 	var art []compiler.Artifact
@@ -266,7 +333,7 @@ func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact
 		// We want to include packages that are ONLY referenced in the tree.
 		// the ones which aren't should be deleted. (TODO: by another cli command?)
 		if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil {
-			Info(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
+			Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.",
 				artifact.GetCompileSpec().GetPackage().HumanReadableString()))
 			return nil
 		}
@@ -306,6 +373,13 @@ func (r *LuetSystemRepository) SetType(p string) {
 	r.LuetRepository.Type = p
 }
 
+func (r *LuetSystemRepository) GetBackend() compiler.CompilerBackend {
+	return r.Backend
+}
+func (r *LuetSystemRepository) SetBackend(b compiler.CompilerBackend) {
+	r.Backend = b
+}
+
 func (r *LuetSystemRepository) SetName(p string) {
 	r.LuetRepository.Name = p
 }
@@ -400,7 +474,7 @@ func (r *LuetSystemRepository) ReadSpecFile(file string, removeFile bool) (Repos
 	return repo, err
 }
 
-func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
+func (r *LuetSystemRepository) genLocalRepo(dst string, resetRevision bool) error {
 	err := os.MkdirAll(dst, os.ModePerm)
 	if err != nil {
 		return err
@@ -522,15 +596,195 @@ func (r *LuetSystemRepository) Write(dst string, resetRevision bool) error {
 		Repo: *r,
 		Path: dst,
 	})
-
 	return nil
 }
 
+func (r *LuetSystemRepository) genDockerRepo(imagePrefix string, resetRevision, force bool) error {
+	// - Iterate over meta, build final images, push them if necessary
+	//   - while pushing, check if image already exists, and if exist push them only if --force is supplied
+	// - Generate final images for metadata and push
+
+	imageRepository := fmt.Sprintf("%s%s", imagePrefix, "repository")
+
+	r.LastUpdate = strconv.FormatInt(time.Now().Unix(), 10)
+
+	repoTemp, err := config.LuetCfg.GetSystem().TempDir("repo")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for repository")
+	}
+	defer os.RemoveAll(repoTemp) // clean up
+
+	if r.GetBackend().ImageAvailable(imageRepository) {
+		if err := r.GetBackend().DownloadImage(compiler.CompilerBackendOptions{ImageName: imageRepository}); err != nil {
+			return errors.Wrapf(err, "while downloading '%s'", imageRepository)
+		}
+
+		if err := r.GetBackend().ExtractRootfs(compiler.CompilerBackendOptions{ImageName: imageRepository, Destination: repoTemp}, false); err != nil {
+			return errors.Wrapf(err, "while extracting '%s'", imageRepository)
+		}
+	}
+
+	repospec := filepath.Join(repoTemp, REPOSITORY_SPECFILE)
+	if resetRevision {
+		r.Revision = 0
+	} else {
+		if _, err := os.Stat(repospec); !os.IsNotExist(err) {
+			// Read existing file for retrieve revision
+			spec, err := r.ReadSpecFile(repospec, false)
+			if err != nil {
+				return err
+			}
+			r.Revision = spec.GetRevision()
+		}
+	}
+	r.Revision++
+
+	Info(fmt.Sprintf(
+		"For repository %s creating revision %d and last update %s...",
+		r.Name, r.Revision, r.LastUpdate,
+	))
+
+	bus.Manager.Publish(bus.EventRepositoryPreBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: imageRepository,
+	})
+
+	// Create tree and repository file
+	archive, err := config.LuetCfg.GetSystem().TempDir("archive")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for archive")
+	}
+	defer os.RemoveAll(archive) // clean up
+	err = r.GetTree().Save(archive)
+	if err != nil {
+		return errors.Wrap(err, "Error met while saving the tree")
+	}
+
+	treeFile := NewDefaultTreeRepositoryFile()
+	a := compiler.NewPackageArtifact(filepath.Join(repoTemp, treeFile.GetFileName()))
+	a.SetCompressionType(treeFile.GetCompressionType())
+	err = a.Compress(archive, 1)
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating package archive")
+	}
+
+	// Update the tree name with the name created by compression selected.
+	treeFile.SetFileName(path.Base(a.GetPath()))
+	err = a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for tree")
+	}
+	treeFile.SetChecksums(a.GetChecksums())
+	r.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile)
+
+	imageTree := fmt.Sprintf("%s%s", imagePrefix, REPOFILE_TREE_KEY)
+	Debug("Generating image", imageTree)
+	if opts, err := a.GenerateFinalImage(imageTree, r.GetBackend(), false); err != nil {
+		return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName)
+	}
+	// TODO: Push imageTree
+
+	// Create Metadata struct and serialized repository
+	meta, serialized := r.Serialize()
+
+	// Create metadata file and repository file
+	metaTmpDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for metadata")
+	}
+	defer os.RemoveAll(metaTmpDir) // clean up
+
+	metaFile, err := r.GetRepositoryFile(REPOFILE_META_KEY)
+	if err != nil {
+		metaFile = NewDefaultMetaRepositoryFile()
+		r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
+	}
+
+	repoMetaSpec := filepath.Join(metaTmpDir, REPOSITORY_METAFILE)
+	// Create repository.meta.yaml file
+	err = meta.WriteFile(repoMetaSpec)
+	if err != nil {
+		return err
+	}
+
+	// create temp dir for metafile
+	metaDir, err := config.LuetCfg.GetSystem().TempDir("metadata")
+	if err != nil {
+		return errors.Wrap(err, "Error met while creating tempdir for metadata")
+	}
+	defer os.RemoveAll(metaDir) // clean up
+
+	a = compiler.NewPackageArtifact(filepath.Join(metaDir, metaFile.GetFileName()))
+	a.SetCompressionType(metaFile.GetCompressionType())
+	err = a.Compress(metaTmpDir, 1)
+	if err != nil {
+		return errors.Wrap(err, "Error met while archiving repository metadata")
+	}
+
+	metaFile.SetFileName(path.Base(a.GetPath()))
+	r.SetRepositoryFile(REPOFILE_META_KEY, metaFile)
+	err = a.Hash()
+	if err != nil {
+		return errors.Wrap(err, "Failed generating checksums for metadata")
+	}
+	metaFile.SetChecksums(a.GetChecksums())
+
+	imageMetaTree := fmt.Sprintf("%s%s", imagePrefix, REPOFILE_META_KEY)
+	if opts, err := a.GenerateFinalImage(imageMetaTree, r.GetBackend(), false); err != nil {
+		return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName)
+	}
+
+	// TODO: Push image meta tree
+	data, err := yaml.Marshal(serialized)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(repospec, data, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	tempRepoFile := filepath.Join(metaDir, REPOSITORY_SPECFILE+".tar")
+	if err := helpers.Tar(repospec, tempRepoFile); err != nil {
+		return errors.Wrap(err, "Error met while archiving repository file")
+	}
+
+	a = compiler.NewPackageArtifact(tempRepoFile)
+	imageRepo := fmt.Sprintf("%s%s", imagePrefix, "repository")
+	if opts, err := a.GenerateFinalImage(imageRepo, r.GetBackend(), false); err != nil {
+		return errors.Wrap(err, "Failed generating repository image"+opts.ImageName)
+	}
+	// TODO: Push image meta tree
+
+	bus.Manager.Publish(bus.EventRepositoryPostBuild, struct {
+		Repo LuetSystemRepository
+		Path string
+	}{
+		Repo: *r,
+		Path: imagePrefix,
+	})
+	return nil
+}
+
+// Write writes the repository metadata to the supplied destination
+func (r *LuetSystemRepository) Write(dst string, resetRevision, force bool) error {
+	switch r.GetType() {
+	case DiskRepositoryType, HttpRepositoryType:
+		return r.genLocalRepo(dst, resetRevision)
+	case DockerRepositoryType:
+		return r.genDockerRepo(dst, resetRevision, force)
+	}
+	return errors.New("invalid repository type")
+}
+
 func (r *LuetSystemRepository) Client() Client {
 	switch r.GetType() {
-	case "disk":
+	case DiskRepositoryType:
 		return client.NewLocalClient(client.RepoData{Urls: r.GetUrls()})
-	case "http":
+	case HttpRepositoryType:
 		return client.NewHttpClient(
 			client.RepoData{
 				Urls:           r.GetUrls(),
@@ -540,6 +794,7 @@ func (r *LuetSystemRepository) Client() Client {
 
 	return nil
 }
+
 func (r *LuetSystemRepository) Sync(force bool) (Repository, error) {
 	var repoUpdated bool = false
 	var treefs, metafs string

pkg/installer/repository_test.go

@@ -90,7 +90,7 @@ var _ = Describe("Repository", func() {
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())
@@ -172,7 +172,7 @@ var _ = Describe("Repository", func() {
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(TREE_TARBALL + ".gz"))).ToNot(BeTrue())
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_METAFILE + ".tar"))).ToNot(BeTrue())
-			err = repo.Write(tmpdir, false)
+			err = repo.Write(tmpdir, false, true)
 			Expect(err).ToNot(HaveOccurred())
 
 			Expect(helpers.Exists(spec.Rel(REPOSITORY_SPECFILE))).To(BeTrue())