Allow create-repo to source trees from remote repositories

This makes possible to create the repositorie from external ones,
It also required to address #26.

Fixes #26

cmd/create-repo.go

@@ -91,6 +91,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		metaName := viper.GetString("meta-filename")
 		source_repo := viper.GetString("repo")
 		backendType := viper.GetString("backend")
+		fromRepo, _ := cmd.Flags().GetBool("from-repositories")
 
 		treeFile := installer.NewDefaultTreeRepositoryFile()
 		metaFile := installer.NewDefaultMetaRepositoryFile()
@@ -102,9 +103,7 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		if source_repo != "" {
 			// Search for system repository
 			lrepo, err := LuetCfg.GetSystemRepository(source_repo)
-			if err != nil {
-				Fatal("Error: " + err.Error())
-			}
+			helpers.CheckErr(err)
 
 			if len(treePaths) <= 0 {
 				treePaths = []string{lrepo.TreePath}
@@ -120,15 +119,19 @@ Create a repository from the metadata description defined in the luet.yaml confi
 				lrepo.Priority,
 				packages,
 				treePaths,
-				pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force)
+				pkg.NewInMemoryDatabase(false),
+				compilerBackend,
+				dst,
+				imagePush,
+				force,
+				fromRepo,
+				LuetCfg)
+			helpers.CheckErr(err)
 
 		} else {
 			repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages,
-				treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force)
-		}
-
-		if err != nil {
-			Fatal("Error: " + err.Error())
+				treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force, fromRepo, LuetCfg)
+			helpers.CheckErr(err)
 		}
 
 		if treetype != "" {
@@ -151,17 +154,15 @@ Create a repository from the metadata description defined in the luet.yaml confi
 		repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile)
 
 		err = repo.Write(dst, reset, true)
-		if err != nil {
-			Fatal("Error: " + err.Error())
-		}
+		helpers.CheckErr(err)
+
 	},
 }
 
 func init() {
 	path, err := os.Getwd()
-	if err != nil {
-		Fatal(err)
-	}
+	helpers.CheckErr(err)
+
 	createrepoCmd.Flags().String("packages", filepath.Join(path, "build"), "Packages folder (output from build)")
 	createrepoCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the source trees to use.")
 	createrepoCmd.Flags().String("output", filepath.Join(path, "build"), "Destination for generated archives. With 'docker' repository type, it should be an image reference (e.g 'foo/bar')")
@@ -180,6 +181,7 @@ func init() {
 	createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename")
 	createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd")
 	createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename")
+	createrepoCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from")
 
 	RootCmd.AddCommand(createrepoCmd)
 }

pkg/compiler/compiler.go

@@ -752,7 +752,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 		Assert:      *targetAssertion,
 	})
 
-	// Update compilespec build options
+	// Update compilespec build options - it will be then serialized into the compilation metadata file
 	p.SetBuildOptions(cs.Options)
 
 	// - If image is set we just generate a plain dockerfile

pkg/compiler/types/artifact/artifact.go

@@ -156,14 +156,20 @@ COPY . /`
 
 // CreateArtifactForFile creates a new artifact from the given file
 func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) {
-
+	if _, err := os.Stat(s); os.IsNotExist(err) {
+		return nil, errors.Wrap(err, "artifact path doesn't exist")
+	}
 	fileName := path.Base(s)
 	archive, err := LuetCfg.GetSystem().TempDir("archive")
 	if err != nil {
 		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)
 	}
 	defer os.RemoveAll(archive) // clean up
-	helpers.CopyFile(s, filepath.Join(archive, fileName))
+	dst := filepath.Join(archive, fileName)
+	if err := helpers.CopyFile(s, dst); err != nil {
+		return nil, errors.Wrapf(err, "error while copying %s to %s", s, dst)
+	}
+
 	artifact, err := LuetCfg.GetSystem().TempDir("artifact")
 	if err != nil {
 		return nil, errors.Wrap(err, "error met while creating tempdir for "+s)

pkg/installer/installer_test.go

@@ -44,7 +44,7 @@ func stubRepo(tmpdir, tree string) (*LuetSystemRepository, error) {
 		1,
 		tmpdir,
 		[]string{tree},
-		pkg.NewInMemoryDatabase(false), nil, "", false, false)
+		pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
 }
 
 var _ = Describe("Installer", func() {
@@ -342,7 +342,7 @@ urls:
 				[]string{tmpdir}, 1,
 				tmpdir,
 				[]string{"../../tests/fixtures/buildable"},
-				pkg.NewInMemoryDatabase(false), nil, "", false, false)
+				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())
@@ -468,7 +468,7 @@ urls:
 				1,
 				tmpdir,
 				[]string{"../../tests/fixtures/buildable"},
-				pkg.NewInMemoryDatabase(false), nil, "", false, false)
+				pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(repo.GetName()).To(Equal("test"))
 			Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue())

pkg/installer/repository.go

@@ -254,13 +254,18 @@ func (f *LuetRepositoryFile) GetChecksums() artifact.Checksums {
 // In case the repository is local, it will build the package Index
 func GenerateRepository(name, descr, t string, urls []string,
 	priority int, src string, treesDir []string, db pkg.PackageDatabase,
-	b compiler.CompilerBackend, imagePrefix string, pushImages, force bool) (*LuetSystemRepository, error) {
+	b compiler.CompilerBackend, imagePrefix string, pushImages, force, fromRepo bool, c *config.LuetConfig) (*LuetSystemRepository, error) {
+
+	// 1: First filter the runtime db to only the metadata we actually have
 
-	tr := tree.NewInstallerRecipe(db)
 	btr := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
+	runtimeTree := pkg.NewInMemoryDatabase(false)
+
+	tempTree := pkg.NewInMemoryDatabase(false)
+	temptr := tree.NewInstallerRecipe(tempTree)
 
 	for _, treeDir := range treesDir {
-		if err := tr.Load(treeDir); err != nil {
+		if err := temptr.Load(treeDir); err != nil {
 			return nil, err
 		}
 		if err := btr.Load(treeDir); err != nil {
@@ -268,9 +273,33 @@ func GenerateRepository(name, descr, t string, urls []string,
 		}
 	}
 
+	// 2: if fromRepo, build a new tree like the compiler is doing and use it to source the above specs,
+	// instead of local tree
+
+	repodb := pkg.NewInMemoryDatabase(false)
+	generalRecipe := tree.NewCompilerRecipe(repodb)
+
+	if fromRepo {
+		if err := LoadBuildTree(generalRecipe, repodb, c); err != nil {
+			Warning("errors while loading trees from repositories", err.Error())
+		}
+
+		if err := repodb.Clone(tempTree); err != nil {
+			Warning("errors while cloning trees from repositories", err.Error())
+		}
+
+	}
+
+	// Pick only atoms in db which have a real metadata for runtime db (tr)
+	for _, p := range tempTree.World() {
+		if _, err := os.Stat(filepath.Join(src, p.GetMetadataFilePath())); err == nil {
+			runtimeTree.CreatePackage(p)
+		}
+	}
+
 	repo := &LuetSystemRepository{
 		LuetRepository:  config.NewLuetRepository(name, t, descr, urls, priority, true, false),
-		Tree:            tr,
+		Tree:            tree.NewInstallerRecipe(runtimeTree),
 		BuildTree:       btr,
 		RepositoryFiles: map[string]LuetRepositoryFile{},
 		PushImages:      pushImages,
@@ -918,7 +947,6 @@ func (r Repositories) SyncDatabase(d pkg.PackageDatabase) {
 			}
 		}
 	}
-
 }
 
 type PackageMatch struct {

pkg/installer/repository_docker.go

@@ -52,15 +52,17 @@ func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDataba
 			Debug("Skipping", info.Name(), err.Error())
 			return nil
 		}
+		if info.IsDir() {
+			Debug("Skipping directories")
+			return nil
+		}
 
-		if strings.HasSuffix(info.Name(), ".metadata.yaml") {
-			a := artifact.NewPackageArtifact(info.Name())
-			imageRepo := fmt.Sprintf("%s:%s", l.imagePrefix, filepath.Base(info.Name()))
+		if !strings.HasSuffix(info.Name(), ".metadata.yaml") {
+			return nil
+		}
 
-			if err := l.pushFileFromArtifact(a, imageRepo); err != nil {
-				return errors.Wrap(err, "while pushing file from artifact")
-			}
-			return nil // Skip with no errors
+		if err := l.pushImageFromArtifact(artifact.NewPackageArtifact(currentpath), l.b); err != nil {
+			return errors.Wrap(err, "while pushing metadata file associated to the artifact")
 		}
 
 		dat, err := ioutil.ReadFile(currentpath)

pkg/installer/repository_test.go

@@ -47,7 +47,7 @@ func dockerStubRepo(tmpdir, tree, image string, push, force bool) (*LuetSystemRe
 		1,
 		tmpdir,
 		[]string{tree},
-		pkg.NewInMemoryDatabase(false), backend.NewSimpleDockerBackend(), image, push, force)
+		pkg.NewInMemoryDatabase(false), backend.NewSimpleDockerBackend(), image, push, force, false, nil)
 }
 
 var _ = Describe("Repository", func() {

tests/integration/28_nobuildtreedocker.sh

@@ -0,0 +1,146 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testConfig() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "docker"
+     enable: true
+     urls:
+       - "${TEST_DOCKER_IMAGE}"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testBuild() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    mkdir $tmpdir/testbuild
+    mkdir $tmpdir/empty
+    luet build --tree "$tmpdir/empty" --config $tmpdir/luet.yaml --from-repositories --destination $tmpdir/testbuild --compression zstd test/c@1.0 > /dev/null
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.zst' ]"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]"
+}
+
+testRepo() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet create-repo \
+    --output "${TEST_DOCKER_IMAGE}-2" \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --tree-compression zstd \
+    --tree-filename foo.tar \
+    --tree "$tmpdir/empty" --config $tmpdir/luet.yaml --from-repositories \
+    --meta-filename repository.meta.tar \
+    --meta-compression zstd \
+    --type docker --push-images --force-push --debug
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+}
+
+testConfigClient() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    cat <<EOF > $tmpdir/luet-client.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+config_from_host: true
+repositories:
+   - name: "main"
+     type: "docker"
+     enable: true
+     urls:
+       - "${TEST_DOCKER_IMAGE}-2"
+EOF
+    luet config --config $tmpdir/luet-client.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet install -y --config $tmpdir/luet-client.yaml test/c@1.0
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+}
+
+testReInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    output=$(luet install -y --config $tmpdir/luet-client.yaml  test/c@1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertContains 'contains warning' "$output" 'No packages to install'
+}
+
+testUnInstall() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet uninstall -y --config $tmpdir/luet-client.yaml test/c@1.0
+    installst=$?
+    assertEquals 'uninstall test successfully' "$installst" "0"
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+}
+
+testInstallAgain() {
+    # Disable tests which require a DOCKER registry
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]"
+    output=$(luet install -y --config $tmpdir/luet-client.yaml test/c@1.0)
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertNotContains 'contains warning' "$output" 'No packages to install'
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]"
+    assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.zst' ]"
+}
+
+testCleanup() {
+    [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping
+
+    luet cleanup --config $tmpdir/luet-client.yaml
+    installst=$?
+    assertEquals 'cleanup test successfully' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+