diff --git a/cmd/create-repo.go b/cmd/create-repo.go index 8b699285..efa54477 100644 --- a/cmd/create-repo.go +++ b/cmd/create-repo.go @@ -91,6 +91,7 @@ Create a repository from the metadata description defined in the luet.yaml confi metaName := viper.GetString("meta-filename") source_repo := viper.GetString("repo") backendType := viper.GetString("backend") + fromRepo, _ := cmd.Flags().GetBool("from-repositories") treeFile := installer.NewDefaultTreeRepositoryFile() metaFile := installer.NewDefaultMetaRepositoryFile() @@ -102,9 +103,7 @@ Create a repository from the metadata description defined in the luet.yaml confi if source_repo != "" { // Search for system repository lrepo, err := LuetCfg.GetSystemRepository(source_repo) - if err != nil { - Fatal("Error: " + err.Error()) - } + helpers.CheckErr(err) if len(treePaths) <= 0 { treePaths = []string{lrepo.TreePath} @@ -120,15 +119,19 @@ Create a repository from the metadata description defined in the luet.yaml confi lrepo.Priority, packages, treePaths, - pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force) + pkg.NewInMemoryDatabase(false), + compilerBackend, + dst, + imagePush, + force, + fromRepo, + LuetCfg) + helpers.CheckErr(err) } else { repo, err = installer.GenerateRepository(name, descr, t, urls, 1, packages, - treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force) - } - - if err != nil { - Fatal("Error: " + err.Error()) + treePaths, pkg.NewInMemoryDatabase(false), compilerBackend, dst, imagePush, force, fromRepo, LuetCfg) + helpers.CheckErr(err) } if treetype != "" { @@ -151,17 +154,15 @@ Create a repository from the metadata description defined in the luet.yaml confi repo.SetRepositoryFile(installer.REPOFILE_META_KEY, metaFile) err = repo.Write(dst, reset, true) - if err != nil { - Fatal("Error: " + err.Error()) - } + helpers.CheckErr(err) + }, } func init() { path, err := os.Getwd() - if err != nil { - Fatal(err) - } + helpers.CheckErr(err) + createrepoCmd.Flags().String("packages", filepath.Join(path, "build"), "Packages folder (output from build)") createrepoCmd.Flags().StringSliceP("tree", "t", []string{path}, "Path of the source trees to use.") createrepoCmd.Flags().String("output", filepath.Join(path, "build"), "Destination for generated archives. With 'docker' repository type, it should be an image reference (e.g 'foo/bar')") @@ -180,6 +181,7 @@ func init() { createrepoCmd.Flags().String("tree-filename", installer.TREE_TARBALL, "Repository tree filename") createrepoCmd.Flags().String("meta-compression", "none", "Compression alg: none, gzip, zstd") createrepoCmd.Flags().String("meta-filename", installer.REPOSITORY_METAFILE+".tar", "Repository metadata filename") + createrepoCmd.Flags().Bool("from-repositories", false, "Consume the user-defined repositories to pull specfiles from") RootCmd.AddCommand(createrepoCmd) } diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 16d928f0..e17dbeca 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -752,7 +752,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil Assert: *targetAssertion, }) - // Update compilespec build options + // Update compilespec build options - it will be then serialized into the compilation metadata file p.SetBuildOptions(cs.Options) // - If image is set we just generate a plain dockerfile diff --git a/pkg/compiler/types/artifact/artifact.go b/pkg/compiler/types/artifact/artifact.go index b9e27ace..18a4a5a3 100644 --- a/pkg/compiler/types/artifact/artifact.go +++ b/pkg/compiler/types/artifact/artifact.go @@ -156,14 +156,20 @@ COPY . /` // CreateArtifactForFile creates a new artifact from the given file func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageArtifact, error) { - + if _, err := os.Stat(s); os.IsNotExist(err) { + return nil, errors.Wrap(err, "artifact path doesn't exist") + } fileName := path.Base(s) archive, err := LuetCfg.GetSystem().TempDir("archive") if err != nil { return nil, errors.Wrap(err, "error met while creating tempdir for "+s) } defer os.RemoveAll(archive) // clean up - helpers.CopyFile(s, filepath.Join(archive, fileName)) + dst := filepath.Join(archive, fileName) + if err := helpers.CopyFile(s, dst); err != nil { + return nil, errors.Wrapf(err, "error while copying %s to %s", s, dst) + } + artifact, err := LuetCfg.GetSystem().TempDir("artifact") if err != nil { return nil, errors.Wrap(err, "error met while creating tempdir for "+s) diff --git a/pkg/installer/installer_test.go b/pkg/installer/installer_test.go index 8c0656f0..4a33de28 100644 --- a/pkg/installer/installer_test.go +++ b/pkg/installer/installer_test.go @@ -44,7 +44,7 @@ func stubRepo(tmpdir, tree string) (*LuetSystemRepository, error) { 1, tmpdir, []string{tree}, - pkg.NewInMemoryDatabase(false), nil, "", false, false) + pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil) } var _ = Describe("Installer", func() { @@ -342,7 +342,7 @@ urls: []string{tmpdir}, 1, tmpdir, []string{"../../tests/fixtures/buildable"}, - pkg.NewInMemoryDatabase(false), nil, "", false, false) + pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil) Expect(err).ToNot(HaveOccurred()) Expect(repo.GetName()).To(Equal("test")) Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue()) @@ -468,7 +468,7 @@ urls: 1, tmpdir, []string{"../../tests/fixtures/buildable"}, - pkg.NewInMemoryDatabase(false), nil, "", false, false) + pkg.NewInMemoryDatabase(false), nil, "", false, false, false, nil) Expect(err).ToNot(HaveOccurred()) Expect(repo.GetName()).To(Equal("test")) Expect(helpers.Exists(spec.Rel("repository.yaml"))).ToNot(BeTrue()) diff --git a/pkg/installer/repository.go b/pkg/installer/repository.go index 83c5c6c5..6e0ee923 100644 --- a/pkg/installer/repository.go +++ b/pkg/installer/repository.go @@ -254,13 +254,18 @@ func (f *LuetRepositoryFile) GetChecksums() artifact.Checksums { // In case the repository is local, it will build the package Index func GenerateRepository(name, descr, t string, urls []string, priority int, src string, treesDir []string, db pkg.PackageDatabase, - b compiler.CompilerBackend, imagePrefix string, pushImages, force bool) (*LuetSystemRepository, error) { + b compiler.CompilerBackend, imagePrefix string, pushImages, force, fromRepo bool, c *config.LuetConfig) (*LuetSystemRepository, error) { + + // 1: First filter the runtime db to only the metadata we actually have - tr := tree.NewInstallerRecipe(db) btr := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false)) + runtimeTree := pkg.NewInMemoryDatabase(false) + + tempTree := pkg.NewInMemoryDatabase(false) + temptr := tree.NewInstallerRecipe(tempTree) for _, treeDir := range treesDir { - if err := tr.Load(treeDir); err != nil { + if err := temptr.Load(treeDir); err != nil { return nil, err } if err := btr.Load(treeDir); err != nil { @@ -268,9 +273,33 @@ func GenerateRepository(name, descr, t string, urls []string, } } + // 2: if fromRepo, build a new tree like the compiler is doing and use it to source the above specs, + // instead of local tree + + repodb := pkg.NewInMemoryDatabase(false) + generalRecipe := tree.NewCompilerRecipe(repodb) + + if fromRepo { + if err := LoadBuildTree(generalRecipe, repodb, c); err != nil { + Warning("errors while loading trees from repositories", err.Error()) + } + + if err := repodb.Clone(tempTree); err != nil { + Warning("errors while cloning trees from repositories", err.Error()) + } + + } + + // Pick only atoms in db which have a real metadata for runtime db (tr) + for _, p := range tempTree.World() { + if _, err := os.Stat(filepath.Join(src, p.GetMetadataFilePath())); err == nil { + runtimeTree.CreatePackage(p) + } + } + repo := &LuetSystemRepository{ LuetRepository: config.NewLuetRepository(name, t, descr, urls, priority, true, false), - Tree: tr, + Tree: tree.NewInstallerRecipe(runtimeTree), BuildTree: btr, RepositoryFiles: map[string]LuetRepositoryFile{}, PushImages: pushImages, @@ -918,7 +947,6 @@ func (r Repositories) SyncDatabase(d pkg.PackageDatabase) { } } } - } type PackageMatch struct { diff --git a/pkg/installer/repository_docker.go b/pkg/installer/repository_docker.go index 8d0b8a06..3e6a2fe5 100644 --- a/pkg/installer/repository_docker.go +++ b/pkg/installer/repository_docker.go @@ -52,15 +52,17 @@ func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDataba Debug("Skipping", info.Name(), err.Error()) return nil } + if info.IsDir() { + Debug("Skipping directories") + return nil + } - if strings.HasSuffix(info.Name(), ".metadata.yaml") { - a := artifact.NewPackageArtifact(info.Name()) - imageRepo := fmt.Sprintf("%s:%s", l.imagePrefix, filepath.Base(info.Name())) + if !strings.HasSuffix(info.Name(), ".metadata.yaml") { + return nil + } - if err := l.pushFileFromArtifact(a, imageRepo); err != nil { - return errors.Wrap(err, "while pushing file from artifact") - } - return nil // Skip with no errors + if err := l.pushImageFromArtifact(artifact.NewPackageArtifact(currentpath), l.b); err != nil { + return errors.Wrap(err, "while pushing metadata file associated to the artifact") } dat, err := ioutil.ReadFile(currentpath) diff --git a/pkg/installer/repository_test.go b/pkg/installer/repository_test.go index 33efad9f..5b63c9da 100644 --- a/pkg/installer/repository_test.go +++ b/pkg/installer/repository_test.go @@ -47,7 +47,7 @@ func dockerStubRepo(tmpdir, tree, image string, push, force bool) (*LuetSystemRe 1, tmpdir, []string{tree}, - pkg.NewInMemoryDatabase(false), backend.NewSimpleDockerBackend(), image, push, force) + pkg.NewInMemoryDatabase(false), backend.NewSimpleDockerBackend(), image, push, force, false, nil) } var _ = Describe("Repository", func() { diff --git a/tests/integration/28_nobuildtreedocker.sh b/tests/integration/28_nobuildtreedocker.sh new file mode 100755 index 00000000..f20c6ff0 --- /dev/null +++ b/tests/integration/28_nobuildtreedocker.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +export LUET_NOLOCK=true + +oneTimeSetUp() { +export tmpdir="$(mktemp -d)" +} + +oneTimeTearDown() { + rm -rf "$tmpdir" +} + +testConfig() { + [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping + + mkdir $tmpdir/testrootfs + cat < $tmpdir/luet.yaml +general: + debug: true +system: + rootfs: $tmpdir/testrootfs + database_path: "/" + database_engine: "boltdb" +config_from_host: true +repositories: + - name: "main" + type: "docker" + enable: true + urls: + - "${TEST_DOCKER_IMAGE}" +EOF + luet config --config $tmpdir/luet.yaml + res=$? + assertEquals 'config test successfully' "$res" "0" +} + +testBuild() { + [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping + + mkdir $tmpdir/testbuild + mkdir $tmpdir/empty + luet build --tree "$tmpdir/empty" --config $tmpdir/luet.yaml --from-repositories --destination $tmpdir/testbuild --compression zstd test/c@1.0 > /dev/null + buildst=$? + assertEquals 'builds successfully' "$buildst" "0" + assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.zst' ]" + assertTrue 'create package' "[ -e '$tmpdir/testbuild/c-test-1.0.package.tar.zst' ]" +} + +testRepo() { + # Disable tests which require a DOCKER registry + [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping + + luet create-repo \ + --output "${TEST_DOCKER_IMAGE}-2" \ + --packages $tmpdir/testbuild \ + --name "test" \ + --descr "Test Repo" \ + --urls $tmpdir/testrootfs \ + --tree-compression zstd \ + --tree-filename foo.tar \ + --tree "$tmpdir/empty" --config $tmpdir/luet.yaml --from-repositories \ + --meta-filename repository.meta.tar \ + --meta-compression zstd \ + --type docker --push-images --force-push --debug + + createst=$? + assertEquals 'create repo successfully' "$createst" "0" +} + +testConfigClient() { + [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping + + cat < $tmpdir/luet-client.yaml +general: + debug: true +system: + rootfs: $tmpdir/testrootfs + database_path: "/" + database_engine: "boltdb" +config_from_host: true +repositories: + - name: "main" + type: "docker" + enable: true + urls: + - "${TEST_DOCKER_IMAGE}-2" +EOF + luet config --config $tmpdir/luet-client.yaml + res=$? + assertEquals 'config test successfully' "$res" "0" +} + +testInstall() { + # Disable tests which require a DOCKER registry + [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping + + luet install -y --config $tmpdir/luet-client.yaml test/c@1.0 + installst=$? + assertEquals 'install test successfully' "$installst" "0" + assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]" +} + +testReInstall() { + # Disable tests which require a DOCKER registry + [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping + + output=$(luet install -y --config $tmpdir/luet-client.yaml test/c@1.0) + installst=$? + assertEquals 'install test successfully' "$installst" "0" + assertContains 'contains warning' "$output" 'No packages to install' +} + +testUnInstall() { + # Disable tests which require a DOCKER registry + [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping + + luet uninstall -y --config $tmpdir/luet-client.yaml test/c@1.0 + installst=$? + assertEquals 'uninstall test successfully' "$installst" "0" + assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]" +} + +testInstallAgain() { + # Disable tests which require a DOCKER registry + [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping + + assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/c' ]" + output=$(luet install -y --config $tmpdir/luet-client.yaml test/c@1.0) + installst=$? + assertEquals 'install test successfully' "$installst" "0" + assertNotContains 'contains warning' "$output" 'No packages to install' + assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/c' ]" + assertTrue 'package in cache' "[ -e '$tmpdir/testrootfs/packages/c-test-1.0.package.tar.zst' ]" +} + +testCleanup() { + [ -z "${TEST_DOCKER_IMAGE:-}" ] && startSkipping + + luet cleanup --config $tmpdir/luet-client.yaml + installst=$? + assertEquals 'cleanup test successfully' "$installst" "0" +} + +# Load shUnit2. +. "$ROOT_DIR/tests/integration/shunit2"/shunit2 +