diff --git a/cmd/build.go b/cmd/build.go index 0e82f6e1..e8489af3 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -133,11 +133,16 @@ Build packages specifying multiple definition trees: db = pkg.NewInMemoryDatabase(false) defer db.Clean() + runtimeDB := pkg.NewInMemoryDatabase(false) + defer runtimeDB.Clean() + + installerRecipe := tree.NewInstallerRecipe(runtimeDB) generalRecipe := tree.NewCompilerRecipe(db) for _, src := range treePaths { util.DefaultContext.Info("Loading tree", src) helpers.CheckErr(generalRecipe.Load(src)) + helpers.CheckErr(installerRecipe.Load(src)) } if fromRepo { @@ -172,6 +177,7 @@ Build packages specifying multiple definition trees: options.WithTemplateFolder(templateFolders), options.WithSolverOptions(opts), options.Wait(wait), + options.WithRuntimeDatabase(installerRecipe.GetDatabase()), options.OnlyTarget(onlyTarget), options.PullFirst(pull), options.KeepImg(keepImages), diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 139bb6ce..b7898758 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -805,15 +805,15 @@ func (cs *LuetCompiler) FromDatabase(db types.PackageDatabase, minimum bool, dst } } -func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (types.PackagesAssertions, error) { - s := solver.NewResolver(cs.Options.SolverOptions.SolverOptions, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), solver.NewSolverFromOptions(cs.Options.SolverOptions)) +func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec, db types.PackageDatabase) (types.PackagesAssertions, error) { + s := solver.NewResolver(cs.Options.SolverOptions.SolverOptions, pkg.NewInMemoryDatabase(false), db, pkg.NewInMemoryDatabase(false), solver.NewSolverFromOptions(cs.Options.SolverOptions)) solution, err := s.Install(types.Packages{p.GetPackage()}) if err != nil { return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString()) } - dependencies, err := solution.Order(cs.Database, p.GetPackage().GetFingerPrint()) + dependencies, err := solution.Order(db, p.GetPackage().GetFingerPrint()) if err != nil { return nil, errors.Wrap(err, "While order a solution for "+p.GetPackage().HumanReadableString()) } @@ -831,7 +831,7 @@ func (cs *LuetCompiler) BuildTree(compilerSpecs compilerspec.LuetCompilationspec bt := &BuildTree{} for _, sp := range compilerSpecs.All() { - ass, err := cs.ComputeDepTree(sp) + ass, err := cs.ComputeDepTree(sp, cs.Database) if err != nil { return nil, err } @@ -843,7 +843,7 @@ func (cs *LuetCompiler) BuildTree(compilerSpecs compilerspec.LuetCompilationspec if err != nil { return nil, err } - ass, err := cs.ComputeDepTree(spec) + ass, err := cs.ComputeDepTree(spec, cs.Database) if err != nil { return nil, err } @@ -872,7 +872,7 @@ func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompi allDependencies := types.PackagesAssertions{} // Get all packages that will be in deps result := []*compilerspec.LuetCompilationSpec{} for _, spec := range p { - sol, err := cs.ComputeDepTree(spec) + sol, err := cs.ComputeDepTree(spec, cs.Database) if err != nil { return nil, errors.Wrap(err, "failed querying hashtree") } @@ -951,30 +951,33 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool cs.Options.Context.Info(joinTag, "Generating a parent image from final packages") - //fromPackages = p.Package.GetRequires() // (first level only) - pTarget := p + if cs.Options.RuntimeDatabase != nil { + // Create a fake db from runtime which has the target entry as the compiler view + db := pkg.NewInMemoryDatabase(false) + cs.Options.RuntimeDatabase.Clone(db) + defer db.Clean() - runtime, err := p.Package.GetRuntimePackage() - if err == nil { - spec, err := cs.FromPackage(runtime) - if err == nil { - cs.Options.Context.Info(joinTag, "Using runtime package for deptree computation") - pTarget = spec - } - } - // resolve deptree of runtime of p and use it in fromPackages - t, err := cs.ComputeDepTree(pTarget) - if err != nil { - return errors.Wrap(err, "failed querying hashtree") - } - - for _, a := range t { - if !a.Value || a.Package.Matches(p.Package) { - continue + if err := db.UpdatePackage(p.Package); err != nil { + return err } - fromPackages = append(fromPackages, a.Package) - cs.Options.Context.Infof("Adding dependency '%s'.", a.Package.HumanReadableString()) + // resolve deptree of runtime of p and use it in fromPackages + t, err := cs.ComputeDepTree(p, db) + if err != nil { + return errors.Wrap(err, "failed querying hashtree") + } + + for _, a := range t { + if !a.Value || a.Package.Matches(p.Package) { + continue + } + + fromPackages = append(fromPackages, a.Package) + cs.Options.Context.Infof("Adding dependency '%s'.", a.Package.HumanReadableString()) + } + } else { + cs.Options.Context.Info(joinTag, "No runtime db present, first level join only") + fromPackages = p.Package.GetRequires() // first level only } // First compute a hash and check if image is available. if it is, then directly consume that diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go index 49238371..c22b272c 100644 --- a/pkg/compiler/compiler_test.go +++ b/pkg/compiler/compiler_test.go @@ -1108,10 +1108,14 @@ var _ = Describe("Compiler", func() { Context("final images", func() { It("reuses final images", func() { generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false)) + installerRecipe := tree.NewInstallerRecipe(pkg.NewInMemoryDatabase(false)) err := generalRecipe.Load("../../tests/fixtures/join_complex") Expect(err).ToNot(HaveOccurred()) + err = installerRecipe.Load("../../tests/fixtures/join_complex") + Expect(err).ToNot(HaveOccurred()) + Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(6)) logdir, err := ioutil.TempDir("", "log") Expect(err).ToNot(HaveOccurred()) @@ -1139,7 +1143,7 @@ var _ = Describe("Compiler", func() { b := sd.NewSimpleDockerBackend(ctx) - joinImage := "luet/cache:586b36482e3f238c76d3536e7ca12cc4" //resulting join image + joinImage := "luet/cache:08738767caa9a7397fad70ae53db85fa" //resulting join image allImages := []string{ joinImage, "test/test:c-test-1.2"} @@ -1155,6 +1159,7 @@ var _ = Describe("Compiler", func() { compiler := NewLuetCompiler(b, generalRecipe.GetDatabase(), options.WithFinalRepository("test/test"), options.EnableGenerateFinalImages, + options.WithRuntimeDatabase(installerRecipe.GetDatabase()), options.PullFirst(true), options.WithContext(c)) @@ -1181,6 +1186,7 @@ var _ = Describe("Compiler", func() { )) Expect(log).ToNot(And( + ContainSubstring("No runtime db present, first level join only"), ContainSubstring("Final image already found test/test:c-test-1.2"), )) @@ -1199,11 +1205,13 @@ var _ = Describe("Compiler", func() { readLogs() Expect(log).To(And( - ContainSubstring("Final image already found test/test:a-test-1.2"), + ContainSubstring("Final image already found test/test:f-test-1.2"), )) Expect(log).ToNot(And( + ContainSubstring("No runtime db present, first level join only"), ContainSubstring("build test/c-1.2 compilation starts"), ContainSubstring("Final image not found for test/c-1.2"), + ContainSubstring("a-test-1.2"), )) }) }) diff --git a/pkg/compiler/imagehashtree.go b/pkg/compiler/imagehashtree.go index d32035dc..9f992dc0 100644 --- a/pkg/compiler/imagehashtree.go +++ b/pkg/compiler/imagehashtree.go @@ -121,7 +121,7 @@ func (ht *ImageHashTree) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, // resolve computes the dependency tree of a compilation spec and returns solver assertions // in order to be able to compile the spec. func (ht *ImageHashTree) resolve(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (types.PackagesAssertions, error) { - dependencies, err := cs.ComputeDepTree(p) + dependencies, err := cs.ComputeDepTree(p, cs.Database) if err != nil { return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString()) } diff --git a/pkg/compiler/types/options/compiler_options.go b/pkg/compiler/types/options/compiler_options.go index d58a3dee..9621ce36 100644 --- a/pkg/compiler/types/options/compiler_options.go +++ b/pkg/compiler/types/options/compiler_options.go @@ -54,6 +54,7 @@ type Compiler struct { // Image repository to push to PushFinalImagesRepository string + RuntimeDatabase types.PackageDatabase Context types.Context } @@ -93,6 +94,13 @@ func WithOptions(opt *Compiler) func(cfg *Compiler) error { } } +func WithRuntimeDatabase(db types.PackageDatabase) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.RuntimeDatabase = db + return nil + } +} + // WithFinalRepository Sets the final repository where to push // images of built artifacts func WithFinalRepository(r string) func(cfg *Compiler) error { diff --git a/tests/fixtures/join_complex/c/c1/definition.yaml b/tests/fixtures/join_complex/c/c1/definition.yaml index 7c0f2ed8..95869273 100644 --- a/tests/fixtures/join_complex/c/c1/definition.yaml +++ b/tests/fixtures/join_complex/c/c1/definition.yaml @@ -1,3 +1,7 @@ category: "test" name: "c" version: "1.2" +requires: +- name: "b" + category: "test" + version: ">=0" \ No newline at end of file diff --git a/tests/integration/31_join_complex.sh b/tests/integration/31_join_complex.sh index a629ecf8..f0effeac 100755 --- a/tests/integration/31_join_complex.sh +++ b/tests/integration/31_join_complex.sh @@ -22,7 +22,7 @@ testBuild() { buildst=$? assertEquals 'builds successfully' "$buildst" "0" assertTrue 'create package z' "[ -e '$tmpdir/testbuild/z-test-0.1.package.tar.gz' ]" - assertTrue 'create package z' "[ -e '$tmpdir/testbuild/x-test-0.1.package.tar.gz' ]" + assertTrue 'create package x' "[ -e '$tmpdir/testbuild/x-test-0.1.package.tar.gz' ]" mkdir $tmpdir/extract tar -xvf $tmpdir/testbuild/x-test-0.1.package.tar.gz -C $tmpdir/extract