From 16e9d7b20c5fcd9a27f05f59588202a3d771f30d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 23 Jun 2020 18:44:29 +0200 Subject: [PATCH] Use packageImage as builder image fingerprint This allows to have an unique identifier for the builder image id against different depgraphs combinations. The package fingerprint is not enough, as an atom could have a difference deptree depending on the requires constraints. TODO: Don't use the full image name, but only the hash as a salt (currently the salt contains ALSO a reference of the image-repository, as such it doesn't allow to port a tree in a different docker registry) --- cmd/tree/validate.go | 4 ++-- pkg/compiler/compiler.go | 21 ++++++++++++++++++--- pkg/package/package.go | 6 +++--- pkg/package/package_test.go | 4 ++-- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/cmd/tree/validate.go b/cmd/tree/validate.go index 2cb09fd7..53b62b7d 100644 --- a/cmd/tree/validate.go +++ b/cmd/tree/validate.go @@ -184,7 +184,7 @@ func validateWorker(i int, )) // Check if the solver is already been done for the deep - _, err := cacheDeps.Get(r.HashFingerprint()) + _, err := cacheDeps.Get(r.HashFingerprint("")) if err == nil { Debug(" :direct_hit: Cache Hit for dep", fmt.Sprintf("%s/%s-%s", r.GetCategory(), r.GetName(), r.GetVersion())) @@ -218,7 +218,7 @@ func validateWorker(i int, } // Register the key - cacheDeps.Set(r.HashFingerprint(), "1") + cacheDeps.Set(r.HashFingerprint(""), "1") } } diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index a6ad398a..9f89ce27 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -229,13 +229,30 @@ func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string } func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec) (Artifact, error) { - fp := p.GetPackage().HashFingerprint() + + pkgTag := ":package: " + p.GetPackage().GetName() + + // Use packageImage as salt into the fp being used + // so the hash is unique also in cases where + // some package deps does have completely different + // depgraphs + // TODO: As the salt contains the packageImage ( in registry/organization/imagename:tag format) + // the images hashes are broken with registry mirrors. + // We should use the image tag, or pass by the package assertion hash which is unique + // and identifies the deptree of the package. + + fp := p.GetPackage().HashFingerprint(packageImage) + if buildertaggedImage == "" { buildertaggedImage = cs.ImageRepository + "-" + fp + "-builder" + Debug(pkgTag, "Creating intermediary image", buildertaggedImage, "from", image) } + + // TODO: Cleanup, not actually hit if packageImage == "" { packageImage = cs.ImageRepository + "-" + fp } + if !cs.Clean { exists := cs.Backend.ImageExists(buildertaggedImage) && cs.Backend.ImageExists(packageImage) if art, err := LoadArtifactFromYaml(p); err == nil && exists { @@ -243,7 +260,6 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage return art, err } } - pkgTag := ":package: " + p.GetPackage().GetName() p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver, // and we build all the images first. @@ -513,7 +529,6 @@ func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssert for _, assertion := range dependencies { //highly dependent on the order if assertion.Value { nthsolution := dependencies.Cut(assertion.Package) - assertion.Hash = solver.PackageHash{ BuildHash: nthsolution.HashFrom(assertion.Package), PackageHash: nthsolution.AssertionHash(), diff --git a/pkg/package/package.go b/pkg/package/package.go index 4d9cf7e5..e0c63c28 100644 --- a/pkg/package/package.go +++ b/pkg/package/package.go @@ -105,7 +105,7 @@ type Package interface { String() string HumanReadableString() string - HashFingerprint() string + HashFingerprint(string) string Clone() Package } @@ -205,9 +205,9 @@ func (p *DefaultPackage) GetFingerPrint() string { return fmt.Sprintf("%s-%s-%s", p.Name, p.Category, p.Version) } -func (p *DefaultPackage) HashFingerprint() string { +func (p *DefaultPackage) HashFingerprint(salt string) string { h := md5.New() - io.WriteString(h, p.GetFingerPrint()) + io.WriteString(h, fmt.Sprintf("%s-%s",p.GetFingerPrint(),salt)) return fmt.Sprintf("%x", h.Sum(nil)) } diff --git a/pkg/package/package_test.go b/pkg/package/package_test.go index 9b786bcf..27988612 100644 --- a/pkg/package/package_test.go +++ b/pkg/package/package_test.go @@ -36,8 +36,8 @@ var _ = Describe("Package", func() { }) It("Generates packages fingerprint's hashes", func() { - Expect(a.HashFingerprint()).ToNot(Equal(a1.HashFingerprint())) - Expect(a.HashFingerprint()).To(Equal("c64caa391b79adb598ad98e261aa79a0")) + Expect(a.HashFingerprint("")).ToNot(Equal(a1.HashFingerprint(""))) + Expect(a.HashFingerprint("")).To(Equal("76972ef6991ec6102f33b401105c1351")) }) })