Multi-stage builds from hash images

Implement multi-stage copy from images part of the build cache of a
package.

Note, this is not the final images where are we copying files from, but
the underlying build container.

Skipping the test on img backend because it fails when pulling external
images during multi-stage build...

Fixes #190

pkg/compiler/compiler.go

@@ -89,7 +89,7 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan *c
 	defer wg.Done()
 
 	for s := range cspecs {
-		ar, err := cs.compile(concurrency, keepPermissions, s)
+		ar, err := cs.compile(concurrency, keepPermissions, nil, s)
 		if err != nil {
 			errors <- err
 		}
@@ -704,7 +704,7 @@ func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompi
 // Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates
 // an artifact
 func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
-	return cs.compile(cs.Options.Concurrency, keepPermissions, p)
+	return cs.compile(cs.Options.Concurrency, keepPermissions, nil, p)
 }
 
 func genImageList(refs []string, hash string) []string {
@@ -733,9 +733,45 @@ func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationS
 	Debug(p.GetPackage().HumanReadableString(), "Build options after inherit", p.BuildOptions)
 }
 
-func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
+func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
+	resolvedCopyFields := []compilerspec.CopyField{}
+	if len(p.Copy) != 0 {
+		Info("Package has multi-stage copy, generating required images")
+	}
+	for _, c := range p.Copy {
+		if c.Package != nil && c.Package.Name != "" && c.Package.Version != "" {
+			Info(" :droplet: generating multi-stage images for", c.Package.HumanReadableString())
+			spec, err := cs.FromPackage(c.Package)
+			if err != nil {
+				return errors.Wrap(err, "while generating images to copy from")
+			}
+			noArtifact := false
+			artifact, err := cs.compile(concurrency, keepPermissions, &noArtifact, spec)
+
+			if err != nil {
+				return errors.Wrap(err, "failed building multi-stage image")
+			}
+
+			resolvedCopyFields = append(resolvedCopyFields, compilerspec.CopyField{
+				Image:       cs.resolveExistingImageHash(artifact.PackageCacheImage, spec),
+				Source:      c.Source,
+				Destination: c.Destination,
+			})
+		} else {
+			resolvedCopyFields = append(resolvedCopyFields, c)
+		}
+	}
+	p.Copy = resolvedCopyFields
+	return nil
+}
+
+func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateArtifact *bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
 	Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
 
+	if err := cs.resolveMultiStageImages(concurrency, keepPermissions, p); err != nil {
+		return nil, errors.Wrap(err, "while resolving multi-stage images")
+	}
+
 	Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage()))
 	if !p.HasImageSource() && !p.EmptyPackage() {
 		return nil,
@@ -772,7 +808,17 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 	// - If image is set we just generate a plain dockerfile
 	// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
 	if p.GetImage() != "" {
-		return cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
+		localGenerateArtifact := true
+		if generateArtifact != nil {
+			localGenerateArtifact = *generateArtifact
+		}
+		a, err := cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, localGenerateArtifact)
+		if err != nil {
+			return nil, errors.Wrap(err, "building direct image")
+		}
+		a.SourceAssertion = p.GetSourceAssertion()
+		a.PackageCacheImage = targetAssertion.Hash.PackageHash
+		return a, nil
 	}
 
 	// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
@@ -785,6 +831,10 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 	currentN := 0
 
 	packageDeps := !cs.Options.PackageTargetOnly
+	if generateArtifact != nil {
+		packageDeps = *generateArtifact
+	}
+
 	buildDeps := !cs.Options.NoDeps
 	buildTarget := !cs.Options.OnlyDeps
 
@@ -851,6 +901,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 				return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
 			}
 
+			a.PackageCacheImage = assertion.Hash.PackageHash
+
 			Info(pkgTag, ":white_check_mark: Done")
 
 			bus.Manager.Publish(bus.EventPackagePostBuild, struct {
@@ -866,16 +918,20 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil
 	}
 
 	if buildTarget {
+		localGenerateArtifact := true
+		if generateArtifact != nil {
+			localGenerateArtifact = *generateArtifact
+		}
 		resolvedSourceImage := cs.resolveExistingImageHash(packageHashTree.SourceHash, p)
 		Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
 		Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedSourceImage)
-		a, err := cs.compileWithImage(resolvedSourceImage, packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
+		a, err := cs.compileWithImage(resolvedSourceImage, packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, localGenerateArtifact)
 		if err != nil {
 			return a, err
 		}
 		a.Dependencies = departifacts
 		a.SourceAssertion = p.GetSourceAssertion()
-
+		a.PackageCacheImage = targetAssertion.Hash.PackageHash
 		bus.Manager.Publish(bus.EventPackagePostBuild, struct {
 			CompileSpec *compilerspec.LuetCompilationSpec
 			Artifact    *artifact.PackageArtifact

pkg/compiler/types/artifact/artifact.go

@@ -61,6 +61,7 @@ type PackageArtifact struct {
 	SourceAssertion   solver.PackagesAssertions         `json:"-"`
 	CompressionType   compression.Implementation        `json:"compressiontype"`
 	Files             []string                          `json:"files"`
+	PackageCacheImage string                            `json:"package_cacheimage"`
 }
 
 func (p *PackageArtifact) ShallowCopy() *PackageArtifact {

pkg/compiler/types/spec/spec.go

@@ -89,8 +89,8 @@ func (specs *LuetCompilationspecs) Unique() *LuetCompilationspecs {
 type CopyField struct {
 	Package     *pkg.DefaultPackage `json:"package"`
 	Image       string              `json:"image"`
-	Source      string              `json:"src"`
-	Destination string              `json:"dst"`
+	Source      string              `json:"source"`
+	Destination string              `json:"destination"`
 }
 
 type LuetCompilationSpec struct {
@@ -268,7 +268,8 @@ ADD ` + s + ` /luetbuild/`
 
 	for _, c := range cs.Copy {
 		if c.Image != "" {
-			spec = spec + fmt.Sprintf("\nCOPY --from=%s %s %s\n", c.Image, c.Source, c.Destination)
+			copyLine := fmt.Sprintf("\nCOPY --from=%s %s %s\n", c.Image, c.Source, c.Destination)
+			spec = spec + copyLine
 		}
 	}
 

tests/fixtures/copy/c/a/build.yaml

@@ -0,0 +1,17 @@
+image: "alpine"
+
+copy:
+  - package:
+      name: "a"
+      category: "test"
+      version: ">=0"
+    source: /test3
+    destination: /test3
+  - image: "busybox"
+    source: /bin/busybox
+    destination: /busybox
+
+steps:
+- mkdir /bina
+- cp /test3 /result
+- cp -rf /busybox /bina/busybox

tests/fixtures/copy/c/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "c"
+version: "1.2"

tests/fixtures/copy/cat/a/a/build.yaml

@@ -0,0 +1,7 @@
+image: "alpine"
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact3 > /test3
+  - echo artifact4 > /test4

tests/fixtures/copy/cat/a/a/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "a"
+version: "1.2"

tests/fixtures/copy/cat/b-1.1/build.yaml

@@ -0,0 +1,13 @@
+requires:
+- category: "test"
+  name: "a"
+  version: ">=0"
+
+prelude:
+  - echo foo > /test
+  - echo bar > /test2
+steps:
+  - echo artifact5 > /newc
+  - echo artifact6 > /newnewc
+  - chmod +x generate.sh
+  - ./generate.sh

tests/fixtures/copy/cat/b-1.1/definition.yaml

@@ -0,0 +1,3 @@
+category: "test"
+name: "b"
+version: "1.1"

tests/fixtures/copy/cat/b-1.1/generate.sh

@@ -0,0 +1 @@
+echo generated > /sonewc

tests/integration/30_copy.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+    export tmpdir="$(mktemp -d)"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+    docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi
+}
+
+testBuild() {
+    [ "$LUET_BACKEND" == "img" ] && startSkipping
+    cat <<EOF > $tmpdir/default.yaml
+extra: "bar"
+foo: "baz"
+EOF
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/copy" \
+               --destination $tmpdir/testbuild --concurrency 1 \
+               --compression gzip --values $tmpdir/default.yaml \
+               test/c
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package c' "[ -e '$tmpdir/testbuild/c-test-1.2.package.tar.gz' ]"
+    mkdir $tmpdir/extract
+    tar -xvf $tmpdir/testbuild/c-test-1.2.package.tar.gz -C $tmpdir/extract
+    assertTrue 'create result in package c' "[ -e '$tmpdir/extract/result' ]"
+    assertTrue 'create busybox in package c' "[ -f '$tmpdir/extract/bina/busybox' ]"
+}
+
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+