diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 29fec6d1..4666a199 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -89,7 +89,7 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan *c defer wg.Done() for s := range cspecs { - ar, err := cs.compile(concurrency, keepPermissions, s) + ar, err := cs.compile(concurrency, keepPermissions, nil, s) if err != nil { errors <- err } @@ -704,7 +704,7 @@ func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompi // Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates // an artifact func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) { - return cs.compile(cs.Options.Concurrency, keepPermissions, p) + return cs.compile(cs.Options.Concurrency, keepPermissions, nil, p) } func genImageList(refs []string, hash string) []string { @@ -733,9 +733,45 @@ func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationS Debug(p.GetPackage().HumanReadableString(), "Build options after inherit", p.BuildOptions) } -func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) { +func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error { + resolvedCopyFields := []compilerspec.CopyField{} + if len(p.Copy) != 0 { + Info("Package has multi-stage copy, generating required images") + } + for _, c := range p.Copy { + if c.Package != nil && c.Package.Name != "" && c.Package.Version != "" { + Info(" :droplet: generating multi-stage images for", c.Package.HumanReadableString()) + spec, err := cs.FromPackage(c.Package) + if err != nil { + return errors.Wrap(err, "while generating images to copy from") + } + noArtifact := false + artifact, err := cs.compile(concurrency, keepPermissions, &noArtifact, spec) + + if err != nil { + return errors.Wrap(err, "failed building multi-stage image") + } + + resolvedCopyFields = append(resolvedCopyFields, compilerspec.CopyField{ + Image: cs.resolveExistingImageHash(artifact.PackageCacheImage, spec), + Source: c.Source, + Destination: c.Destination, + }) + } else { + resolvedCopyFields = append(resolvedCopyFields, c) + } + } + p.Copy = resolvedCopyFields + return nil +} + +func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateArtifact *bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) { Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:") + if err := cs.resolveMultiStageImages(concurrency, keepPermissions, p); err != nil { + return nil, errors.Wrap(err, "while resolving multi-stage images") + } + Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage())) if !p.HasImageSource() && !p.EmptyPackage() { return nil, @@ -772,7 +808,17 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil // - If image is set we just generate a plain dockerfile // Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above if p.GetImage() != "" { - return cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true) + localGenerateArtifact := true + if generateArtifact != nil { + localGenerateArtifact = *generateArtifact + } + a, err := cs.compileWithImage(p.GetImage(), packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, localGenerateArtifact) + if err != nil { + return nil, errors.Wrap(err, "building direct image") + } + a.SourceAssertion = p.GetSourceAssertion() + a.PackageCacheImage = targetAssertion.Hash.PackageHash + return a, nil } // - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based @@ -785,6 +831,10 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil currentN := 0 packageDeps := !cs.Options.PackageTargetOnly + if generateArtifact != nil { + packageDeps = *generateArtifact + } + buildDeps := !cs.Options.NoDeps buildTarget := !cs.Options.OnlyDeps @@ -851,6 +901,8 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString()) } + a.PackageCacheImage = assertion.Hash.PackageHash + Info(pkgTag, ":white_check_mark: Done") bus.Manager.Publish(bus.EventPackagePostBuild, struct { @@ -866,16 +918,20 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compil } if buildTarget { + localGenerateArtifact := true + if generateArtifact != nil { + localGenerateArtifact = *generateArtifact + } resolvedSourceImage := cs.resolveExistingImageHash(packageHashTree.SourceHash, p) Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString()) Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedSourceImage) - a, err := cs.compileWithImage(resolvedSourceImage, packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true) + a, err := cs.compileWithImage(resolvedSourceImage, packageHashTree.BuilderImageHash, targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, localGenerateArtifact) if err != nil { return a, err } a.Dependencies = departifacts a.SourceAssertion = p.GetSourceAssertion() - + a.PackageCacheImage = targetAssertion.Hash.PackageHash bus.Manager.Publish(bus.EventPackagePostBuild, struct { CompileSpec *compilerspec.LuetCompilationSpec Artifact *artifact.PackageArtifact diff --git a/pkg/compiler/types/artifact/artifact.go b/pkg/compiler/types/artifact/artifact.go index b70b42e7..87c991e8 100644 --- a/pkg/compiler/types/artifact/artifact.go +++ b/pkg/compiler/types/artifact/artifact.go @@ -55,12 +55,13 @@ import ( type PackageArtifact struct { Path string `json:"path"` - Dependencies []*PackageArtifact `json:"dependencies"` - CompileSpec *compilerspec.LuetCompilationSpec `json:"compilationspec"` - Checksums Checksums `json:"checksums"` - SourceAssertion solver.PackagesAssertions `json:"-"` - CompressionType compression.Implementation `json:"compressiontype"` - Files []string `json:"files"` + Dependencies []*PackageArtifact `json:"dependencies"` + CompileSpec *compilerspec.LuetCompilationSpec `json:"compilationspec"` + Checksums Checksums `json:"checksums"` + SourceAssertion solver.PackagesAssertions `json:"-"` + CompressionType compression.Implementation `json:"compressiontype"` + Files []string `json:"files"` + PackageCacheImage string `json:"package_cacheimage"` } func (p *PackageArtifact) ShallowCopy() *PackageArtifact { diff --git a/pkg/compiler/types/spec/spec.go b/pkg/compiler/types/spec/spec.go index 46f79bb7..d1b07cb1 100644 --- a/pkg/compiler/types/spec/spec.go +++ b/pkg/compiler/types/spec/spec.go @@ -89,8 +89,8 @@ func (specs *LuetCompilationspecs) Unique() *LuetCompilationspecs { type CopyField struct { Package *pkg.DefaultPackage `json:"package"` Image string `json:"image"` - Source string `json:"src"` - Destination string `json:"dst"` + Source string `json:"source"` + Destination string `json:"destination"` } type LuetCompilationSpec struct { @@ -268,7 +268,8 @@ ADD ` + s + ` /luetbuild/` for _, c := range cs.Copy { if c.Image != "" { - spec = spec + fmt.Sprintf("\nCOPY --from=%s %s %s\n", c.Image, c.Source, c.Destination) + copyLine := fmt.Sprintf("\nCOPY --from=%s %s %s\n", c.Image, c.Source, c.Destination) + spec = spec + copyLine } } diff --git a/tests/fixtures/copy/c/a/build.yaml b/tests/fixtures/copy/c/a/build.yaml new file mode 100644 index 00000000..0e8df864 --- /dev/null +++ b/tests/fixtures/copy/c/a/build.yaml @@ -0,0 +1,17 @@ +image: "alpine" + +copy: + - package: + name: "a" + category: "test" + version: ">=0" + source: /test3 + destination: /test3 + - image: "busybox" + source: /bin/busybox + destination: /busybox + +steps: +- mkdir /bina +- cp /test3 /result +- cp -rf /busybox /bina/busybox \ No newline at end of file diff --git a/tests/fixtures/copy/c/a/definition.yaml b/tests/fixtures/copy/c/a/definition.yaml new file mode 100644 index 00000000..7c0f2ed8 --- /dev/null +++ b/tests/fixtures/copy/c/a/definition.yaml @@ -0,0 +1,3 @@ +category: "test" +name: "c" +version: "1.2" diff --git a/tests/fixtures/copy/cat/a/a/build.yaml b/tests/fixtures/copy/cat/a/a/build.yaml new file mode 100644 index 00000000..7d5f1aa1 --- /dev/null +++ b/tests/fixtures/copy/cat/a/a/build.yaml @@ -0,0 +1,7 @@ +image: "alpine" +prelude: + - echo foo > /test + - echo bar > /test2 +steps: + - echo artifact3 > /test3 + - echo artifact4 > /test4 diff --git a/tests/fixtures/copy/cat/a/a/definition.yaml b/tests/fixtures/copy/cat/a/a/definition.yaml new file mode 100644 index 00000000..006dd943 --- /dev/null +++ b/tests/fixtures/copy/cat/a/a/definition.yaml @@ -0,0 +1,3 @@ +category: "test" +name: "a" +version: "1.2" diff --git a/tests/fixtures/copy/cat/b-1.1/build.yaml b/tests/fixtures/copy/cat/b-1.1/build.yaml new file mode 100644 index 00000000..62d26ddc --- /dev/null +++ b/tests/fixtures/copy/cat/b-1.1/build.yaml @@ -0,0 +1,13 @@ +requires: +- category: "test" + name: "a" + version: ">=0" + +prelude: + - echo foo > /test + - echo bar > /test2 +steps: + - echo artifact5 > /newc + - echo artifact6 > /newnewc + - chmod +x generate.sh + - ./generate.sh diff --git a/tests/fixtures/copy/cat/b-1.1/definition.yaml b/tests/fixtures/copy/cat/b-1.1/definition.yaml new file mode 100644 index 00000000..e695c6c9 --- /dev/null +++ b/tests/fixtures/copy/cat/b-1.1/definition.yaml @@ -0,0 +1,3 @@ +category: "test" +name: "b" +version: "1.1" diff --git a/tests/fixtures/copy/cat/b-1.1/generate.sh b/tests/fixtures/copy/cat/b-1.1/generate.sh new file mode 100644 index 00000000..3b2dc1b2 --- /dev/null +++ b/tests/fixtures/copy/cat/b-1.1/generate.sh @@ -0,0 +1 @@ +echo generated > /sonewc diff --git a/tests/integration/30_copy.sh b/tests/integration/30_copy.sh new file mode 100755 index 00000000..e8706bea --- /dev/null +++ b/tests/integration/30_copy.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +export LUET_NOLOCK=true + +oneTimeSetUp() { + export tmpdir="$(mktemp -d)" + docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi +} + +oneTimeTearDown() { + rm -rf "$tmpdir" + docker images --filter='reference=luet/cache' --format='{{.Repository}}:{{.Tag}}' | xargs -r docker rmi +} + +testBuild() { + [ "$LUET_BACKEND" == "img" ] && startSkipping + cat < $tmpdir/default.yaml +extra: "bar" +foo: "baz" +EOF + mkdir $tmpdir/testbuild + luet build --tree "$ROOT_DIR/tests/fixtures/copy" \ + --destination $tmpdir/testbuild --concurrency 1 \ + --compression gzip --values $tmpdir/default.yaml \ + test/c + buildst=$? + assertEquals 'builds successfully' "$buildst" "0" + assertTrue 'create package c' "[ -e '$tmpdir/testbuild/c-test-1.2.package.tar.gz' ]" + mkdir $tmpdir/extract + tar -xvf $tmpdir/testbuild/c-test-1.2.package.tar.gz -C $tmpdir/extract + assertTrue 'create result in package c' "[ -e '$tmpdir/extract/result' ]" + assertTrue 'create busybox in package c' "[ -f '$tmpdir/extract/bina/busybox' ]" +} + + +# Load shUnit2. +. "$ROOT_DIR/tests/integration/shunit2"/shunit2 +