Add retrieve to compilation spec

It allows to copy build artifact during buildtime. In this way
artifacts between different seed images can be shared

```
retrieve:
- a-test-1.0.package.*
- https://...
```

They will be available under WORKDIR
This commit is contained in:
Ettore Di Giacinto 2020-02-13 14:15:43 +01:00
parent 20bc250470
commit f329e1d5e0
No known key found for this signature in database
GPG Key ID: 1ADA699B145A2D1C
11 changed files with 246 additions and 0 deletions

View File

@ -256,6 +256,15 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
return nil, errors.Wrap(err, "Could not copy package sources")
}
// Copy file into the build context, the compilespec might have requested to do so.
if len(p.GetRetrieve()) > 0 {
err := p.CopyRetrieves(buildDir)
if err != nil {
Warning("Failed copying retrieves", err.Error())
}
}
if buildertaggedImage == "" {
buildertaggedImage = cs.ImageRepository + "-" + p.GetPackage().GetFingerPrint() + "-builder"
}

View File

@ -154,6 +154,9 @@ type CompilationSpec interface {
GetSourceAssertion() solver.PackagesAssertions
SetSourceAssertion(as solver.PackagesAssertions)
GetRetrieve() []string
CopyRetrieves(dest string) error
}
type CompilationSpecs interface {

View File

@ -21,6 +21,7 @@ import (
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/solver"
"github.com/otiai10/copy"
yaml "gopkg.in/yaml.v2"
)
@ -95,6 +96,8 @@ type LuetCompilationSpec struct {
Package *pkg.DefaultPackage `json:"package"`
SourceAssertion solver.PackagesAssertions `json:"-"`
Retrieve []string `json:"retrieve"`
OutputPath string `json:"-"` // Where the build processfiles go
Unpack bool `json:"unpack"`
Includes []string `json:"includes"`
@ -136,6 +139,10 @@ func (cs *LuetCompilationSpec) GetIncludes() []string {
return cs.Includes
}
func (cs *LuetCompilationSpec) GetRetrieve() []string {
return cs.Retrieve
}
func (cs *LuetCompilationSpec) GetSeedImage() string {
return cs.Seed
}
@ -164,6 +171,24 @@ func (cs *LuetCompilationSpec) SetSeedImage(s string) {
cs.Seed = s
}
func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
var err error
if len(cs.Retrieve) > 0 {
for _, s := range cs.Retrieve {
matches, err := filepath.Glob(cs.Rel(s))
if err != nil {
continue
}
for _, m := range matches {
err = copy.Copy(m, filepath.Join(dest, filepath.Base(m)))
}
}
}
return err
}
// TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
spec := `
@ -174,6 +199,19 @@ ENV PACKAGE_NAME=` + cs.Package.GetName() + `
ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()
if len(cs.Retrieve) > 0 {
for _, s := range cs.Retrieve {
//var file string
// if helpers.IsValidUrl(s) {
// file = s
// } else {
// file = cs.Rel(s)
// }
spec = spec + `
ADD ` + s + ` /luetbuild/`
}
}
for _, s := range cs.Env {
spec = spec + `
ENV ` + s

View File

@ -105,4 +105,74 @@ RUN echo bar > /test2`))
})
})
It("Renders retrieve and env fields", func() {
generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
err := generalRecipe.Load("../../tests/fixtures/retrieve")
Expect(err).ToNot(HaveOccurred())
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
lspec, ok := spec.(*LuetCompilationSpec)
Expect(ok).To(BeTrue())
Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
Expect(lspec.Image).To(Equal("luet/base"))
Expect(lspec.Seed).To(Equal("alpine"))
tmpdir, err := ioutil.TempDir("", "tree")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tmpdir) // clean up
err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM alpine
COPY . /luetbuild
WORKDIR /luetbuild
ENV PACKAGE_NAME=a
ENV PACKAGE_VERSION=1.0
ENV PACKAGE_CATEGORY=test
ADD test /luetbuild/
ADD http://www.google.com /luetbuild/
ENV test=1`))
lspec.SetOutputPath("/foo/bar")
err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM alpine
COPY . /luetbuild
WORKDIR /luetbuild
ENV PACKAGE_NAME=a
ENV PACKAGE_VERSION=1.0
ENV PACKAGE_CATEGORY=test
ADD /foo/bar/test /luetbuild/
ADD http://www.google.com /luetbuild/
ENV test=1`))
err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM luet/base
ENV PACKAGE_NAME=a
ENV PACKAGE_VERSION=1.0
ENV PACKAGE_CATEGORY=test
ENV test=1
RUN echo foo > /test
RUN echo bar > /test2`))
})
})

View File

@ -0,0 +1,3 @@
image: "alpine"
steps:
- echo a > /a

View File

@ -0,0 +1,3 @@
name: "a"
version: "1.0"
category: "test"

View File

@ -0,0 +1,10 @@
steps:
- tar xvf a-test-1.0.package.* -C ./
- mv a /b
requires:
- name: "a"
version: "1.0"
category: "test"
retrieve:
- a-test-1.0.package.*

View File

@ -0,0 +1,7 @@
name: "b"
version: "1.0"
category: "test"
requires:
- name: "a"
version: "1.0"
category: "test"

10
tests/fixtures/retrieve/a/build.yaml vendored Normal file
View File

@ -0,0 +1,10 @@
image: "luet/base"
seed: "alpine"
steps:
- echo foo > /test
- echo bar > /test2
retrieve:
- test
- http://www.google.com
env:
- test=1

View File

@ -0,0 +1,3 @@
name: "a"
version: "1.0"
category: "test"

View File

@ -0,0 +1,90 @@
#!/bin/bash
export LUET_NOLOCK=true
oneTimeSetUp() {
export tmpdir="$(mktemp -d)"
}
oneTimeTearDown() {
rm -rf "$tmpdir"
}
testBuild() {
mkdir $tmpdir/testbuild
luet build --tree "$ROOT_DIR/tests/fixtures/retrieve-integration" --destination $tmpdir/testbuild --compression gzip test/b
buildst=$?
assertEquals 'builds successfully' "$buildst" "0"
assertTrue 'create package dep B' "[ -e '$tmpdir/testbuild/b-test-1.0.package.tar.gz' ]"
assertTrue 'create package' "[ -e '$tmpdir/testbuild/a-test-1.0.package.tar.gz' ]"
}
testRepo() {
assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
luet create-repo --tree "$ROOT_DIR/tests/fixtures/retrieve-integration" \
--output $tmpdir/testbuild \
--packages $tmpdir/testbuild \
--name "test" \
--descr "Test Repo" \
--urls $tmpdir/testrootfs \
--type disk > /dev/null
createst=$?
assertEquals 'create repo successfully' "$createst" "0"
assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
}
testConfig() {
mkdir $tmpdir/testrootfs
cat <<EOF > $tmpdir/luet.yaml
general:
debug: true
system:
rootfs: $tmpdir/testrootfs
database_path: "/"
database_engine: "boltdb"
repositories:
- name: "main"
type: "disk"
enable: true
urls:
- "$tmpdir/testbuild"
EOF
luet config --config $tmpdir/luet.yaml
res=$?
assertEquals 'config test successfully' "$res" "0"
}
testInstall() {
luet install --config $tmpdir/luet.yaml test/b
#luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
installst=$?
assertEquals 'install test successfully' "$installst" "0"
assertTrue 'package B installed' "[ -e '$tmpdir/testrootfs/b' ]"
val=$(cat "$tmpdir/testrootfs/b")
assertEquals 'package B content comes from a' "$val" "a"
assertTrue 'package A installed' "[ -e '$tmpdir/testrootfs/a' ]"
}
testUnInstall() {
luet uninstall --config $tmpdir/luet.yaml test/b
installst=$?
assertEquals 'uninstall test successfully' "$installst" "0"
assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/b' ]"
assertTrue 'package uninstalled' "[ ! -e '$tmpdir/testrootfs/a' ]"
}
testCleanup() {
luet cleanup --config $tmpdir/luet.yaml
installst=$?
assertEquals 'install test successfully' "$installst" "0"
assertTrue 'package installed' "[ ! -e '$tmpdir/testrootfs/packages/b-test-1.0.package.tar.gz' ]"
}
# Load shUnit2.
. "$ROOT_DIR/tests/integration/shunit2"/shunit2