kairos/luet
1
0
Fork 0
mirror of https://github.com/mudler/luet.git synced 2025-09-16 07:10:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add 2 new events for image unpacking (#226)

Browse Source 
* Reduce possibility of circular dependency

Just by adding an import for bus to anything in the helper dir, we would
run into a circular dependency due to how things are structured. That
means that we cannot set any events for unpacking or docker helper
pulling an image.

This commit tries to work around this by doing several things.
 - Remove full imports of the helper module by segmentating some modules
   into their own submodule, like docker or match so just using a small match
   function doesnt bring the whole module
 - Removing a simple function to check if a dir exists from importing
   the full helper module and instead write the function (5 lines)
 - Using logrus in the bus module instead of logger, which avoids a
   circular dependency

Signed-off-by: Itxaka <igarcia@suse.com>

* Add two new events for unpacking an image

Both pre and post unpacking an image

Signed-off-by: Itxaka <igarcia@suse.com>
This commit is contained in:
Itxaka
2021-06-01 16:43:31 +02:00
committed by GitHub
parent b12c7678d4
commit 1bd4d520a4
38 changed files with 458 additions and 419 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/compiler/backend/simpledocker.go
View File

@@ -24,6 +24,7 @@ import (
"strings"
bus "github.com/mudler/luet/pkg/bus"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
capi "github.com/mudler/docker-companion/api"
@@ -237,7 +238,7 @@ func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
return errors.Wrap(err, "Error met while unpacking rootfs")
}
manifest, err := helpers.Read(filepath.Join(rootfs, "manifest.json"))
manifest, err := fileHelper.Read(filepath.Join(rootfs, "manifest.json"))
if err != nil {
return errors.Wrap(err, "Error met while reading image manifest")
}

12
pkg/compiler/backend/simpledocker_test.go
View File

@@ -21,12 +21,12 @@ import (
"github.com/mudler/luet/pkg/compiler/backend"
. "github.com/mudler/luet/pkg/compiler/backend"
"github.com/mudler/luet/pkg/compiler/types/artifact"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
"io/ioutil"
"os"
"path/filepath"
helpers "github.com/mudler/luet/pkg/helpers"
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/tree"
. "github.com/onsi/ginkgo"
@@ -60,7 +60,7 @@ var _ = Describe("Docker backend", func() {
err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM alpine
@@ -79,11 +79,11 @@ ENV PACKAGE_CATEGORY=app-admin`))
Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
Expect(fileHelper.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err = helpers.Read(filepath.Join(tmpdir, "LuetDockerfile"))
dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "LuetDockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM luet/base
@@ -103,7 +103,7 @@ RUN echo bar > /test2`))
Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
artifacts := []artifact.ArtifactNode{{
Name: "/luetbuild/LuetDockerfile",
@@ -132,7 +132,7 @@ RUN echo bar > /test2`))
}
Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred())
Expect(helpers.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
Expect(fileHelper.Exists(filepath.Join(tmpdir, "output3.tar"))).To(BeTrue())
Expect(b.ImageExists(opts2.ImageName)).To(BeFalse())
})

5
pkg/compiler/compiler.go
View File

@@ -36,6 +36,7 @@ import (
"github.com/mudler/luet/pkg/compiler/types/options"
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
"github.com/mudler/luet/pkg/helpers"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
. "github.com/mudler/luet/pkg/logger"
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/solver"
@@ -319,7 +320,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
defer os.RemoveAll(buildDir) // clean up
// First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow)
err = helpers.CopyDir(p.GetPackage().GetPath(), buildDir)
err = fileHelper.CopyDir(p.GetPackage().GetPath(), buildDir)
if err != nil {
return builderOpts, runnerOpts, errors.Wrap(err, "Could not copy package sources")
}
@@ -1173,7 +1174,7 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack pkg.
if err != nil {
return nil, errors.Wrap(err, "while marshalling values file")
}
f := filepath.Join(valuesdir, helpers.RandStringRunes(20))
f := filepath.Join(valuesdir, fileHelper.RandStringRunes(20))
if err := ioutil.WriteFile(f, out, os.ModePerm); err != nil {
return nil, errors.Wrap(err, "while writing temporary values file")
}

233
pkg/compiler/compiler_test.go
View File

@@ -25,6 +25,7 @@ import (
"github.com/mudler/luet/pkg/compiler/types/options"
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
helpers "github.com/mudler/luet/pkg/helpers"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/tree"
. "github.com/onsi/ginkgo"
@@ -59,15 +60,15 @@ var _ = Describe("Compiler", func() {
artifact, err := compiler.Compile(false, spec)
Expect(err).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
content1, err := helpers.Read(spec.Rel("test5"))
content1, err := fileHelper.Read(spec.Rel("test5"))
Expect(err).ToNot(HaveOccurred())
content2, err := helpers.Read(spec.Rel("test6"))
content2, err := fileHelper.Read(spec.Rel("test6"))
Expect(err).ToNot(HaveOccurred())
Expect(content1).To(Equal("artifact5\n"))
Expect(content2).To(Equal("artifact6\n"))
@@ -99,11 +100,11 @@ var _ = Describe("Compiler", func() {
artifact, err := compiler.Compile(false, spec)
Expect(err).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("result"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("bina/busybox"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("result"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("bina/busybox"))).To(BeTrue())
})
It("Compiles it correctly with Join", func() {
@@ -129,11 +130,11 @@ var _ = Describe("Compiler", func() {
artifact, err := compiler.Compile(false, spec)
Expect(err).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("newc"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("newc"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
})
})
@@ -164,7 +165,7 @@ var _ = Describe("Compiler", func() {
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
Expect(errs).To(BeNil())
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
@@ -227,23 +228,23 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts)).To(Equal(3))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
content1, err := helpers.Read(spec.Rel("c"))
content1, err := fileHelper.Read(spec.Rel("c"))
Expect(err).ToNot(HaveOccurred())
content2, err := helpers.Read(spec.Rel("cd"))
content2, err := fileHelper.Read(spec.Rel("cd"))
Expect(err).ToNot(HaveOccurred())
Expect(content1).To(Equal("c\n"))
Expect(content2).To(Equal("c\n"))
content1, err = helpers.Read(spec.Rel("d"))
content1, err = fileHelper.Read(spec.Rel("d"))
Expect(err).ToNot(HaveOccurred())
content2, err = helpers.Read(spec.Rel("dd"))
content2, err = fileHelper.Read(spec.Rel("dd"))
Expect(err).ToNot(HaveOccurred())
Expect(content1).To(Equal("s\n"))
Expect(content2).To(Equal("dd\n"))
@@ -277,17 +278,17 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts2)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
for _, artifact := range artifacts2 {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("etc/hosts"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("etc/hosts"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test1"))).To(BeTrue())
})
It("Compiles and includes ony wanted files", func() {
@@ -316,12 +317,12 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test6"))).ToNot(BeTrue())
})
It("Compiles and excludes files", func() {
@@ -350,13 +351,13 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
})
It("Compiles includes and excludes files", func() {
@@ -385,13 +386,13 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("marvot"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test6"))).ToNot(BeTrue())
})
It("Compiles and excludes ony wanted files also from unpacked packages", func() {
@@ -419,12 +420,12 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
})
It("Compiles includes and excludes ony wanted files also from unpacked packages", func() {
@@ -452,12 +453,12 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
})
It("Compiles and includes ony wanted files also from unpacked packages", func() {
@@ -485,16 +486,16 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test5"))).ToNot(BeTrue())
Expect(helpers.Exists(spec.Rel("test6"))).ToNot(BeTrue())
Expect(helpers.Exists(spec.Rel("test"))).ToNot(BeTrue())
Expect(helpers.Exists(spec.Rel("test2"))).ToNot(BeTrue())
Expect(helpers.Exists(spec.Rel("lib/firmware"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("marvin"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test5"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test6"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test2"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec.Rel("lib/firmware"))).ToNot(BeTrue())
})
It("Compiles a more complex tree", func() {
@@ -523,18 +524,18 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("extra-layer"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("extra-layer"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("base-layer-0.1.metadata.yaml"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("extra-layer-0.1.metadata.yaml"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("base-layer-0.1.metadata.yaml"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.metadata.yaml"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
})
It("Compiles with provides support", func() {
@@ -564,19 +565,19 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("d"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("dd"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("c"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("cd"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("d"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("dd"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("c"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("cd"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
})
It("Compiles with provides and selectors support", func() {
@@ -607,19 +608,19 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("d"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("dd"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("c"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("cd"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("d"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("dd"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("c"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("cd"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("d-test-1.0.metadata.yaml"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("c-test-1.0.metadata.yaml"))).To(BeTrue())
})
It("Compiles revdeps", func() {
generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
@@ -647,16 +648,16 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts)).To(Equal(2))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("extra-layer"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("extra-layer"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("usr/bin/pkgs-checker"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("base-layer-0.1.package.tar"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("extra-layer-0.1.package.tar"))).To(BeTrue())
})
It("Compiles complex dependencies trees with best matches", func() {
@@ -685,17 +686,17 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(6))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Untar(spec.Rel("vhba-sys-fs-5.4.2-20190410.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("build-layer-0.1.package.tar"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("build-sabayon-overlay-layer-0.20191212.package.tar"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("build-sabayon-overlays-layer-0.1.package.tar"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("linux-sabayon-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("sabayon-sources-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("vhba"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("build-layer-0.1.package.tar"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("build-sabayon-overlay-layer-0.20191212.package.tar"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("build-sabayon-overlays-layer-0.1.package.tar"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("linux-sabayon-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("sabayon-sources-sys-kernel-5.4.2.package.tar"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("vhba"))).To(BeTrue())
})
It("Compiles revdeps with seeds", func() {
@@ -720,31 +721,31 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts)).To(Equal(4))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
// A deps on B, so A artifacts are here:
Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
// B
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("artifact42"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("artifact42"))).To(BeTrue())
// C depends on B, so B is here
content1, err := helpers.Read(spec.Rel("c"))
content1, err := fileHelper.Read(spec.Rel("c"))
Expect(err).ToNot(HaveOccurred())
content2, err := helpers.Read(spec.Rel("cd"))
content2, err := fileHelper.Read(spec.Rel("cd"))
Expect(err).ToNot(HaveOccurred())
Expect(content1).To(Equal("c\n"))
Expect(content2).To(Equal("c\n"))
// D is here as it requires C, and C was recompiled
content1, err = helpers.Read(spec.Rel("d"))
content1, err = fileHelper.Read(spec.Rel("d"))
Expect(err).ToNot(HaveOccurred())
content2, err = helpers.Read(spec.Rel("dd"))
content2, err = fileHelper.Read(spec.Rel("dd"))
Expect(err).ToNot(HaveOccurred())
Expect(content1).To(Equal("s\n"))
Expect(content2).To(Equal("dd\n"))
@@ -777,19 +778,19 @@ var _ = Describe("Compiler", func() {
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
for _, d := range artifact.Dependencies {
Expect(helpers.Exists(d.Path)).To(BeTrue())
Expect(fileHelper.Exists(d.Path)).To(BeTrue())
Expect(helpers.Untar(d.Path, tmpdir, false)).ToNot(HaveOccurred())
}
}
Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test4"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test3"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test4"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test6"))).To(BeTrue())
})
})
@@ -821,8 +822,8 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("var"))).ToNot(BeTrue())
})
})
@@ -869,13 +870,13 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts[0].Dependencies)).To(Equal(0))
Expect(helpers.Untar(spec.Rel("dironly-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test2"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test1"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("test2"))).To(BeTrue())
Expect(helpers.Untar(spec2.Rel("dironly_filter-test-1.0.package.tar"), tmpdir2, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec2.Rel("test5"))).To(BeTrue())
Expect(helpers.Exists(spec2.Rel("test6"))).ToNot(BeTrue())
Expect(helpers.Exists(spec2.Rel("artifact42"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec2.Rel("test5"))).To(BeTrue())
Expect(fileHelper.Exists(spec2.Rel("test6"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec2.Rel("artifact42"))).ToNot(BeTrue())
})
})
@@ -905,12 +906,12 @@ var _ = Describe("Compiler", func() {
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
Expect(fileHelper.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
Expect(artifacts[0].Unpack(tmpdir, false)).ToNot(HaveOccurred())
// Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
Expect(fileHelper.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("var"))).ToNot(BeTrue())
})
})
@@ -961,7 +962,7 @@ var _ = Describe("Compiler", func() {
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
Expect(artifacts[0].Files).To(ContainElement("bin/busybox"))
Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
Expect(fileHelper.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
art, err := LoadArtifactFromYaml(spec)
Expect(err).ToNot(HaveOccurred())

13
pkg/compiler/types/artifact/artifact.go
View File

@@ -42,6 +42,7 @@ import (
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
. "github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/helpers"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
. "github.com/mudler/luet/pkg/logger"
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/solver"
@@ -168,7 +169,7 @@ func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageAr
}
defer os.RemoveAll(archive) // clean up
dst := filepath.Join(archive, fileName)
if err := helpers.CopyFile(s, dst); err != nil {
if err := fileHelper.CopyFile(s, dst); err != nil {
return nil, errors.Wrapf(err, "error while copying %s to %s", s, dst)
}
@@ -209,7 +210,7 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b ImageBuilder, k
return builderOpts, errors.Wrap(err, "error met while uncompressing artifact "+a.Path)
}
empty, err := helpers.DirectoryIsEmpty(uncompressedFiles)
empty, err := fileHelper.DirectoryIsEmpty(uncompressedFiles)
if err != nil {
return builderOpts, errors.Wrap(err, "error met while checking if directory is empty "+uncompressedFiles)
}
@@ -218,7 +219,7 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b ImageBuilder, k
// We can't generate FROM scratch empty images. Docker will refuse to export them
// workaround: Inject a .virtual empty file
if empty {
helpers.Touch(filepath.Join(uncompressedFiles, ".virtual"))
fileHelper.Touch(filepath.Join(uncompressedFiles, ".virtual"))
}
data := a.genDockerfile()
@@ -401,12 +402,12 @@ func tarModifierWrapperFunc(dst, path string, header *tar.Header, content io.Rea
// We want to protect file only if the hash of the files are differing OR the file size are
differs := (existingHash != "" && existingHash != tarHash) || (err != nil && f != nil && header.Size != f.Size())
// Check if exists
if helpers.Exists(destPath) && differs {
if fileHelper.Exists(destPath) && differs {
for i := 1; i < 1000; i++ {
name := filepath.Join(filepath.Join(filepath.Dir(path),
fmt.Sprintf("._cfg%04d_%s", i, filepath.Base(path))))
if helpers.Exists(name) {
if fileHelper.Exists(name) {
continue
}
Info(fmt.Sprintf("Found protected file %s. Creating %s.", destPath,
@@ -628,7 +629,7 @@ func worker(i int, wg *sync.WaitGroup, s <-chan CopyJob) {
_, err := os.Lstat(job.Dst)
if err != nil {
Debug("Copying ", job.Src)
if err := helpers.DeepCopyFile(job.Src, job.Dst); err != nil {
if err := fileHelper.DeepCopyFile(job.Src, job.Dst); err != nil {
Warning("Error copying", job, err)
}
}

23
pkg/compiler/types/artifact/artifact_test.go
View File

@@ -29,6 +29,7 @@ import (
. "github.com/mudler/luet/pkg/compiler"
helpers "github.com/mudler/luet/pkg/helpers"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/tree"
. "github.com/onsi/ginkgo"
@@ -71,7 +72,7 @@ var _ = Describe("Artifact", func() {
err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM alpine
@@ -89,12 +90,12 @@ ENV PACKAGE_CATEGORY=app-admin`))
}
Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
Expect(b.ExportImage(opts)).ToNot(HaveOccurred())
Expect(helpers.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
Expect(fileHelper.Exists(filepath.Join(tmpdir2, "output1.tar"))).To(BeTrue())
Expect(b.BuildImage(opts)).ToNot(HaveOccurred())
err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "LuetDockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err = helpers.Read(filepath.Join(tmpdir, "LuetDockerfile"))
dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "LuetDockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM luet/base
@@ -113,7 +114,7 @@ RUN echo bar > /test2`))
}
Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
Expect(fileHelper.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
diffs, err := compiler.GenerateChanges(b, opts, opts2)
Expect(err).ToNot(HaveOccurred())
@@ -140,15 +141,15 @@ RUN echo bar > /test2`))
a, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None)
Expect(err).ToNot(HaveOccurred())
Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
Expect(fileHelper.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
err = helpers.Untar(a.Path, unpacked, false)
Expect(err).ToNot(HaveOccurred())
Expect(helpers.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
Expect(helpers.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
content1, err := helpers.Read(filepath.Join(unpacked, "test"))
Expect(fileHelper.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
Expect(fileHelper.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
content1, err := fileHelper.Read(filepath.Join(unpacked, "test"))
Expect(err).ToNot(HaveOccurred())
Expect(content1).To(Equal("foo\n"))
content2, err := helpers.Read(filepath.Join(unpacked, "test2"))
content2, err := fileHelper.Read(filepath.Join(unpacked, "test2"))
Expect(err).ToNot(HaveOccurred())
Expect(content2).To(Equal("bar\n"))
@@ -156,7 +157,7 @@ RUN echo bar > /test2`))
Expect(err).ToNot(HaveOccurred())
err = a.Verify()
Expect(err).ToNot(HaveOccurred())
Expect(helpers.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
Expect(fileHelper.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
err = a.Verify()
Expect(err).To(HaveOccurred())
@@ -244,7 +245,7 @@ RUN echo bar > /test2`))
err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
Expect(err).ToNot(HaveOccurred())
Expect(helpers.DirectoryIsEmpty(result)).To(BeFalse())
Expect(fileHelper.DirectoryIsEmpty(result)).To(BeFalse())
content, err := ioutil.ReadFile(filepath.Join(result, ".virtual"))
Expect(err).ToNot(HaveOccurred())

12
pkg/compiler/types/spec/spec_test.go
View File

@@ -22,9 +22,9 @@ import (
options "github.com/mudler/luet/pkg/compiler/types/options"
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
. "github.com/mudler/luet/pkg/compiler"
helpers "github.com/mudler/luet/pkg/helpers"
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/tree"
. "github.com/onsi/ginkgo"
@@ -154,7 +154,7 @@ var _ = Describe("Spec", func() {
lspec.Env = []string{"test=1"}
err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM alpine
@@ -167,7 +167,7 @@ ENV test=1`))
err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM luet/base
@@ -205,7 +205,7 @@ RUN echo bar > /test2`))
err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err := helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM alpine
@@ -222,7 +222,7 @@ ENV test=1`))
err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM alpine
@@ -237,7 +237,7 @@ ENV test=1`))
err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err = helpers.Read(filepath.Join(tmpdir, "Dockerfile"))
dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`