Tag 0.9.16

As now we build only when necessary, we need to make sure the images are
built similarly. The discrepancies between the two are less now, and
they can share the same logic.

This fixes a regresion where when no prelude is defined, the build
context isn't copied over

README.md

@@ -22,6 +22,7 @@ It is written entirely in Golang and where used as package manager, it can run i
 - [It uses SAT solving techniques to solve the deptree](https://luet-lab.github.io/docs/docs/concepts/constraints/) ( Inspired by [OPIUM](https://ranjitjhala.github.io/static/opium.pdf) )
 - Support for [collections](https://luet-lab.github.io/docs/docs/concepts/packages/collections/) and [templated package definitions](https://luet-lab.github.io/docs/docs/concepts/packages/templates/)
 - [Can be extended with Plugins and Extensions](https://luet-lab.github.io/docs/docs/concepts/plugins-and-extensions/)
+- [Can build packages in Kubernetes (experimental)](https://github.com/mudler/luet-k8s)
 
 ## Install
 

cmd/root.go

@@ -40,7 +40,7 @@ var Verbose bool
 var LockedCommands = []string{"install", "uninstall", "upgrade"}
 
 const (
-	LuetCLIVersion = "0.9.14"
+	LuetCLIVersion = "0.9.16"
 	LuetEnvPrefix  = "LUET"
 )
 

cmd/uninstall.go

@@ -81,14 +81,15 @@ var uninstallCmd = &cobra.Command{
 			installer.LoadConfigProtectConfs(LuetCfg)
 
 			inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-				Concurrency:        LuetCfg.GetGeneral().Concurrency,
-				SolverOptions:      *LuetCfg.GetSolverOptions(),
-				NoDeps:             nodeps,
-				Force:              force,
-				FullUninstall:      full,
-				FullCleanUninstall: fullClean,
-				CheckConflicts:     checkconflicts,
-				Ask:                !yes,
+				Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+				SolverOptions:               *LuetCfg.GetSolverOptions(),
+				NoDeps:                      nodeps,
+				Force:                       force,
+				FullUninstall:               full,
+				FullCleanUninstall:          fullClean,
+				CheckConflicts:              checkconflicts,
+				Ask:                         !yes,
+				PreserveSystemEssentialData: true,
 			})
 
 			if LuetCfg.GetSystem().DatabaseEngine == "boltdb" {

cmd/upgrade.go

@@ -84,15 +84,16 @@ var upgradeCmd = &cobra.Command{
 		installer.LoadConfigProtectConfs(LuetCfg)
 
 		inst := installer.NewLuetInstaller(installer.LuetInstallerOptions{
-			Concurrency:                LuetCfg.GetGeneral().Concurrency,
-			SolverOptions:              *LuetCfg.GetSolverOptions(),
-			Force:                      force,
-			FullUninstall:              full,
-			NoDeps:                     nodeps,
-			SolverUpgrade:              universe,
-			RemoveUnavailableOnUpgrade: clean,
-			UpgradeNewRevisions:        sync,
-			Ask:                        !yes,
+			Concurrency:                 LuetCfg.GetGeneral().Concurrency,
+			SolverOptions:               *LuetCfg.GetSolverOptions(),
+			Force:                       force,
+			FullUninstall:               full,
+			NoDeps:                      nodeps,
+			SolverUpgrade:               universe,
+			RemoveUnavailableOnUpgrade:  clean,
+			UpgradeNewRevisions:         sync,
+			PreserveSystemEssentialData: true,
+			Ask:                         !yes,
 		})
 		inst.Repositories(repos)
 

pkg/bus/events.go

@@ -16,9 +16,9 @@ var (
 
 	// EventPackagePreBuild is the event fired before a package is being built
 	EventPackagePreBuild pluggable.EventType = "package.pre.build"
-	// EventPackagePreBuildArtifact is the event fired before a package artifact is being built
+	// EventPackagePreBuildArtifact is the event fired before a package tarball is being generated
 	EventPackagePreBuildArtifact pluggable.EventType = "package.pre.build_artifact"
-	// EventPackagePostBuildArtifact is the event fired after a package artifact was built
+	// EventPackagePostBuildArtifact is the event fired after a package tarball is generated
 	EventPackagePostBuildArtifact pluggable.EventType = "package.post.build_artifact"
 	// EventPackagePostBuild is the event fired after a package was built
 	EventPackagePostBuild pluggable.EventType = "package.post.build"

pkg/compiler/artifact_test.go

@@ -96,6 +96,7 @@ ENV PACKAGE_CATEGORY=app-admin`))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
 WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
@@ -113,7 +114,10 @@ RUN echo bar > /test2`))
 			diffs, err := b.Changes(opts, opts2)
 			Expect(err).ToNot(HaveOccurred())
 
-			artifacts := []ArtifactNode{}
+			artifacts := []ArtifactNode{{
+				Name: "/luetbuild/LuetDockerfile",
+				Size: 175,
+			}}
 			if os.Getenv("DOCKER_BUILDKIT") == "1" {
 				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
 			}

pkg/compiler/backend/simpledocker_test.go

@@ -87,6 +87,7 @@ ENV PACKAGE_CATEGORY=app-admin`))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
 WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
@@ -102,7 +103,10 @@ RUN echo bar > /test2`))
 			Expect(b.ImageDefinitionToTar(opts2)).ToNot(HaveOccurred())
 			Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
 
-			artifacts := []ArtifactNode{}
+			artifacts := []ArtifactNode{{
+				Name: "/luetbuild/LuetDockerfile",
+				Size: 175,
+			}}
 			if os.Getenv("DOCKER_BUILDKIT") == "1" {
 				artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
 			}

pkg/compiler/backend/simpleimg.go

@@ -59,7 +59,7 @@ func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
 	defer SpinnerStop()
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed removing image: "+string(out))
 	}
 
 	Info(":tea: Image " + name + " removed")
@@ -75,7 +75,7 @@ func (*SimpleImg) DownloadImage(opts compiler.CompilerBackendOptions) error {
 	cmd := exec.Command("img", buildarg...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed downloading image: "+string(out))
 	}
 
 	Info(":tea: Image " + name + " downloaded")
@@ -124,7 +124,7 @@ func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error {
 	Debug(":tea: Saving image " + name)
 	out, err := exec.Command("img", buildarg...).CombinedOutput()
 	if err != nil {
-		return errors.Wrap(err, "Failed building image: "+string(out))
+		return errors.Wrap(err, "Failed exporting image: "+string(out))
 	}
 	Info(":tea: Image " + name + " saved")
 	return nil

pkg/compiler/compiler_test.go

@@ -50,8 +50,8 @@ var _ = Describe("Compiler", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			compiler.SetConcurrency(2)

pkg/compiler/spec.go

@@ -203,10 +203,9 @@ func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
 	return err
 }
 
-// TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
-func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
+func (cs *LuetCompilationSpec) genDockerfile(image string, steps []string) string {
 	spec := `
-FROM ` + cs.GetSeedImage() + `
+FROM ` + image + `
 COPY . /luetbuild
 WORKDIR /luetbuild
 ENV PACKAGE_NAME=` + cs.Package.GetName() + `
@@ -231,45 +230,22 @@ ADD ` + s + ` /luetbuild/`
 ENV ` + s
 	}
 
-	for _, s := range cs.GetPreBuildSteps() {
+	for _, s := range steps {
 		spec = spec + `
 RUN ` + s
 	}
-	return spec, nil
+	return spec
 }
 
-// TODO: docker build image first. Then a backend can be used to actually spin up a container with it and run the steps within
+// RenderBuildImage renders the dockerfile of the image used as a pre-build step
+func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
+	return cs.genDockerfile(cs.GetSeedImage(), cs.GetPreBuildSteps()), nil
+
+}
+
+// RenderStepImage renders the dockerfile used for the image used for building the package
 func (cs *LuetCompilationSpec) RenderStepImage(image string) (string, error) {
-	spec := `
-FROM ` + image + `
-WORKDIR /luetbuild
-ENV PACKAGE_NAME=` + cs.Package.GetName() + `
-ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
-ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()
-
-	if len(cs.Retrieve) > 0 {
-		for _, s := range cs.Retrieve {
-			//var file string
-			// if helpers.IsValidUrl(s) {
-			// 	file = s
-			// } else {
-			// 	file = cs.Rel(s)
-			// }
-			spec = spec + `
-ADD ` + s + ` /luetbuild/`
-		}
-	}
-
-	for _, s := range cs.Env {
-		spec = spec + `
-ENV ` + s
-	}
-	for _, s := range cs.BuildSteps() {
-		spec = spec + `
-RUN ` + s
-	}
-
-	return spec, nil
+	return cs.genDockerfile(image, cs.BuildSteps()), nil
 }
 
 func (cs *LuetCompilationSpec) WriteBuildImageDefinition(path string) error {

pkg/compiler/spec_test.go

@@ -96,6 +96,7 @@ ENV test=1`))
 			Expect(err).ToNot(HaveOccurred())
 			Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
 WORKDIR /luetbuild
 ENV PACKAGE_NAME=enman
 ENV PACKAGE_VERSION=1.4.0
@@ -169,6 +170,7 @@ ENV test=1`))
 
 		Expect(dockerfile).To(Equal(`
 FROM luet/base
+COPY . /luetbuild
 WORKDIR /luetbuild
 ENV PACKAGE_NAME=a
 ENV PACKAGE_VERSION=1.0

pkg/installer/installer.go

@@ -587,7 +587,7 @@ func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, err
 	}
 
 	err = artifact.Verify()
-	if err != nil && !l.Options.Force {
+	if err != nil {
 		return nil, errors.Wrap(err, "Artifact integrity check failure")
 	}
 	return artifact, nil
@@ -621,15 +621,11 @@ func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan Artif
 	for p := range c {
 		// TODO: Keep trace of what was added from the tar, and save it into system
 		_, err := l.downloadPackage(p)
-		if err != nil && !l.Options.Force {
-			//TODO: Uninstall, rollback.
-			Fatal("Failed installing package "+p.Package.GetName(), err.Error())
-			return errors.Wrap(err, "Failed installing package "+p.Package.GetName())
-		}
-		if err == nil {
-			Info("\n:package: Package ", p.Package.HumanReadableString(), "downloaded")
-		} else if err != nil && l.Options.Force {
-			Info("\n:package: ", p.Package.HumanReadableString(), "downloaded with failures (force download)")
+		if err != nil {
+			Fatal("Failed downloading package "+p.Package.GetName(), err.Error())
+			return errors.Wrap(err, "Failed downloading package "+p.Package.GetName())
+		} else {
+			Info(":package: Package ", p.Package.HumanReadableString(), "downloaded")
 		}
 	}
 

pkg/installer/installer_test.go

@@ -60,8 +60,8 @@ var _ = Describe("Installer", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			c.SetConcurrency(2)
@@ -176,8 +176,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			c.SetConcurrency(2)
@@ -294,8 +294,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			c.SetConcurrency(2)
@@ -412,8 +412,8 @@ urls:
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			c.SetConcurrency(2)

pkg/installer/repository_test.go

@@ -60,8 +60,8 @@ var _ = Describe("Repository", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			compiler.SetConcurrency(1)
@@ -133,8 +133,8 @@ var _ = Describe("Repository", func() {
 			Expect(err).ToNot(HaveOccurred())
 			defer os.RemoveAll(tmpdir) // clean up
 
-			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "./generate.sh"}))
-			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2", "chmod +x generate.sh"}))
+			Expect(spec.BuildSteps()).To(Equal([]string{"echo artifact5 > /test5", "echo artifact6 > /test6", "chmod +x generate.sh", "./generate.sh"}))
+			Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
 
 			spec.SetOutputPath(tmpdir)
 			spec2.SetOutputPath(tmpdir)

tests/fixtures/buildable/cat/b/build.yaml

@@ -2,8 +2,8 @@ image: "alpine"
 prelude:
   - echo foo > /test
   - echo bar > /test2
-  - chmod +x generate.sh
 steps:
   - echo artifact5 > /test5
   - echo artifact6 > /test6
+  - chmod +x generate.sh
   - ./generate.sh

tests/fixtures/buildableseed/cat/b/build.yaml

@@ -2,8 +2,8 @@ image: "alpine"
 prelude:
   - echo foo > /test
   - echo bar > /test2
-  - chmod +x generate.sh
 steps:
   - echo artifact5 > /test5
   - echo artifact6 > /test6
+  - chmod +x generate.sh
   - ./generate.sh

tests/fixtures/expansion/cat/b/build.yaml

@@ -2,8 +2,8 @@ image: "alpine"
 prelude:
   - echo foo > /test
   - echo bar > /test2
-  - chmod +x generate.sh
 steps:
   - echo artifact5 > /test5
   - echo artifact6 > /test6
+  - chmod +x generate.sh
   - ./generate.sh

tests/fixtures/package_dir/dironly_filter/build.yaml

@@ -1,10 +1,10 @@
 image: "alpine"
 prelude:
   - mkdir /foo
-  - chmod +x generate.sh
 steps:
   - echo artifact5 > /foo/test5
   - echo artifact6 > /foo/test6
+  - chmod +x generate.sh
   - ./generate.sh
 package_dir: /foo
 includes:

tests/fixtures/upgrade/cat/b-1.1/build.yaml

@@ -2,8 +2,8 @@ image: "alpine"
 prelude:
   - echo foo > /test
   - echo bar > /test2
-  - chmod +x generate.sh
 steps:
   - echo artifact5 > /newc
   - echo artifact6 > /newnewc
+  - chmod +x generate.sh
   - ./generate.sh

tests/fixtures/upgrade/cat/b/build.yaml

@@ -2,8 +2,8 @@ image: "alpine"
 prelude:
   - echo foo > /test
   - echo bar > /test2
-  - chmod +x generate.sh
 steps:
   - echo artifact5 > /test5
   - echo artifact6 > /test6
+  - chmod +x generate.sh
   - ./generate.sh

tests/fixtures/upgrade_integration/cat/b-1.1/build.yaml

@@ -2,8 +2,8 @@ image: "alpine"
 prelude:
   - echo foo > /test
   - echo bar > /test2
-  - chmod +x generate.sh
 steps:
   - echo artifact5 > /newc
   - echo artifact6 > /newnewc
+  - chmod +x generate.sh
   - ./generate.sh

tests/fixtures/upgrade_integration/cat/b/build.yaml

@@ -2,8 +2,8 @@ image: "alpine"
 prelude:
   - echo foo > /test
   - echo bar > /test2
-  - chmod +x generate.sh
 steps:
   - echo artifact5 > /test5
   - echo artifact6 > /test6
+  - chmod +x generate.sh
   - ./generate.sh

tests/fixtures/upgrade_new_repo/cat/b-1.1/build.yaml

@@ -2,8 +2,8 @@ image: "alpine"
 prelude:
   - echo foo > /test
   - echo bar > /test2
-  - chmod +x generate.sh
 steps:
   - echo artifact5 > /newc
   - echo artifact6 > /newnewc
+  - chmod +x generate.sh
   - ./generate.sh

tests/fixtures/upgrade_old_repo/cat/b/build.yaml

@@ -2,8 +2,8 @@ image: "alpine"
 prelude:
   - echo foo > /test
   - echo bar > /test2
-  - chmod +x generate.sh
 steps:
   - echo artifact5 > /test5
   - echo artifact6 > /test6
+  - chmod +x generate.sh
   - ./generate.sh

tests/fixtures/upgrade_old_repo_revision/cat/b-1.1/build.yaml

@@ -2,8 +2,8 @@ image: "alpine"
 prelude:
   - echo foo > /test
   - echo bar > /test2
-  - chmod +x generate.sh
 steps:
   - echo artifact5 > /newc
   - echo artifact6 > /newnewc
+  - chmod +x generate.sh
   - ./generate.sh