diff --git a/cmd/build.go b/cmd/build.go index 29f4a81c..6972f6de 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -36,6 +36,7 @@ var buildCmd = &cobra.Command{ Short: "build a package or a tree", Long: `build packages or trees from luet tree definitions. Packages are in [category]/[name]-[version] form`, PreRun: func(cmd *cobra.Command, args []string) { + viper.BindPFlag("clean", cmd.Flags().Lookup("clean")) viper.BindPFlag("tree", cmd.Flags().Lookup("tree")) viper.BindPFlag("destination", cmd.Flags().Lookup("destination")) viper.BindPFlag("backend", cmd.Flags().Lookup("backend")) @@ -47,6 +48,7 @@ var buildCmd = &cobra.Command{ }, Run: func(cmd *cobra.Command, args []string) { + clean := viper.GetBool("clean") src := viper.GetString("tree") dst := viper.GetString("destination") concurrency := LuetCfg.GetGeneral().Concurrency @@ -90,7 +92,9 @@ var buildCmd = &cobra.Command{ if err != nil { Fatal("Error: " + err.Error()) } - luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase()) + opts := compiler.NewDefaultCompilerOptions() + opts.Clean = clean + luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts) luetCompiler.SetConcurrency(concurrency) luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType)) if !all { @@ -163,6 +167,7 @@ func init() { if err != nil { Fatal(err) } + buildCmd.Flags().Bool("clean", true, "Build all packages without considering the packages present in the build directory") buildCmd.Flags().String("tree", path, "Source luet tree") buildCmd.Flags().String("backend", "docker", "backend used (docker,img)") buildCmd.Flags().Bool("privileged", false, "Privileged (Keep permissions)") diff --git a/pkg/compiler/artifact.go b/pkg/compiler/artifact.go index c28a8b0c..44ac8f2c 100644 --- a/pkg/compiler/artifact.go +++ b/pkg/compiler/artifact.go @@ -92,6 +92,22 @@ func NewPackageArtifactFromYaml(data []byte) (Artifact, error) { return p, err } +func LoadArtifactFromYaml(spec CompilationSpec) (Artifact, error) { + + metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml" + dat, err := ioutil.ReadFile(spec.Rel(metaFile)) + if err != nil { + return nil, errors.Wrap(err, "Error reading file "+metaFile) + } + art, err := NewPackageArtifactFromYaml(dat) + if err != nil { + return nil, errors.Wrap(err, "Error writing file "+metaFile) + } + // It is relative, set it back to abs + art.SetPath(spec.Rel(art.GetPath())) + return art, nil +} + func (a *PackageArtifact) SetCompressionType(t CompressionImplementation) { a.CompressionType = t } diff --git a/pkg/compiler/artifact_test.go b/pkg/compiler/artifact_test.go index d5b243cc..5237d7a6 100644 --- a/pkg/compiler/artifact_test.go +++ b/pkg/compiler/artifact_test.go @@ -41,7 +41,7 @@ var _ = Describe("Artifact", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1)) - compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"}) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/compiler/backend/simpledocker_test.go b/pkg/compiler/backend/simpledocker_test.go index 05961f04..07a9d341 100644 --- a/pkg/compiler/backend/simpledocker_test.go +++ b/pkg/compiler/backend/simpledocker_test.go @@ -40,7 +40,7 @@ var _ = Describe("Docker backend", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1)) - compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"}) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index 717f2606..dc494b45 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -21,7 +21,6 @@ import ( "os" "path/filepath" "regexp" - "runtime" "strings" "sync" @@ -37,15 +36,15 @@ const BuildFile = "build.yaml" type LuetCompiler struct { *tree.CompilerRecipe - Backend CompilerBackend - Database pkg.PackageDatabase - ImageRepository string - PullFirst, KeepImg bool - Concurrency int - CompressionType CompressionImplementation + Backend CompilerBackend + Database pkg.PackageDatabase + ImageRepository string + PullFirst, KeepImg, Clean bool + Concurrency int + CompressionType CompressionImplementation } -func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase) Compiler { +func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions) Compiler { // The CompilerRecipe will gives us a tree with only build deps listed. return &LuetCompiler{ Backend: backend, @@ -53,11 +52,12 @@ func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase) Compiler { tree.Recipe{Database: db}, }, Database: db, - ImageRepository: "luet/cache", - PullFirst: true, - CompressionType: None, - KeepImg: true, - Concurrency: runtime.NumCPU(), + ImageRepository: opt.ImageRepository, + PullFirst: opt.PullFirst, + CompressionType: opt.CompressionType, + KeepImg: opt.KeepImg, + Concurrency: opt.Concurrency, + Clean: opt.Clean, } } @@ -227,6 +227,12 @@ func (cs *LuetCompiler) stripIncludesFromRootfs(includes []string, rootfs string } func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec) (Artifact, error) { + if !cs.Clean { + if art, err := LoadArtifactFromYaml(p); err == nil { + Debug("Artifact reloaded. Skipping build") + return art, err + } + } pkgTag := ":package: " + p.GetPackage().GetName() p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver, @@ -380,6 +386,12 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage } func (cs *LuetCompiler) packageFromImage(p CompilationSpec, tag string, keepPermissions, keepImg bool, concurrency int) (Artifact, error) { + if !cs.Clean { + if art, err := LoadArtifactFromYaml(p); err == nil { + Debug("Artifact reloaded. Skipping build") + return art, err + } + } pkgTag := ":package: " + p.GetPackage().GetName() Info(pkgTag, " 🍩 Build starts 🔨 🔨 🔨 ") diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go index 1eea1f46..6f4817c1 100644 --- a/pkg/compiler/compiler_test.go +++ b/pkg/compiler/compiler_test.go @@ -38,7 +38,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -82,7 +82,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -120,7 +120,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -177,7 +177,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -219,7 +219,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -254,7 +254,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -292,7 +292,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"}) Expect(err).ToNot(HaveOccurred()) @@ -333,7 +333,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -377,7 +377,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -419,7 +419,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"}) Expect(err).ToNot(HaveOccurred()) @@ -457,7 +457,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"}) Expect(err).ToNot(HaveOccurred()) @@ -496,7 +496,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) @@ -548,7 +548,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -591,7 +591,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"}) Expect(err).ToNot(HaveOccurred()) @@ -624,7 +624,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"}) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/compiler/interface.go b/pkg/compiler/interface.go index d4fff3ca..fbe334da 100644 --- a/pkg/compiler/interface.go +++ b/pkg/compiler/interface.go @@ -16,6 +16,8 @@ package compiler import ( + "runtime" + pkg "github.com/mudler/luet/pkg/package" "github.com/mudler/luet/pkg/solver" ) @@ -40,6 +42,25 @@ type CompilerBackendOptions struct { Destination string } +type CompilerOptions struct { + ImageRepository string + PullFirst, KeepImg bool + Concurrency int + CompressionType CompressionImplementation + Clean bool +} + +func NewDefaultCompilerOptions() *CompilerOptions { + return &CompilerOptions{ + ImageRepository: "luet/cache", + PullFirst: true, + CompressionType: None, + KeepImg: true, + Concurrency: runtime.NumCPU(), + Clean: true, + } +} + type CompilerBackend interface { BuildImage(CompilerBackendOptions) error ExportImage(CompilerBackendOptions) error diff --git a/pkg/compiler/spec_test.go b/pkg/compiler/spec_test.go index d6c15eb0..c0ce591a 100644 --- a/pkg/compiler/spec_test.go +++ b/pkg/compiler/spec_test.go @@ -61,7 +61,7 @@ var _ = Describe("Spec", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1)) - compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase()) + compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"}) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/installer/installer_test.go b/pkg/installer/installer_test.go index 23eccc26..3a01ca2f 100644 --- a/pkg/installer/installer_test.go +++ b/pkg/installer/installer_test.go @@ -47,7 +47,7 @@ var _ = Describe("Installer", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -160,7 +160,7 @@ urls: Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -279,7 +279,7 @@ urls: Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4)) - c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions()) spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -393,7 +393,7 @@ urls: Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4)) - c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions()) spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/installer/repository_test.go b/pkg/installer/repository_test.go index 4fe68cc4..403d5046 100644 --- a/pkg/installer/repository_test.go +++ b/pkg/installer/repository_test.go @@ -47,7 +47,7 @@ var _ = Describe("Repository", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) + compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred())