diff --git a/cmd/build.go b/cmd/build.go index 715ee341..4be9372a 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -23,7 +23,11 @@ import ( "github.com/ghodss/yaml" helpers "github.com/mudler/luet/cmd/helpers" "github.com/mudler/luet/pkg/compiler" - "github.com/mudler/luet/pkg/compiler/backend" + "github.com/mudler/luet/pkg/compiler/types/artifact" + compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" + + "github.com/mudler/luet/pkg/compiler/types/compression" + "github.com/mudler/luet/pkg/compiler/types/options" . "github.com/mudler/luet/pkg/config" . "github.com/mudler/luet/pkg/logger" pkg "github.com/mudler/luet/pkg/package" @@ -118,10 +122,11 @@ Build packages specifying multiple definition trees: LuetCfg.GetLogging().SetLogLevel("error") } pretend, _ := cmd.Flags().GetBool("pretend") - compilerSpecs := compiler.NewLuetCompilationspecs() + compilerSpecs := compilerspec.NewLuetCompilationspecs() var db pkg.PackageDatabase - compilerBackend := backend.NewBackend(backendType) + compilerBackend, err := compiler.NewBackend(backendType) + helpers.CheckErr(err) switch databaseType { case "memory": @@ -129,9 +134,7 @@ Build packages specifying multiple definition trees: case "boltdb": tmpdir, err := ioutil.TempDir("", "package") - if err != nil { - Fatal(err) - } + helpers.CheckErr(err) db = pkg.NewBoltDatabase(tmpdir) } @@ -141,11 +144,7 @@ Build packages specifying multiple definition trees: for _, src := range treePaths { Info("Loading tree", src) - - err := generalRecipe.Load(src) - if err != nil { - Fatal("Error: " + err.Error()) - } + helpers.CheckErr(generalRecipe.Load(src)) } Info("Building in", dst) @@ -156,38 +155,41 @@ Build packages specifying multiple definition trees: attempts := LuetCfg.Viper.GetInt("solver.max_attempts") pullRepo, _ := cmd.Flags().GetStringArray("pull-repository") - LuetCfg.GetSolverOptions().Type = stype - LuetCfg.GetSolverOptions().LearnRate = float32(rate) - LuetCfg.GetSolverOptions().Discount = float32(discount) - LuetCfg.GetSolverOptions().MaxAttempts = attempts - LuetCfg.GetGeneral().ShowBuildOutput = LuetCfg.Viper.GetBool("general.show_build_output") - Debug("Solver", LuetCfg.GetSolverOptions().CompactString()) - - opts := compiler.NewDefaultCompilerOptions() - opts.SolverOptions = *LuetCfg.GetSolverOptions() - opts.PushImageRepository = imageRepository - opts.PullImageRepository = pullRepo - opts.PullFirst = pull - opts.KeepImg = keepImages - opts.Push = push - opts.OnlyDeps = onlydeps - opts.NoDeps = nodeps - opts.Wait = wait - opts.PackageTargetOnly = onlyTarget - opts.BuildValuesFile = values - var solverOpts solver.Options - if concurrent { - solverOpts = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency} - } else { - solverOpts = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency} + opts := &LuetSolverOptions{ + Type: stype, + LearnRate: float32(rate), + Discount: float32(discount), + MaxAttempts: attempts, } - luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), opts, solverOpts) - luetCompiler.SetBackendArgs(backendArgs) - luetCompiler.SetConcurrency(concurrency) - luetCompiler.SetCompressionType(compiler.CompressionImplementation(compressionType)) + Debug("Solver", opts.CompactString()) + + if concurrent { + opts.Options = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency} + } else { + opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: concurrency} + } + + luetCompiler := compiler.NewLuetCompiler(compilerBackend, generalRecipe.GetDatabase(), + options.NoDeps(nodeps), + options.WithBackendType(backendType), + options.PushImages(push), + options.WithBuildValues(values), + options.WithPullRepositories(pullRepo), + options.WithPushRepository(imageRepository), + options.WithSolverOptions(*opts), + options.Wait(wait), + options.OnlyTarget(onlyTarget), + options.PullFirst(pull), + options.KeepImg(keepImages), + options.OnlyDeps(onlydeps), + options.BackendArgs(backendArgs), + options.Concurrency(concurrency), + options.WithCompressionType(compression.Implementation(compressionType)), + ) + if full { specs, err := luetCompiler.FromDatabase(generalRecipe.GetDatabase(), true, dst) if err != nil { @@ -228,13 +230,13 @@ Build packages specifying multiple definition trees: } } - var artifact []compiler.Artifact + var artifact []*artifact.PackageArtifact var errs []error if revdeps { artifact, errs = luetCompiler.CompileWithReverseDeps(privileged, compilerSpecs) } else if pretend { - toCalculate := []compiler.CompilationSpec{} + toCalculate := []*compilerspec.LuetCompilationSpec{} if full { var err error toCalculate, err = luetCompiler.ComputeMinimumCompilableSet(compilerSpecs.All()...) @@ -284,6 +286,7 @@ Build packages specifying multiple definition trees: } } } else { + artifact, errs = luetCompiler.CompileParallel(privileged, compilerSpecs) } if len(errs) != 0 { @@ -293,7 +296,7 @@ Build packages specifying multiple definition trees: Fatal("Bailing out") } for _, a := range artifact { - Info("Artifact generated:", a.GetPath()) + Info("Artifact generated:", a.Path) } }, } diff --git a/cmd/create-repo.go b/cmd/create-repo.go index 4c71096f..8b699285 100644 --- a/cmd/create-repo.go +++ b/cmd/create-repo.go @@ -18,8 +18,9 @@ import ( "os" "path/filepath" + helpers "github.com/mudler/luet/cmd/helpers" "github.com/mudler/luet/pkg/compiler" - "github.com/mudler/luet/pkg/compiler/backend" + "github.com/mudler/luet/pkg/compiler/types/compression" . "github.com/mudler/luet/pkg/config" installer "github.com/mudler/luet/pkg/installer" . "github.com/mudler/luet/pkg/logger" @@ -74,7 +75,7 @@ Create a repository from the metadata description defined in the luet.yaml confi }, Run: func(cmd *cobra.Command, args []string) { var err error - var repo installer.Repository + var repo *installer.LuetSystemRepository treePaths := viper.GetStringSlice("tree") dst := viper.GetString("output") @@ -93,7 +94,8 @@ Create a repository from the metadata description defined in the luet.yaml confi treeFile := installer.NewDefaultTreeRepositoryFile() metaFile := installer.NewDefaultMetaRepositoryFile() - compilerBackend := backend.NewBackend(backendType) + compilerBackend, err := compiler.NewBackend(backendType) + helpers.CheckErr(err) force := viper.GetBool("force-push") imagePush := viper.GetBool("push-images") @@ -130,7 +132,7 @@ Create a repository from the metadata description defined in the luet.yaml confi } if treetype != "" { - treeFile.SetCompressionType(compiler.CompressionImplementation(treetype)) + treeFile.SetCompressionType(compression.Implementation(treetype)) } if treeName != "" { @@ -138,7 +140,7 @@ Create a repository from the metadata description defined in the luet.yaml confi } if metatype != "" { - metaFile.SetCompressionType(compiler.CompressionImplementation(metatype)) + metaFile.SetCompressionType(compression.Implementation(metatype)) } if metaName != "" { diff --git a/cmd/database/create.go b/cmd/database/create.go index e7c17f04..7911d46b 100644 --- a/cmd/database/create.go +++ b/cmd/database/create.go @@ -18,7 +18,8 @@ package cmd_database import ( "io/ioutil" - "github.com/mudler/luet/pkg/compiler" + artifact "github.com/mudler/luet/pkg/compiler/types/artifact" + . "github.com/mudler/luet/pkg/logger" pkg "github.com/mudler/luet/pkg/package" @@ -66,21 +67,21 @@ For reference, inspect a "metadata.yaml" file generated while running "luet buil if err != nil { Fatal("Failed reading ", a, ": ", err.Error()) } - art, err := compiler.NewPackageArtifactFromYaml(dat) + art, err := artifact.NewPackageArtifactFromYaml(dat) if err != nil { Fatal("Failed reading yaml ", a, ": ", err.Error()) } - files := art.GetFiles() + files := art.Files - if _, err := systemDB.CreatePackage(art.GetCompileSpec().GetPackage()); err != nil { + if _, err := systemDB.CreatePackage(art.CompileSpec.GetPackage()); err != nil { Fatal("Failed to create ", a, ": ", err.Error()) } - if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.GetCompileSpec().GetPackage().GetFingerPrint(), Files: files}); err != nil { + if err := systemDB.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: art.CompileSpec.GetPackage().GetFingerPrint(), Files: files}); err != nil { Fatal("Failed setting package files for ", a, ": ", err.Error()) } - Info(art.GetCompileSpec().GetPackage().HumanReadableString(), " created") + Info(art.CompileSpec.GetPackage().HumanReadableString(), " created") } }, diff --git a/cmd/helpers/cli.go b/cmd/helpers/cli.go index d2694fba..c5a633a3 100644 --- a/cmd/helpers/cli.go +++ b/cmd/helpers/cli.go @@ -22,6 +22,8 @@ import ( "regexp" "strings" + . "github.com/mudler/luet/pkg/logger" + _gentoo "github.com/Sabayon/pkgs-checker/pkg/gentoo" pkg "github.com/mudler/luet/pkg/package" version "github.com/mudler/luet/pkg/versioner" @@ -112,3 +114,9 @@ func ParsePackageStr(p string) (*pkg.DefaultPackage, error) { return pack, nil } + +func CheckErr(err error) { + if err != nil { + Fatal(err) + } +} diff --git a/cmd/pack.go b/cmd/pack.go index 560baa0c..fbea9a38 100644 --- a/cmd/pack.go +++ b/cmd/pack.go @@ -20,7 +20,9 @@ import ( "time" helpers "github.com/mudler/luet/cmd/helpers" - "github.com/mudler/luet/pkg/compiler" + "github.com/mudler/luet/pkg/compiler/types/artifact" + "github.com/mudler/luet/pkg/compiler/types/compression" + compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" . "github.com/mudler/luet/pkg/config" . "github.com/mudler/luet/pkg/logger" @@ -64,21 +66,21 @@ Afterwards, you can use the content generated and associate it with a tree and a Fatal("Invalid package string ", packageName, ": ", err.Error()) } - spec := &compiler.LuetCompilationSpec{Package: p} - artifact := compiler.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar")) - artifact.SetCompressionType(compiler.CompressionImplementation(compressionType)) - err = artifact.Compress(sourcePath, concurrency) + spec := &compilerspec.LuetCompilationSpec{Package: p} + a := artifact.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar")) + a.CompressionType = compression.Implementation(compressionType) + err = a.Compress(sourcePath, concurrency) if err != nil { Fatal("failed compressing ", packageName, ": ", err.Error()) } - artifact.SetCompileSpec(spec) - filelist, err := artifact.FileList() + a.CompileSpec = spec + filelist, err := a.FileList() if err != nil { Fatal("failed generating file list for ", packageName, ": ", err.Error()) } - artifact.SetFiles(filelist) - artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String()) - err = artifact.WriteYaml(dst) + a.Files = filelist + a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String()) + err = a.WriteYaml(dst) if err != nil { Fatal("failed writing metadata yaml file for ", packageName, ": ", err.Error()) } diff --git a/cmd/repo/list.go b/cmd/repo/list.go index e7b93ae9..c1c31f32 100644 --- a/cmd/repo/list.go +++ b/cmd/repo/list.go @@ -72,7 +72,7 @@ func NewRepoListCommand() *cobra.Command { if repo.Cached { r := installer.NewSystemRepository(repo) - localRepo, _ := r.(*installer.LuetSystemRepository).ReadSpecFile(filepath.Join(repobasedir, + localRepo, _ := r.ReadSpecFile(filepath.Join(repobasedir, installer.REPOSITORY_SPECFILE)) if localRepo != nil { tsec, _ := strconv.ParseInt(localRepo.GetLastUpdate(), 10, 64) diff --git a/cmd/search.go b/cmd/search.go index badd935c..b2704196 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -172,7 +172,7 @@ func searchOnline(term string, l list.Writer, t table.Writer, label, labelMatch, Hidden: m.Package.IsHidden(), } if m.Artifact != nil { - r.Files = m.Artifact.GetFiles() + r.Files = m.Artifact.Files } results.Packages = append(results.Packages, *r) } @@ -190,7 +190,7 @@ func searchOnline(term string, l list.Writer, t table.Writer, label, labelMatch, Hidden: revdep.IsHidden(), } if m.Artifact != nil { - r.Files = m.Artifact.GetFiles() + r.Files = m.Artifact.Files } results.Packages = append(results.Packages, *r) } @@ -262,7 +262,7 @@ func searchFiles(term string, l list.Writer, t table.Writer) Results { Category: m.Package.GetCategory(), Repository: m.Repo.GetName(), Hidden: m.Package.IsHidden(), - Files: m.Artifact.GetFiles(), + Files: m.Artifact.Files, }) } return results diff --git a/cmd/tree/images.go b/cmd/tree/images.go index f8a4df4d..f590f75b 100644 --- a/cmd/tree/images.go +++ b/cmd/tree/images.go @@ -25,6 +25,7 @@ import ( helpers "github.com/mudler/luet/cmd/helpers" "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/compiler/backend" + "github.com/mudler/luet/pkg/compiler/types/options" . "github.com/mudler/luet/pkg/config" . "github.com/mudler/luet/pkg/logger" pkg "github.com/mudler/luet/pkg/package" @@ -74,13 +75,15 @@ func NewTreeImageCommand() *cobra.Command { } compilerBackend := backend.NewSimpleDockerBackend() - opts := compiler.NewDefaultCompilerOptions() - opts.SolverOptions = *LuetCfg.GetSolverOptions() - opts.PushImageRepository = imageRepository - opts.PullImageRepository = pullRepo - - solverOpts := solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1} - luetCompiler := compiler.NewLuetCompiler(compilerBackend, reciper.GetDatabase(), opts, solverOpts) + opts := *LuetCfg.GetSolverOptions() + opts.Options = solver.Options{Type: solver.SingleCoreSimple, Concurrency: 1} + luetCompiler := compiler.NewLuetCompiler( + compilerBackend, + reciper.GetDatabase(), + options.WithPushRepository(imageRepository), + options.WithPullRepositories(pullRepo), + options.WithSolverOptions(opts), + ) a := args[0] diff --git a/pkg/compiler/backend/diff.go b/pkg/compiler/backend.go similarity index 50% rename from pkg/compiler/backend/diff.go rename to pkg/compiler/backend.go index 7663b348..7cbb844a 100644 --- a/pkg/compiler/backend/diff.go +++ b/pkg/compiler/backend.go @@ -1,33 +1,52 @@ -// Copyright © 2020 Ettore Di Giacinto -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, see . - -package backend +package compiler import ( + "fmt" "io/ioutil" "os" "path/filepath" "strings" - . "github.com/mudler/luet/pkg/logger" + artifact "github.com/mudler/luet/pkg/compiler/types/artifact" - "github.com/mudler/luet/pkg/compiler" + "github.com/mudler/luet/pkg/compiler/backend" "github.com/mudler/luet/pkg/config" "github.com/pkg/errors" + + . "github.com/mudler/luet/pkg/logger" ) +func NewBackend(s string) (CompilerBackend, error) { + var compilerBackend CompilerBackend + + switch s { + case backend.ImgBackend: + compilerBackend = backend.NewSimpleImgBackend() + case backend.DockerBackend: + compilerBackend = backend.NewSimpleDockerBackend() + default: + return nil, errors.New("invalid backend. Unsupported") + } + + return compilerBackend, nil +} + +type CompilerBackend interface { + BuildImage(backend.Options) error + ExportImage(backend.Options) error + RemoveImage(backend.Options) error + ImageDefinitionToTar(backend.Options) error + ExtractRootfs(opts backend.Options, keepPerms bool) error + + CopyImage(string, string) error + DownloadImage(opts backend.Options) error + + Push(opts backend.Options) error + ImageAvailable(string) bool + + ImageExists(string) bool +} + // GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods // example of json return: [ // { @@ -54,46 +73,46 @@ import ( // } // } // ] -func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) { +func GenerateChanges(b CompilerBackend, fromImage, toImage backend.Options) ([]artifact.ArtifactLayer, error) { - res := compiler.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName} + res := artifact.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName} tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction") if err != nil { - return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs") + return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs") } defer os.RemoveAll(tmpdiffs) // clean up srcRootFS, err := ioutil.TempDir(tmpdiffs, "src") if err != nil { - return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs") + return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs") } defer os.RemoveAll(srcRootFS) // clean up dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst") if err != nil { - return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs") + return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs") } defer os.RemoveAll(dstRootFS) // clean up - srcImageExtract := compiler.CompilerBackendOptions{ + srcImageExtract := backend.Options{ ImageName: fromImage.ImageName, Destination: srcRootFS, } Debug("Extracting source image", fromImage.ImageName) err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs if err != nil { - return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName) + return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName) } - dstImageExtract := compiler.CompilerBackendOptions{ + dstImageExtract := backend.Options{ ImageName: toImage.ImageName, Destination: dstRootFS, } Debug("Extracting destination image", toImage.ImageName) err = b.ExtractRootfs(dstImageExtract, false) if err != nil { - return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName) + return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName) } // Get Additions/Changes. dst -> src @@ -114,7 +133,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com if sizeA != sizeB { // fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath)) - res.Diffs.Changes = append(res.Diffs.Changes, compiler.ArtifactNode{ + res.Diffs.Changes = append(res.Diffs.Changes, artifact.ArtifactNode{ Name: filepath.Join("/", realpath), Size: int(sizeB), }) @@ -127,7 +146,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil { sizeB = s.Size() } - res.Diffs.Additions = append(res.Diffs.Additions, compiler.ArtifactNode{ + res.Diffs.Additions = append(res.Diffs.Additions, artifact.ArtifactNode{ Name: filepath.Join("/", realpath), Size: int(sizeB), }) @@ -138,7 +157,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com return nil }) if err != nil { - return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination") + return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination") } // Get deletions. src -> dst @@ -150,7 +169,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com realpath := strings.Replace(path, srcRootFS, "", -1) if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil { // fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath)) - res.Diffs.Deletions = append(res.Diffs.Deletions, compiler.ArtifactNode{ + res.Diffs.Deletions = append(res.Diffs.Deletions, artifact.ArtifactNode{ Name: filepath.Join("/", realpath), }) } @@ -158,8 +177,71 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com return nil }) if err != nil { - return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source") + return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source") } - return []compiler.ArtifactLayer{res}, nil + diffs := []artifact.ArtifactLayer{res} + + if config.LuetCfg.GetGeneral().Debug { + summary := ComputeArtifactLayerSummary(diffs) + for _, l := range summary.Layers { + Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)", + l.FromImage, l.ToImage, + l.AddFiles, l.AddSizes, + l.DelFiles, l.DelSizes, + l.ChangeFiles, l.ChangeSizes)) + } + } + + return diffs, nil +} + +type ArtifactLayerSummary struct { + FromImage string `json:"image1"` + ToImage string `json:"image2"` + AddFiles int `json:"add_files"` + AddSizes int64 `json:"add_sizes"` + DelFiles int `json:"del_files"` + DelSizes int64 `json:"del_sizes"` + ChangeFiles int `json:"change_files"` + ChangeSizes int64 `json:"change_sizes"` +} + +type ArtifactLayersSummary struct { + Layers []ArtifactLayerSummary `json:"summary"` +} + +func ComputeArtifactLayerSummary(diffs []artifact.ArtifactLayer) ArtifactLayersSummary { + + ans := ArtifactLayersSummary{ + Layers: make([]ArtifactLayerSummary, 0), + } + + for _, layer := range diffs { + sum := ArtifactLayerSummary{ + FromImage: layer.FromImage, + ToImage: layer.ToImage, + AddFiles: 0, + AddSizes: 0, + DelFiles: 0, + DelSizes: 0, + ChangeFiles: 0, + ChangeSizes: 0, + } + for _, a := range layer.Diffs.Additions { + sum.AddFiles++ + sum.AddSizes += int64(a.Size) + } + for _, d := range layer.Diffs.Deletions { + sum.DelFiles++ + sum.DelSizes += int64(d.Size) + } + for _, c := range layer.Diffs.Changes { + sum.ChangeFiles++ + sum.ChangeSizes += int64(c.Size) + } + ans.Layers = append(ans.Layers, sum) + } + + return ans } diff --git a/pkg/compiler/backend/common.go b/pkg/compiler/backend/common.go index 0c6f80cb..aadd6b21 100644 --- a/pkg/compiler/backend/common.go +++ b/pkg/compiler/backend/common.go @@ -18,7 +18,6 @@ package backend import ( "os/exec" - "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/config" . "github.com/mudler/luet/pkg/logger" @@ -36,16 +35,13 @@ func imageAvailable(image string) bool { return err == nil } -func NewBackend(s string) compiler.CompilerBackend { - var compilerBackend compiler.CompilerBackend - - switch s { - case ImgBackend: - compilerBackend = NewSimpleImgBackend() - case DockerBackend: - compilerBackend = NewSimpleDockerBackend() - } - return compilerBackend +type Options struct { + ImageName string + SourcePath string + DockerFileName string + Destination string + Context string + BackendArgs []string } func runCommand(cmd *exec.Cmd) error { @@ -75,7 +71,7 @@ func runCommand(cmd *exec.Cmd) error { return nil } -func genBuildCommand(opts compiler.CompilerBackendOptions) []string { +func genBuildCommand(opts Options) []string { context := opts.Context if context == "" { diff --git a/pkg/compiler/backend/simpledocker.go b/pkg/compiler/backend/simpledocker.go index 4bf541c7..6504a79d 100644 --- a/pkg/compiler/backend/simpledocker.go +++ b/pkg/compiler/backend/simpledocker.go @@ -17,7 +17,6 @@ package backend import ( "encoding/json" - "fmt" "io/ioutil" "os" "os/exec" @@ -29,8 +28,6 @@ import ( docker "github.com/fsouza/go-dockerclient" capi "github.com/mudler/docker-companion/api" - "github.com/mudler/luet/pkg/compiler" - "github.com/mudler/luet/pkg/config" "github.com/mudler/luet/pkg/helpers" . "github.com/mudler/luet/pkg/logger" @@ -39,12 +36,12 @@ import ( type SimpleDocker struct{} -func NewSimpleDockerBackend() compiler.CompilerBackend { +func NewSimpleDockerBackend() *SimpleDocker { return &SimpleDocker{} } // TODO: Missing still: labels, and build args expansion -func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error { +func (*SimpleDocker) BuildImage(opts Options) error { name := opts.ImageName bus.Manager.Publish(bus.EventImagePreBuild, opts) @@ -93,7 +90,7 @@ func (*SimpleDocker) CopyImage(src, dst string) error { return nil } -func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error { +func (*SimpleDocker) DownloadImage(opts Options) error { name := opts.ImageName bus.Manager.Publish(bus.EventImagePrePull, opts) @@ -132,7 +129,7 @@ func (*SimpleDocker) ImageAvailable(imagename string) bool { return imageAvailable(imagename) } -func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error { +func (*SimpleDocker) RemoveImage(opts Options) error { name := opts.ImageName buildarg := []string{"rmi", name} out, err := exec.Command("docker", buildarg...).CombinedOutput() @@ -144,7 +141,7 @@ func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error { return nil } -func (*SimpleDocker) Push(opts compiler.CompilerBackendOptions) error { +func (*SimpleDocker) Push(opts Options) error { name := opts.ImageName pusharg := []string{"push", name} bus.Manager.Publish(bus.EventImagePrePush, opts) @@ -163,7 +160,7 @@ func (*SimpleDocker) Push(opts compiler.CompilerBackendOptions) error { return nil } -func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error { +func (s *SimpleDocker) ImageDefinitionToTar(opts Options) error { if err := s.BuildImage(opts); err != nil { return errors.Wrap(err, "Failed building image") } @@ -176,7 +173,7 @@ func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions return nil } -func (*SimpleDocker) ExportImage(opts compiler.CompilerBackendOptions) error { +func (*SimpleDocker) ExportImage(opts Options) error { name := opts.ImageName path := opts.Destination @@ -199,7 +196,7 @@ type ManifestEntry struct { Layers []string `json:"Layers"` } -func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error { +func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error { name := opts.ImageName dst := opts.Destination @@ -214,7 +211,7 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP Spinner(22) defer SpinnerStop() - if err := b.ExportImage(compiler.CompilerBackendOptions{ImageName: name, Destination: imageExport}); err != nil { + if err := b.ExportImage(Options{ImageName: name, Destination: imageExport}); err != nil { return errors.Wrap(err, "failed while extracting rootfs for "+name) } @@ -229,7 +226,7 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP } defer os.RemoveAll(tempUnpack) // clean up imageExport := filepath.Join(tempUnpack, "image.tar") - if err := b.ExportImage(compiler.CompilerBackendOptions{ImageName: opts.ImageName, Destination: imageExport}); err != nil { + if err := b.ExportImage(Options{ImageName: opts.ImageName, Destination: imageExport}); err != nil { return errors.Wrap(err, "while exporting image before extraction") } src = imageExport @@ -293,21 +290,3 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP return nil } - -// Changes retrieves changes between image layers -func (d *SimpleDocker) Changes(fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) { - diffs, err := GenerateChanges(d, fromImage, toImage) - - if config.LuetCfg.GetGeneral().Debug { - summary := compiler.ComputeArtifactLayerSummary(diffs) - for _, l := range summary.Layers { - Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)", - l.FromImage, l.ToImage, - l.AddFiles, l.AddSizes, - l.DelFiles, l.DelSizes, - l.ChangeFiles, l.ChangeSizes)) - } - } - - return diffs, err -} diff --git a/pkg/compiler/backend/simpledocker_test.go b/pkg/compiler/backend/simpledocker_test.go index dac767dd..f7369427 100644 --- a/pkg/compiler/backend/simpledocker_test.go +++ b/pkg/compiler/backend/simpledocker_test.go @@ -16,9 +16,11 @@ package backend_test import ( + "github.com/mudler/luet/pkg/compiler" . "github.com/mudler/luet/pkg/compiler" + "github.com/mudler/luet/pkg/compiler/backend" . "github.com/mudler/luet/pkg/compiler/backend" - "github.com/mudler/luet/pkg/solver" + "github.com/mudler/luet/pkg/compiler/types/artifact" "io/ioutil" "os" @@ -41,13 +43,10 @@ var _ = Describe("Docker backend", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1)) - compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) - spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"}) + cc := NewLuetCompiler(nil, generalRecipe.GetDatabase()) + lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"}) Expect(err).ToNot(HaveOccurred()) - lspec, ok := spec.(*LuetCompilationSpec) - Expect(ok).To(BeTrue()) - Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"})) Expect(lspec.Image).To(Equal("luet/base")) Expect(lspec.Seed).To(Equal("alpine")) @@ -71,7 +70,7 @@ ENV PACKAGE_NAME=enman ENV PACKAGE_VERSION=1.4.0 ENV PACKAGE_CATEGORY=app-admin`)) b := NewSimpleDockerBackend() - opts := CompilerBackendOptions{ + opts := backend.Options{ ImageName: "luet/base", SourcePath: tmpdir, DockerFileName: "Dockerfile", @@ -95,7 +94,7 @@ ENV PACKAGE_VERSION=1.4.0 ENV PACKAGE_CATEGORY=app-admin RUN echo foo > /test RUN echo bar > /test2`)) - opts2 := CompilerBackendOptions{ + opts2 := backend.Options{ ImageName: "test", SourcePath: tmpdir, DockerFileName: "LuetDockerfile", @@ -106,26 +105,26 @@ RUN echo bar > /test2`)) Expect(b.ExportImage(opts2)).ToNot(HaveOccurred()) Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue()) - artifacts := []ArtifactNode{{ + artifacts := []artifact.ArtifactNode{{ Name: "/luetbuild/LuetDockerfile", Size: 175, }} if os.Getenv("DOCKER_BUILDKIT") == "1" { - artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0}) + artifacts = append(artifacts, artifact.ArtifactNode{Name: "/etc/resolv.conf", Size: 0}) } - artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4}) - artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4}) + artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test", Size: 4}) + artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test2", Size: 4}) - Expect(b.Changes(opts, opts2)).To(Equal( - []ArtifactLayer{{ + Expect(compiler.GenerateChanges(b, opts, opts2)).To(Equal( + []artifact.ArtifactLayer{{ FromImage: "luet/base", ToImage: "test", - Diffs: ArtifactDiffs{ + Diffs: artifact.ArtifactDiffs{ Additions: artifacts, }, }})) - opts2 = CompilerBackendOptions{ + opts2 = backend.Options{ ImageName: "test", SourcePath: tmpdir, DockerFileName: "LuetDockerfile", diff --git a/pkg/compiler/backend/simpleimg.go b/pkg/compiler/backend/simpleimg.go index 0ce5eff6..5c052771 100644 --- a/pkg/compiler/backend/simpleimg.go +++ b/pkg/compiler/backend/simpleimg.go @@ -22,7 +22,6 @@ import ( bus "github.com/mudler/luet/pkg/bus" - "github.com/mudler/luet/pkg/compiler" . "github.com/mudler/luet/pkg/logger" "github.com/pkg/errors" @@ -30,12 +29,12 @@ import ( type SimpleImg struct{} -func NewSimpleImgBackend() compiler.CompilerBackend { +func NewSimpleImgBackend() *SimpleImg { return &SimpleImg{} } // TODO: Missing still: labels, and build args expansion -func (*SimpleImg) BuildImage(opts compiler.CompilerBackendOptions) error { +func (*SimpleImg) BuildImage(opts Options) error { name := opts.ImageName bus.Manager.Publish(bus.EventImagePreBuild, opts) @@ -56,7 +55,7 @@ func (*SimpleImg) BuildImage(opts compiler.CompilerBackendOptions) error { return nil } -func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error { +func (*SimpleImg) RemoveImage(opts Options) error { name := opts.ImageName buildarg := []string{"rm", name} Spinner(22) @@ -70,7 +69,7 @@ func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error { return nil } -func (*SimpleImg) DownloadImage(opts compiler.CompilerBackendOptions) error { +func (*SimpleImg) DownloadImage(opts Options) error { name := opts.ImageName bus.Manager.Publish(bus.EventImagePrePull, opts) @@ -123,7 +122,7 @@ func (*SimpleImg) ImageExists(imagename string) bool { return false } -func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error { +func (s *SimpleImg) ImageDefinitionToTar(opts Options) error { if err := s.BuildImage(opts); err != nil { return errors.Wrap(err, "Failed building image") } @@ -136,7 +135,7 @@ func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) e return nil } -func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error { +func (*SimpleImg) ExportImage(opts Options) error { name := opts.ImageName path := opts.Destination buildarg := []string{"save", "-o", path, name} @@ -154,7 +153,7 @@ func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error { } // ExtractRootfs extracts the docker image content inside the destination -func (s *SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error { +func (s *SimpleImg) ExtractRootfs(opts Options, keepPerms bool) error { name := opts.ImageName path := opts.Destination @@ -180,13 +179,7 @@ func (s *SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerm return nil } -// TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs -// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs -func (i *SimpleImg) Changes(fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) { - return GenerateChanges(i, fromImage, toImage) -} - -func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error { +func (*SimpleImg) Push(opts Options) error { name := opts.ImageName bus.Manager.Publish(bus.EventImagePrePush, opts) diff --git a/pkg/compiler/backend/diff_test.go b/pkg/compiler/backend_test.go similarity index 86% rename from pkg/compiler/backend/diff_test.go rename to pkg/compiler/backend_test.go index 1d3c56ea..768d452e 100644 --- a/pkg/compiler/backend/diff_test.go +++ b/pkg/compiler/backend_test.go @@ -13,10 +13,10 @@ // You should have received a copy of the GNU General Public License along // with this program; if not, see . -package backend_test +package compiler_test import ( - "github.com/mudler/luet/pkg/compiler" + . "github.com/mudler/luet/pkg/compiler" . "github.com/mudler/luet/pkg/compiler/backend" . "github.com/onsi/ginkgo" @@ -24,7 +24,7 @@ import ( ) var _ = Describe("Docker image diffs", func() { - var b compiler.CompilerBackend + var b CompilerBackend BeforeEach(func() { b = NewSimpleDockerBackend() @@ -32,7 +32,7 @@ var _ = Describe("Docker image diffs", func() { Context("Generate diffs from docker images", func() { It("Detect no changes", func() { - opts := compiler.CompilerBackendOptions{ + opts := Options{ ImageName: "alpine:latest", } err := b.DownloadImage(opts) @@ -47,18 +47,18 @@ var _ = Describe("Docker image diffs", func() { }) It("Detects additions and changed files", func() { - err := b.DownloadImage(compiler.CompilerBackendOptions{ + err := b.DownloadImage(Options{ ImageName: "quay.io/mocaccino/micro", }) Expect(err).ToNot(HaveOccurred()) - err = b.DownloadImage(compiler.CompilerBackendOptions{ + err = b.DownloadImage(Options{ ImageName: "quay.io/mocaccino/extra", }) Expect(err).ToNot(HaveOccurred()) - layers, err := GenerateChanges(b, compiler.CompilerBackendOptions{ + layers, err := GenerateChanges(b, Options{ ImageName: "quay.io/mocaccino/micro", - }, compiler.CompilerBackendOptions{ + }, Options{ ImageName: "quay.io/mocaccino/extra", }) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go index a193f8c9..a51cd6f2 100644 --- a/pkg/compiler/compiler.go +++ b/pkg/compiler/compiler.go @@ -19,6 +19,7 @@ import ( "fmt" "io/ioutil" "os" + "path" "path/filepath" "regexp" @@ -27,13 +28,14 @@ import ( "time" bus "github.com/mudler/luet/pkg/bus" - yaml "gopkg.in/yaml.v2" - + "github.com/mudler/luet/pkg/compiler/backend" + artifact "github.com/mudler/luet/pkg/compiler/types/artifact" + "github.com/mudler/luet/pkg/compiler/types/options" + compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" "github.com/mudler/luet/pkg/helpers" . "github.com/mudler/luet/pkg/logger" pkg "github.com/mudler/luet/pkg/package" "github.com/mudler/luet/pkg/solver" - "github.com/mudler/luet/pkg/tree" "github.com/pkg/errors" ) @@ -41,63 +43,55 @@ const BuildFile = "build.yaml" const DefinitionFile = "definition.yaml" const CollectionFile = "collection.yaml" -type LuetCompiler struct { - *tree.CompilerRecipe - Backend CompilerBackend - Database pkg.PackageDatabase - PushImageRepository string - PullImageRepository []string +type ArtifactIndex []*artifact.PackageArtifact - PullFirst, KeepImg, Clean bool - Concurrency int - CompressionType CompressionImplementation - Options CompilerOptions - SolverOptions solver.Options - BackedArgs []string +func (i ArtifactIndex) CleanPath() ArtifactIndex { + newIndex := ArtifactIndex{} + for _, art := range i { + // FIXME: This is a dup and makes difficult to add attributes to artifacts + newIndex = append(newIndex, &artifact.PackageArtifact{ + Path: path.Base(art.Path), + SourceAssertion: art.SourceAssertion, + CompileSpec: art.CompileSpec, + Dependencies: art.Dependencies, + CompressionType: art.CompressionType, + Checksums: art.Checksums, + Files: art.Files, + }) + } + return newIndex + //Update if exists, otherwise just create } -func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions, solvopts solver.Options) Compiler { +type LuetCompiler struct { + //*tree.CompilerRecipe + Backend CompilerBackend + Database pkg.PackageDatabase + Options options.Compiler +} + +func NewCompiler(p ...options.Option) *LuetCompiler { + c := options.NewDefaultCompiler() + c.Apply(p...) + + return &LuetCompiler{Options: *c} +} + +func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, compilerOpts ...options.Option) *LuetCompiler { // The CompilerRecipe will gives us a tree with only build deps listed. - if len(opt.PullImageRepository) == 0 { - opt.PullImageRepository = []string{opt.PushImageRepository} - } + c := NewCompiler(compilerOpts...) + // c.Options.BackendType + c.Backend = backend + c.Database = db + // c.CompilerRecipe = &tree.CompilerRecipe{ + // Recipe: tree.Recipe{Database: db}, + // } - return &LuetCompiler{ - Backend: backend, - CompilerRecipe: &tree.CompilerRecipe{ - Recipe: tree.Recipe{Database: db}, - }, - Database: db, - PushImageRepository: opt.PushImageRepository, - PullImageRepository: opt.PullImageRepository, - PullFirst: opt.PullFirst, - CompressionType: opt.CompressionType, - KeepImg: opt.KeepImg, - Concurrency: opt.Concurrency, - Options: *opt, - SolverOptions: solvopts, - } + return c } -// SetBackendArgs sets arbitrary backend arguments. -// Those for example can be commands passed to the docker daemon during build phase, -// as build-args, etc. -func (cs *LuetCompiler) SetBackendArgs(args []string) { - cs.BackedArgs = args -} - -// SetConcurrency sets the compiler concurrency -func (cs *LuetCompiler) SetConcurrency(i int) { - cs.Concurrency = i -} - -// SetCompressionType sets the compiler compression type for resulting artifacts -func (cs *LuetCompiler) SetCompressionType(t CompressionImplementation) { - cs.CompressionType = t -} - -func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan CompilationSpec, a *[]Artifact, m *sync.Mutex, concurrency int, keepPermissions bool, errors chan error) { +func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan *compilerspec.LuetCompilationSpec, a *[]*artifact.PackageArtifact, m *sync.Mutex, concurrency int, keepPermissions bool, errors chan error) { defer wg.Done() for s := range cspecs { @@ -113,17 +107,17 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan Co } // CompileWithReverseDeps compiles the supplied compilationspecs and their reverse dependencies -func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) { +func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps *compilerspec.LuetCompilationspecs) ([]*artifact.PackageArtifact, []error) { artifacts, err := cs.CompileParallel(keepPermissions, ps) if len(err) != 0 { return artifacts, err } Info(":ant: Resolving reverse dependencies") - toCompile := NewLuetCompilationspecs() + toCompile := compilerspec.NewLuetCompilationspecs() for _, a := range artifacts { - revdeps := a.GetCompileSpec().GetPackage().Revdeps(cs.Database) + revdeps := a.CompileSpec.GetPackage().Revdeps(cs.Database) for _, r := range revdeps { spec, asserterr := cs.FromPackage(r) if err != nil { @@ -146,15 +140,15 @@ func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps Compilat // CompileParallel compiles the supplied compilationspecs in parallel // to note, no specific heuristic is implemented, and the specs are run in parallel as they are. -func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) { - all := make(chan CompilationSpec) - artifacts := []Artifact{} +func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps *compilerspec.LuetCompilationspecs) ([]*artifact.PackageArtifact, []error) { + all := make(chan *compilerspec.LuetCompilationSpec) + artifacts := []*artifact.PackageArtifact{} mutex := &sync.Mutex{} errors := make(chan error, ps.Len()) var wg = new(sync.WaitGroup) - for i := 0; i < cs.Concurrency; i++ { + for i := 0; i < cs.Options.Concurrency; i++ { wg.Add(1) - go cs.compilerWorker(i, wg, all, &artifacts, mutex, cs.Concurrency, keepPermissions, errors) + go cs.compilerWorker(i, wg, all, &artifacts, mutex, cs.Options.Concurrency, keepPermissions, errors) } for _, p := range ps.All() { @@ -233,7 +227,7 @@ func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, includ return nil } -func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p CompilationSpec, runnerOpts CompilerBackendOptions) (Artifact, error) { +func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec, runnerOpts backend.Options) (*artifact.PackageArtifact, error) { rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs") if err != nil { @@ -241,7 +235,7 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p Compil } defer os.RemoveAll(rootfs) // clean up - err = cs.Backend.ExtractRootfs(CompilerBackendOptions{ + err = cs.Backend.ExtractRootfs(backend.Options{ ImageName: runnerOpts.ImageName, Destination: rootfs}, keepPermissions) if err != nil { return nil, errors.Wrap(err, "Could not extract rootfs") @@ -260,18 +254,18 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p Compil // strip from excludes cs.stripFromRootfs(p.GetExcludes(), rootfs, false) } - artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")) - artifact.SetCompressionType(cs.CompressionType) + a := artifact.NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar")) + a.CompressionType = cs.Options.CompressionType - if err := artifact.Compress(rootfs, concurrency); err != nil { + if err := a.Compress(rootfs, concurrency); err != nil { return nil, errors.Wrap(err, "Error met while creating package archive") } - artifact.SetCompileSpec(p) - return artifact, nil + a.CompileSpec = p + return a, nil } -func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p CompilationSpec, builderOpts, runnerOpts CompilerBackendOptions) (Artifact, error) { +func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec, builderOpts, runnerOpts backend.Options) (*artifact.PackageArtifact, error) { rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs") if err != nil { @@ -288,30 +282,30 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p Com } Info(pkgTag, ":hammer: Generating delta") - diffs, err := cs.Backend.Changes(builderOpts, runnerOpts) + diffs, err := GenerateChanges(cs.Backend, builderOpts, runnerOpts) if err != nil { return nil, errors.Wrap(err, "Could not generate changes from layers") } Debug("Extracting image to grab files from delta") - if err := cs.Backend.ExtractRootfs(CompilerBackendOptions{ + if err := cs.Backend.ExtractRootfs(backend.Options{ ImageName: runnerOpts.ImageName, Destination: rootfs}, keepPermissions); err != nil { return nil, errors.Wrap(err, "Could not extract rootfs") } - artifact, err := ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.CompressionType) + artifact, err := artifact.ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.Options.CompressionType) if err != nil { return nil, errors.Wrap(err, "Could not generate deltas") } - artifact.SetCompileSpec(p) + artifact.CompileSpec = p return artifact, nil } func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions bool, - p CompilationSpec) (CompilerBackendOptions, CompilerBackendOptions, error) { + p *compilerspec.LuetCompilationSpec) (backend.Options, backend.Options, error) { - var runnerOpts, builderOpts CompilerBackendOptions + var runnerOpts, builderOpts backend.Options pkgTag := ":package: " + p.GetPackage().HumanReadableString() @@ -325,7 +319,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag fp := p.GetPackage().HashFingerprint(helpers.StripRegistryFromImage(packageImage)) if buildertaggedImage == "" { - buildertaggedImage = cs.PushImageRepository + ":builder-" + fp + buildertaggedImage = cs.Options.PushImageRepository + ":builder-" + fp Debug(pkgTag, "Creating intermediary image", buildertaggedImage, "from", image) } @@ -375,22 +369,22 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition") } - builderOpts = CompilerBackendOptions{ + builderOpts = backend.Options{ ImageName: buildertaggedImage, SourcePath: buildDir, DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile", Destination: p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"), - BackendArgs: cs.BackedArgs, + BackendArgs: cs.Options.BackendArgs, } - runnerOpts = CompilerBackendOptions{ + runnerOpts = backend.Options{ ImageName: packageImage, SourcePath: buildDir, DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile", Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"), - BackendArgs: cs.BackedArgs, + BackendArgs: cs.Options.BackendArgs, } - buildAndPush := func(opts CompilerBackendOptions) error { + buildAndPush := func(opts backend.Options) error { buildImage := true if cs.Options.PullFirst { err := cs.Backend.DownloadImage(opts) @@ -430,10 +424,10 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag return builderOpts, runnerOpts, nil } -func (cs *LuetCompiler) genArtifact(p CompilationSpec, builderOpts, runnerOpts CompilerBackendOptions, concurrency int, keepPermissions bool) (Artifact, error) { +func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builderOpts, runnerOpts backend.Options, concurrency int, keepPermissions bool) (*artifact.PackageArtifact, error) { - // generate Artifact - var artifact Artifact + // generate *artifact.PackageArtifact + var a *artifact.PackageArtifact var rootfs string var err error pkgTag := ":package: " + p.GetPackage().HumanReadableString() @@ -448,53 +442,53 @@ func (cs *LuetCompiler) genArtifact(p CompilationSpec, builderOpts, runnerOpts C } defer os.RemoveAll(rootfs) // clean up - artifact := NewPackageArtifact(fakePackage) - artifact.SetCompressionType(cs.CompressionType) + a := artifact.NewPackageArtifact(fakePackage) + a.CompressionType = cs.Options.CompressionType - if err := artifact.Compress(rootfs, concurrency); err != nil { + if err := a.Compress(rootfs, concurrency); err != nil { return nil, errors.Wrap(err, "Error met while creating package archive") } - artifact.SetCompileSpec(p) - artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String()) + a.CompileSpec = p + a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String()) - err = artifact.WriteYaml(p.GetOutputPath()) + err = a.WriteYaml(p.GetOutputPath()) if err != nil { - return artifact, errors.Wrap(err, "Failed while writing metadata file") + return a, errors.Wrap(err, "Failed while writing metadata file") } Info(pkgTag, " :white_check_mark: done (empty virtual package)") - return artifact, nil + return a, nil } if p.UnpackedPackage() { // Take content of container as a base for our package files - artifact, err = cs.unpackFs(concurrency, keepPermissions, p, runnerOpts) + a, err = cs.unpackFs(concurrency, keepPermissions, p, runnerOpts) if err != nil { return nil, errors.Wrap(err, "Error met while extracting image") } } else { // Generate delta between the two images - artifact, err = cs.unpackDelta(concurrency, keepPermissions, p, builderOpts, runnerOpts) + a, err = cs.unpackDelta(concurrency, keepPermissions, p, builderOpts, runnerOpts) if err != nil { return nil, errors.Wrap(err, "Error met while generating delta") } } - filelist, err := artifact.FileList() + filelist, err := a.FileList() if err != nil { - return artifact, errors.Wrap(err, "Failed getting package list") + return a, errors.Wrap(err, "Failed getting package list") } - artifact.SetFiles(filelist) - artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String()) + a.Files = filelist + a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String()) - err = artifact.WriteYaml(p.GetOutputPath()) + err = a.WriteYaml(p.GetOutputPath()) if err != nil { - return artifact, errors.Wrap(err, "Failed while writing metadata file") + return a, errors.Wrap(err, "Failed while writing metadata file") } Info(pkgTag, " :white_check_mark: Done") - return artifact, nil + return a, nil } func (cs *LuetCompiler) waitForImages(images []string) { @@ -532,8 +526,8 @@ func oneOfImagesAvailable(images []string, b CompilerBackend) (bool, string) { func (cs *LuetCompiler) resolveExistingImageHash(imageHash string) string { var resolvedImage string - toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.PushImageRepository, imageHash)}, - genImageList(cs.PullImageRepository, imageHash)...) + toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)}, + genImageList(cs.Options.PullImageRepository, imageHash)...) if exists, which := oneOfImagesExists(toChecklist, cs.Backend); exists { resolvedImage = which } @@ -544,28 +538,43 @@ func (cs *LuetCompiler) resolveExistingImageHash(imageHash string) string { } if resolvedImage == "" { - resolvedImage = fmt.Sprintf("%s:%s", cs.PushImageRepository, imageHash) + resolvedImage = fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash) } return resolvedImage } -func (cs *LuetCompiler) getImageArtifact(hash string, p CompilationSpec) (Artifact, error) { +func LoadArtifactFromYaml(spec *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) { + metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml" + dat, err := ioutil.ReadFile(spec.Rel(metaFile)) + if err != nil { + return nil, errors.Wrap(err, "Error reading file "+metaFile) + } + art, err := artifact.NewPackageArtifactFromYaml(dat) + if err != nil { + return nil, errors.Wrap(err, "Error writing file "+metaFile) + } + // It is relative, set it back to abs + art.Path = spec.Rel(art.Path) + return art, nil +} + +func (cs *LuetCompiler) getImageArtifact(hash string, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) { // we check if there is an available image with the given hash and // we return a full artifact if can be loaded locally. - toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.PushImageRepository, hash)}, - genImageList(cs.PullImageRepository, hash)...) + toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, hash)}, + genImageList(cs.Options.PullImageRepository, hash)...) exists, _ := oneOfImagesExists(toChecklist, cs.Backend) if art, err := LoadArtifactFromYaml(p); err == nil && exists { // If YAML is correctly loaded, and both images exists, no reason to rebuild. - Debug("Artifact reloaded from YAML. Skipping build") + Debug("Package reloaded from YAML. Skipping build") return art, nil } cs.waitForImages(toChecklist) available, _ := oneOfImagesAvailable(toChecklist, cs.Backend) if exists || (cs.Options.PullFirst && available) { Debug("Image available, returning empty artifact") - return &PackageArtifact{}, nil + return &artifact.PackageArtifact{}, nil } return nil, errors.New("artifact not found") @@ -578,13 +587,13 @@ func (cs *LuetCompiler) getImageArtifact(hash string, p CompilationSpec) (Artifa func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage string, packageTagHash string, concurrency int, keepPermissions, keepImg bool, - p CompilationSpec, generateArtifact bool) (Artifact, error) { + p *compilerspec.LuetCompilationSpec, generateArtifact bool) (*artifact.PackageArtifact, error) { // If it is a virtual, check if we have to generate an empty artifact or not. if generateArtifact && p.IsVirtual() { - return cs.genArtifact(p, CompilerBackendOptions{}, CompilerBackendOptions{}, concurrency, keepPermissions) + return cs.genArtifact(p, backend.Options{}, backend.Options{}, concurrency, keepPermissions) } else if p.IsVirtual() { - return &PackageArtifact{}, nil + return &artifact.PackageArtifact{}, nil } if !generateArtifact { @@ -597,7 +606,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage string, packa } // always going to point at the destination from the repo defined - packageImage := fmt.Sprintf("%s:%s", cs.PushImageRepository, packageTagHash) + packageImage := fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, packageTagHash) builderOpts, runnerOpts, err := cs.buildPackageImage(image, buildertaggedImage, packageImage, concurrency, keepPermissions, p) if err != nil { return nil, errors.Wrap(err, "failed building package image") @@ -616,7 +625,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage string, packa } if !generateArtifact { - return &PackageArtifact{}, nil + return &artifact.PackageArtifact{}, nil } return cs.genArtifact(p, builderOpts, runnerOpts, concurrency, keepPermissions) @@ -624,8 +633,8 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage string, packa // FromDatabase returns all the available compilation specs from a database. If the minimum flag is returned // it will be computed a minimal subset that will guarantees that all packages are built ( if not targeting a single package explictly ) -func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error) { - compilerSpecs := NewLuetCompilationspecs() +func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]*compilerspec.LuetCompilationSpec, error) { + compilerSpecs := compilerspec.NewLuetCompilationspecs() w := db.World() @@ -649,11 +658,11 @@ func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst s } // ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs -func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error) { +func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) { // Generate a set with all the deps of the provided specs // we will use that set to remove the deps from the list of provided compilation specs allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps - result := []CompilationSpec{} + result := []*compilerspec.LuetCompilationSpec{} for _, spec := range p { ass, err := cs.ComputeDepTree(spec) if err != nil { @@ -673,9 +682,9 @@ func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...CompilationSpec) ([]Com // ComputeDepTree computes the dependency tree of a compilation spec and returns solver assertions // in order to be able to compile the spec. -func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) { +func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) { - s := solver.NewResolver(cs.SolverOptions, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver()) + s := solver.NewResolver(cs.Options.SolverOptions.Options, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver()) solution, err := s.Install(pkg.Packages{p.GetPackage()}) if err != nil { @@ -704,13 +713,13 @@ func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssert // Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates // an artifact -func (cs *LuetCompiler) Compile(keepPermissions bool, p CompilationSpec) (Artifact, error) { +func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) { asserts, err := cs.ComputeDepTree(p) if err != nil { panic(err) } p.SetSourceAssertion(asserts) - return cs.compile(cs.Concurrency, keepPermissions, p) + return cs.compile(cs.Options.Concurrency, keepPermissions, p) } func genImageList(refs []string, hash string) []string { @@ -721,7 +730,7 @@ func genImageList(refs []string, hash string) []string { return res } -func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) { +func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) { Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:") Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage())) @@ -736,17 +745,20 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint()) bus.Manager.Publish(bus.EventPackagePreBuild, struct { - CompileSpec CompilationSpec + CompileSpec *compilerspec.LuetCompilationSpec Assert solver.PackageAssert }{ CompileSpec: p, Assert: *targetAssertion, }) + // Update compilespec build options + p.SetBuildOptions(cs.Options) + // - If image is set we just generate a plain dockerfile // Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above if p.GetImage() != "" { - return cs.compileWithImage(p.GetImage(), "", targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.KeepImg, p, true) + return cs.compileWithImage(p.GetImage(), "", targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true) } // - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based @@ -755,7 +767,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila // - We later on compute an hash used to identify the image, so each similar deptree keeps the same build image. dependencies := p.GetSourceAssertion().Drop(p.GetPackage()) // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already) - departifacts := []Artifact{} // TODO: Return this somehow + departifacts := []*artifact.PackageArtifact{} // TODO: Return this somehow var lastHash string depsN := 0 currentN := 0 @@ -781,7 +793,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila Debug(pkgTag, " :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash) bus.Manager.Publish(bus.EventPackagePreBuild, struct { - CompileSpec CompilationSpec + CompileSpec *compilerspec.LuetCompilationSpec Assert solver.PackageAssert }{ CompileSpec: compileSpec, @@ -796,30 +808,30 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila if compileSpec.GetImage() != "" { Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image") - artifact, err := cs.compileWithImage(compileSpec.GetImage(), resolvedBuildImage, assertion.Hash.PackageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps) + a, err := cs.compileWithImage(compileSpec.GetImage(), resolvedBuildImage, assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps) if err != nil { return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString()) } - departifacts = append(departifacts, artifact) + departifacts = append(departifacts, a) Info(pkgTag, ":white_check_mark: Done") continue } Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree") - artifact, err := cs.compileWithImage(resolvedBuildImage, "", assertion.Hash.PackageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps) + a, err := cs.compileWithImage(resolvedBuildImage, "", assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps) if err != nil { return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString()) } bus.Manager.Publish(bus.EventPackagePostBuild, struct { - CompileSpec CompilationSpec - Artifact Artifact + CompileSpec *compilerspec.LuetCompilationSpec + Artifact *artifact.PackageArtifact }{ CompileSpec: compileSpec, - Artifact: artifact, + Artifact: a, }) - departifacts = append(departifacts, artifact) + departifacts = append(departifacts, a) Info(pkgTag, ":white_check_mark: Done") } @@ -831,22 +843,22 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila resolvedBuildImage := cs.resolveExistingImageHash(lastHash) Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString()) Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedBuildImage) - artifact, err := cs.compileWithImage(resolvedBuildImage, "", targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.KeepImg, p, true) + a, err := cs.compileWithImage(resolvedBuildImage, "", targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true) if err != nil { - return artifact, err + return a, err } - artifact.SetDependencies(departifacts) - artifact.SetSourceAssertion(p.GetSourceAssertion()) + a.Dependencies = departifacts + a.SourceAssertion = p.GetSourceAssertion() bus.Manager.Publish(bus.EventPackagePostBuild, struct { - CompileSpec CompilationSpec - Artifact Artifact + CompileSpec *compilerspec.LuetCompilationSpec + Artifact *artifact.PackageArtifact }{ CompileSpec: p, - Artifact: artifact, + Artifact: a, }) - return artifact, err + return a, err } else { return departifacts[len(departifacts)-1], nil } @@ -857,8 +869,15 @@ type templatedata map[string]interface{} func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) { var dataresult []byte - val := pack.Rel(DefinitionFile) + + // Update processed build values + dst, err := helpers.UnMarshalValues(cs.Options.BuildValuesFile) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling values") + } + cs.Options.BuildValues = []map[string]interface{}{(map[string]interface{})(dst)} + if _, err := os.Stat(pack.Rel(CollectionFile)); err == nil { val = pack.Rel(CollectionFile) @@ -879,11 +898,6 @@ func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) { raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion()) - dst, err := helpers.UnMarshalValues(cs.Options.BuildValuesFile) - if err != nil { - return nil, errors.Wrap(err, "unmarshalling values") - } - dat, err := helpers.RenderHelm(string(dataBuild), raw, dst) if err != nil { return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile)) @@ -896,12 +910,12 @@ func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) { } dataresult = []byte(out) } - return dataresult, nil + return dataresult, nil } // FromPackage returns a compilation spec from a package definition -func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) { +func (cs *LuetCompiler) FromPackage(p pkg.Package) (*compilerspec.LuetCompilationSpec, error) { pack, err := cs.Database.FindPackageCandidate(p) if err != nil { @@ -913,7 +927,7 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) { return nil, errors.Wrap(err, "while rendering package template") } - return NewLuetCompilationSpec(bytes, pack) + return compilerspec.NewLuetCompilationSpec(bytes, pack) } // GetBackend returns the current compilation backend diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go index 82befbd1..682ba9b3 100644 --- a/pkg/compiler/compiler_test.go +++ b/pkg/compiler/compiler_test.go @@ -21,9 +21,11 @@ import ( . "github.com/mudler/luet/pkg/compiler" sd "github.com/mudler/luet/pkg/compiler/backend" + "github.com/mudler/luet/pkg/compiler/types/compression" + "github.com/mudler/luet/pkg/compiler/types/options" + compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" helpers "github.com/mudler/luet/pkg/helpers" pkg "github.com/mudler/luet/pkg/package" - "github.com/mudler/luet/pkg/solver" "github.com/mudler/luet/pkg/tree" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -39,7 +41,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2)) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -54,12 +56,11 @@ var _ = Describe("Compiler", func() { Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"})) spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(2) artifact, err := compiler.Compile(false, spec) Expect(err).ToNot(HaveOccurred()) - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue()) @@ -83,7 +84,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(1)) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -98,12 +99,11 @@ var _ = Describe("Compiler", func() { spec.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdir) - compiler.SetConcurrency(2) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2)) Expect(errs).To(BeNil()) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } }) @@ -118,7 +118,7 @@ var _ = Describe("Compiler", func() { err = generalRecipe.Load("../../tests/fixtures/templates") Expect(err).ToNot(HaveOccurred()) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1)) pkg, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) @@ -142,7 +142,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2)) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -159,15 +159,14 @@ var _ = Describe("Compiler", func() { spec.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdir) spec3.SetOutputPath(tmpdir) - compiler.SetConcurrency(2) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2, spec3)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(3)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue()) @@ -199,7 +198,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(1)) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -207,23 +206,22 @@ var _ = Describe("Compiler", func() { Expect(err).ToNot(HaveOccurred()) spec.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) - artifacts2, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec2)) + artifacts2, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec2)) Expect(errs).To(BeNil()) Expect(len(artifacts2)).To(Equal(1)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } for _, artifact := range artifacts2 { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } Expect(helpers.Exists(spec.Rel("etc/hosts"))).To(BeTrue()) @@ -241,7 +239,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -250,15 +248,14 @@ var _ = Describe("Compiler", func() { // Expect(err).ToNot(HaveOccurred()) spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue()) @@ -276,7 +273,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -285,15 +282,14 @@ var _ = Describe("Compiler", func() { // Expect(err).ToNot(HaveOccurred()) spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue()) @@ -312,7 +308,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -321,15 +317,14 @@ var _ = Describe("Compiler", func() { // Expect(err).ToNot(HaveOccurred()) spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue()) @@ -348,7 +343,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -357,14 +352,13 @@ var _ = Describe("Compiler", func() { // Expect(err).ToNot(HaveOccurred()) spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue()) Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) @@ -382,7 +376,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -391,14 +385,13 @@ var _ = Describe("Compiler", func() { // Expect(err).ToNot(HaveOccurred()) spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue()) Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) @@ -416,7 +409,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -425,14 +418,13 @@ var _ = Describe("Compiler", func() { // Expect(err).ToNot(HaveOccurred()) spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } Expect(helpers.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue()) @@ -454,7 +446,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"}) Expect(err).ToNot(HaveOccurred()) @@ -463,15 +455,14 @@ var _ = Describe("Compiler", func() { // Expect(err).ToNot(HaveOccurred()) spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred()) @@ -495,7 +486,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -504,16 +495,15 @@ var _ = Describe("Compiler", func() { // Expect(err).ToNot(HaveOccurred()) spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) - Expect(len(artifacts[0].GetDependencies())).To(Equal(1)) + Expect(len(artifacts[0].Dependencies)).To(Equal(1)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred()) @@ -539,7 +529,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -548,16 +538,15 @@ var _ = Describe("Compiler", func() { // Expect(err).ToNot(HaveOccurred()) spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) - Expect(len(artifacts[0].GetDependencies())).To(Equal(1)) + Expect(len(artifacts[0].Dependencies)).To(Equal(1)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred()) @@ -581,7 +570,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"}) Expect(err).ToNot(HaveOccurred()) @@ -591,13 +580,13 @@ var _ = Describe("Compiler", func() { spec.SetOutputPath(tmpdir) - artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileWithReverseDeps(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(2)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred()) @@ -619,7 +608,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"}) Expect(err).ToNot(HaveOccurred()) @@ -629,13 +618,13 @@ var _ = Describe("Compiler", func() { spec.SetOutputPath(tmpdir) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) - Expect(len(artifacts[0].GetDependencies())).To(Equal(6)) + Expect(len(artifacts[0].Dependencies)).To(Equal(6)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } Expect(helpers.Untar(spec.Rel("vhba-sys-fs-5.4.2-20190410.package.tar"), tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue()) @@ -658,19 +647,19 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) spec.SetOutputPath(tmpdir) - artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileWithReverseDeps(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(4)) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) } // A deps on B, so A artifacts are here: @@ -710,7 +699,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2)) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -722,17 +711,16 @@ var _ = Describe("Compiler", func() { defer os.RemoveAll(tmpdir) // clean up spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(2) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) for _, artifact := range artifacts { - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) - for _, d := range artifact.GetDependencies() { - Expect(helpers.Exists(d.GetPath())).To(BeTrue()) - Expect(helpers.Untar(d.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + for _, d := range artifact.Dependencies { + Expect(helpers.Exists(d.Path)).To(BeTrue()) + Expect(helpers.Untar(d.Path, tmpdir, false)).ToNot(HaveOccurred()) } } @@ -753,7 +741,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"}) Expect(err).ToNot(HaveOccurred()) @@ -765,12 +753,11 @@ var _ = Describe("Compiler", func() { defer os.RemoveAll(tmpdir) // clean up spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) - Expect(len(artifacts[0].GetDependencies())).To(Equal(1)) + Expect(len(artifacts[0].Dependencies)).To(Equal(1)) Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue()) @@ -786,7 +773,7 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{ Name: "dironly", @@ -814,12 +801,10 @@ var _ = Describe("Compiler", func() { spec.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdir2) - compiler.SetConcurrency(1) - - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(2)) - Expect(len(artifacts[0].GetDependencies())).To(Equal(0)) + Expect(len(artifacts[0].Dependencies)).To(Equal(0)) Expect(helpers.Untar(spec.Rel("dironly-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue()) @@ -841,11 +826,11 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"}) Expect(err).ToNot(HaveOccurred()) - compiler.SetCompressionType(GZip) + compiler.Options.CompressionType = compression.GZip Expect(spec.GetPackage().GetPath()).ToNot(Equal("")) tmpdir, err := ioutil.TempDir("", "tree") @@ -853,12 +838,11 @@ var _ = Describe("Compiler", func() { defer os.RemoveAll(tmpdir) // clean up spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) - Expect(len(artifacts[0].GetDependencies())).To(Equal(1)) + Expect(len(artifacts[0].Dependencies)).To(Equal(1)) Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse()) Expect(artifacts[0].Unpack(tmpdir, false)).ToNot(HaveOccurred()) @@ -877,7 +861,7 @@ var _ = Describe("Compiler", func() { err := generalRecipe.Load("../../tests/fixtures/includeimage") Expect(err).ToNot(HaveOccurred()) Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) specs, err := compiler.FromDatabase(generalRecipe.GetDatabase(), true, "") Expect(err).ToNot(HaveOccurred()) @@ -896,11 +880,11 @@ var _ = Describe("Compiler", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2)) - compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"}) Expect(err).ToNot(HaveOccurred()) - compiler.SetCompressionType(GZip) + compiler.Options.CompressionType = compression.GZip Expect(spec.GetPackage().GetPath()).ToNot(Equal("")) tmpdir, err := ioutil.TempDir("", "tree") @@ -908,20 +892,19 @@ var _ = Describe("Compiler", func() { defer os.RemoveAll(tmpdir) // clean up spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec)) + artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeNil()) Expect(len(artifacts)).To(Equal(1)) - Expect(len(artifacts[0].GetDependencies())).To(Equal(1)) - Expect(artifacts[0].GetFiles()).To(ContainElement("bin/busybox")) + Expect(len(artifacts[0].Dependencies)).To(Equal(1)) + Expect(artifacts[0].Files).To(ContainElement("bin/busybox")) Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue()) art, err := LoadArtifactFromYaml(spec) Expect(err).ToNot(HaveOccurred()) - files := art.GetFiles() + files := art.Files Expect(files).To(ContainElement("bin/busybox")) }) }) diff --git a/pkg/compiler/interface.go b/pkg/compiler/interface.go deleted file mode 100644 index a5d4554d..00000000 --- a/pkg/compiler/interface.go +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright © 2019 Ettore Di Giacinto -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, see . - -package compiler - -import ( - "runtime" - - "github.com/mudler/luet/pkg/config" - pkg "github.com/mudler/luet/pkg/package" - "github.com/mudler/luet/pkg/solver" -) - -type Compiler interface { - Compile(bool, CompilationSpec) (Artifact, error) - CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) - CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) - ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) - ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error) - SetConcurrency(i int) - FromPackage(pkg.Package) (CompilationSpec, error) - FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error) - SetBackend(CompilerBackend) - GetBackend() CompilerBackend - - SetBackendArgs([]string) - SetCompressionType(t CompressionImplementation) -} - -type CompilerBackendOptions struct { - ImageName string - SourcePath string - DockerFileName string - Destination string - Context string - BackendArgs []string -} - -type CompilerOptions struct { - PushImageRepository string - PullImageRepository []string - PullFirst, KeepImg, Push bool - Concurrency int - CompressionType CompressionImplementation - - Wait bool - OnlyDeps bool - NoDeps bool - SolverOptions config.LuetSolverOptions - BuildValuesFile string - - PackageTargetOnly bool -} - -func NewDefaultCompilerOptions() *CompilerOptions { - return &CompilerOptions{ - PushImageRepository: "luet/cache", - PullFirst: false, - Push: false, - CompressionType: None, - KeepImg: true, - Concurrency: runtime.NumCPU(), - OnlyDeps: false, - NoDeps: false, - } -} - -type CompilerBackend interface { - BuildImage(CompilerBackendOptions) error - ExportImage(CompilerBackendOptions) error - RemoveImage(CompilerBackendOptions) error - Changes(fromImage, toImage CompilerBackendOptions) ([]ArtifactLayer, error) - ImageDefinitionToTar(CompilerBackendOptions) error - ExtractRootfs(opts CompilerBackendOptions, keepPerms bool) error - - CopyImage(string, string) error - DownloadImage(opts CompilerBackendOptions) error - - Push(opts CompilerBackendOptions) error - ImageAvailable(string) bool - - ImageExists(string) bool -} - -type Artifact interface { - GetPath() string - SetPath(string) - GetDependencies() []Artifact - SetDependencies(d []Artifact) - GetSourceAssertion() solver.PackagesAssertions - SetSourceAssertion(as solver.PackagesAssertions) - - SetCompileSpec(as CompilationSpec) - GetCompileSpec() CompilationSpec - WriteYaml(dst string) error - Unpack(dst string, keepPerms bool) error - Compress(src string, concurrency int) error - SetCompressionType(t CompressionImplementation) - FileList() ([]string, error) - Hash() error - Verify() error - - SetFiles(f []string) - GetFiles() []string - GetFileName() string - - GetChecksums() Checksums - SetChecksums(c Checksums) - - GenerateFinalImage(string, CompilerBackend, bool) (CompilerBackendOptions, error) - GetUncompressedName() string -} - -type ArtifactNode struct { - Name string `json:"Name"` - Size int `json:"Size"` -} -type ArtifactDiffs struct { - Additions []ArtifactNode `json:"Adds"` - Deletions []ArtifactNode `json:"Dels"` - Changes []ArtifactNode `json:"Mods"` -} -type ArtifactLayer struct { - FromImage string `json:"Image1"` - ToImage string `json:"Image2"` - Diffs ArtifactDiffs `json:"Diff"` -} -type ArtifactLayerSummary struct { - FromImage string `json:"image1"` - ToImage string `json:"image2"` - AddFiles int `json:"add_files"` - AddSizes int64 `json:"add_sizes"` - DelFiles int `json:"del_files"` - DelSizes int64 `json:"del_sizes"` - ChangeFiles int `json:"change_files"` - ChangeSizes int64 `json:"change_sizes"` -} -type ArtifactLayersSummary struct { - Layers []ArtifactLayerSummary `json:"summary"` -} - -// CompilationSpec represent a compilation specification derived from a package -type CompilationSpec interface { - ImageUnpack() bool // tells if the definition is just an image - GetIncludes() []string - GetExcludes() []string - - RenderBuildImage() (string, error) - WriteBuildImageDefinition(string) error - - RenderStepImage(image string) (string, error) - WriteStepImageDefinition(fromimage, path string) error - - GetPackage() pkg.Package - BuildSteps() []string - - GetSeedImage() string - SetSeedImage(string) - - GetImage() string - SetImage(string) - - SetOutputPath(string) - GetOutputPath() string - Rel(string) string - - GetPreBuildSteps() []string - - GetSourceAssertion() solver.PackagesAssertions - SetSourceAssertion(as solver.PackagesAssertions) - - GetRetrieve() []string - CopyRetrieves(dest string) error - - SetPackageDir(string) - GetPackageDir() string - - EmptyPackage() bool - UnpackedPackage() bool - HasImageSource() bool - IsVirtual() bool -} - -type CompilationSpecs interface { - Unique() CompilationSpecs - Len() int - All() []CompilationSpec - Add(CompilationSpec) - Remove(s CompilationSpecs) CompilationSpecs -} diff --git a/pkg/compiler/artifact.go b/pkg/compiler/types/artifact/artifact.go similarity index 77% rename from pkg/compiler/artifact.go rename to pkg/compiler/types/artifact/artifact.go index f1f10194..ed0506dd 100644 --- a/pkg/compiler/artifact.go +++ b/pkg/compiler/types/artifact/artifact.go @@ -13,7 +13,7 @@ // You should have received a copy of the GNU General Public License along // with this program; if not, see . -package compiler +package artifact import ( "archive/tar" @@ -36,6 +36,9 @@ import ( "sync" bus "github.com/mudler/luet/pkg/bus" + backend "github.com/mudler/luet/pkg/compiler/backend" + compression "github.com/mudler/luet/pkg/compiler/types/compression" + compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" . "github.com/mudler/luet/pkg/config" "github.com/mudler/luet/pkg/helpers" . "github.com/mudler/luet/pkg/logger" @@ -45,54 +48,25 @@ import ( yaml "gopkg.in/yaml.v2" ) -type CompressionImplementation string - -const ( - None CompressionImplementation = "none" // e.g. tar for standard packages - GZip CompressionImplementation = "gzip" - Zstandard CompressionImplementation = "zstd" -) - -type ArtifactIndex []Artifact - -func (i ArtifactIndex) CleanPath() ArtifactIndex { - newIndex := ArtifactIndex{} - for _, n := range i { - art := n.(*PackageArtifact) - // FIXME: This is a dup and makes difficult to add attributes to artifacts - newIndex = append(newIndex, &PackageArtifact{ - Path: path.Base(n.GetPath()), - SourceAssertion: art.SourceAssertion, - CompileSpec: art.CompileSpec, - Dependencies: art.Dependencies, - CompressionType: art.CompressionType, - Checksums: art.Checksums, - Files: art.Files, - }) - } - return newIndex - //Update if exists, otherwise just create -} - // When compiling, we write also a fingerprint.metadata.yaml file with PackageArtifact. In this way we can have another command to create the repository // which will consist in just of an repository.yaml which is just the repository structure with the list of package artifact. // In this way a generic client can fetch the packages and, after unpacking the tree, performing queries to install packages. type PackageArtifact struct { Path string `json:"path"` - Dependencies []*PackageArtifact `json:"dependencies"` - CompileSpec *LuetCompilationSpec `json:"compilationspec"` - Checksums Checksums `json:"checksums"` - SourceAssertion solver.PackagesAssertions `json:"-"` - CompressionType CompressionImplementation `json:"compressiontype"` - Files []string `json:"files"` + Dependencies []*PackageArtifact `json:"dependencies"` + CompileSpec *compilerspec.LuetCompilationSpec `json:"compilationspec"` + Checksums Checksums `json:"checksums"` + SourceAssertion solver.PackagesAssertions `json:"-"` + CompressionType compression.Implementation `json:"compressiontype"` + Files []string `json:"files"` } -func NewPackageArtifact(path string) Artifact { - return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: None} +func NewPackageArtifact(path string) *PackageArtifact { + return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: compression.None} } -func NewPackageArtifactFromYaml(data []byte) (Artifact, error) { +func NewPackageArtifactFromYaml(data []byte) (*PackageArtifact, error) { p := &PackageArtifact{Checksums: Checksums{}} err := yaml.Unmarshal(data, &p) if err != nil { @@ -102,42 +76,6 @@ func NewPackageArtifactFromYaml(data []byte) (Artifact, error) { return p, err } -func LoadArtifactFromYaml(spec CompilationSpec) (Artifact, error) { - - metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml" - dat, err := ioutil.ReadFile(spec.Rel(metaFile)) - if err != nil { - return nil, errors.Wrap(err, "Error reading file "+metaFile) - } - art, err := NewPackageArtifactFromYaml(dat) - if err != nil { - return nil, errors.Wrap(err, "Error writing file "+metaFile) - } - // It is relative, set it back to abs - art.SetPath(spec.Rel(art.GetPath())) - return art, nil -} - -func (a *PackageArtifact) SetCompressionType(t CompressionImplementation) { - a.CompressionType = t -} - -func (a *PackageArtifact) GetChecksums() Checksums { - return a.Checksums -} - -func (a *PackageArtifact) SetChecksums(c Checksums) { - a.Checksums = c -} - -func (a *PackageArtifact) SetFiles(f []string) { - a.Files = f -} - -func (a *PackageArtifact) GetFiles() []string { - return a.Files -} - func (a *PackageArtifact) Hash() error { return a.Checksums.Generate(a) } @@ -181,8 +119,8 @@ func (a *PackageArtifact) WriteYaml(dst string) error { } //p := a.CompileSpec.GetPackage().GetPath() - mangle.GetCompileSpec().GetPackage().SetPath("") - for _, ass := range mangle.GetCompileSpec().GetSourceAssertion() { + mangle.CompileSpec.GetPackage().SetPath("") + for _, ass := range mangle.CompileSpec.GetSourceAssertion() { ass.Package.SetPath("") } @@ -191,7 +129,7 @@ func (a *PackageArtifact) WriteYaml(dst string) error { return errors.Wrap(err, "While marshalling for PackageArtifact YAML") } - err = ioutil.WriteFile(filepath.Join(dst, a.GetCompileSpec().GetPackage().GetFingerPrint()+".metadata.yaml"), data, os.ModePerm) + err = ioutil.WriteFile(filepath.Join(dst, a.CompileSpec.GetPackage().GetFingerPrint()+".metadata.yaml"), data, os.ModePerm) if err != nil { return errors.Wrap(err, "While writing PackageArtifact YAML") } @@ -201,48 +139,8 @@ func (a *PackageArtifact) WriteYaml(dst string) error { return nil } -func (a *PackageArtifact) GetSourceAssertion() solver.PackagesAssertions { - return a.SourceAssertion -} - -func (a *PackageArtifact) SetCompileSpec(as CompilationSpec) { - a.CompileSpec = as.(*LuetCompilationSpec) -} - -func (a *PackageArtifact) GetCompileSpec() CompilationSpec { - return a.CompileSpec -} - -func (a *PackageArtifact) SetSourceAssertion(as solver.PackagesAssertions) { - a.SourceAssertion = as -} - -func (a *PackageArtifact) GetDependencies() []Artifact { - ret := []Artifact{} - for _, d := range a.Dependencies { - ret = append(ret, d) - } - return ret -} - -func (a *PackageArtifact) SetDependencies(d []Artifact) { - ret := []*PackageArtifact{} - for _, dd := range d { - ret = append(ret, dd.(*PackageArtifact)) - } - a.Dependencies = ret -} - -func (a *PackageArtifact) GetPath() string { - return a.Path -} - func (a *PackageArtifact) GetFileName() string { - return path.Base(a.GetPath()) -} - -func (a *PackageArtifact) SetPath(p string) { - a.Path = p + return path.Base(a.Path) } func (a *PackageArtifact) genDockerfile() string { @@ -274,9 +172,13 @@ func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageAr return a, a.Compress(archive, 1) } +type ImageBuilder interface { + BuildImage(backend.Options) error +} + // GenerateFinalImage takes an artifact and builds a Docker image with its content -func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend, keepPerms bool) (CompilerBackendOptions, error) { - builderOpts := CompilerBackendOptions{} +func (a *PackageArtifact) GenerateFinalImage(imageName string, b ImageBuilder, keepPerms bool) (backend.Options, error) { + builderOpts := backend.Options{} archive, err := LuetCfg.GetSystem().TempDir("archive") if err != nil { return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path) @@ -311,7 +213,7 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend return builderOpts, errors.Wrap(err, "error met while rendering artifact dockerfile "+a.Path) } - builderOpts = CompilerBackendOptions{ + builderOpts = backend.Options{ ImageName: imageName, SourcePath: archive, DockerFileName: dockerFile, @@ -326,7 +228,7 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend func (a *PackageArtifact) Compress(src string, concurrency int) error { switch a.CompressionType { - case Zstandard: + case compression.Zstandard: err := helpers.Tar(src, a.Path) if err != nil { return err @@ -364,7 +266,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error { a.Path = zstdFile return nil - case GZip: + case compression.GZip: err := helpers.Tar(src, a.Path) if err != nil { return err @@ -409,10 +311,10 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error { func (a *PackageArtifact) getCompressedName() string { switch a.CompressionType { - case Zstandard: + case compression.Zstandard: return a.Path + ".zst" - case GZip: + case compression.GZip: return a.Path + ".gz" } return a.Path @@ -421,7 +323,7 @@ func (a *PackageArtifact) getCompressedName() string { // GetUncompressedName returns the artifact path without the extension suffix func (a *PackageArtifact) GetUncompressedName() string { switch a.CompressionType { - case Zstandard, GZip: + case compression.Zstandard, compression.GZip: return strings.TrimSuffix(a.Path, filepath.Ext(a.Path)) } return a.Path @@ -509,13 +411,13 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error { tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc) switch a.CompressionType { - case Zstandard: + case compression.Zstandard: // Create the uncompressed archive - archive, err := os.Create(a.GetPath() + ".uncompressed") + archive, err := os.Create(a.Path + ".uncompressed") if err != nil { return err } - defer os.RemoveAll(a.GetPath() + ".uncompressed") + defer os.RemoveAll(a.Path + ".uncompressed") defer archive.Close() original, err := os.Open(a.Path) @@ -534,22 +436,22 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error { _, err = io.Copy(archive, d) if err != nil { - return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed") + return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed") } - err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst, + err = helpers.UntarProtect(a.Path+".uncompressed", dst, LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier) if err != nil { return err } return nil - case GZip: + case compression.GZip: // Create the uncompressed archive - archive, err := os.Create(a.GetPath() + ".uncompressed") + archive, err := os.Create(a.Path + ".uncompressed") if err != nil { return err } - defer os.RemoveAll(a.GetPath() + ".uncompressed") + defer os.RemoveAll(a.Path + ".uncompressed") defer archive.Close() original, err := os.Open(a.Path) @@ -567,10 +469,10 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error { _, err = io.Copy(archive, r) if err != nil { - return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed") + return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed") } - err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst, + err = helpers.UntarProtect(a.Path+".uncompressed", dst, LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier) if err != nil { return err @@ -578,7 +480,7 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error { return nil // Defaults to tar only (covers when "none" is supplied) default: - return helpers.UntarProtect(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner, + return helpers.UntarProtect(a.Path, dst, LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier) } return errors.New("Compression type must be supplied") @@ -588,12 +490,12 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error { func (a *PackageArtifact) FileList() ([]string, error) { var tr *tar.Reader switch a.CompressionType { - case Zstandard: - archive, err := os.Create(a.GetPath() + ".uncompressed") + case compression.Zstandard: + archive, err := os.Create(a.Path + ".uncompressed") if err != nil { return []string{}, err } - defer os.RemoveAll(a.GetPath() + ".uncompressed") + defer os.RemoveAll(a.Path + ".uncompressed") defer archive.Close() original, err := os.Open(a.Path) @@ -609,13 +511,13 @@ func (a *PackageArtifact) FileList() ([]string, error) { } defer r.Close() tr = tar.NewReader(r) - case GZip: + case compression.GZip: // Create the uncompressed archive - archive, err := os.Create(a.GetPath() + ".uncompressed") + archive, err := os.Create(a.Path + ".uncompressed") if err != nil { return []string{}, err } - defer os.RemoveAll(a.GetPath() + ".uncompressed") + defer os.RemoveAll(a.Path + ".uncompressed") defer archive.Close() original, err := os.Open(a.Path) @@ -634,7 +536,7 @@ func (a *PackageArtifact) FileList() ([]string, error) { // Defaults to tar only (covers when "none" is supplied) default: - tarFile, err := os.Open(a.GetPath()) + tarFile, err := os.Open(a.Path) if err != nil { return []string{}, errors.Wrap(err, "Could not open package archive") } @@ -729,8 +631,24 @@ func compileRegexes(regexes []string) []*regexp.Regexp { return result } +type ArtifactNode struct { + Name string `json:"Name"` + Size int `json:"Size"` +} +type ArtifactDiffs struct { + Additions []ArtifactNode `json:"Adds"` + Deletions []ArtifactNode `json:"Dels"` + Changes []ArtifactNode `json:"Mods"` +} + +type ArtifactLayer struct { + FromImage string `json:"Image1"` + ToImage string `json:"Image2"` + Diffs ArtifactDiffs `json:"Diff"` +} + // ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format -func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t CompressionImplementation) (Artifact, error) { +func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t compression.Implementation) (*PackageArtifact, error) { archive, err := LuetCfg.GetSystem().TempDir("archive") if err != nil { @@ -852,45 +770,10 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren wg.Wait() a := NewPackageArtifact(dst) - a.SetCompressionType(t) + a.CompressionType = t err = a.Compress(archive, concurrency) if err != nil { return nil, errors.Wrap(err, "Error met while creating package archive") } return a, nil } - -func ComputeArtifactLayerSummary(diffs []ArtifactLayer) ArtifactLayersSummary { - - ans := ArtifactLayersSummary{ - Layers: make([]ArtifactLayerSummary, 0), - } - - for _, layer := range diffs { - sum := ArtifactLayerSummary{ - FromImage: layer.FromImage, - ToImage: layer.ToImage, - AddFiles: 0, - AddSizes: 0, - DelFiles: 0, - DelSizes: 0, - ChangeFiles: 0, - ChangeSizes: 0, - } - for _, a := range layer.Diffs.Additions { - sum.AddFiles++ - sum.AddSizes += int64(a.Size) - } - for _, d := range layer.Diffs.Deletions { - sum.DelFiles++ - sum.DelSizes += int64(d.Size) - } - for _, c := range layer.Diffs.Changes { - sum.ChangeFiles++ - sum.ChangeSizes += int64(c.Size) - } - ans.Layers = append(ans.Layers, sum) - } - - return ans -} diff --git a/pkg/compiler/artifact_test.go b/pkg/compiler/types/artifact/artifact_test.go similarity index 82% rename from pkg/compiler/artifact_test.go rename to pkg/compiler/types/artifact/artifact_test.go index 3d015532..2b846763 100644 --- a/pkg/compiler/artifact_test.go +++ b/pkg/compiler/types/artifact/artifact_test.go @@ -13,7 +13,7 @@ // You should have received a copy of the GNU General Public License along // with this program; if not, see . -package compiler_test +package artifact_test import ( "io/ioutil" @@ -22,7 +22,10 @@ import ( "github.com/mudler/luet/pkg/compiler" . "github.com/mudler/luet/pkg/compiler/backend" - "github.com/mudler/luet/pkg/solver" + backend "github.com/mudler/luet/pkg/compiler/backend" + . "github.com/mudler/luet/pkg/compiler/types/artifact" + compression "github.com/mudler/luet/pkg/compiler/types/compression" + compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" . "github.com/mudler/luet/pkg/compiler" helpers "github.com/mudler/luet/pkg/helpers" @@ -43,13 +46,10 @@ var _ = Describe("Artifact", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1)) - compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) - spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"}) + cc := NewLuetCompiler(nil, generalRecipe.GetDatabase()) + lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"}) Expect(err).ToNot(HaveOccurred()) - lspec, ok := spec.(*LuetCompilationSpec) - Expect(ok).To(BeTrue()) - Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"})) Expect(lspec.Image).To(Equal("luet/base")) Expect(lspec.Seed).To(Equal("alpine")) @@ -81,7 +81,7 @@ ENV PACKAGE_NAME=enman ENV PACKAGE_VERSION=1.4.0 ENV PACKAGE_CATEGORY=app-admin`)) b := NewSimpleDockerBackend() - opts := CompilerBackendOptions{ + opts := backend.Options{ ImageName: "luet/base", SourcePath: tmpdir, DockerFileName: "Dockerfile", @@ -105,7 +105,7 @@ ENV PACKAGE_VERSION=1.4.0 ENV PACKAGE_CATEGORY=app-admin RUN echo foo > /test RUN echo bar > /test2`)) - opts2 := CompilerBackendOptions{ + opts2 := backend.Options{ ImageName: "test", SourcePath: tmpdir, DockerFileName: "LuetDockerfile", @@ -114,7 +114,7 @@ RUN echo bar > /test2`)) Expect(b.BuildImage(opts2)).ToNot(HaveOccurred()) Expect(b.ExportImage(opts2)).ToNot(HaveOccurred()) Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue()) - diffs, err := b.Changes(opts, opts2) + diffs, err := compiler.GenerateChanges(b, opts, opts2) Expect(err).ToNot(HaveOccurred()) artifacts := []ArtifactNode{{ @@ -135,13 +135,13 @@ RUN echo bar > /test2`)) Additions: artifacts, }, }})) - err = b.ExtractRootfs(CompilerBackendOptions{ImageName: "test", Destination: rootfs}, false) + err = b.ExtractRootfs(backend.Options{ImageName: "test", Destination: rootfs}, false) Expect(err).ToNot(HaveOccurred()) - artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, None) + a, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None) Expect(err).ToNot(HaveOccurred()) Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue()) - err = helpers.Untar(artifact.GetPath(), unpacked, false) + err = helpers.Untar(a.Path, unpacked, false) Expect(err).ToNot(HaveOccurred()) Expect(helpers.Exists(filepath.Join(unpacked, "test"))).To(BeTrue()) Expect(helpers.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue()) @@ -152,13 +152,13 @@ RUN echo bar > /test2`)) Expect(err).ToNot(HaveOccurred()) Expect(content2).To(Equal("bar\n")) - err = artifact.Hash() + err = a.Hash() Expect(err).ToNot(HaveOccurred()) - err = artifact.Verify() + err = a.Verify() Expect(err).ToNot(HaveOccurred()) Expect(helpers.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred()) - err = artifact.Verify() + err = a.Verify() Expect(err).To(HaveOccurred()) }) @@ -183,13 +183,13 @@ RUN echo bar > /test2`)) err = ioutil.WriteFile(filepath.Join(tmpdir, "foo", "bar", "test"), testString, 0644) Expect(err).ToNot(HaveOccurred()) - artifact := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar")) - artifact.SetCompileSpec(&LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}}) + a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar")) + a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}} - err = artifact.Compress(tmpdir, 1) + err = a.Compress(tmpdir, 1) Expect(err).ToNot(HaveOccurred()) resultingImage := imageprefix + "foo--1.0" - opts, err := artifact.GenerateFinalImage(resultingImage, b, false) + opts, err := a.GenerateFinalImage(resultingImage, b, false) Expect(err).ToNot(HaveOccurred()) Expect(opts.ImageName).To(Equal(resultingImage)) @@ -199,7 +199,7 @@ RUN echo bar > /test2`)) Expect(err).ToNot(HaveOccurred()) defer os.RemoveAll(result) // clean up - err = b.ExtractRootfs(CompilerBackendOptions{ImageName: resultingImage, Destination: result}, false) + err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false) Expect(err).ToNot(HaveOccurred()) content, err := ioutil.ReadFile(filepath.Join(result, "test")) @@ -225,13 +225,13 @@ RUN echo bar > /test2`)) Expect(err).ToNot(HaveOccurred()) defer os.RemoveAll(tmpWork) // clean up - artifact := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar")) - artifact.SetCompileSpec(&LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}}) + a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar")) + a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}} - err = artifact.Compress(tmpdir, 1) + err = a.Compress(tmpdir, 1) Expect(err).ToNot(HaveOccurred()) resultingImage := imageprefix + "foo--1.0" - opts, err := artifact.GenerateFinalImage(resultingImage, b, false) + opts, err := a.GenerateFinalImage(resultingImage, b, false) Expect(err).ToNot(HaveOccurred()) Expect(opts.ImageName).To(Equal(resultingImage)) @@ -241,7 +241,7 @@ RUN echo bar > /test2`)) Expect(err).ToNot(HaveOccurred()) defer os.RemoveAll(result) // clean up - err = b.ExtractRootfs(CompilerBackendOptions{ImageName: resultingImage, Destination: result}, false) + err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false) Expect(err).ToNot(HaveOccurred()) Expect(helpers.DirectoryIsEmpty(result)).To(BeFalse()) @@ -253,15 +253,15 @@ RUN echo bar > /test2`)) It("Retrieves uncompressed name", func() { a := NewPackageArtifact("foo.tar.gz") - a.SetCompressionType(compiler.GZip) + a.CompressionType = (compression.GZip) Expect(a.GetUncompressedName()).To(Equal("foo.tar")) a = NewPackageArtifact("foo.tar.zst") - a.SetCompressionType(compiler.Zstandard) + a.CompressionType = compression.Zstandard Expect(a.GetUncompressedName()).To(Equal("foo.tar")) a = NewPackageArtifact("foo.tar") - a.SetCompressionType(compiler.None) + a.CompressionType = compression.None Expect(a.GetUncompressedName()).To(Equal("foo.tar")) }) }) diff --git a/pkg/compiler/checksum.go b/pkg/compiler/types/artifact/checksum.go similarity index 86% rename from pkg/compiler/checksum.go rename to pkg/compiler/types/artifact/checksum.go index 286b1e4c..6d213143 100644 --- a/pkg/compiler/checksum.go +++ b/pkg/compiler/types/artifact/checksum.go @@ -13,7 +13,7 @@ // You should have received a copy of the GNU General Public License along // with this program; if not, see . -package compiler +package artifact import ( @@ -43,7 +43,7 @@ type HashOptions struct { } // Generate generates all Checksums supported for the artifact -func (c *Checksums) Generate(a Artifact) error { +func (c *Checksums) Generate(a *PackageArtifact) error { return c.generateSHA256(a) } @@ -56,13 +56,13 @@ func (c Checksums) Compare(d Checksums) error { return nil } -func (c *Checksums) generateSHA256(a Artifact) error { +func (c *Checksums) generateSHA256(a *PackageArtifact) error { return c.generateSum(a, HashOptions{Hasher: sha256.New(), Type: SHA256}) } -func (c *Checksums) generateSum(a Artifact, opts HashOptions) error { +func (c *Checksums) generateSum(a *PackageArtifact, opts HashOptions) error { - f, err := os.Open(a.GetPath()) + f, err := os.Open(a.Path) if err != nil { return err } diff --git a/pkg/compiler/checksum_test.go b/pkg/compiler/types/artifact/checksum_test.go similarity index 96% rename from pkg/compiler/checksum_test.go rename to pkg/compiler/types/artifact/checksum_test.go index 20c7f4f4..8895ad7b 100644 --- a/pkg/compiler/checksum_test.go +++ b/pkg/compiler/types/artifact/checksum_test.go @@ -13,13 +13,13 @@ // You should have received a copy of the GNU General Public License along // with this program; if not, see . -package compiler_test +package artifact_test import ( "io/ioutil" "os" - . "github.com/mudler/luet/pkg/compiler" + . "github.com/mudler/luet/pkg/compiler/types/artifact" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" diff --git a/pkg/compiler/types/compression/compression.go b/pkg/compiler/types/compression/compression.go new file mode 100644 index 00000000..e14ecad3 --- /dev/null +++ b/pkg/compiler/types/compression/compression.go @@ -0,0 +1,9 @@ +package compression + +type Implementation string + +const ( + None Implementation = "none" // e.g. tar for standard packages + GZip Implementation = "gzip" + Zstandard Implementation = "zstd" +) diff --git a/pkg/compiler/types/options/compiler_options.go b/pkg/compiler/types/options/compiler_options.go new file mode 100644 index 00000000..cef26cac --- /dev/null +++ b/pkg/compiler/types/options/compiler_options.go @@ -0,0 +1,191 @@ +// Copyright © 2019-2021 Ettore Di Giacinto +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see . + +package options + +import ( + "runtime" + + "github.com/mudler/luet/pkg/compiler/types/compression" + "github.com/mudler/luet/pkg/config" + "github.com/mudler/luet/pkg/solver" +) + +type Compiler struct { + PushImageRepository string + PullImageRepository []string + PullFirst, KeepImg, Push bool + Concurrency int + CompressionType compression.Implementation + + Wait bool + OnlyDeps bool + NoDeps bool + SolverOptions config.LuetSolverOptions + BuildValuesFile []string + BuildValues []map[string]interface{} + + PackageTargetOnly bool + + BackendArgs []string + + BackendType string +} + +func NewDefaultCompiler() *Compiler { + return &Compiler{ + PushImageRepository: "luet/cache", + PullFirst: false, + Push: false, + CompressionType: compression.None, + KeepImg: true, + Concurrency: runtime.NumCPU(), + OnlyDeps: false, + NoDeps: false, + SolverOptions: config.LuetSolverOptions{Options: solver.Options{Concurrency: 1, Type: solver.SingleCoreSimple}}, + } +} + +type Option func(cfg *Compiler) error + +func (cfg *Compiler) Apply(opts ...Option) error { + for _, opt := range opts { + if opt == nil { + continue + } + if err := opt(cfg); err != nil { + return err + } + } + return nil +} + +func WithOptions(opt *Compiler) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg = opt + return nil + } +} + +func WithBackendType(r string) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.BackendType = r + return nil + } +} + +func WithBuildValues(r []string) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.BuildValuesFile = r + return nil + } +} + +func WithPullRepositories(r []string) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.PullImageRepository = r + return nil + } +} + +func WithPushRepository(r string) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + if len(cfg.PullImageRepository) == 0 { + cfg.PullImageRepository = []string{cfg.PushImageRepository} + } + cfg.PushImageRepository = r + return nil + } +} + +func BackendArgs(r []string) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.BackendArgs = r + return nil + } +} + +func PullFirst(b bool) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.PullFirst = b + return nil + } +} + +func KeepImg(b bool) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.KeepImg = b + return nil + } +} + +func PushImages(b bool) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.Push = b + return nil + } +} + +func Wait(b bool) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.Wait = b + return nil + } +} + +func OnlyDeps(b bool) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.OnlyDeps = b + return nil + } +} + +func OnlyTarget(b bool) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.PackageTargetOnly = b + return nil + } +} + +func NoDeps(b bool) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.NoDeps = b + return nil + } +} + +func Concurrency(i int) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + if i == 0 { + i = runtime.NumCPU() + } + cfg.Concurrency = i + return nil + } +} + +func WithCompressionType(t compression.Implementation) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.CompressionType = t + return nil + } +} + +func WithSolverOptions(c config.LuetSolverOptions) func(cfg *Compiler) error { + return func(cfg *Compiler) error { + cfg.SolverOptions = c + return nil + } +} diff --git a/pkg/compiler/spec.go b/pkg/compiler/types/spec/spec.go similarity index 90% rename from pkg/compiler/spec.go rename to pkg/compiler/types/spec/spec.go index 97929931..ba433bc8 100644 --- a/pkg/compiler/spec.go +++ b/pkg/compiler/types/spec/spec.go @@ -13,12 +13,14 @@ // You should have received a copy of the GNU General Public License along // with this program; if not, see . -package compiler +package compilerspec import ( "io/ioutil" "path/filepath" + options "github.com/mudler/luet/pkg/compiler/types/options" + pkg "github.com/mudler/luet/pkg/package" "github.com/mudler/luet/pkg/solver" "github.com/otiai10/copy" @@ -27,7 +29,7 @@ import ( type LuetCompilationspecs []LuetCompilationSpec -func NewLuetCompilationspecs(s ...CompilationSpec) CompilationSpecs { +func NewLuetCompilationspecs(s ...*LuetCompilationSpec) *LuetCompilationspecs { all := LuetCompilationspecs{} for _, spec := range s { @@ -40,7 +42,7 @@ func (specs LuetCompilationspecs) Len() int { return len(specs) } -func (specs *LuetCompilationspecs) Remove(s CompilationSpecs) CompilationSpecs { +func (specs *LuetCompilationspecs) Remove(s *LuetCompilationspecs) *LuetCompilationspecs { newSpecs := LuetCompilationspecs{} SPECS: for _, spec := range specs.All() { @@ -54,16 +56,12 @@ SPECS: return &newSpecs } -func (specs *LuetCompilationspecs) Add(s CompilationSpec) { - c, ok := s.(*LuetCompilationSpec) - if !ok { - panic("LuetCompilationspecs supports only []LuetCompilationSpec") - } - *specs = append(*specs, *c) +func (specs *LuetCompilationspecs) Add(s *LuetCompilationSpec) { + *specs = append(*specs, *s) } -func (specs *LuetCompilationspecs) All() []CompilationSpec { - var cspecs []CompilationSpec +func (specs *LuetCompilationspecs) All() []*LuetCompilationSpec { + var cspecs []*LuetCompilationSpec for i, _ := range *specs { f := (*specs)[i] cspecs = append(cspecs, &f) @@ -72,7 +70,7 @@ func (specs *LuetCompilationspecs) All() []CompilationSpec { return cspecs } -func (specs *LuetCompilationspecs) Unique() CompilationSpecs { +func (specs *LuetCompilationspecs) Unique() *LuetCompilationspecs { newSpecs := LuetCompilationspecs{} seen := map[string]bool{} @@ -103,9 +101,11 @@ type LuetCompilationSpec struct { Unpack bool `json:"unpack"` Includes []string `json:"includes"` Excludes []string `json:"excludes"` + + BuildOptions options.Compiler `json:"build_options"` } -func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) { +func NewLuetCompilationSpec(b []byte, p pkg.Package) (*LuetCompilationSpec, error) { var spec LuetCompilationSpec err := yaml.Unmarshal(b, &spec) if err != nil { @@ -118,6 +118,10 @@ func (cs *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions { return cs.SourceAssertion } +func (cs *LuetCompilationSpec) SetBuildOptions(b options.Compiler) { + cs.BuildOptions = b +} + func (cs *LuetCompilationSpec) SetSourceAssertion(as solver.PackagesAssertions) { cs.SourceAssertion = as } diff --git a/pkg/compiler/spec_test.go b/pkg/compiler/types/spec/spec_test.go similarity index 74% rename from pkg/compiler/spec_test.go rename to pkg/compiler/types/spec/spec_test.go index ff35b9cf..7b54dd1d 100644 --- a/pkg/compiler/spec_test.go +++ b/pkg/compiler/types/spec/spec_test.go @@ -13,17 +13,18 @@ // You should have received a copy of the GNU General Public License along // with this program; if not, see . -package compiler_test +package compilerspec_test import ( "io/ioutil" "os" "path/filepath" + compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" + . "github.com/mudler/luet/pkg/compiler" helpers "github.com/mudler/luet/pkg/helpers" pkg "github.com/mudler/luet/pkg/package" - "github.com/mudler/luet/pkg/solver" "github.com/mudler/luet/pkg/tree" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -32,41 +33,41 @@ import ( var _ = Describe("Spec", func() { Context("Luet specs", func() { It("Allows normal operations", func() { - testSpec := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}} - testSpec2 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "bar", Category: "a", Version: "0"}} - testSpec3 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "baz", Category: "a", Version: "0"}} - testSpec4 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}} + testSpec := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}} + testSpec2 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "bar", Category: "a", Version: "0"}} + testSpec3 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "baz", Category: "a", Version: "0"}} + testSpec4 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}} - specs := NewLuetCompilationspecs(testSpec, testSpec2) + specs := compilerspec.NewLuetCompilationspecs(testSpec, testSpec2) Expect(specs.Len()).To(Equal(2)) - Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2})) + Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2})) specs.Add(testSpec3) - Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3})) + Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3})) specs.Add(testSpec4) - Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3, testSpec4})) + Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3, testSpec4})) newSpec := specs.Unique() - Expect(newSpec.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3})) + Expect(newSpec.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3})) - newSpec2 := specs.Remove(NewLuetCompilationspecs(testSpec, testSpec2)) - Expect(newSpec2.All()).To(Equal([]CompilationSpec{testSpec3})) + newSpec2 := specs.Remove(compilerspec.NewLuetCompilationspecs(testSpec, testSpec2)) + Expect(newSpec2.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec3})) }) Context("virtuals", func() { When("is empty", func() { It("is virtual", func() { - spec := &LuetCompilationSpec{} + spec := &compilerspec.LuetCompilationSpec{} Expect(spec.IsVirtual()).To(BeTrue()) }) }) When("has defined steps", func() { It("is not a virtual", func() { - spec := &LuetCompilationSpec{Steps: []string{"foo"}} + spec := &compilerspec.LuetCompilationSpec{Steps: []string{"foo"}} Expect(spec.IsVirtual()).To(BeFalse()) }) }) When("has defined image", func() { It("is not a virtual", func() { - spec := &LuetCompilationSpec{Image: "foo"} + spec := &compilerspec.LuetCompilationSpec{Image: "foo"} Expect(spec.IsVirtual()).To(BeFalse()) }) }) @@ -82,13 +83,10 @@ var _ = Describe("Spec", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1)) - compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) - spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"}) + compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase()) + lspec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"}) Expect(err).ToNot(HaveOccurred()) - lspec, ok := spec.(*LuetCompilationSpec) - Expect(ok).To(BeTrue()) - Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"})) Expect(lspec.Image).To(Equal("luet/base")) Expect(lspec.Seed).To(Equal("alpine")) @@ -137,13 +135,10 @@ RUN echo bar > /test2`)) Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1)) - compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) - spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"}) + compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase()) + lspec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) - lspec, ok := spec.(*LuetCompilationSpec) - Expect(ok).To(BeTrue()) - Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"})) Expect(lspec.Image).To(Equal("luet/base")) Expect(lspec.Seed).To(Equal("alpine")) diff --git a/pkg/config/config.go b/pkg/config/config.go index 21d53335..211ab95c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -65,6 +65,7 @@ type LuetGeneralConfig struct { } type LuetSolverOptions struct { + solver.Options Type string `mapstructure:"type"` LearnRate float32 `mapstructure:"rate"` Discount float32 `mapstructure:"discount"` @@ -159,8 +160,8 @@ type LuetRepository struct { Enable bool `json:"enable" yaml:"enable" mapstructure:"enable"` Cached bool `json:"cached,omitempty" yaml:"cached,omitempty" mapstructure:"cached,omitempty"` Authentication map[string]string `json:"auth,omitempty" yaml:"auth,omitempty" mapstructure:"auth,omitempty"` - TreePath string `json:"tree_path,omitempty" yaml:"tree_path,omitempty" mapstructure:"tree_path"` - MetaPath string `json:"meta_path,omitempty" yaml:"meta_path,omitempty" mapstructure:"meta_path"` + TreePath string `json:"treepath,omitempty" yaml:"treepath,omitempty" mapstructure:"treepath"` + MetaPath string `json:"metapath,omitempty" yaml:"metapath,omitempty" mapstructure:"metapath"` Verify bool `json:"verify,omitempty" yaml:"verify,omitempty" mapstructure:"verify"` // Serialized options not used in repository configuration diff --git a/pkg/helpers/helm.go b/pkg/helpers/helm.go index d708cece..d89fc85b 100644 --- a/pkg/helpers/helm.go +++ b/pkg/helpers/helm.go @@ -2,7 +2,6 @@ package helpers import ( "io/ioutil" - "sort" "github.com/imdario/mergo" "github.com/pkg/errors" @@ -39,13 +38,13 @@ func RenderHelm(template string, values, d map[string]interface{}) (string, erro type templatedata map[string]interface{} +// UnMarshalValues unmarshal values files and joins them into a unique templatedata +// the join happens from right to left, so any rightmost value file overwrites the content of the ones before it. func UnMarshalValues(values []string) (templatedata, error) { dst := templatedata{} if len(values) > 0 { - allbv := values - sort.Sort(sort.Reverse(sort.StringSlice(allbv))) - for _, bv := range allbv { - current := map[string]interface{}{} + for _, bv := range reverse(values) { + current := templatedata{} defBuild, err := ioutil.ReadFile(bv) if err != nil { @@ -63,6 +62,13 @@ func UnMarshalValues(values []string) (templatedata, error) { return dst, nil } +func reverse(s []string) []string { + for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { + s[i], s[j] = s[j], s[i] + } + return s +} + func RenderFiles(toTemplate, valuesFile string, defaultFile ...string) (string, error) { raw, err := ioutil.ReadFile(toTemplate) if err != nil { diff --git a/pkg/helpers/helm_test.go b/pkg/helpers/helm_test.go index 9d0c62e5..f45150ea 100644 --- a/pkg/helpers/helm_test.go +++ b/pkg/helpers/helm_test.go @@ -114,11 +114,46 @@ foo: "bar" Expect(err).ToNot(HaveOccurred()) - res, err := RenderFiles(toTemplate, values, "") + res, err := RenderFiles(toTemplate, values) Expect(err).ToNot(HaveOccurred()) Expect(res).To(Equal("bar")) }) + It("Render files merging defaults", func() { + testDir, err := ioutil.TempDir(os.TempDir(), "test") + Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(testDir) + + toTemplate := filepath.Join(testDir, "totemplate.yaml") + values := filepath.Join(testDir, "values.yaml") + d := filepath.Join(testDir, "default.yaml") + d2 := filepath.Join(testDir, "default2.yaml") + + writeFile(toTemplate, `{{.Values.foo}}{{.Values.bar}}{{.Values.b}}`) + writeFile(values, ` +foo: "bar" +b: "f" +`) + writeFile(d, ` +foo: "baz" +`) + + writeFile(d2, ` +foo: "do" +bar: "nei" +`) + + Expect(err).ToNot(HaveOccurred()) + + res, err := RenderFiles(toTemplate, values, d2, d) + Expect(err).ToNot(HaveOccurred()) + Expect(res).To(Equal("bazneif")) + + res, err = RenderFiles(toTemplate, values, d, d2) + Expect(err).ToNot(HaveOccurred()) + Expect(res).To(Equal("doneif")) + }) + It("doesn't interpolate if no one provides the values", func() { testDir, err := ioutil.TempDir(os.TempDir(), "test") Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/installer/client/docker.go b/pkg/installer/client/docker.go index 49723a72..0fa86021 100644 --- a/pkg/installer/client/docker.go +++ b/pkg/installer/client/docker.go @@ -26,7 +26,7 @@ import ( "github.com/docker/go-units" "github.com/pkg/errors" - "github.com/mudler/luet/pkg/compiler" + "github.com/mudler/luet/pkg/compiler/types/artifact" "github.com/mudler/luet/pkg/config" "github.com/mudler/luet/pkg/helpers" . "github.com/mudler/luet/pkg/logger" @@ -51,7 +51,7 @@ func NewDockerClient(r RepoData) *DockerClient { return &DockerClient{RepoData: r, auth: auth} } -func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) { +func (c *DockerClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) { //var u *url.URL = nil var err error var temp string @@ -59,8 +59,8 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar Spinner(22) defer SpinnerStop() - var resultingArtifact compiler.Artifact - artifactName := path.Base(artifact.GetPath()) + var resultingArtifact *artifact.PackageArtifact + artifactName := path.Base(a.Path) cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName) Debug("Cache file", cacheFile) if err := helpers.EnsureDir(cacheFile); err != nil { @@ -78,9 +78,9 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar // Check if file is already in cache if helpers.Exists(cacheFile) { Debug("Cache hit for artifact", artifactName) - resultingArtifact = artifact - resultingArtifact.SetPath(cacheFile) - resultingArtifact.SetChecksums(compiler.Checksums{}) + resultingArtifact = a + resultingArtifact.Path = cacheFile + resultingArtifact.Checksums = artifact.Checksums{} } else { temp, err = config.LuetCfg.GetSystem().TempDir("tree") @@ -91,7 +91,7 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar for _, uri := range c.RepoData.Urls { - imageName := fmt.Sprintf("%s:%s", uri, artifact.GetCompileSpec().GetPackage().ImageID()) + imageName := fmt.Sprintf("%s:%s", uri, a.CompileSpec.GetPackage().ImageID()) Info("Downloading image", imageName) contentstore, err := config.LuetCfg.GetSystem().TempDir("contentstore") @@ -111,11 +111,11 @@ func (c *DockerClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Ar Info(fmt.Sprintf("Size: %s", units.BytesSize(float64(info.ContentSize)))) Debug("\nCompressing result ", filepath.Join(temp), "to", cacheFile) - newart := artifact + newart := a // We discard checksum, that are checked while during pull and unpack - newart.SetChecksums(compiler.Checksums{}) - newart.SetPath(cacheFile) // First set to cache file - newart.SetPath(newart.GetUncompressedName()) // Calculate the real path from cacheFile + newart.Checksums = artifact.Checksums{} + newart.Path = cacheFile // First set to cache file + newart.Path = newart.GetUncompressedName() // Calculate the real path from cacheFile err = newart.Compress(temp, 1) if err != nil { Error(fmt.Sprintf("Failed compressing package %s: %s", imageName, err.Error())) diff --git a/pkg/installer/client/docker_test.go b/pkg/installer/client/docker_test.go index 012b8169..0a2f8b30 100644 --- a/pkg/installer/client/docker_test.go +++ b/pkg/installer/client/docker_test.go @@ -20,7 +20,9 @@ import ( "os" "path/filepath" - compiler "github.com/mudler/luet/pkg/compiler" + "github.com/mudler/luet/pkg/compiler/types/artifact" + compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" + helpers "github.com/mudler/luet/pkg/helpers" pkg "github.com/mudler/luet/pkg/package" @@ -54,9 +56,9 @@ var _ = Describe("Docker client", func() { }) It("Downloads artifacts", func() { - f, err := c.DownloadArtifact(&compiler.PackageArtifact{ + f, err := c.DownloadArtifact(&artifact.PackageArtifact{ Path: "test.tar", - CompileSpec: &compiler.LuetCompilationSpec{ + CompileSpec: &compilerspec.LuetCompilationSpec{ Package: &pkg.DefaultPackage{ Name: "c", Category: "test", @@ -71,7 +73,7 @@ var _ = Describe("Docker client", func() { Expect(f.Unpack(tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Read(filepath.Join(tmpdir, "c"))).To(Equal("c\n")) Expect(helpers.Read(filepath.Join(tmpdir, "cd"))).To(Equal("c\n")) - os.RemoveAll(f.GetPath()) + os.RemoveAll(f.Path) }) }) }) diff --git a/pkg/installer/client/http.go b/pkg/installer/client/http.go index 412203ea..0f79f084 100644 --- a/pkg/installer/client/http.go +++ b/pkg/installer/client/http.go @@ -24,9 +24,9 @@ import ( "path/filepath" "time" + "github.com/mudler/luet/pkg/compiler/types/artifact" . "github.com/mudler/luet/pkg/logger" - "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/config" "github.com/mudler/luet/pkg/helpers" @@ -66,13 +66,13 @@ func Round(input float64) float64 { return math.Floor(input + 0.5) } -func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) { +func (c *HttpClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) { var u *url.URL = nil var err error var req *grab.Request var temp string - artifactName := path.Base(artifact.GetPath()) + artifactName := path.Base(a.Path) cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName) ok := false @@ -168,8 +168,8 @@ func (c *HttpClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Arti } } - newart := artifact - newart.SetPath(cacheFile) + newart := a + newart.Path = cacheFile return newart, nil } diff --git a/pkg/installer/client/http_test.go b/pkg/installer/client/http_test.go index 59057754..5dc92a2c 100644 --- a/pkg/installer/client/http_test.go +++ b/pkg/installer/client/http_test.go @@ -22,7 +22,7 @@ import ( "os" "path/filepath" - compiler "github.com/mudler/luet/pkg/compiler" + "github.com/mudler/luet/pkg/compiler/types/artifact" helpers "github.com/mudler/luet/pkg/helpers" . "github.com/mudler/luet/pkg/installer/client" @@ -63,10 +63,10 @@ var _ = Describe("Http client", func() { Expect(err).ToNot(HaveOccurred()) c := NewHttpClient(RepoData{Urls: []string{ts.URL}}) - path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"}) + path, err := c.DownloadArtifact(&artifact.PackageArtifact{Path: "test.txt"}) Expect(err).ToNot(HaveOccurred()) - Expect(helpers.Read(path.GetPath())).To(Equal("test")) - os.RemoveAll(path.GetPath()) + Expect(helpers.Read(path.Path)).To(Equal("test")) + os.RemoveAll(path.Path) }) }) diff --git a/pkg/installer/client/local.go b/pkg/installer/client/local.go index b58dd58f..200e4701 100644 --- a/pkg/installer/client/local.go +++ b/pkg/installer/client/local.go @@ -20,10 +20,10 @@ import ( "path" "path/filepath" + "github.com/mudler/luet/pkg/compiler/types/artifact" "github.com/mudler/luet/pkg/config" . "github.com/mudler/luet/pkg/logger" - "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/helpers" ) @@ -35,11 +35,11 @@ func NewLocalClient(r RepoData) *LocalClient { return &LocalClient{RepoData: r} } -func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Artifact, error) { +func (c *LocalClient) DownloadArtifact(a *artifact.PackageArtifact) (*artifact.PackageArtifact, error) { var err error rootfs := "" - artifactName := path.Base(artifact.GetPath()) + artifactName := path.Base(a.Path) cacheFile := filepath.Join(config.LuetCfg.GetSystem().GetSystemPkgsCacheDirPath(), artifactName) if !config.LuetCfg.ConfigFromHost { @@ -74,8 +74,8 @@ func (c *LocalClient) DownloadArtifact(artifact compiler.Artifact) (compiler.Art } } - newart := artifact - newart.SetPath(cacheFile) + newart := a + newart.Path = cacheFile return newart, nil } diff --git a/pkg/installer/client/local_test.go b/pkg/installer/client/local_test.go index 3d1521d2..83fc9422 100644 --- a/pkg/installer/client/local_test.go +++ b/pkg/installer/client/local_test.go @@ -20,7 +20,7 @@ import ( "os" "path/filepath" - compiler "github.com/mudler/luet/pkg/compiler" + "github.com/mudler/luet/pkg/compiler/types/artifact" helpers "github.com/mudler/luet/pkg/helpers" . "github.com/mudler/luet/pkg/installer/client" @@ -56,10 +56,10 @@ var _ = Describe("Local client", func() { Expect(err).ToNot(HaveOccurred()) c := NewLocalClient(RepoData{Urls: []string{tmpdir}}) - path, err := c.DownloadArtifact(&compiler.PackageArtifact{Path: "test.txt"}) + path, err := c.DownloadArtifact(&artifact.PackageArtifact{Path: "test.txt"}) Expect(err).ToNot(HaveOccurred()) - Expect(helpers.Read(path.GetPath())).To(Equal("test")) - os.RemoveAll(path.GetPath()) + Expect(helpers.Read(path.Path)).To(Equal("test")) + os.RemoveAll(path.Path) }) }) diff --git a/pkg/installer/installer.go b/pkg/installer/installer.go index 48b25f14..05536ace 100644 --- a/pkg/installer/installer.go +++ b/pkg/installer/installer.go @@ -24,9 +24,10 @@ import ( "strings" "sync" + artifact "github.com/mudler/luet/pkg/compiler/types/artifact" + . "github.com/logrusorgru/aurora" "github.com/mudler/luet/pkg/bus" - compiler "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/config" "github.com/mudler/luet/pkg/helpers" . "github.com/mudler/luet/pkg/logger" @@ -58,11 +59,11 @@ type LuetInstaller struct { type ArtifactMatch struct { Package pkg.Package - Artifact compiler.Artifact - Repository Repository + Artifact *artifact.PackageArtifact + Repository *LuetSystemRepository } -func NewLuetInstaller(opts LuetInstallerOptions) Installer { +func NewLuetInstaller(opts LuetInstallerOptions) *LuetInstaller { return &LuetInstaller{Options: opts} } @@ -106,11 +107,11 @@ func (l *LuetInstaller) computeUpgrade(syncedRepos Repositories, s *System) (pkg continue } for _, artefact := range matches[0].Repo.GetIndex() { - if artefact.GetCompileSpec().GetPackage() == nil { + if artefact.CompileSpec.GetPackage() == nil { return uninstall, toInstall, errors.New("Package in compilespec empty") } - if artefact.GetCompileSpec().GetPackage().Matches(p) && artefact.GetCompileSpec().GetPackage().GetBuildTimestamp() != p.GetBuildTimestamp() { + if artefact.CompileSpec.GetPackage().Matches(p) && artefact.CompileSpec.GetPackage().GetBuildTimestamp() != p.GetBuildTimestamp() { toInstall = append(toInstall, matches[0].Package).Unique() uninstall = append(uninstall, p).Unique() } @@ -406,12 +407,12 @@ func (l *LuetInstaller) Reclaim(s *System) error { for _, repo := range syncedRepos { for _, artefact := range repo.GetIndex() { Debug("Checking if", - artefact.GetCompileSpec().GetPackage().HumanReadableString(), + artefact.CompileSpec.GetPackage().HumanReadableString(), "from", repo.GetName(), "is installed") FILES: - for _, f := range artefact.GetFiles() { + for _, f := range artefact.Files { if helpers.Exists(filepath.Join(s.Target, f)) { - p, err := repo.GetTree().GetDatabase().FindPackage(artefact.GetCompileSpec().GetPackage()) + p, err := repo.GetTree().GetDatabase().FindPackage(artefact.CompileSpec.GetPackage()) if err != nil { return err } @@ -435,7 +436,7 @@ func (l *LuetInstaller) Reclaim(s *System) error { if err != nil && !l.Options.Force { return errors.Wrap(err, "Failed creating package") } - s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: pack.GetFingerPrint(), Files: match.Artifact.GetFiles()}) + s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: pack.GetFingerPrint(), Files: match.Artifact.Files}) Info(":zap:Reclaimed package:", pack.HumanReadableString()) } Info("Done!") @@ -512,11 +513,11 @@ func (l *LuetInstaller) computeInstall(syncedRepos Repositories, cp pkg.Packages } A: for _, artefact := range matches[0].Repo.GetIndex() { - if artefact.GetCompileSpec().GetPackage() == nil { + if artefact.CompileSpec.GetPackage() == nil { return toInstall, p, solution, allRepos, errors.New("Package in compilespec empty") } - if matches[0].Package.Matches(artefact.GetCompileSpec().GetPackage()) { - currentPack.SetBuildTimestamp(artefact.GetCompileSpec().GetPackage().GetBuildTimestamp()) + if matches[0].Package.Matches(artefact.CompileSpec.GetPackage()) { + currentPack.SetBuildTimestamp(artefact.CompileSpec.GetPackage().GetBuildTimestamp()) // Filter out already installed if _, err := s.Database.FindPackage(currentPack); err != nil { toInstall[currentPack.GetFingerPrint()] = ArtifactMatch{Package: currentPack, Artifact: artefact, Repository: matches[0].Repo} @@ -602,7 +603,7 @@ func (l *LuetInstaller) install(syncedRepos Repositories, toInstall map[string]A return s.ExecuteFinalizers(toFinalize) } -func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, error) { +func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (*artifact.PackageArtifact, error) { artifact, err := a.Repository.Client().DownloadArtifact(a.Artifact) if err != nil { @@ -616,26 +617,26 @@ func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, err return artifact, nil } -func (l *LuetInstaller) installPackage(a ArtifactMatch, s *System) error { +func (l *LuetInstaller) installPackage(m ArtifactMatch, s *System) error { - artifact, err := l.downloadPackage(a) + a, err := l.downloadPackage(m) if err != nil && !l.Options.Force { return errors.Wrap(err, "Failed downloading package") } - files, err := artifact.FileList() + files, err := a.FileList() if err != nil && !l.Options.Force { return errors.Wrap(err, "Could not open package archive") } - err = artifact.Unpack(s.Target, true) + err = a.Unpack(s.Target, true) if err != nil && !l.Options.Force { return errors.Wrap(err, "Error met while unpacking rootfs") } // First create client and download // Then unpack to system - return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: a.Package.GetFingerPrint(), Files: files}) + return s.Database.SetPackageFiles(&pkg.PackageFile{PackageFingerprint: m.Package.GetFingerPrint(), Files: files}) } func (l *LuetInstaller) downloadWorker(i int, wg *sync.WaitGroup, c <-chan ArtifactMatch) error { @@ -855,4 +856,4 @@ func (l *LuetInstaller) Uninstall(s *System, packs ...pkg.Package) error { return uninstall() } -func (l *LuetInstaller) Repositories(r []Repository) { l.PackageRepositories = r } +func (l *LuetInstaller) Repositories(r []*LuetSystemRepository) { l.PackageRepositories = r } diff --git a/pkg/installer/installer_test.go b/pkg/installer/installer_test.go index 04a3aac9..8c0656f0 100644 --- a/pkg/installer/installer_test.go +++ b/pkg/installer/installer_test.go @@ -23,9 +23,10 @@ import ( // . "github.com/mudler/luet/pkg/installer" compiler "github.com/mudler/luet/pkg/compiler" backend "github.com/mudler/luet/pkg/compiler/backend" + compression "github.com/mudler/luet/pkg/compiler/types/compression" + "github.com/mudler/luet/pkg/compiler/types/options" + compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" "github.com/mudler/luet/pkg/helpers" - "github.com/mudler/luet/pkg/installer" - solver "github.com/mudler/luet/pkg/solver" . "github.com/mudler/luet/pkg/installer" pkg "github.com/mudler/luet/pkg/package" @@ -34,7 +35,7 @@ import ( . "github.com/onsi/gomega" ) -func stubRepo(tmpdir, tree string) (installer.Repository, error) { +func stubRepo(tmpdir, tree string) (*LuetSystemRepository, error) { return GenerateRepository( "test", "description", @@ -47,6 +48,7 @@ func stubRepo(tmpdir, tree string) (installer.Repository, error) { } var _ = Describe("Installer", func() { + Context("Writes a repository definition", func() { It("Writes a repo and can install packages from it", func() { //repo:=NewLuetSystemRepository() @@ -62,7 +64,9 @@ var _ = Describe("Installer", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), + generalRecipe.GetDatabase(), + options.Concurrency(2)) spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -77,12 +81,11 @@ var _ = Describe("Installer", func() { Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"})) spec.SetOutputPath(tmpdir) - c.SetConcurrency(2) - artifact, err := c.Compile(false, spec) + a, err := c.Compile(false, spec) Expect(err).ToNot(HaveOccurred()) - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(a.Path)).To(BeTrue()) + Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue()) @@ -178,7 +181,7 @@ urls: Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), - generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + generalRecipe.GetDatabase(), options.Concurrency(2)) spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -193,12 +196,11 @@ urls: Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"})) spec.SetOutputPath(tmpdir) - c.SetConcurrency(2) artifact, err := c.Compile(false, spec) Expect(err).ToNot(HaveOccurred()) - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue()) @@ -217,7 +219,7 @@ urls: Expect(err).ToNot(HaveOccurred()) treeFile := NewDefaultTreeRepositoryFile() - treeFile.SetCompressionType(compiler.None) + treeFile.SetCompressionType(compression.None) repo.SetRepositoryFile(REPOFILE_TREE_KEY, treeFile) Expect(err).ToNot(HaveOccurred()) Expect(repo.GetName()).To(Equal("test")) @@ -298,7 +300,8 @@ urls: Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), + options.Concurrency(2)) spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -313,12 +316,11 @@ urls: Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"})) spec.SetOutputPath(tmpdir) - c.SetConcurrency(2) artifact, err := c.Compile(false, spec) Expect(err).ToNot(HaveOccurred()) - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue()) @@ -423,7 +425,8 @@ urls: Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), + options.Concurrency(2)) spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -438,12 +441,11 @@ urls: Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"})) spec.SetOutputPath(tmpdir) - c.SetConcurrency(2) artifact, err := c.Compile(false, spec) Expect(err).ToNot(HaveOccurred()) - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue()) @@ -518,7 +520,7 @@ urls: Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1)) - c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), options.Concurrency(2)) spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -526,11 +528,10 @@ urls: Expect(spec.GetPackage().GetPath()).ToNot(Equal("")) spec.SetOutputPath(tmpdir2) - c.SetConcurrency(2) artifact, err = c.Compile(false, spec) Expect(err).ToNot(HaveOccurred()) - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) repo, err = stubRepo(tmpdir2, "../../tests/fixtures/alpine") Expect(err).ToNot(HaveOccurred()) @@ -577,7 +578,7 @@ urls: Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4)) - c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2)) spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -595,9 +596,8 @@ urls: spec.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdir) spec3.SetOutputPath(tmpdir) - c.SetConcurrency(2) - _, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3)) + _, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3)) Expect(errs).To(BeEmpty()) @@ -695,8 +695,8 @@ urls: Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) Expect(len(generalRecipeNewRepo.GetDatabase().GetPackages())).To(Equal(3)) - c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) - c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2)) + c2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipeNewRepo.GetDatabase()) spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -718,13 +718,12 @@ urls: spec.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdirnewrepo) spec3.SetOutputPath(tmpdir) - c.SetConcurrency(2) - _, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec3)) + _, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec3)) Expect(errs).To(BeEmpty()) - _, errs = c2.CompileParallel(false, compiler.NewLuetCompilationspecs(spec2)) + _, errs = c2.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec2)) Expect(errs).To(BeEmpty()) repo, err := stubRepo(tmpdir, "../../tests/fixtures/upgrade_old_repo") @@ -827,7 +826,12 @@ urls: Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4)) - c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + c := compiler.NewLuetCompiler( + backend.NewSimpleDockerBackend(), + generalRecipe.GetDatabase(), + options.Concurrency(2), + options.WithCompressionType(compression.GZip), + ) spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -844,9 +848,8 @@ urls: spec.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdir) spec3.SetOutputPath(tmpdir) - c.SetConcurrency(2) - c.SetCompressionType(compiler.GZip) - _, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3)) + + _, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3)) Expect(errs).To(BeEmpty()) @@ -985,7 +988,9 @@ urls: Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4)) - c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), + options.Concurrency(2), + options.WithCompressionType(compression.GZip)) spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -1002,9 +1007,7 @@ urls: spec.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdir) spec3.SetOutputPath(tmpdir) - c.SetConcurrency(2) - c.SetCompressionType(compiler.GZip) - _, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec2, spec3)) + _, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3)) Expect(errs).To(BeEmpty()) @@ -1090,7 +1093,8 @@ urls: Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + c := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), + options.WithCompressionType(compression.GZip)) spec, err := c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -1104,9 +1108,7 @@ urls: defer os.RemoveAll(tmpdir) // clean up spec.SetOutputPath(tmpdir) spec3.SetOutputPath(tmpdir) - c.SetConcurrency(1) - c.SetCompressionType(compiler.GZip) - _, errs := c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec, spec3)) + _, errs := c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec3)) Expect(errs).To(BeEmpty()) @@ -1182,7 +1184,7 @@ urls: Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(3)) - c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + c = compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase()) spec, err = c.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.1"}) Expect(err).ToNot(HaveOccurred()) @@ -1192,7 +1194,7 @@ urls: defer os.RemoveAll(tmpdir2) // clean up spec.SetOutputPath(tmpdir2) - _, errs = c.CompileParallel(false, compiler.NewLuetCompilationspecs(spec)) + _, errs = c.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec)) Expect(errs).To(BeEmpty()) diff --git a/pkg/installer/interface.go b/pkg/installer/interface.go index 3c758f19..b7293982 100644 --- a/pkg/installer/interface.go +++ b/pkg/installer/interface.go @@ -16,67 +16,13 @@ package installer import ( - compiler "github.com/mudler/luet/pkg/compiler" - pkg "github.com/mudler/luet/pkg/package" - "github.com/mudler/luet/pkg/tree" + artifact "github.com/mudler/luet/pkg/compiler/types/artifact" //"github.com/mudler/luet/pkg/solver" ) -type Installer interface { - Install(pkg.Packages, *System) error - Uninstall(*System, ...pkg.Package) error - Upgrade(s *System) error - Reclaim(s *System) error - - Repositories([]Repository) - SyncRepositories(bool) (Repositories, error) - Swap(pkg.Packages, pkg.Packages, *System) error -} - type Client interface { - DownloadArtifact(compiler.Artifact) (compiler.Artifact, error) + DownloadArtifact(*artifact.PackageArtifact) (*artifact.PackageArtifact, error) DownloadFile(string) (string, error) } -type Repositories []Repository - -type Repository interface { - GetName() string - GetDescription() string - GetUrls() []string - SetUrls([]string) - AddUrl(string) - GetPriority() int - GetIndex() compiler.ArtifactIndex - SetIndex(i compiler.ArtifactIndex) - GetTree() tree.Builder - SetTree(tree.Builder) - Write(path string, resetRevision, force bool) error - Sync(bool) (Repository, error) - GetTreePath() string - SetTreePath(string) - GetMetaPath() string - SetMetaPath(string) - GetType() string - SetType(string) - SetAuthentication(map[string]string) - GetAuthentication() map[string]string - GetRevision() int - IncrementRevision() - GetLastUpdate() string - SetLastUpdate(string) - Client() Client - - SetPriority(int) - GetRepositoryFile(string) (LuetRepositoryFile, error) - SetRepositoryFile(string, LuetRepositoryFile) - SetName(p string) - Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized) - GetBackend() compiler.CompilerBackend - SetBackend(b compiler.CompilerBackend) - FileSearch(pattern string) (pkg.Packages, error) - SearchArtefact(p pkg.Package) (compiler.Artifact, error) - - SetVerify(bool) - GetVerify() bool -} +type Repositories []*LuetSystemRepository diff --git a/pkg/installer/repository.go b/pkg/installer/repository.go index 81746830..9f4d48f4 100644 --- a/pkg/installer/repository.go +++ b/pkg/installer/repository.go @@ -25,6 +25,9 @@ import ( "strconv" "time" + artifact "github.com/mudler/luet/pkg/compiler/types/artifact" + compression "github.com/mudler/luet/pkg/compiler/types/compression" + "github.com/mudler/luet/pkg/compiler" "github.com/mudler/luet/pkg/config" "github.com/mudler/luet/pkg/helpers" @@ -53,9 +56,9 @@ const ( ) type LuetRepositoryFile struct { - FileName string `json:"filename"` - CompressionType compiler.CompressionImplementation `json:"compressiontype,omitempty"` - Checksums compiler.Checksums `json:"checksums,omitempty"` + FileName string `json:"filename"` + CompressionType compression.Implementation `json:"compressiontype,omitempty"` + Checksums artifact.Checksums `json:"checksums,omitempty"` } type LuetSystemRepository struct { @@ -71,22 +74,8 @@ type LuetSystemRepository struct { imagePrefix string } -type LuetSystemRepositorySerialized struct { - Name string `json:"name"` - Description string `json:"description,omitempty"` - Urls []string `json:"urls"` - Priority int `json:"priority"` - Type string `json:"type"` - Revision int `json:"revision,omitempty"` - LastUpdate string `json:"last_update,omitempty"` - TreePath string `json:"treepath"` - MetaPath string `json:"metapath"` - RepositoryFiles map[string]LuetRepositoryFile `json:"repo_files"` - Verify bool `json:"verify"` -} - type LuetSystemRepositoryMetadata struct { - Index []*compiler.PackageArtifact `json:"index,omitempty"` + Index []*artifact.PackageArtifact `json:"index,omitempty"` } type LuetSearchModeType int @@ -156,21 +145,21 @@ func (m *LuetSystemRepositoryMetadata) ToArtifactIndex() (ans compiler.ArtifactI func NewDefaultTreeRepositoryFile() LuetRepositoryFile { return LuetRepositoryFile{ FileName: TREE_TARBALL, - CompressionType: compiler.GZip, + CompressionType: compression.GZip, } } func NewDefaultCompilerTreeRepositoryFile() LuetRepositoryFile { return LuetRepositoryFile{ FileName: COMPILERTREE_TARBALL, - CompressionType: compiler.GZip, + CompressionType: compression.GZip, } } func NewDefaultMetaRepositoryFile() LuetRepositoryFile { return LuetRepositoryFile{ FileName: REPOSITORY_METAFILE + ".tar", - CompressionType: compiler.None, + CompressionType: compression.None, } } @@ -191,28 +180,28 @@ func (f *LuetRepositoryFile) GetFileName() string { // SetCompressionType sets the compression type of the repository file. // Each repository can ship arbitrary file that will be downloaded by the client // in case of need, this sets the compression type that the client will use to uncompress the artifact -func (f *LuetRepositoryFile) SetCompressionType(c compiler.CompressionImplementation) { +func (f *LuetRepositoryFile) SetCompressionType(c compression.Implementation) { f.CompressionType = c } // GetCompressionType gets the compression type of the repository file. // Each repository can ship arbitrary file that will be downloaded by the client // in case of need, this gets the compression type that the client will use to uncompress the artifact -func (f *LuetRepositoryFile) GetCompressionType() compiler.CompressionImplementation { +func (f *LuetRepositoryFile) GetCompressionType() compression.Implementation { return f.CompressionType } // SetChecksums sets the checksum of the repository file. // Each repository can ship arbitrary file that will be downloaded by the client // in case of need, this sets the checksums that the client will use to verify the artifact -func (f *LuetRepositoryFile) SetChecksums(c compiler.Checksums) { +func (f *LuetRepositoryFile) SetChecksums(c artifact.Checksums) { f.Checksums = c } // GetChecksums gets the checksum of the repository file. // Each repository can ship arbitrary file that will be downloaded by the client // in case of need, this gets the checksums that the client will use to verify the artifact -func (f *LuetRepositoryFile) GetChecksums() compiler.Checksums { +func (f *LuetRepositoryFile) GetChecksums() artifact.Checksums { return f.Checksums } @@ -221,7 +210,7 @@ func (f *LuetRepositoryFile) GetChecksums() compiler.Checksums { // In case the repository is local, it will build the package Index func GenerateRepository(name, descr, t string, urls []string, priority int, src string, treesDir []string, db pkg.PackageDatabase, - b compiler.CompilerBackend, imagePrefix string, pushImages, force bool) (Repository, error) { + b compiler.CompilerBackend, imagePrefix string, pushImages, force bool) (*LuetSystemRepository, error) { tr := tree.NewInstallerRecipe(db) btr := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false)) @@ -253,44 +242,23 @@ func GenerateRepository(name, descr, t string, urls []string, return repo, nil } -func NewSystemRepository(repo config.LuetRepository) Repository { +func NewSystemRepository(repo config.LuetRepository) *LuetSystemRepository { return &LuetSystemRepository{ LuetRepository: &repo, RepositoryFiles: map[string]LuetRepositoryFile{}, } } -func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (Repository, error) { - var p *LuetSystemRepositorySerialized +func NewLuetSystemRepositoryFromYaml(data []byte, db pkg.PackageDatabase) (*LuetSystemRepository, error) { + var p *LuetSystemRepository err := yaml.Unmarshal(data, &p) if err != nil { return nil, err } - repo := config.NewLuetRepository( - p.Name, - p.Type, - p.Description, - p.Urls, - p.Priority, - true, - false, - ) - repo.Verify = p.Verify - r := &LuetSystemRepository{ - LuetRepository: repo, - RepositoryFiles: p.RepositoryFiles, - } + p.Tree = tree.NewInstallerRecipe(db) - if p.Revision > 0 { - r.Revision = p.Revision - } - if p.LastUpdate != "" { - r.LastUpdate = p.LastUpdate - } - r.Tree = tree.NewInstallerRecipe(db) - - return r, err + return p, err } func (r *LuetSystemRepository) SetPriority(n int) { @@ -320,9 +288,9 @@ func (r *LuetSystemRepository) FileSearch(pattern string) (pkg.Packages, error) } ARTIFACT: for _, a := range r.GetIndex() { - for _, f := range a.GetFiles() { + for _, f := range a.Files { if reg.MatchString(f) { - matches = append(matches, a.GetCompileSpec().GetPackage()) + matches = append(matches, a.CompileSpec.GetPackage()) continue ARTIFACT } } @@ -440,7 +408,7 @@ func (r *LuetSystemRepository) BumpRevision(repospec string, resetRevision bool) // AddMetadata adds the repository serialized content into the metadata key of the repository // It writes the serialized content to repospec, and writes the repository.meta.yaml file into dst -func (r *LuetSystemRepository) AddMetadata(repospec, dst string) (compiler.Artifact, error) { +func (r *LuetSystemRepository) AddMetadata(repospec, dst string) (*artifact.PackageArtifact, error) { // Create Metadata struct and serialized repository meta, serialized := r.Serialize() @@ -477,7 +445,7 @@ func (r *LuetSystemRepository) AddMetadata(repospec, dst string) (compiler.Artif // AddTree adds a tree.Builder with the given key to the repository. // It will generate an artifact which will be then embedded in the repository manifest // It returns the generated artifacts and an error -func (r *LuetSystemRepository) AddTree(t tree.Builder, dst, key string,f LuetRepositoryFile) (compiler.Artifact, error) { +func (r *LuetSystemRepository) AddTree(t tree.Builder, dst, key string, f LuetRepositoryFile) (*artifact.PackageArtifact, error) { // Create tree and repository file archive, err := config.LuetCfg.GetSystem().TempDir("archive") if err != nil { @@ -499,15 +467,15 @@ func (r *LuetSystemRepository) AddTree(t tree.Builder, dst, key string,f LuetRep // AddRepositoryFile adds a path to a key in the repository manifest. // The path will be compressed, and a default File has to be passed in case there is no entry into // the repository manifest -func (r *LuetSystemRepository) AddRepositoryFile(src, fileKey, repositoryRoot string, defaults LuetRepositoryFile) (compiler.Artifact, error) { +func (r *LuetSystemRepository) AddRepositoryFile(src, fileKey, repositoryRoot string, defaults LuetRepositoryFile) (*artifact.PackageArtifact, error) { treeFile, err := r.GetRepositoryFile(fileKey) if err != nil { treeFile = defaults r.SetRepositoryFile(fileKey, treeFile) } - a := compiler.NewPackageArtifact(filepath.Join(repositoryRoot, treeFile.GetFileName())) - a.SetCompressionType(treeFile.GetCompressionType()) + a := artifact.NewPackageArtifact(filepath.Join(repositoryRoot, treeFile.GetFileName())) + a.CompressionType = treeFile.GetCompressionType() err = a.Compress(src, 1) if err != nil { return a, errors.Wrap(err, "Error met while creating package archive") @@ -518,8 +486,8 @@ func (r *LuetSystemRepository) AddRepositoryFile(src, fileKey, repositoryRoot st return a, errors.Wrap(err, "Failed generating checksums for tree") } // Update the tree name with the name created by compression selected. - treeFile.SetChecksums(a.GetChecksums()) - treeFile.SetFileName(path.Base(a.GetPath())) + treeFile.SetChecksums(a.Checksums) + treeFile.SetFileName(path.Base(a.Path)) r.SetRepositoryFile(fileKey, treeFile) @@ -537,12 +505,12 @@ func (r *LuetSystemRepository) SetRepositoryFile(name string, f LuetRepositoryFi r.RepositoryFiles[name] = f } -func (r *LuetSystemRepository) ReadSpecFile(file string) (Repository, error) { +func (r *LuetSystemRepository) ReadSpecFile(file string) (*LuetSystemRepository, error) { dat, err := ioutil.ReadFile(file) if err != nil { return nil, errors.Wrap(err, "Error reading file "+file) } - var repo Repository + var repo *LuetSystemRepository repo, err = NewLuetSystemRepositoryFromYaml(dat, pkg.NewInMemoryDatabase(false)) if err != nil { return nil, errors.Wrap(err, "Error reading repository from file "+file) @@ -563,7 +531,7 @@ func (r *LuetSystemRepository) ReadSpecFile(file string) (Repository, error) { type RepositoryGenerator interface { Generate(*LuetSystemRepository, string, bool) error - Initialize(string, pkg.PackageDatabase) ([]compiler.Artifact, error) + Initialize(string, pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) } func (r *LuetSystemRepository) getGenerator() (RepositoryGenerator, error) { @@ -616,9 +584,9 @@ func (r *LuetSystemRepository) Client() Client { return nil } -func (r *LuetSystemRepository) SearchArtefact(p pkg.Package) (compiler.Artifact, error) { +func (r *LuetSystemRepository) SearchArtefact(p pkg.Package) (*artifact.PackageArtifact, error) { for _, a := range r.GetIndex() { - if a.GetCompileSpec().GetPackage().Matches(p) { + if a.CompileSpec.GetPackage().Matches(p) { return a, nil } } @@ -626,7 +594,7 @@ func (r *LuetSystemRepository) SearchArtefact(p pkg.Package) (compiler.Artifact, return nil, errors.New("Not found") } -func (r *LuetSystemRepository) Sync(force bool) (Repository, error) { +func (r *LuetSystemRepository) Sync(force bool) (*LuetSystemRepository, error) { var repoUpdated bool = false var treefs, metafs string aurora := GetAurora() @@ -699,9 +667,9 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) { defer os.Remove(downloadedTreeFile) // Treat the file as artifact, in order to verify it - treeFileArtifact := compiler.NewPackageArtifact(downloadedTreeFile) - treeFileArtifact.SetChecksums(treeFile.GetChecksums()) - treeFileArtifact.SetCompressionType(treeFile.GetCompressionType()) + treeFileArtifact := artifact.NewPackageArtifact(downloadedTreeFile) + treeFileArtifact.Checksums = treeFile.GetChecksums() + treeFileArtifact.CompressionType = treeFile.GetCompressionType() err = treeFileArtifact.Verify() if err != nil { @@ -717,9 +685,9 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) { } defer os.Remove(downloadedMeta) - metaFileArtifact := compiler.NewPackageArtifact(downloadedMeta) - metaFileArtifact.SetChecksums(metaFile.GetChecksums()) - metaFileArtifact.SetCompressionType(metaFile.GetCompressionType()) + metaFileArtifact := artifact.NewPackageArtifact(downloadedMeta) + metaFileArtifact.Checksums = metaFile.GetChecksums() + metaFileArtifact.CompressionType = metaFile.GetCompressionType() err = metaFileArtifact.Verify() if err != nil { @@ -806,30 +774,20 @@ func (r *LuetSystemRepository) Sync(force bool) (Repository, error) { return repo, nil } -func (r *LuetSystemRepository) Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepositorySerialized) { +func (r *LuetSystemRepository) Serialize() (*LuetSystemRepositoryMetadata, LuetSystemRepository) { - serialized := LuetSystemRepositorySerialized{ - Name: r.Name, - Description: r.Description, - Urls: r.Urls, - Priority: r.Priority, - Type: r.Type, - Revision: r.Revision, - LastUpdate: r.LastUpdate, - RepositoryFiles: r.RepositoryFiles, - Verify: r.Verify, - } + serialized := *r + serialized.Authentication = nil // Check if is needed set the index or simply use // value returned by CleanPath - r.Index = r.Index.CleanPath() + serialized.Index = serialized.Index.CleanPath() meta := &LuetSystemRepositoryMetadata{ - Index: []*compiler.PackageArtifact{}, + Index: []*artifact.PackageArtifact{}, } for _, a := range r.Index { - art := a.(*compiler.PackageArtifact) - meta.Index = append(meta.Index, art) + meta.Index = append(meta.Index, a) } return meta, serialized @@ -877,8 +835,8 @@ func (r Repositories) SyncDatabase(d pkg.PackageDatabase) { } type PackageMatch struct { - Repo Repository - Artifact compiler.Artifact + Repo *LuetSystemRepository + Artifact *artifact.PackageArtifact Package pkg.Package } diff --git a/pkg/installer/repository_docker.go b/pkg/installer/repository_docker.go index 56202bde..131f1d0b 100644 --- a/pkg/installer/repository_docker.go +++ b/pkg/installer/repository_docker.go @@ -24,6 +24,9 @@ import ( "strings" "time" + artifact "github.com/mudler/luet/pkg/compiler/types/artifact" + + "github.com/mudler/luet/pkg/compiler/backend" . "github.com/mudler/luet/pkg/logger" pkg "github.com/mudler/luet/pkg/package" @@ -40,7 +43,7 @@ type dockerRepositoryGenerator struct { imagePush, force bool } -func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) { +func (l *dockerRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) { return generatePackageImages(l.b, l.imagePrefix, path, db, l.imagePush, l.force) } @@ -49,12 +52,12 @@ func pushImage(b compiler.CompilerBackend, image string, force bool) error { Debug("Image", image, "already present, skipping") return nil } - return b.Push(compiler.CompilerBackendOptions{ImageName: image}) + return b.Push(backend.Options{ImageName: image}) } -func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase, imagePush, force bool) ([]compiler.Artifact, error) { +func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, db pkg.PackageDatabase, imagePush, force bool) ([]*artifact.PackageArtifact, error) { Info("Generating docker images for packages in", imagePrefix) - var art []compiler.Artifact + var art []*artifact.PackageArtifact var ff = func(currentpath string, info os.FileInfo, err error) error { if !strings.HasSuffix(info.Name(), ".metadata.yaml") { @@ -66,30 +69,30 @@ func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, return errors.Wrap(err, "Error reading file "+currentpath) } - artifact, err := compiler.NewPackageArtifactFromYaml(dat) + a, err := artifact.NewPackageArtifactFromYaml(dat) if err != nil { return errors.Wrap(err, "Error reading yaml "+currentpath) } // Set the path relative to the file. // The metadata contains the full path where the file was located during buildtime. - artifact.SetPath(filepath.Join(filepath.Dir(currentpath), filepath.Base(artifact.GetPath()))) + a.Path = filepath.Join(filepath.Dir(currentpath), filepath.Base(a.Path)) // We want to include packages that are ONLY referenced in the tree. // the ones which aren't should be deleted. (TODO: by another cli command?) - if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil { + if _, notfound := db.FindPackage(a.CompileSpec.Package); notfound != nil { Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.", - artifact.GetCompileSpec().GetPackage().HumanReadableString())) + a.CompileSpec.Package.HumanReadableString())) return nil } - packageImage := fmt.Sprintf("%s:%s", imagePrefix, artifact.GetCompileSpec().GetPackage().ImageID()) + packageImage := fmt.Sprintf("%s:%s", imagePrefix, a.CompileSpec.GetPackage().ImageID()) if imagePush && b.ImageAvailable(packageImage) && !force { Info("Image", packageImage, "already present, skipping. use --force-push to override") } else { Info("Generating final image", packageImage, - "for package ", artifact.GetCompileSpec().GetPackage().HumanReadableString()) - if opts, err := artifact.GenerateFinalImage(packageImage, b, true); err != nil { + "for package ", a.CompileSpec.GetPackage().HumanReadableString()) + if opts, err := a.GenerateFinalImage(packageImage, b, true); err != nil { return errors.Wrap(err, "Failed generating metadata tree"+opts.ImageName) } } @@ -99,7 +102,7 @@ func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, } } - art = append(art, artifact) + art = append(art, a) return nil } @@ -112,7 +115,7 @@ func generatePackageImages(b compiler.CompilerBackend, imagePrefix, path string, return art, nil } -func (d *dockerRepositoryGenerator) pushFileFromArtifact(a compiler.Artifact, imageTree string, r *LuetSystemRepository) error { +func (d *dockerRepositoryGenerator) pushFileFromArtifact(a *artifact.PackageArtifact, imageTree string, r *LuetSystemRepository) error { Debug("Generating image", imageTree) if opts, err := a.GenerateFinalImage(imageTree, r.GetBackend(), false); err != nil { return errors.Wrap(err, "Failed generating metadata tree "+opts.ImageName) @@ -138,7 +141,7 @@ func (d *dockerRepositoryGenerator) pushRepoMetadata(repospec string, r *LuetSys return errors.Wrap(err, "Error met while archiving repository file") } - a := compiler.NewPackageArtifact(tempRepoFile) + a := artifact.NewPackageArtifact(tempRepoFile) imageRepo := fmt.Sprintf("%s:%s", d.imagePrefix, REPOSITORY_SPECFILE) if err := d.pushFileFromArtifact(a, imageRepo, r); err != nil { @@ -147,10 +150,10 @@ func (d *dockerRepositoryGenerator) pushRepoMetadata(repospec string, r *LuetSys return nil } -func (d *dockerRepositoryGenerator) pushImageFromArtifact(a compiler.Artifact, r *LuetSystemRepository) error { +func (d *dockerRepositoryGenerator) pushImageFromArtifact(a *artifact.PackageArtifact, r *LuetSystemRepository) error { // we generate a new archive containing the required compressed file. // TODO: Bundle all the extra files in 1 docker image only, instead of an image for each file - treeArchive, err := compiler.CreateArtifactForFile(a.GetPath()) + treeArchive, err := artifact.CreateArtifactForFile(a.Path) if err != nil { return errors.Wrap(err, "failed generating checksums for tree") } @@ -176,11 +179,11 @@ func (d *dockerRepositoryGenerator) Generate(r *LuetSystemRepository, imagePrefi defer os.RemoveAll(repoTemp) // clean up if r.GetBackend().ImageAvailable(imageRepository) { - if err := r.GetBackend().DownloadImage(compiler.CompilerBackendOptions{ImageName: imageRepository}); err != nil { + if err := r.GetBackend().DownloadImage(backend.Options{ImageName: imageRepository}); err != nil { return errors.Wrapf(err, "while downloading '%s'", imageRepository) } - if err := r.GetBackend().ExtractRootfs(compiler.CompilerBackendOptions{ImageName: imageRepository, Destination: repoTemp}, false); err != nil { + if err := r.GetBackend().ExtractRootfs(backend.Options{ImageName: imageRepository, Destination: repoTemp}, false); err != nil { return errors.Wrapf(err, "while extracting '%s'", imageRepository) } } diff --git a/pkg/installer/repository_local.go b/pkg/installer/repository_local.go index 75d7355b..d834bbc2 100644 --- a/pkg/installer/repository_local.go +++ b/pkg/installer/repository_local.go @@ -24,23 +24,23 @@ import ( "strings" "time" + artifact "github.com/mudler/luet/pkg/compiler/types/artifact" . "github.com/mudler/luet/pkg/logger" pkg "github.com/mudler/luet/pkg/package" "github.com/mudler/luet/pkg/bus" - compiler "github.com/mudler/luet/pkg/compiler" "github.com/pkg/errors" ) type localRepositoryGenerator struct{} -func (l *localRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) { +func (l *localRepositoryGenerator) Initialize(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) { return buildPackageIndex(path, db) } -func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact, error) { +func buildPackageIndex(path string, db pkg.PackageDatabase) ([]*artifact.PackageArtifact, error) { - var art []compiler.Artifact + var art []*artifact.PackageArtifact var ff = func(currentpath string, info os.FileInfo, err error) error { if !strings.HasSuffix(info.Name(), ".metadata.yaml") { @@ -52,20 +52,20 @@ func buildPackageIndex(path string, db pkg.PackageDatabase) ([]compiler.Artifact return errors.Wrap(err, "Error reading file "+currentpath) } - artifact, err := compiler.NewPackageArtifactFromYaml(dat) + a, err := artifact.NewPackageArtifactFromYaml(dat) if err != nil { return errors.Wrap(err, "Error reading yaml "+currentpath) } // We want to include packages that are ONLY referenced in the tree. // the ones which aren't should be deleted. (TODO: by another cli command?) - if _, notfound := db.FindPackage(artifact.GetCompileSpec().GetPackage()); notfound != nil { + if _, notfound := db.FindPackage(a.CompileSpec.GetPackage()); notfound != nil { Debug(fmt.Sprintf("Package %s not found in tree. Ignoring it.", - artifact.GetCompileSpec().GetPackage().HumanReadableString())) + a.CompileSpec.GetPackage().HumanReadableString())) return nil } - art = append(art, artifact) + art = append(art, a) return nil } diff --git a/pkg/installer/repository_test.go b/pkg/installer/repository_test.go index c5385371..33efad9f 100644 --- a/pkg/installer/repository_test.go +++ b/pkg/installer/repository_test.go @@ -26,18 +26,19 @@ import ( "github.com/mudler/luet/pkg/compiler" backend "github.com/mudler/luet/pkg/compiler/backend" + compilerspec "github.com/mudler/luet/pkg/compiler/types/spec" + + artifact "github.com/mudler/luet/pkg/compiler/types/artifact" config "github.com/mudler/luet/pkg/config" "github.com/mudler/luet/pkg/helpers" - "github.com/mudler/luet/pkg/installer" . "github.com/mudler/luet/pkg/installer" pkg "github.com/mudler/luet/pkg/package" - "github.com/mudler/luet/pkg/solver" "github.com/mudler/luet/pkg/tree" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) -func dockerStubRepo(tmpdir, tree, image string, push, force bool) (installer.Repository, error) { +func dockerStubRepo(tmpdir, tree, image string, push, force bool) (*LuetSystemRepository, error) { return GenerateRepository( "test", "description", @@ -64,7 +65,7 @@ var _ = Describe("Repository", func() { Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -79,12 +80,11 @@ var _ = Describe("Repository", func() { Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"})) spec.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) artifact, err := compiler.Compile(false, spec) Expect(err).ToNot(HaveOccurred()) - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue()) @@ -132,11 +132,11 @@ var _ = Describe("Repository", func() { Expect(len(generalRecipe2.GetDatabase().GetPackages())).To(Equal(1)) Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler2 := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe2.GetDatabase()) spec2, err := compiler2.FromPackage(&pkg.DefaultPackage{Name: "alpine", Category: "seed", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) - compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + compiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -153,18 +153,16 @@ var _ = Describe("Repository", func() { spec.SetOutputPath(tmpdir) spec2.SetOutputPath(tmpdir) - compiler.SetConcurrency(1) - compiler2.SetConcurrency(1) artifact, err := compiler.Compile(false, spec) Expect(err).ToNot(HaveOccurred()) - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred()) artifact2, err := compiler2.Compile(false, spec2) Expect(err).ToNot(HaveOccurred()) - Expect(helpers.Exists(artifact2.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact2.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(artifact2.Path)).To(BeTrue()) + Expect(helpers.Untar(artifact2.Path, tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue()) @@ -254,7 +252,7 @@ urls: Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3)) - localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := localcompiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"}) Expect(err).ToNot(HaveOccurred()) @@ -266,12 +264,11 @@ urls: defer os.RemoveAll(tmpdir) // clean up spec.SetOutputPath(tmpdir) - localcompiler.SetConcurrency(1) - artifact, err := localcompiler.Compile(false, spec) + a, err := localcompiler.Compile(false, spec) Expect(err).ToNot(HaveOccurred()) - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(a.Path)).To(BeTrue()) + Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred()) Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue()) Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue()) @@ -300,9 +297,9 @@ urls: Expect(err).ToNot(HaveOccurred()) Expect(helpers.Read(f)).To(ContainSubstring("name: test")) - a, err := c.DownloadArtifact(&compiler.PackageArtifact{ + a, err = c.DownloadArtifact(&artifact.PackageArtifact{ Path: "test.tar", - CompileSpec: &compiler.LuetCompilationSpec{ + CompileSpec: &compilerspec.LuetCompilationSpec{ Package: &pkg.DefaultPackage{ Name: "b", Category: "test", @@ -329,7 +326,7 @@ urls: Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(5)) - localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), compiler.NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple}) + localcompiler := compiler.NewLuetCompiler(backend.NewSimpleDockerBackend(), generalRecipe.GetDatabase()) spec, err := localcompiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.99"}) Expect(err).ToNot(HaveOccurred()) @@ -341,12 +338,11 @@ urls: defer os.RemoveAll(tmpdir) // clean up spec.SetOutputPath(tmpdir) - localcompiler.SetConcurrency(1) - artifact, err := localcompiler.Compile(false, spec) + a, err := localcompiler.Compile(false, spec) Expect(err).ToNot(HaveOccurred()) - Expect(helpers.Exists(artifact.GetPath())).To(BeTrue()) - Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred()) + Expect(helpers.Exists(a.Path)).To(BeTrue()) + Expect(helpers.Untar(a.Path, tmpdir, false)).ToNot(HaveOccurred()) repo, err := dockerStubRepo(tmpdir, "../../tests/fixtures/virtuals", repoImage, true, true) Expect(err).ToNot(HaveOccurred()) @@ -372,9 +368,9 @@ urls: Expect(err).ToNot(HaveOccurred()) Expect(helpers.Read(f)).To(ContainSubstring("name: test")) - a, err := c.DownloadArtifact(&compiler.PackageArtifact{ + a, err = c.DownloadArtifact(&artifact.PackageArtifact{ Path: "test.tar", - CompileSpec: &compiler.LuetCompilationSpec{ + CompileSpec: &compilerspec.LuetCompilationSpec{ Package: &pkg.DefaultPackage{ Name: "a", Category: "test", @@ -397,14 +393,14 @@ urls: repos := Repositories{ &LuetSystemRepository{ Index: compiler.ArtifactIndex{ - &compiler.PackageArtifact{ - CompileSpec: &compiler.LuetCompilationSpec{ + &artifact.PackageArtifact{ + CompileSpec: &compilerspec.LuetCompilationSpec{ Package: &pkg.DefaultPackage{}, }, Path: "bar", Files: []string{"boo"}, }, - &compiler.PackageArtifact{ + &artifact.PackageArtifact{ Path: "d", Files: []string{"baz"}, }, @@ -414,15 +410,15 @@ urls: matches := repos.SearchPackages("bo", FileSearch) Expect(len(matches)).To(Equal(1)) - Expect(matches[0].Artifact.GetPath()).To(Equal("bar")) + Expect(matches[0].Artifact.Path).To(Equal("bar")) }) It("Searches packages", func() { repo := &LuetSystemRepository{ Index: compiler.ArtifactIndex{ - &compiler.PackageArtifact{ + &artifact.PackageArtifact{ Path: "foo", - CompileSpec: &compiler.LuetCompilationSpec{ + CompileSpec: &compilerspec.LuetCompilationSpec{ Package: &pkg.DefaultPackage{ Name: "foo", Category: "bar", @@ -430,9 +426,9 @@ urls: }, }, }, - &compiler.PackageArtifact{ + &artifact.PackageArtifact{ Path: "baz", - CompileSpec: &compiler.LuetCompilationSpec{ + CompileSpec: &compilerspec.LuetCompilationSpec{ Package: &pkg.DefaultPackage{ Name: "foo", Category: "baz", @@ -449,7 +445,7 @@ urls: Version: "1.0", }) Expect(err).ToNot(HaveOccurred()) - Expect(a.GetPath()).To(Equal("baz")) + Expect(a.Path).To(Equal("baz")) a, err = repo.SearchArtefact(&pkg.DefaultPackage{ Name: "foo", @@ -457,7 +453,7 @@ urls: Version: "1.0", }) Expect(err).ToNot(HaveOccurred()) - Expect(a.GetPath()).To(Equal("foo")) + Expect(a.Path).To(Equal("foo")) // Doesn't exist. so must fail _, err = repo.SearchArtefact(&pkg.DefaultPackage{ diff --git a/pkg/installer/system.go b/pkg/installer/system.go index 913b329a..5cea19ad 100644 --- a/pkg/installer/system.go +++ b/pkg/installer/system.go @@ -24,7 +24,7 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package) error { executedFinalizer := map[string]bool{} for _, p := range packs { if helpers.Exists(p.Rel(tree.FinalizerFile)) { - out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile), "") + out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile)) if err != nil { Warning("Failed rendering finalizer for ", p.HumanReadableString(), err.Error()) errs = multierror.Append(errs, err) diff --git a/pkg/tree/compiler_recipe.go b/pkg/tree/compiler_recipe.go index c9e4bd86..3057d80a 100644 --- a/pkg/tree/compiler_recipe.go +++ b/pkg/tree/compiler_recipe.go @@ -103,7 +103,7 @@ func (r *CompilerRecipe) Load(path string) error { compileDefPath := pack.Rel(CompilerDefinitionFile) if helpers.Exists(compileDefPath) { - dat, err := helpers.RenderFiles(compileDefPath, currentpath, "") + dat, err := helpers.RenderFiles(compileDefPath, currentpath) if err != nil { return errors.Wrap(err, "Error templating file "+CompilerDefinitionFile+" from "+