kairos/luet
1
0
Fork 0
mirror of https://github.com/mudler/luet.git synced 2025-09-13 13:50:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

⚙️ Add ability to build from Dockerfiles directly

Browse Source
This commit is contained in:
Ettore Di Giacinto
2022-04-26 19:24:31 +02:00
committed by mudler
parent 4e2a2adfc1
commit e70a543f42
457 changed files with 33148 additions and 4999 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
pkg/compiler/compiler.go
View File

@@ -16,6 +16,7 @@
package compiler
import (
"bytes"
"crypto/md5"
"fmt"
"io"
@@ -29,6 +30,8 @@ import (
"sync"
"time"
dockerfile "github.com/asottile/dockerfile"
"github.com/imdario/mergo"
bus "github.com/mudler/luet/pkg/api/core/bus"
"github.com/mudler/luet/pkg/api/core/context"
"github.com/mudler/luet/pkg/api/core/image"
@@ -36,14 +39,10 @@ import (
"github.com/mudler/luet/pkg/api/core/types"
artifact "github.com/mudler/luet/pkg/api/core/types/artifact"
"github.com/mudler/luet/pkg/compiler/backend"
"github.com/mudler/luet/pkg/compiler/types/options"
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
pkg "github.com/mudler/luet/pkg/database"
"github.com/mudler/luet/pkg/helpers"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
"github.com/mudler/luet/pkg/solver"
"github.com/imdario/mergo"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"
)
@@ -68,17 +67,17 @@ type LuetCompiler struct {
//*tree.CompilerRecipe
Backend CompilerBackend
Database types.PackageDatabase
Options options.Compiler
Options types.CompilerOptions
}
func NewCompiler(p ...options.Option) *LuetCompiler {
c := options.NewDefaultCompiler()
func NewCompiler(p ...types.CompilerOption) *LuetCompiler {
c := newDefaultCompiler()
c.Apply(p...)
return &LuetCompiler{Options: *c}
}
func NewLuetCompiler(backend CompilerBackend, db types.PackageDatabase, compilerOpts ...options.Option) *LuetCompiler {
func NewLuetCompiler(backend CompilerBackend, db types.PackageDatabase, compilerOpts ...types.CompilerOption) *LuetCompiler {
// The CompilerRecipe will gives us a tree with only build deps listed.
c := NewCompiler(compilerOpts...)
@@ -92,7 +91,7 @@ func NewLuetCompiler(backend CompilerBackend, db types.PackageDatabase, compiler
return c
}
func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan *compilerspec.LuetCompilationSpec, a *[]*artifact.PackageArtifact, m *sync.Mutex, concurrency int, keepPermissions bool, errors chan error) {
func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan *types.LuetCompilationSpec, a *[]*artifact.PackageArtifact, m *sync.Mutex, concurrency int, keepPermissions bool, errors chan error) {
defer wg.Done()
for s := range cspecs {
@@ -108,14 +107,14 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan *c
}
// CompileWithReverseDeps compiles the supplied compilationspecs and their reverse dependencies
func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps *compilerspec.LuetCompilationspecs) ([]*artifact.PackageArtifact, []error) {
func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps *types.LuetCompilationspecs) ([]*artifact.PackageArtifact, []error) {
artifacts, err := cs.CompileParallel(keepPermissions, ps)
if len(err) != 0 {
return artifacts, err
}
cs.Options.Context.Info(":ant: Resolving reverse dependencies")
toCompile := compilerspec.NewLuetCompilationspecs()
toCompile := types.NewLuetCompilationspecs()
for _, a := range artifacts {
revdeps := a.CompileSpec.GetPackage().Revdeps(cs.Database)
@@ -141,8 +140,8 @@ func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps *compile
// CompileParallel compiles the supplied compilationspecs in parallel
// to note, no specific heuristic is implemented, and the specs are run in parallel as they are.
func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps *compilerspec.LuetCompilationspecs) ([]*artifact.PackageArtifact, []error) {
all := make(chan *compilerspec.LuetCompilationSpec)
func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps *types.LuetCompilationspecs) ([]*artifact.PackageArtifact, []error) {
all := make(chan *types.LuetCompilationSpec)
artifacts := []*artifact.PackageArtifact{}
mutex := &sync.Mutex{}
errors := make(chan error, ps.Len())
@@ -227,7 +226,7 @@ func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, includ
return nil
}
func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec, runnerOpts backend.Options) (*artifact.PackageArtifact, error) {
func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *types.LuetCompilationSpec, runnerOpts backend.Options) (*artifact.PackageArtifact, error) {
if !cs.Backend.ImageExists(runnerOpts.ImageName) {
if err := cs.Backend.DownloadImage(runnerOpts); err != nil {
@@ -274,7 +273,7 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compi
return a, nil
}
func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec, builderOpts, runnerOpts backend.Options) (*artifact.PackageArtifact, error) {
func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *types.LuetCompilationSpec, builderOpts, runnerOpts backend.Options) (*artifact.PackageArtifact, error) {
rootfs, err := cs.Options.Context.TempDir("rootfs")
if err != nil {
@@ -339,7 +338,7 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *co
func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,
concurrency int, keepPermissions bool,
p *compilerspec.LuetCompilationSpec) (backend.Options, backend.Options, error) {
p *types.LuetCompilationSpec) (backend.Options, backend.Options, error) {
var runnerOpts, builderOpts backend.Options
@@ -449,7 +448,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
return builderOpts, runnerOpts, nil
}
func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builderOpts, runnerOpts backend.Options, concurrency int, keepPermissions bool) (*artifact.PackageArtifact, error) {
func (cs *LuetCompiler) genArtifact(p *types.LuetCompilationSpec, builderOpts, runnerOpts backend.Options, concurrency int, keepPermissions bool) (*artifact.PackageArtifact, error) {
// generate *artifact.PackageArtifact
var a *artifact.PackageArtifact
@@ -526,7 +525,7 @@ func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builder
}
// finalizeImages finalizes images and generates final artifacts (push them as well if necessary).
func (cs *LuetCompiler) finalizeImages(a *artifact.PackageArtifact, p *compilerspec.LuetCompilationSpec, keepPermissions bool) error {
func (cs *LuetCompiler) finalizeImages(a *artifact.PackageArtifact, p *types.LuetCompilationSpec, keepPermissions bool) error {
// TODO: This is a small readaptation of repository_docker.go pushImageFromArtifact().
// Maybe can be moved to a common place.
@@ -626,7 +625,7 @@ func oneOfImagesAvailable(images []string, b CompilerBackend) (bool, string) {
return false, ""
}
func (cs *LuetCompiler) findImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
func (cs *LuetCompiler) findImageHash(imageHash string, p *types.LuetCompilationSpec) string {
var resolvedImage string
cs.Options.Context.Debug("Resolving image hash for", p.Package.HumanReadableString(), "hash", imageHash, "Pull repositories", p.BuildOptions.PullImageRepository)
toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)},
@@ -646,7 +645,7 @@ func (cs *LuetCompiler) findImageHash(imageHash string, p *compilerspec.LuetComp
return resolvedImage
}
func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilerspec.LuetCompilationSpec) string {
func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *types.LuetCompilationSpec) string {
resolvedImage := cs.findImageHash(imageHash, p)
if resolvedImage == "" {
@@ -655,7 +654,7 @@ func (cs *LuetCompiler) resolveExistingImageHash(imageHash string, p *compilersp
return resolvedImage
}
func LoadArtifactFromYaml(spec *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
func LoadArtifactFromYaml(spec *types.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
metaFile := spec.GetPackage().GetMetadataFilePath()
dat, err := ioutil.ReadFile(spec.Rel(metaFile))
if err != nil {
@@ -670,7 +669,7 @@ func LoadArtifactFromYaml(spec *compilerspec.LuetCompilationSpec) (*artifact.Pac
return art, nil
}
func (cs *LuetCompiler) getImageArtifact(hash string, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
func (cs *LuetCompiler) getImageArtifact(hash string, p *types.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
// we check if there is an available image with the given hash and
// we return a full artifact if can be loaded locally.
cs.Options.Context.Debug("Get image artifact for", p.Package.HumanReadableString(), "hash", hash, "Pull repositories", p.BuildOptions.PullImageRepository)
@@ -700,7 +699,7 @@ func (cs *LuetCompiler) getImageArtifact(hash string, p *compilerspec.LuetCompil
func (cs *LuetCompiler) compileWithImage(image, builderHash string, packageTagHash string,
concurrency int,
keepPermissions, keepImg bool,
p *compilerspec.LuetCompilationSpec, generateArtifact bool) (*artifact.PackageArtifact, error) {
p *types.LuetCompilationSpec, generateArtifact bool) (*artifact.PackageArtifact, error) {
// If it is a virtual, check if we have to generate an empty artifact or not.
if generateArtifact && p.IsVirtual() {
@@ -781,8 +780,8 @@ func (cs *LuetCompiler) compileWithImage(image, builderHash string, packageTagHa
// FromDatabase returns all the available compilation specs from a database. If the minimum flag is returned
// it will be computed a minimal subset that will guarantees that all packages are built ( if not targeting a single package explictly )
func (cs *LuetCompiler) FromDatabase(db types.PackageDatabase, minimum bool, dst string) ([]*compilerspec.LuetCompilationSpec, error) {
compilerSpecs := compilerspec.NewLuetCompilationspecs()
func (cs *LuetCompiler) FromDatabase(db types.PackageDatabase, minimum bool, dst string) ([]*types.LuetCompilationSpec, error) {
compilerSpecs := types.NewLuetCompilationspecs()
w := db.World()
@@ -805,7 +804,7 @@ func (cs *LuetCompiler) FromDatabase(db types.PackageDatabase, minimum bool, dst
}
}
func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec, db types.PackageDatabase) (types.PackagesAssertions, error) {
func (cs *LuetCompiler) ComputeDepTree(p *types.LuetCompilationSpec, db types.PackageDatabase) (types.PackagesAssertions, error) {
s := solver.NewResolver(cs.Options.SolverOptions.SolverOptions, pkg.NewInMemoryDatabase(false), db, pkg.NewInMemoryDatabase(false), solver.NewSolverFromOptions(cs.Options.SolverOptions))
solution, err := s.Install(types.Packages{p.GetPackage()})
@@ -826,7 +825,7 @@ func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec, db t
// for _, l := range bt.AllLevels() {
// fmt.Println(strings.Join(bt.AllInLevel(l), " "))
// }
func (cs *LuetCompiler) BuildTree(compilerSpecs compilerspec.LuetCompilationspecs) (*BuildTree, error) {
func (cs *LuetCompiler) BuildTree(compilerSpecs types.LuetCompilationspecs) (*BuildTree, error) {
compilationTree := map[string]map[string]interface{}{}
bt := &BuildTree{}
@@ -866,11 +865,11 @@ func (cs *LuetCompiler) BuildTree(compilerSpecs compilerspec.LuetCompilationspec
}
// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) {
func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*types.LuetCompilationSpec) ([]*types.LuetCompilationSpec, error) {
// Generate a set with all the deps of the provided specs
// we will use that set to remove the deps from the list of provided compilation specs
allDependencies := types.PackagesAssertions{} // Get all packages that will be in deps
result := []*compilerspec.LuetCompilationSpec{}
result := []*types.LuetCompilationSpec{}
for _, spec := range p {
sol, err := cs.ComputeDepTree(spec, cs.Database)
if err != nil {
@@ -889,7 +888,7 @@ func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompi
// Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates
// an artifact
func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
func (cs *LuetCompiler) Compile(keepPermissions bool, p *types.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
return cs.compile(cs.Options.Concurrency, keepPermissions, nil, nil, p)
}
@@ -901,7 +900,7 @@ func genImageList(refs []string, hash string) []string {
return res
}
func (cs *LuetCompiler) inheritSpecBuildOptions(p *compilerspec.LuetCompilationSpec) {
func (cs *LuetCompiler) inheritSpecBuildOptions(p *types.LuetCompilationSpec) {
cs.Options.Context.Debug(p.GetPackage().HumanReadableString(), "Build options before inherit", p.BuildOptions)
// Append push repositories from buildpsec buildoptions as pull if found.
@@ -940,7 +939,7 @@ func (cs *LuetCompiler) getSpecHash(pkgs types.Packages, salt string) (string, e
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool, p *types.LuetCompilationSpec) error {
if !p.RequiresFinalImages {
return nil
}
@@ -1102,8 +1101,8 @@ func (cs *LuetCompiler) resolveFinalImages(concurrency int, keepPermissions bool
return nil
}
func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) error {
resolvedCopyFields := []compilerspec.CopyField{}
func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions bool, p *types.LuetCompilationSpec) error {
resolvedCopyFields := []types.CopyField{}
copyTag := ">:droplet: copy<"
if len(p.Copy) != 0 {
@@ -1131,7 +1130,7 @@ func (cs *LuetCompiler) resolveMultiStageImages(concurrency int, keepPermissions
return errors.Wrap(err, "failed building multi-stage image")
}
resolvedCopyFields = append(resolvedCopyFields, compilerspec.CopyField{
resolvedCopyFields = append(resolvedCopyFields, types.CopyField{
Image: cs.resolveExistingImageHash(artifact.PackageCacheImage, spec),
Source: c.Source,
Destination: c.Destination,
@@ -1175,7 +1174,7 @@ func CompilerFinalImages(cs *LuetCompiler) (*LuetCompiler, error) {
return copy, nil
}
func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateFinalArtifact *bool, generateDependenciesFinalArtifact *bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateFinalArtifact *bool, generateDependenciesFinalArtifact *bool, p *types.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
cs.Options.Context.Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
//Before multistage : join - same as multistage, but keep artifacts, join them, create a new one and generate a final image.
@@ -1212,7 +1211,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
targetAssertion := packageHashTree.Target
bus.Manager.Publish(bus.EventPackagePreBuild, struct {
CompileSpec *compilerspec.LuetCompilationSpec
CompileSpec *types.LuetCompilationSpec
Assert types.PackageAssert
PackageHashTree *PackageImageHashTree
}{
@@ -1282,7 +1281,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
compileSpec.SetOutputPath(p.GetOutputPath())
bus.Manager.Publish(bus.EventPackagePreBuild, struct {
CompileSpec *compilerspec.LuetCompilationSpec
CompileSpec *types.LuetCompilationSpec
Assert types.PackageAssert
}{
CompileSpec: compileSpec,
@@ -1337,7 +1336,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
cs.Options.Context.Success(pkgTag, ":white_check_mark: Done")
bus.Manager.Publish(bus.EventPackagePostBuild, struct {
CompileSpec *compilerspec.LuetCompilationSpec
CompileSpec *types.LuetCompilationSpec
Artifact *artifact.PackageArtifact
}{
CompileSpec: compileSpec,
@@ -1364,7 +1363,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, generateF
a.SourceAssertion = p.GetSourceAssertion()
a.PackageCacheImage = targetAssertion.Hash.PackageHash
bus.Manager.Publish(bus.EventPackagePostBuild, struct {
CompileSpec *compilerspec.LuetCompilationSpec
CompileSpec *types.LuetCompilationSpec
Artifact *artifact.PackageArtifact
}{
CompileSpec: p,
@@ -1463,14 +1462,14 @@ func (cs *LuetCompiler) templatePackage(vals []map[string]interface{}, pack *typ
}
// FromPackage returns a compilation spec from a package definition
func (cs *LuetCompiler) FromPackage(p *types.Package) (*compilerspec.LuetCompilationSpec, error) {
func (cs *LuetCompiler) FromPackage(p *types.Package) (*types.LuetCompilationSpec, error) {
// This would be nice to move it out from the compiler, but it is strictly tight to it given the build options
pack, err := cs.Database.FindPackageCandidate(p)
if err != nil {
return nil, err
}
opts := options.Compiler{}
opts := types.CompilerOptions{}
artifactMetadataFile := filepath.Join(pack.GetTreeDir(), "..", pack.GetMetadataFilePath())
cs.Options.Context.Debug("Checking if metadata file is present", artifactMetadataFile)
@@ -1498,6 +1497,31 @@ func (cs *LuetCompiler) FromPackage(p *types.Package) (*compilerspec.LuetCompila
cs.Options.Context.Debug("metadata file not present, skipping", artifactMetadataFile)
}
// If the input is a dockerfile, just consume it and parse any image source from it
if pack.OriginDockerfile != "" {
img := ""
// TODO: Carry this info and parse Dockerfile from somewhere else?
cmds, err := dockerfile.ParseReader(bytes.NewBufferString(pack.OriginDockerfile))
if err != nil {
return nil, errors.Wrap(err, "could not decode Dockerfile")
}
for _, c := range cmds {
if c.Cmd == "FROM" &&
len(c.Value) > 0 && !strings.Contains(strings.ToLower(fmt.Sprint(c.Value)), "as") {
img = c.Value[0]
}
}
compilationSpec := &types.LuetCompilationSpec{
Image: img,
Package: pack,
BuildOptions: &types.CompilerOptions{},
}
cs.inheritSpecBuildOptions(compilationSpec)
return compilationSpec, nil
}
// Update processed build values
dst, err := template.UnMarshalValues(cs.Options.BuildValuesFile)
if err != nil {
@@ -1510,7 +1534,7 @@ func (cs *LuetCompiler) FromPackage(p *types.Package) (*compilerspec.LuetCompila
return nil, errors.Wrap(err, "while rendering package template")
}
newSpec, err := compilerspec.NewLuetCompilationSpec(bytes, pack)
newSpec, err := types.NewLuetCompilationSpec(bytes, pack)
if err != nil {
return nil, err
}

68
pkg/compiler/compiler_test.go
View File

@@ -32,9 +32,7 @@ import (
"github.com/mudler/luet/pkg/api/core/types/artifact"
. "github.com/mudler/luet/pkg/compiler"
sd "github.com/mudler/luet/pkg/compiler/backend"
"github.com/mudler/luet/pkg/compiler/types/compression"
"github.com/mudler/luet/pkg/compiler/types/options"
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
pkg "github.com/mudler/luet/pkg/database"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
"github.com/mudler/luet/pkg/tree"
@@ -174,7 +172,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
spec2.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec, spec2))
Expect(errs).To(BeNil())
for _, artifact := range artifacts {
Expect(fileHelper.Exists(artifact.Path)).To(BeTrue())
@@ -235,7 +233,7 @@ var _ = Describe("Compiler", func() {
spec2.SetOutputPath(tmpdir)
spec3.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec, spec2, spec3))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(3))
@@ -281,11 +279,11 @@ var _ = Describe("Compiler", func() {
Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
spec2.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
artifacts2, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec2))
artifacts2, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec2))
Expect(errs).To(BeNil())
Expect(len(artifacts2)).To(Equal(1))
@@ -324,7 +322,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
@@ -357,7 +355,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
@@ -392,7 +390,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
@@ -426,7 +424,7 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
@@ -459,7 +457,7 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
@@ -492,7 +490,7 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
@@ -530,7 +528,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
@@ -570,7 +568,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
@@ -613,7 +611,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
@@ -654,7 +652,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileWithReverseDeps(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileWithReverseDeps(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(2))
@@ -683,7 +681,7 @@ var _ = Describe("Compiler", func() {
spec, err := compiler.FromPackage(&types.Package{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
Expect(err).ToNot(HaveOccurred())
bt, err := compiler.BuildTree(compilerspec.LuetCompilationspecs{*spec})
bt, err := compiler.BuildTree(types.LuetCompilationspecs{*spec})
Expect(err).ToNot(HaveOccurred())
Expect(bt.AllLevels()).To(Equal([]int{0, 1, 2, 3, 4, 5}))
@@ -716,7 +714,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(6))
@@ -751,7 +749,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileWithReverseDeps(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileWithReverseDeps(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(4))
@@ -810,7 +808,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
for _, a := range artifacts {
Expect(fileHelper.Exists(a.Path)).To(BeTrue())
@@ -852,7 +850,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
@@ -889,7 +887,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec, spec2))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(2))
//Expect(len(artifacts[0].Dependencies)).To(Equal(1))
@@ -961,7 +959,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
spec2.SetOutputPath(tmpdir2)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec, spec2))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(2))
Expect(len(artifacts[0].Dependencies)).To(Equal(0))
@@ -990,7 +988,7 @@ var _ = Describe("Compiler", func() {
spec, err := compiler.FromPackage(&types.Package{Name: "runtime", Category: "layer", Version: "0.1"})
Expect(err).ToNot(HaveOccurred())
compiler.Options.CompressionType = compression.GZip
compiler.Options.CompressionType = types.GZip
Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
tmpdir, err := ioutil.TempDir("", "tree")
@@ -999,7 +997,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
@@ -1044,7 +1042,7 @@ var _ = Describe("Compiler", func() {
spec, err := compiler.FromPackage(&types.Package{Name: "runtime", Category: "layer", Version: "0.1"})
Expect(err).ToNot(HaveOccurred())
compiler.Options.CompressionType = compression.GZip
compiler.Options.CompressionType = types.GZip
Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
tmpdir, err := ioutil.TempDir("", "tree")
@@ -1053,7 +1051,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
@@ -1080,7 +1078,7 @@ var _ = Describe("Compiler", func() {
spec, err := compiler.FromPackage(&types.Package{Name: "runtime", Category: "layer", Version: "0.1"})
Expect(err).ToNot(HaveOccurred())
compiler.Options.CompressionType = compression.GZip
compiler.Options.CompressionType = types.GZip
Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
tmpdir, err := ioutil.TempDir("", "tree")
@@ -1089,7 +1087,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
@@ -1143,7 +1141,7 @@ var _ = Describe("Compiler", func() {
b := sd.NewSimpleDockerBackend(ctx)
joinImage := "luet/cache:08738767caa9a7397fad70ae53db85fa" //resulting join image
joinImage := "luet/cache:c4224fd8279e077727573703b6db70d4" //resulting join image
allImages := []string{
joinImage,
"test/test:c-test-1.2"}
@@ -1165,7 +1163,7 @@ var _ = Describe("Compiler", func() {
spec, err := compiler.FromPackage(&types.Package{Name: "x", Category: "test", Version: "0.1"})
Expect(err).ToNot(HaveOccurred())
compiler.Options.CompressionType = compression.GZip
compiler.Options.CompressionType = types.GZip
Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
tmpdir, err := ioutil.TempDir("", "tree")
@@ -1174,7 +1172,7 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
@@ -1183,12 +1181,12 @@ var _ = Describe("Compiler", func() {
ContainSubstring("Generating final image for"),
ContainSubstring("Adding dependency"),
ContainSubstring("Final image not found for test/c-1.2"),
))
), log)
Expect(log).ToNot(And(
ContainSubstring("No runtime db present, first level join only"),
ContainSubstring("Final image already found test/test:c-test-1.2"),
))
), log)
os.WriteFile(logPath, []byte{}, os.ModePerm) // cleanup logs
// Remove the join hash so we force using final images
@@ -1197,7 +1195,7 @@ var _ = Describe("Compiler", func() {
//compile again
By("Recompiling")
artifacts, errs = compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
artifacts, errs = compiler.CompileParallel(false, types.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))

7
pkg/compiler/imagehashtree.go
View File

@@ -19,7 +19,6 @@ import (
"fmt"
"github.com/mudler/luet/pkg/api/core/types"
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
"github.com/pkg/errors"
)
@@ -75,7 +74,7 @@ func (ht *PackageImageHashTree) String() string {
// Query takes a compiler and a compilation spec and returns a PackageImageHashTree tied to it.
// PackageImageHashTree contains all the informations to resolve the spec build images in order to
// reproducibly re-build images from packages
func (ht *ImageHashTree) Query(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (*PackageImageHashTree, error) {
func (ht *ImageHashTree) Query(cs *LuetCompiler, p *types.LuetCompilationSpec) (*PackageImageHashTree, error) {
assertions, err := ht.resolve(cs, p)
if err != nil {
return nil, err
@@ -110,7 +109,7 @@ func (ht *ImageHashTree) Query(cs *LuetCompiler, p *compilerspec.LuetCompilation
}, nil
}
func (ht *ImageHashTree) genBuilderImageTag(p *compilerspec.LuetCompilationSpec, packageImage string) string {
func (ht *ImageHashTree) genBuilderImageTag(p *types.LuetCompilationSpec, packageImage string) string {
// Use packageImage as salt into the fp being used
// so the hash is unique also in cases where
// some package deps does have completely different
@@ -120,7 +119,7 @@ func (ht *ImageHashTree) genBuilderImageTag(p *compilerspec.LuetCompilationSpec,
// resolve computes the dependency tree of a compilation spec and returns solver assertions
// in order to be able to compile the spec.
func (ht *ImageHashTree) resolve(cs *LuetCompiler, p *compilerspec.LuetCompilationSpec) (types.PackagesAssertions, error) {
func (ht *ImageHashTree) resolve(cs *LuetCompiler, p *types.LuetCompilationSpec) (types.PackagesAssertions, error) {
dependencies, err := cs.ComputeDepTree(p, cs.Database)
if err != nil {
return nil, errors.Wrap(err, "While computing a solution for "+p.GetPackage().HumanReadableString())

56
pkg/compiler/imagehashtree_test.go
View File

@@ -21,7 +21,6 @@ import (
"github.com/mudler/luet/pkg/api/core/context"
. "github.com/mudler/luet/pkg/compiler"
sd "github.com/mudler/luet/pkg/compiler/backend"
"github.com/mudler/luet/pkg/compiler/types/options"
pkg "github.com/mudler/luet/pkg/database"
"github.com/mudler/luet/pkg/tree"
. "github.com/onsi/ginkgo/v2"
@@ -31,14 +30,14 @@ import (
var _ = Describe("ImageHashTree", func() {
ctx := context.NewContext()
generalRecipe := tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), Concurrency(2))
hashtree := NewHashTree(generalRecipe.GetDatabase())
Context("Simple package definition", func() {
BeforeEach(func() {
generalRecipe = tree.NewCompilerRecipe(pkg.NewInMemoryDatabase(false))
err := generalRecipe.Load("../../tests/fixtures/buildable")
Expect(err).ToNot(HaveOccurred())
compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), Concurrency(2))
hashtree = NewHashTree(generalRecipe.GetDatabase())
})
@@ -50,13 +49,13 @@ var _ = Describe("ImageHashTree", func() {
packageHash, err := hashtree.Query(compiler, spec)
Expect(err).ToNot(HaveOccurred())
Expect(packageHash.Target.Hash.BuildHash).To(Equal("895697a8bb51b219b78ed081fa1b778801e81505bb03f56acafcf3c476620fc1"))
Expect(packageHash.Target.Hash.PackageHash).To(Equal("2a6c3dc0dd7af2902fd8823a24402d89b2030cfbea6e63fe81afb34af8b1a005"))
Expect(packageHash.BuilderImageHash).To(Equal("builder-3a28d240f505d69123735a567beaf80e"))
Expect(packageHash.Target.Hash.BuildHash).To(Equal("bf767dba10e4aa9c25e09f1f61ed9944b8e4736f72b2a1f9ac0125f68a714580"), packageHash.Target.Hash.BuildHash)
Expect(packageHash.Target.Hash.PackageHash).To(Equal("6ce76e1a85f02841db083e59d4f9d3e4ab16154f925c1d81014c4938a6b1b1f9"), packageHash.Target.Hash.PackageHash)
Expect(packageHash.BuilderImageHash).To(Equal("builder-4ba2735d6368f56627776f8fb8ce6a16"), packageHash.BuilderImageHash)
})
})
expectedPackageHash := "4154ad4e5dfa2aea41292b3c49eeb04ef327456ecb6312f12d7b94d18ac8cb64"
expectedPackageHash := "562b4295b87d561af237997e1320560ee9495a02f69c3c77391b783d2e01ced2"
Context("complex package definition", func() {
BeforeEach(func() {
@@ -64,7 +63,7 @@ var _ = Describe("ImageHashTree", func() {
err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision")
Expect(err).ToNot(HaveOccurred())
compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), Concurrency(2))
hashtree = NewHashTree(generalRecipe.GetDatabase())
})
@@ -75,19 +74,19 @@ var _ = Describe("ImageHashTree", func() {
packageHash, err := hashtree.Query(compiler, spec)
Expect(err).ToNot(HaveOccurred())
expectedHash := "b4b61939260263582da1dfa5289182a0a7570ef8658f3b01b1997fe5d8a95e49"
expectedHash := "c5b87e16b2ecafc67e671d8e2c38adf4c4a6eed2a80180229d5892d52e81779b"
Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(expectedPackageHash))
Expect(packageHash.SourceHash).To(Equal(expectedPackageHash))
Expect(packageHash.BuilderImageHash).To(Equal("builder-381bd2ad9abe1ac6c3c26cba8f8cca0b"))
Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(expectedPackageHash), packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash)
Expect(packageHash.SourceHash).To(Equal(expectedPackageHash), packageHash.SourceHash)
Expect(packageHash.BuilderImageHash).To(Equal("builder-d934bd6bbf716f5d598d764532bc585c"), packageHash.BuilderImageHash)
//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
Expect(packageHash.Target.Hash.PackageHash).To(Equal("3a372fcee17b2c7912eabb04b50f7d5a83e75402da0c96c102f7c2e836ebaa10"))
Expect(packageHash.Target.Hash.PackageHash).To(Equal("78cace3ee661d14cb2b6236df3dcdc789e36c26a1701ba3e0213e355540a1174"), packageHash.Target.Hash.PackageHash)
a := &types.Package{Name: "a", Category: "test", Version: "1.1"}
hash, err := packageHash.DependencyBuildImage(a)
Expect(err).ToNot(HaveOccurred())
Expect(hash).To(Equal(expectedHash))
Expect(hash).To(Equal(expectedHash), hash)
assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
Expect(assertionA.Hash.PackageHash).To(Equal(expectedPackageHash))
@@ -98,7 +97,7 @@ var _ = Describe("ImageHashTree", func() {
hashB, err := packageHash.DependencyBuildImage(b)
Expect(err).ToNot(HaveOccurred())
Expect(hashB).To(Equal("fc6fdd4bd62d51fc06c2c22e8bc56543727a2340220972594e28c623ea3a9c6c"))
Expect(hashB).To(Equal("9ece11c782e862e366ab4b42fdaaea9d89abe41ff4d9ed1bd24c81f6041bc9da"), hashB)
})
})
@@ -109,7 +108,7 @@ var _ = Describe("ImageHashTree", func() {
//Definition of A here is slightly changed in the steps build.yaml file (1 character only)
err := generalRecipe.Load("../../tests/fixtures/upgrade_old_repo_revision_content_changed")
Expect(err).ToNot(HaveOccurred())
compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), options.Concurrency(2))
compiler = NewLuetCompiler(sd.NewSimpleDockerBackend(ctx), generalRecipe.GetDatabase(), Concurrency(2))
hashtree = NewHashTree(generalRecipe.GetDatabase())
})
@@ -119,36 +118,35 @@ var _ = Describe("ImageHashTree", func() {
packageHash, err := hashtree.Query(compiler, spec)
Expect(err).ToNot(HaveOccurred())
Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).ToNot(Equal(expectedPackageHash))
sourceHash := "5534399abed19a3c93b0e638811a5ba6d07e68f6782e2b40aaf2b09c408a3154"
Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(sourceHash))
Expect(packageHash.SourceHash).To(Equal(sourceHash))
Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).ToNot(Equal(expectedPackageHash), packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash)
sourceHash := "726635a86f03483c432e33d80ba85443cf30453960826bd813d816786f712bcf"
Expect(packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash).To(Equal(sourceHash), packageHash.Dependencies[len(packageHash.Dependencies)-1].Hash.PackageHash)
Expect(packageHash.SourceHash).To(Equal(sourceHash), packageHash.SourceHash)
Expect(packageHash.SourceHash).ToNot(Equal(expectedPackageHash), packageHash.SourceHash)
Expect(packageHash.SourceHash).ToNot(Equal(expectedPackageHash))
Expect(packageHash.BuilderImageHash).To(Equal("builder-2a3905cf55bdcd1e4cea6b128cbf5b3a"))
Expect(packageHash.BuilderImageHash).To(Equal("builder-d326b367b72ae030a545e8713d45c9aa"), packageHash.BuilderImageHash)
//Expect(packageHash.Target.Hash.BuildHash).To(Equal("79d7107d13d578b362e6a7bf10ec850efce26316405b8d732ce8f9e004d64281"))
Expect(packageHash.Target.Hash.PackageHash).To(Equal("4a13154de2e802fbd250236294562fad8c9f2c51ab8a3fc359323dd1ed064907"))
Expect(packageHash.Target.Hash.PackageHash).To(Equal("e99b996d2ae378e901668b2f56b184af694fe1f1bc92544a2813d6102738098d"), packageHash.Target.Hash.PackageHash)
a := &types.Package{Name: "a", Category: "test", Version: "1.1"}
hash, err := packageHash.DependencyBuildImage(a)
Expect(err).ToNot(HaveOccurred())
Expect(hash).To(Equal("b4b61939260263582da1dfa5289182a0a7570ef8658f3b01b1997fe5d8a95e49"))
Expect(hash).To(Equal("c5b87e16b2ecafc67e671d8e2c38adf4c4a6eed2a80180229d5892d52e81779b"), hash)
assertionA := packageHash.Dependencies.Search(a.GetFingerPrint())
Expect(assertionA.Hash.PackageHash).To(Equal("5534399abed19a3c93b0e638811a5ba6d07e68f6782e2b40aaf2b09c408a3154"))
Expect(assertionA.Hash.PackageHash).ToNot(Equal(expectedPackageHash))
Expect(assertionA.Hash.PackageHash).To(Equal("726635a86f03483c432e33d80ba85443cf30453960826bd813d816786f712bcf"), assertionA.Hash.PackageHash)
Expect(assertionA.Hash.PackageHash).ToNot(Equal(expectedPackageHash), assertionA.Hash.PackageHash)
b := &types.Package{Name: "b", Category: "test", Version: "1.0"}
assertionB := packageHash.Dependencies.Search(b.GetFingerPrint())
Expect(assertionB.Hash.PackageHash).To(Equal("b4b61939260263582da1dfa5289182a0a7570ef8658f3b01b1997fe5d8a95e49"))
Expect(assertionB.Hash.PackageHash).To(Equal("c5b87e16b2ecafc67e671d8e2c38adf4c4a6eed2a80180229d5892d52e81779b"), assertionB.Hash.PackageHash)
hashB, err := packageHash.DependencyBuildImage(b)
Expect(err).ToNot(HaveOccurred())
Expect(hashB).To(Equal("fc6fdd4bd62d51fc06c2c22e8bc56543727a2340220972594e28c623ea3a9c6c"))
Expect(hashB).To(Equal("9ece11c782e862e366ab4b42fdaaea9d89abe41ff4d9ed1bd24c81f6041bc9da"), hashB)
})
})

208
pkg/compiler/options.go Normal file
View File

@@ -0,0 +1,208 @@
// Copyright © 2022 Ettore Di Giacinto <mudler@mocaccino.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package compiler
import (
"runtime"
"github.com/mudler/luet/pkg/api/core/types"
)
func newDefaultCompiler() *types.CompilerOptions {
return &types.CompilerOptions{
PushImageRepository: "luet/cache",
PullFirst: false,
Push: false,
CompressionType: types.None,
KeepImg: true,
Concurrency: runtime.NumCPU(),
OnlyDeps: false,
NoDeps: false,
SolverOptions: types.LuetSolverOptions{SolverOptions: types.SolverOptions{Concurrency: 1, Type: types.SolverSingleCoreSimple}},
}
}
func WithOptions(opt *types.CompilerOptions) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg = opt
return nil
}
}
func WithRuntimeDatabase(db types.PackageDatabase) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.RuntimeDatabase = db
return nil
}
}
// WithFinalRepository Sets the final repository where to push
// images of built artifacts
func WithFinalRepository(r string) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.PushFinalImagesRepository = r
return nil
}
}
func EnableGenerateFinalImages(cfg *types.CompilerOptions) error {
cfg.GenerateFinalImages = true
return nil
}
func EnablePushFinalImages(cfg *types.CompilerOptions) error {
cfg.PushFinalImages = true
return nil
}
func ForcePushFinalImages(cfg *types.CompilerOptions) error {
cfg.PushFinalImagesForce = true
return nil
}
func WithBackendType(r string) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.BackendType = r
return nil
}
}
func WithTemplateFolder(r []string) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.TemplatesFolder = r
return nil
}
}
func WithBuildValues(r []string) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.BuildValuesFile = r
return nil
}
}
func WithPullRepositories(r []string) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.PullImageRepository = r
return nil
}
}
// WithPushRepository Sets the image reference where to push
// cache images
func WithPushRepository(r string) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
if len(cfg.PullImageRepository) == 0 {
cfg.PullImageRepository = []string{cfg.PushImageRepository}
}
cfg.PushImageRepository = r
return nil
}
}
func BackendArgs(r []string) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.BackendArgs = r
return nil
}
}
func PullFirst(b bool) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.PullFirst = b
return nil
}
}
func KeepImg(b bool) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.KeepImg = b
return nil
}
}
func Rebuild(b bool) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.Rebuild = b
return nil
}
}
func PushImages(b bool) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.Push = b
return nil
}
}
func Wait(b bool) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.Wait = b
return nil
}
}
func OnlyDeps(b bool) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.OnlyDeps = b
return nil
}
}
func OnlyTarget(b bool) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.PackageTargetOnly = b
return nil
}
}
func NoDeps(b bool) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.NoDeps = b
return nil
}
}
func Concurrency(i int) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
if i == 0 {
i = runtime.NumCPU()
}
cfg.Concurrency = i
return nil
}
}
func WithCompressionType(t types.CompressionImplementation) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.CompressionType = t
return nil
}
}
func WithSolverOptions(c types.LuetSolverOptions) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.SolverOptions = c
return nil
}
}
func WithContext(c types.Context) func(cfg *types.CompilerOptions) error {
return func(cfg *types.CompilerOptions) error {
cfg.Context = c
return nil
}
}

9
pkg/compiler/types/compression/compression.go
View File

@@ -1,9 +0,0 @@
package compression
type Implementation string
const (
None Implementation = "none" // e.g. tar for standard packages
GZip Implementation = "gzip"
Zstandard Implementation = "zstd"
)

260
pkg/compiler/types/options/compiler_options.go
View File

@@ -1,260 +0,0 @@
// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package options
import (
"runtime"
"github.com/mudler/luet/pkg/api/core/types"
"github.com/mudler/luet/pkg/compiler/types/compression"
)
type Compiler struct {
PushImageRepository string
PullImageRepository []string
PullFirst, KeepImg, Push bool
Concurrency int
CompressionType compression.Implementation
Wait bool
OnlyDeps bool
NoDeps bool
SolverOptions types.LuetSolverOptions
BuildValuesFile []string
BuildValues []map[string]interface{}
PackageTargetOnly bool
Rebuild bool
BackendArgs []string
BackendType string
// TemplatesFolder. should default to tree/templates
TemplatesFolder []string
// Tells wether to push final container images after building
PushFinalImages bool
PushFinalImagesForce bool
GenerateFinalImages bool
// Image repository to push to
PushFinalImagesRepository string
RuntimeDatabase types.PackageDatabase
Context types.Context
}
func NewDefaultCompiler() *Compiler {
return &Compiler{
PushImageRepository: "luet/cache",
PullFirst: false,
Push: false,
CompressionType: compression.None,
KeepImg: true,
Concurrency: runtime.NumCPU(),
OnlyDeps: false,
NoDeps: false,
SolverOptions: types.LuetSolverOptions{SolverOptions: types.SolverOptions{Concurrency: 1, Type: types.SolverSingleCoreSimple}},
}
}
type Option func(cfg *Compiler) error
func (cfg *Compiler) Apply(opts ...Option) error {
for _, opt := range opts {
if opt == nil {
continue
}
if err := opt(cfg); err != nil {
return err
}
}
return nil
}
func WithOptions(opt *Compiler) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg = opt
return nil
}
}
func WithRuntimeDatabase(db types.PackageDatabase) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.RuntimeDatabase = db
return nil
}
}
// WithFinalRepository Sets the final repository where to push
// images of built artifacts
func WithFinalRepository(r string) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.PushFinalImagesRepository = r
return nil
}
}
func EnableGenerateFinalImages(cfg *Compiler) error {
cfg.GenerateFinalImages = true
return nil
}
func EnablePushFinalImages(cfg *Compiler) error {
cfg.PushFinalImages = true
return nil
}
func ForcePushFinalImages(cfg *Compiler) error {
cfg.PushFinalImagesForce = true
return nil
}
func WithBackendType(r string) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.BackendType = r
return nil
}
}
func WithTemplateFolder(r []string) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.TemplatesFolder = r
return nil
}
}
func WithBuildValues(r []string) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.BuildValuesFile = r
return nil
}
}
func WithPullRepositories(r []string) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.PullImageRepository = r
return nil
}
}
// WithPushRepository Sets the image reference where to push
// cache images
func WithPushRepository(r string) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
if len(cfg.PullImageRepository) == 0 {
cfg.PullImageRepository = []string{cfg.PushImageRepository}
}
cfg.PushImageRepository = r
return nil
}
}
func BackendArgs(r []string) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.BackendArgs = r
return nil
}
}
func PullFirst(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.PullFirst = b
return nil
}
}
func KeepImg(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.KeepImg = b
return nil
}
}
func Rebuild(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.Rebuild = b
return nil
}
}
func PushImages(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.Push = b
return nil
}
}
func Wait(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.Wait = b
return nil
}
}
func OnlyDeps(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.OnlyDeps = b
return nil
}
}
func OnlyTarget(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.PackageTargetOnly = b
return nil
}
}
func NoDeps(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.NoDeps = b
return nil
}
}
func Concurrency(i int) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
if i == 0 {
i = runtime.NumCPU()
}
cfg.Concurrency = i
return nil
}
}
func WithCompressionType(t compression.Implementation) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.CompressionType = t
return nil
}
}
func WithSolverOptions(c types.LuetSolverOptions) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.SolverOptions = c
return nil
}
}
func WithContext(c types.Context) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.Context = c
return nil
}
}

385
pkg/compiler/types/spec/spec.go
View File

@@ -1,385 +0,0 @@
// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package compilerspec
import (
"fmt"
"io/ioutil"
"path/filepath"
"github.com/mitchellh/hashstructure/v2"
"github.com/mudler/luet/pkg/api/core/types"
options "github.com/mudler/luet/pkg/compiler/types/options"
"github.com/ghodss/yaml"
"github.com/otiai10/copy"
dirhash "golang.org/x/mod/sumdb/dirhash"
)
type LuetCompilationspecs []LuetCompilationSpec
func NewLuetCompilationspecs(s ...*LuetCompilationSpec) *LuetCompilationspecs {
all := LuetCompilationspecs{}
for _, spec := range s {
all.Add(spec)
}
return &all
}
func (specs LuetCompilationspecs) Len() int {
return len(specs)
}
func (specs *LuetCompilationspecs) Remove(s *LuetCompilationspecs) *LuetCompilationspecs {
newSpecs := LuetCompilationspecs{}
SPECS:
for _, spec := range specs.All() {
for _, target := range s.All() {
if target.GetPackage().Matches(spec.GetPackage()) {
continue SPECS
}
}
newSpecs.Add(spec)
}
return &newSpecs
}
func (specs *LuetCompilationspecs) Add(s *LuetCompilationSpec) {
*specs = append(*specs, *s)
}
func (specs *LuetCompilationspecs) All() []*LuetCompilationSpec {
var cspecs []*LuetCompilationSpec
for i := range *specs {
f := (*specs)[i]
cspecs = append(cspecs, &f)
}
return cspecs
}
func (specs *LuetCompilationspecs) Unique() *LuetCompilationspecs {
newSpecs := LuetCompilationspecs{}
seen := map[string]bool{}
for i := range *specs {
j := (*specs)[i]
_, ok := seen[j.GetPackage().GetFingerPrint()]
if !ok {
seen[j.GetPackage().GetFingerPrint()] = true
newSpecs = append(newSpecs, j)
}
}
return &newSpecs
}
type CopyField struct {
Package *types.Package `json:"package"`
Image string `json:"image"`
Source string `json:"source"`
Destination string `json:"destination"`
}
type LuetCompilationSpec struct {
Steps []string `json:"steps"` // Are run inside a container and the result layer diff is saved
Env []string `json:"env"`
Prelude []string `json:"prelude"` // Are run inside the image which will be our builder
Image string `json:"image"`
Seed string `json:"seed"`
Package *types.Package `json:"package"`
SourceAssertion types.PackagesAssertions `json:"-"`
PackageDir string `json:"package_dir" yaml:"package_dir"`
Retrieve []string `json:"retrieve"`
OutputPath string `json:"-"` // Where the build processfiles go
Unpack bool `json:"unpack"`
Includes []string `json:"includes"`
Excludes []string `json:"excludes"`
BuildOptions *options.Compiler `json:"build_options"`
Copy []CopyField `json:"copy"`
RequiresFinalImages bool `json:"requires_final_images" yaml:"requires_final_images"`
}
// Signature is a portion of the spec that yields a signature for the hash
type Signature struct {
Image string
Steps []string
PackageDir string
Prelude []string
Seed string
Env []string
Retrieve []string
Unpack bool
Includes []string
Excludes []string
Copy []CopyField
Requires types.Packages
RequiresFinalImages bool
}
func (cs *LuetCompilationSpec) signature() Signature {
return Signature{
Image: cs.Image,
Steps: cs.Steps,
PackageDir: cs.PackageDir,
Prelude: cs.Prelude,
Seed: cs.Seed,
Env: cs.Env,
Retrieve: cs.Retrieve,
Unpack: cs.Unpack,
Includes: cs.Includes,
Excludes: cs.Excludes,
Copy: cs.Copy,
Requires: cs.Package.GetRequires(),
RequiresFinalImages: cs.RequiresFinalImages,
}
}
func NewLuetCompilationSpec(b []byte, p *types.Package) (*LuetCompilationSpec, error) {
var spec LuetCompilationSpec
var packageDefinition types.Package
err := yaml.Unmarshal(b, &spec)
if err != nil {
return &spec, err
}
err = yaml.Unmarshal(b, &packageDefinition)
if err != nil {
return &spec, err
}
// Update requires/conflict/provides
// When we have been passed a bytes slice, parse it as a package
// and updates requires/conflicts/provides.
// This is required in order to allow manipulation of such fields with templating
copy := *p
spec.Package = &copy
if len(packageDefinition.GetRequires()) != 0 {
spec.Package.Requires(packageDefinition.GetRequires())
}
if len(packageDefinition.GetConflicts()) != 0 {
spec.Package.Conflicts(packageDefinition.GetConflicts())
}
if len(packageDefinition.GetProvides()) != 0 {
spec.Package.SetProvides(packageDefinition.GetProvides())
}
return &spec, nil
}
func (cs *LuetCompilationSpec) GetSourceAssertion() types.PackagesAssertions {
return cs.SourceAssertion
}
func (cs *LuetCompilationSpec) SetBuildOptions(b options.Compiler) {
cs.BuildOptions = &b
}
func (cs *LuetCompilationSpec) SetSourceAssertion(as types.PackagesAssertions) {
cs.SourceAssertion = as
}
func (cs *LuetCompilationSpec) GetPackage() *types.Package {
return cs.Package
}
func (cs *LuetCompilationSpec) GetPackageDir() string {
return cs.PackageDir
}
func (cs *LuetCompilationSpec) SetPackageDir(s string) {
cs.PackageDir = s
}
func (cs *LuetCompilationSpec) BuildSteps() []string {
return cs.Steps
}
func (cs *LuetCompilationSpec) ImageUnpack() bool {
return cs.Unpack
}
func (cs *LuetCompilationSpec) GetPreBuildSteps() []string {
return cs.Prelude
}
func (cs *LuetCompilationSpec) GetIncludes() []string {
return cs.Includes
}
func (cs *LuetCompilationSpec) GetExcludes() []string {
return cs.Excludes
}
func (cs *LuetCompilationSpec) GetRetrieve() []string {
return cs.Retrieve
}
// IsVirtual returns true if the spec is virtual.
// A spec is virtual if the package is empty, and it has no image source to unpack from.
func (cs *LuetCompilationSpec) IsVirtual() bool {
return cs.EmptyPackage() && !cs.HasImageSource()
}
func (cs *LuetCompilationSpec) GetSeedImage() string {
return cs.Seed
}
func (cs *LuetCompilationSpec) GetImage() string {
return cs.Image
}
func (cs *LuetCompilationSpec) GetOutputPath() string {
return cs.OutputPath
}
func (p *LuetCompilationSpec) Rel(s string) string {
return filepath.Join(p.GetOutputPath(), s)
}
func (cs *LuetCompilationSpec) SetImage(s string) {
cs.Image = s
}
func (cs *LuetCompilationSpec) SetOutputPath(s string) {
cs.OutputPath = s
}
func (cs *LuetCompilationSpec) SetSeedImage(s string) {
cs.Seed = s
}
func (cs *LuetCompilationSpec) EmptyPackage() bool {
return len(cs.BuildSteps()) == 0 && !cs.UnpackedPackage()
}
func (cs *LuetCompilationSpec) UnpackedPackage() bool {
// If package_dir was specified in the spec, we want to treat the content of the directory
// as the root of our archive. ImageUnpack is implied to be true. override it
unpack := cs.ImageUnpack()
if cs.GetPackageDir() != "" {
unpack = true
}
return unpack
}
// HasImageSource returns true when the compilation spec has an image source.
// a compilation spec has an image source when it depends on other packages or have a source image
// explictly supplied
func (cs *LuetCompilationSpec) HasImageSource() bool {
return (cs.Package != nil && len(cs.GetPackage().GetRequires()) != 0) || cs.GetImage() != "" || (cs.RequiresFinalImages && len(cs.Package.GetRequires()) != 0)
}
func (cs *LuetCompilationSpec) Hash() (string, error) {
// build a signature, we want to be part of the hash only the fields that are relevant for build purposes
signature := cs.signature()
h, err := hashstructure.Hash(signature, hashstructure.FormatV2, nil)
if err != nil {
return "", err
}
sum, err := dirhash.HashDir(cs.Package.Path, "", dirhash.DefaultHash)
if err != nil {
return fmt.Sprint(h), err
}
return fmt.Sprint(h, sum), err
}
func (cs *LuetCompilationSpec) CopyRetrieves(dest string) error {
var err error
if len(cs.Retrieve) > 0 {
for _, s := range cs.Retrieve {
matches, err := filepath.Glob(cs.Rel(s))
if err != nil {
continue
}
for _, m := range matches {
err = copy.Copy(m, filepath.Join(dest, filepath.Base(m)))
}
}
}
return err
}
func (cs *LuetCompilationSpec) genDockerfile(image string, steps []string) string {
spec := `
FROM ` + image + `
COPY . /luetbuild
WORKDIR /luetbuild
ENV PACKAGE_NAME=` + cs.Package.GetName() + `
ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()
if len(cs.Retrieve) > 0 {
for _, s := range cs.Retrieve {
//var file string
// if helpers.IsValidUrl(s) {
// file = s
// } else {
// file = cs.Rel(s)
// }
spec = spec + `
ADD ` + s + ` /luetbuild/`
}
}
for _, c := range cs.Copy {
if c.Image != "" {
copyLine := fmt.Sprintf("\nCOPY --from=%s %s %s\n", c.Image, c.Source, c.Destination)
spec = spec + copyLine
}
}
for _, s := range cs.Env {
spec = spec + `
ENV ` + s
}
for _, s := range steps {
spec = spec + `
RUN ` + s
}
return spec
}
// RenderBuildImage renders the dockerfile of the image used as a pre-build step
func (cs *LuetCompilationSpec) RenderBuildImage() (string, error) {
return cs.genDockerfile(cs.GetSeedImage(), cs.GetPreBuildSteps()), nil
}
// RenderStepImage renders the dockerfile used for the image used for building the package
func (cs *LuetCompilationSpec) RenderStepImage(image string) (string, error) {
return cs.genDockerfile(image, cs.BuildSteps()), nil
}
func (cs *LuetCompilationSpec) WriteBuildImageDefinition(path string) error {
data, err := cs.RenderBuildImage()
if err != nil {
return err
}
return ioutil.WriteFile(path, []byte(data), 0644)
}
func (cs *LuetCompilationSpec) WriteStepImageDefinition(fromimage, path string) error {
data, err := cs.RenderStepImage(fromimage)
if err != nil {
return err
}
return ioutil.WriteFile(path, []byte(data), 0644)
}

28
pkg/compiler/types/spec/spec_suite_test.go
View File

@@ -1,28 +0,0 @@
// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package compilerspec_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestSpec(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Spec Suite")
}

259
pkg/compiler/types/spec/spec_test.go
View File

@@ -1,259 +0,0 @@
// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package compilerspec_test
import (
"io/ioutil"
"os"
"path/filepath"
"github.com/mudler/luet/pkg/api/core/types"
options "github.com/mudler/luet/pkg/compiler/types/options"
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
fileHelper "github.com/mudler/luet/pkg/helpers/file"
. "github.com/mudler/luet/pkg/compiler"
pkg "github.com/mudler/luet/pkg/database"
"github.com/mudler/luet/pkg/tree"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("Spec", func() {
Context("Luet specs", func() {
It("Allows normal operations", func() {
testSpec := &compilerspec.LuetCompilationSpec{Package: &types.Package{Name: "foo", Category: "a", Version: "0"}}
testSpec2 := &compilerspec.LuetCompilationSpec{Package: &types.Package{Name: "bar", Category: "a", Version: "0"}}
testSpec3 := &compilerspec.LuetCompilationSpec{Package: &types.Package{Name: "baz", Category: "a", Version: "0"}}
testSpec4 := &compilerspec.LuetCompilationSpec{Package: &types.Package{Name: "foo", Category: "a", Version: "0"}}
specs := compilerspec.NewLuetCompilationspecs(testSpec, testSpec2)
Expect(specs.Len()).To(Equal(2))
Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2}))
specs.Add(testSpec3)
Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3}))
specs.Add(testSpec4)
Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3, testSpec4}))
newSpec := specs.Unique()
Expect(newSpec.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3}))
newSpec2 := specs.Remove(compilerspec.NewLuetCompilationspecs(testSpec, testSpec2))
Expect(newSpec2.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec3}))
})
Context("virtuals", func() {
When("is empty", func() {
It("is virtual", func() {
spec := &compilerspec.LuetCompilationSpec{}
Expect(spec.IsVirtual()).To(BeTrue())
})
})
When("has defined steps", func() {
It("is not a virtual", func() {
spec := &compilerspec.LuetCompilationSpec{Steps: []string{"foo"}}
Expect(spec.IsVirtual()).To(BeFalse())
})
})
When("has defined image", func() {
It("is not a virtual", func() {
spec := &compilerspec.LuetCompilationSpec{Image: "foo"}
Expect(spec.IsVirtual()).To(BeFalse())
})
})
})
})
Context("Image hashing", func() {
It("is stable", func() {
spec1 := &compilerspec.LuetCompilationSpec{
Image: "foo",
BuildOptions: &options.Compiler{BuildValues: []map[string]interface{}{{"foo": "bar", "baz": true}}},
Package: &types.Package{
Name: "foo",
Category: "Bar",
Labels: map[string]string{
"foo": "bar",
"baz": "foo",
},
},
}
spec2 := &compilerspec.LuetCompilationSpec{
Image: "foo",
BuildOptions: &options.Compiler{BuildValues: []map[string]interface{}{{"foo": "bar", "baz": true}}},
Package: &types.Package{
Name: "foo",
Category: "Bar",
Labels: map[string]string{
"foo": "bar",
"baz": "foo",
},
},
}
spec3 := &compilerspec.LuetCompilationSpec{
Image: "foo",
Steps: []string{"foo"},
Package: &types.Package{
Name: "foo",
Category: "Bar",
Labels: map[string]string{
"foo": "bar",
"baz": "foo",
},
},
}
hash, err := spec1.Hash()
Expect(err).ToNot(HaveOccurred())
hash2, err := spec2.Hash()
Expect(err).ToNot(HaveOccurred())
hash3, err := spec3.Hash()
Expect(err).ToNot(HaveOccurred())
Expect(hash).To(Equal(hash2))
hashagain, err := spec2.Hash()
Expect(err).ToNot(HaveOccurred())
Expect(hash).ToNot(Equal(hash3))
Expect(hash).To(Equal(hashagain))
})
})
Context("Simple package build definition", func() {
It("Loads it correctly", func() {
generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
err := generalRecipe.Load("../../../../tests/fixtures/buildtree")
Expect(err).ToNot(HaveOccurred())
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
lspec, err := compiler.FromPackage(&types.Package{Name: "enman", Category: "app-admin", Version: "1.4.0"})
Expect(err).ToNot(HaveOccurred())
Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
Expect(lspec.Image).To(Equal("luet/base"))
Expect(lspec.Seed).To(Equal("alpine"))
tmpdir, err := ioutil.TempDir("", "tree")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tmpdir) // clean up
lspec.Env = []string{"test=1"}
err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM alpine
COPY . /luetbuild
WORKDIR /luetbuild
ENV PACKAGE_NAME=enman
ENV PACKAGE_VERSION=1.4.0
ENV PACKAGE_CATEGORY=app-admin
ENV test=1`))
err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM luet/base
COPY . /luetbuild
WORKDIR /luetbuild
ENV PACKAGE_NAME=enman
ENV PACKAGE_VERSION=1.4.0
ENV PACKAGE_CATEGORY=app-admin
ENV test=1
RUN echo foo > /test
RUN echo bar > /test2`))
})
})
It("Renders retrieve and env fields", func() {
generalRecipe := tree.NewGeneralRecipe(pkg.NewInMemoryDatabase(false))
err := generalRecipe.Load("../../../../tests/fixtures/retrieve")
Expect(err).ToNot(HaveOccurred())
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
lspec, err := compiler.FromPackage(&types.Package{Name: "a", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
Expect(lspec.Image).To(Equal("luet/base"))
Expect(lspec.Seed).To(Equal("alpine"))
tmpdir, err := ioutil.TempDir("", "tree")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tmpdir) // clean up
err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err := fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM alpine
COPY . /luetbuild
WORKDIR /luetbuild
ENV PACKAGE_NAME=a
ENV PACKAGE_VERSION=1.0
ENV PACKAGE_CATEGORY=test
ADD test /luetbuild/
ADD http://www.google.com /luetbuild/
ENV test=1`))
lspec.SetOutputPath("/foo/bar")
err = lspec.WriteBuildImageDefinition(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM alpine
COPY . /luetbuild
WORKDIR /luetbuild
ENV PACKAGE_NAME=a
ENV PACKAGE_VERSION=1.0
ENV PACKAGE_CATEGORY=test
ADD test /luetbuild/
ADD http://www.google.com /luetbuild/
ENV test=1`))
err = lspec.WriteStepImageDefinition(lspec.Image, filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
dockerfile, err = fileHelper.Read(filepath.Join(tmpdir, "Dockerfile"))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM luet/base
COPY . /luetbuild
WORKDIR /luetbuild
ENV PACKAGE_NAME=a
ENV PACKAGE_VERSION=1.0
ENV PACKAGE_CATEGORY=test
ADD test /luetbuild/
ADD http://www.google.com /luetbuild/
ENV test=1
RUN echo foo > /test
RUN echo bar > /test2`))
})
})