Refactor compiler and annotate buildoptions into compiler metadata

This allows to later pick up values used during build of each package
This commit is contained in:
Ettore Di Giacinto
2021-04-12 19:00:36 +02:00
parent 44cae094e8
commit 57c769b4a5
43 changed files with 1130 additions and 1235 deletions

View File

@@ -1,33 +1,52 @@
// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package backend
package compiler
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
. "github.com/mudler/luet/pkg/logger"
artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
"github.com/mudler/luet/pkg/compiler"
"github.com/mudler/luet/pkg/compiler/backend"
"github.com/mudler/luet/pkg/config"
"github.com/pkg/errors"
. "github.com/mudler/luet/pkg/logger"
)
func NewBackend(s string) (CompilerBackend, error) {
var compilerBackend CompilerBackend
switch s {
case backend.ImgBackend:
compilerBackend = backend.NewSimpleImgBackend()
case backend.DockerBackend:
compilerBackend = backend.NewSimpleDockerBackend()
default:
return nil, errors.New("invalid backend. Unsupported")
}
return compilerBackend, nil
}
type CompilerBackend interface {
BuildImage(backend.Options) error
ExportImage(backend.Options) error
RemoveImage(backend.Options) error
ImageDefinitionToTar(backend.Options) error
ExtractRootfs(opts backend.Options, keepPerms bool) error
CopyImage(string, string) error
DownloadImage(opts backend.Options) error
Push(opts backend.Options) error
ImageAvailable(string) bool
ImageExists(string) bool
}
// GenerateChanges generates changes between two images using a backend by leveraging export/extractrootfs methods
// example of json return: [
// {
@@ -54,46 +73,46 @@ import (
// }
// }
// ]
func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
func GenerateChanges(b CompilerBackend, fromImage, toImage backend.Options) ([]artifact.ArtifactLayer, error) {
res := compiler.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName}
res := artifact.ArtifactLayer{FromImage: fromImage.ImageName, ToImage: toImage.ImageName}
tmpdiffs, err := config.LuetCfg.GetSystem().TempDir("extraction")
if err != nil {
return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
}
defer os.RemoveAll(tmpdiffs) // clean up
srcRootFS, err := ioutil.TempDir(tmpdiffs, "src")
if err != nil {
return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
}
defer os.RemoveAll(srcRootFS) // clean up
dstRootFS, err := ioutil.TempDir(tmpdiffs, "dst")
if err != nil {
return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while creating tempdir for rootfs")
}
defer os.RemoveAll(dstRootFS) // clean up
srcImageExtract := compiler.CompilerBackendOptions{
srcImageExtract := backend.Options{
ImageName: fromImage.ImageName,
Destination: srcRootFS,
}
Debug("Extracting source image", fromImage.ImageName)
err = b.ExtractRootfs(srcImageExtract, false) // No need to keep permissions as we just collect file diffs
if err != nil {
return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName)
return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking src image "+fromImage.ImageName)
}
dstImageExtract := compiler.CompilerBackendOptions{
dstImageExtract := backend.Options{
ImageName: toImage.ImageName,
Destination: dstRootFS,
}
Debug("Extracting destination image", toImage.ImageName)
err = b.ExtractRootfs(dstImageExtract, false)
if err != nil {
return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName)
return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while unpacking dst image "+toImage.ImageName)
}
// Get Additions/Changes. dst -> src
@@ -114,7 +133,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
if sizeA != sizeB {
// fmt.Println("File changed", path, filepath.Join(srcRootFS, realpath))
res.Diffs.Changes = append(res.Diffs.Changes, compiler.ArtifactNode{
res.Diffs.Changes = append(res.Diffs.Changes, artifact.ArtifactNode{
Name: filepath.Join("/", realpath),
Size: int(sizeB),
})
@@ -127,7 +146,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
if s, err := os.Lstat(filepath.Join(dstRootFS, realpath)); err == nil {
sizeB = s.Size()
}
res.Diffs.Additions = append(res.Diffs.Additions, compiler.ArtifactNode{
res.Diffs.Additions = append(res.Diffs.Additions, artifact.ArtifactNode{
Name: filepath.Join("/", realpath),
Size: int(sizeB),
})
@@ -138,7 +157,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
return nil
})
if err != nil {
return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image destination")
}
// Get deletions. src -> dst
@@ -150,7 +169,7 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
realpath := strings.Replace(path, srcRootFS, "", -1)
if _, err = os.Lstat(filepath.Join(dstRootFS, realpath)); err != nil {
// fmt.Println("File deleted", path, filepath.Join(srcRootFS, realpath))
res.Diffs.Deletions = append(res.Diffs.Deletions, compiler.ArtifactNode{
res.Diffs.Deletions = append(res.Diffs.Deletions, artifact.ArtifactNode{
Name: filepath.Join("/", realpath),
})
}
@@ -158,8 +177,71 @@ func GenerateChanges(b compiler.CompilerBackend, fromImage, toImage compiler.Com
return nil
})
if err != nil {
return []compiler.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
return []artifact.ArtifactLayer{}, errors.Wrap(err, "Error met while walking image source")
}
return []compiler.ArtifactLayer{res}, nil
diffs := []artifact.ArtifactLayer{res}
if config.LuetCfg.GetGeneral().Debug {
summary := ComputeArtifactLayerSummary(diffs)
for _, l := range summary.Layers {
Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
l.FromImage, l.ToImage,
l.AddFiles, l.AddSizes,
l.DelFiles, l.DelSizes,
l.ChangeFiles, l.ChangeSizes))
}
}
return diffs, nil
}
type ArtifactLayerSummary struct {
FromImage string `json:"image1"`
ToImage string `json:"image2"`
AddFiles int `json:"add_files"`
AddSizes int64 `json:"add_sizes"`
DelFiles int `json:"del_files"`
DelSizes int64 `json:"del_sizes"`
ChangeFiles int `json:"change_files"`
ChangeSizes int64 `json:"change_sizes"`
}
type ArtifactLayersSummary struct {
Layers []ArtifactLayerSummary `json:"summary"`
}
func ComputeArtifactLayerSummary(diffs []artifact.ArtifactLayer) ArtifactLayersSummary {
ans := ArtifactLayersSummary{
Layers: make([]ArtifactLayerSummary, 0),
}
for _, layer := range diffs {
sum := ArtifactLayerSummary{
FromImage: layer.FromImage,
ToImage: layer.ToImage,
AddFiles: 0,
AddSizes: 0,
DelFiles: 0,
DelSizes: 0,
ChangeFiles: 0,
ChangeSizes: 0,
}
for _, a := range layer.Diffs.Additions {
sum.AddFiles++
sum.AddSizes += int64(a.Size)
}
for _, d := range layer.Diffs.Deletions {
sum.DelFiles++
sum.DelSizes += int64(d.Size)
}
for _, c := range layer.Diffs.Changes {
sum.ChangeFiles++
sum.ChangeSizes += int64(c.Size)
}
ans.Layers = append(ans.Layers, sum)
}
return ans
}

View File

@@ -18,7 +18,6 @@ package backend
import (
"os/exec"
"github.com/mudler/luet/pkg/compiler"
"github.com/mudler/luet/pkg/config"
. "github.com/mudler/luet/pkg/logger"
@@ -36,16 +35,13 @@ func imageAvailable(image string) bool {
return err == nil
}
func NewBackend(s string) compiler.CompilerBackend {
var compilerBackend compiler.CompilerBackend
switch s {
case ImgBackend:
compilerBackend = NewSimpleImgBackend()
case DockerBackend:
compilerBackend = NewSimpleDockerBackend()
}
return compilerBackend
type Options struct {
ImageName string
SourcePath string
DockerFileName string
Destination string
Context string
BackendArgs []string
}
func runCommand(cmd *exec.Cmd) error {
@@ -75,7 +71,7 @@ func runCommand(cmd *exec.Cmd) error {
return nil
}
func genBuildCommand(opts compiler.CompilerBackendOptions) []string {
func genBuildCommand(opts Options) []string {
context := opts.Context
if context == "" {

View File

@@ -17,7 +17,6 @@ package backend
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"os/exec"
@@ -29,8 +28,6 @@ import (
docker "github.com/fsouza/go-dockerclient"
capi "github.com/mudler/docker-companion/api"
"github.com/mudler/luet/pkg/compiler"
"github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/helpers"
. "github.com/mudler/luet/pkg/logger"
@@ -39,12 +36,12 @@ import (
type SimpleDocker struct{}
func NewSimpleDockerBackend() compiler.CompilerBackend {
func NewSimpleDockerBackend() *SimpleDocker {
return &SimpleDocker{}
}
// TODO: Missing still: labels, and build args expansion
func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
func (*SimpleDocker) BuildImage(opts Options) error {
name := opts.ImageName
bus.Manager.Publish(bus.EventImagePreBuild, opts)
@@ -93,7 +90,7 @@ func (*SimpleDocker) CopyImage(src, dst string) error {
return nil
}
func (*SimpleDocker) DownloadImage(opts compiler.CompilerBackendOptions) error {
func (*SimpleDocker) DownloadImage(opts Options) error {
name := opts.ImageName
bus.Manager.Publish(bus.EventImagePrePull, opts)
@@ -132,7 +129,7 @@ func (*SimpleDocker) ImageAvailable(imagename string) bool {
return imageAvailable(imagename)
}
func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
func (*SimpleDocker) RemoveImage(opts Options) error {
name := opts.ImageName
buildarg := []string{"rmi", name}
out, err := exec.Command("docker", buildarg...).CombinedOutput()
@@ -144,7 +141,7 @@ func (*SimpleDocker) RemoveImage(opts compiler.CompilerBackendOptions) error {
return nil
}
func (*SimpleDocker) Push(opts compiler.CompilerBackendOptions) error {
func (*SimpleDocker) Push(opts Options) error {
name := opts.ImageName
pusharg := []string{"push", name}
bus.Manager.Publish(bus.EventImagePrePush, opts)
@@ -163,7 +160,7 @@ func (*SimpleDocker) Push(opts compiler.CompilerBackendOptions) error {
return nil
}
func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
func (s *SimpleDocker) ImageDefinitionToTar(opts Options) error {
if err := s.BuildImage(opts); err != nil {
return errors.Wrap(err, "Failed building image")
}
@@ -176,7 +173,7 @@ func (s *SimpleDocker) ImageDefinitionToTar(opts compiler.CompilerBackendOptions
return nil
}
func (*SimpleDocker) ExportImage(opts compiler.CompilerBackendOptions) error {
func (*SimpleDocker) ExportImage(opts Options) error {
name := opts.ImageName
path := opts.Destination
@@ -199,7 +196,7 @@ type ManifestEntry struct {
Layers []string `json:"Layers"`
}
func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
func (b *SimpleDocker) ExtractRootfs(opts Options, keepPerms bool) error {
name := opts.ImageName
dst := opts.Destination
@@ -214,7 +211,7 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP
Spinner(22)
defer SpinnerStop()
if err := b.ExportImage(compiler.CompilerBackendOptions{ImageName: name, Destination: imageExport}); err != nil {
if err := b.ExportImage(Options{ImageName: name, Destination: imageExport}); err != nil {
return errors.Wrap(err, "failed while extracting rootfs for "+name)
}
@@ -229,7 +226,7 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP
}
defer os.RemoveAll(tempUnpack) // clean up
imageExport := filepath.Join(tempUnpack, "image.tar")
if err := b.ExportImage(compiler.CompilerBackendOptions{ImageName: opts.ImageName, Destination: imageExport}); err != nil {
if err := b.ExportImage(Options{ImageName: opts.ImageName, Destination: imageExport}); err != nil {
return errors.Wrap(err, "while exporting image before extraction")
}
src = imageExport
@@ -293,21 +290,3 @@ func (b *SimpleDocker) ExtractRootfs(opts compiler.CompilerBackendOptions, keepP
return nil
}
// Changes retrieves changes between image layers
func (d *SimpleDocker) Changes(fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
diffs, err := GenerateChanges(d, fromImage, toImage)
if config.LuetCfg.GetGeneral().Debug {
summary := compiler.ComputeArtifactLayerSummary(diffs)
for _, l := range summary.Layers {
Debug(fmt.Sprintf("Diff %s -> %s: add %d (%d bytes), del %d (%d bytes), change %d (%d bytes)",
l.FromImage, l.ToImage,
l.AddFiles, l.AddSizes,
l.DelFiles, l.DelSizes,
l.ChangeFiles, l.ChangeSizes))
}
}
return diffs, err
}

View File

@@ -16,9 +16,11 @@
package backend_test
import (
"github.com/mudler/luet/pkg/compiler"
. "github.com/mudler/luet/pkg/compiler"
"github.com/mudler/luet/pkg/compiler/backend"
. "github.com/mudler/luet/pkg/compiler/backend"
"github.com/mudler/luet/pkg/solver"
"github.com/mudler/luet/pkg/compiler/types/artifact"
"io/ioutil"
"os"
@@ -41,13 +43,10 @@ var _ = Describe("Docker backend", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
cc := NewLuetCompiler(nil, generalRecipe.GetDatabase())
lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
Expect(err).ToNot(HaveOccurred())
lspec, ok := spec.(*LuetCompilationSpec)
Expect(ok).To(BeTrue())
Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
Expect(lspec.Image).To(Equal("luet/base"))
Expect(lspec.Seed).To(Equal("alpine"))
@@ -71,7 +70,7 @@ ENV PACKAGE_NAME=enman
ENV PACKAGE_VERSION=1.4.0
ENV PACKAGE_CATEGORY=app-admin`))
b := NewSimpleDockerBackend()
opts := CompilerBackendOptions{
opts := backend.Options{
ImageName: "luet/base",
SourcePath: tmpdir,
DockerFileName: "Dockerfile",
@@ -95,7 +94,7 @@ ENV PACKAGE_VERSION=1.4.0
ENV PACKAGE_CATEGORY=app-admin
RUN echo foo > /test
RUN echo bar > /test2`))
opts2 := CompilerBackendOptions{
opts2 := backend.Options{
ImageName: "test",
SourcePath: tmpdir,
DockerFileName: "LuetDockerfile",
@@ -106,26 +105,26 @@ RUN echo bar > /test2`))
Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
artifacts := []ArtifactNode{{
artifacts := []artifact.ArtifactNode{{
Name: "/luetbuild/LuetDockerfile",
Size: 175,
}}
if os.Getenv("DOCKER_BUILDKIT") == "1" {
artifacts = append(artifacts, ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
artifacts = append(artifacts, artifact.ArtifactNode{Name: "/etc/resolv.conf", Size: 0})
}
artifacts = append(artifacts, ArtifactNode{Name: "/test", Size: 4})
artifacts = append(artifacts, ArtifactNode{Name: "/test2", Size: 4})
artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test", Size: 4})
artifacts = append(artifacts, artifact.ArtifactNode{Name: "/test2", Size: 4})
Expect(b.Changes(opts, opts2)).To(Equal(
[]ArtifactLayer{{
Expect(compiler.GenerateChanges(b, opts, opts2)).To(Equal(
[]artifact.ArtifactLayer{{
FromImage: "luet/base",
ToImage: "test",
Diffs: ArtifactDiffs{
Diffs: artifact.ArtifactDiffs{
Additions: artifacts,
},
}}))
opts2 = CompilerBackendOptions{
opts2 = backend.Options{
ImageName: "test",
SourcePath: tmpdir,
DockerFileName: "LuetDockerfile",

View File

@@ -22,7 +22,6 @@ import (
bus "github.com/mudler/luet/pkg/bus"
"github.com/mudler/luet/pkg/compiler"
. "github.com/mudler/luet/pkg/logger"
"github.com/pkg/errors"
@@ -30,12 +29,12 @@ import (
type SimpleImg struct{}
func NewSimpleImgBackend() compiler.CompilerBackend {
func NewSimpleImgBackend() *SimpleImg {
return &SimpleImg{}
}
// TODO: Missing still: labels, and build args expansion
func (*SimpleImg) BuildImage(opts compiler.CompilerBackendOptions) error {
func (*SimpleImg) BuildImage(opts Options) error {
name := opts.ImageName
bus.Manager.Publish(bus.EventImagePreBuild, opts)
@@ -56,7 +55,7 @@ func (*SimpleImg) BuildImage(opts compiler.CompilerBackendOptions) error {
return nil
}
func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
func (*SimpleImg) RemoveImage(opts Options) error {
name := opts.ImageName
buildarg := []string{"rm", name}
Spinner(22)
@@ -70,7 +69,7 @@ func (*SimpleImg) RemoveImage(opts compiler.CompilerBackendOptions) error {
return nil
}
func (*SimpleImg) DownloadImage(opts compiler.CompilerBackendOptions) error {
func (*SimpleImg) DownloadImage(opts Options) error {
name := opts.ImageName
bus.Manager.Publish(bus.EventImagePrePull, opts)
@@ -123,7 +122,7 @@ func (*SimpleImg) ImageExists(imagename string) bool {
return false
}
func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) error {
func (s *SimpleImg) ImageDefinitionToTar(opts Options) error {
if err := s.BuildImage(opts); err != nil {
return errors.Wrap(err, "Failed building image")
}
@@ -136,7 +135,7 @@ func (s *SimpleImg) ImageDefinitionToTar(opts compiler.CompilerBackendOptions) e
return nil
}
func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error {
func (*SimpleImg) ExportImage(opts Options) error {
name := opts.ImageName
path := opts.Destination
buildarg := []string{"save", "-o", path, name}
@@ -154,7 +153,7 @@ func (*SimpleImg) ExportImage(opts compiler.CompilerBackendOptions) error {
}
// ExtractRootfs extracts the docker image content inside the destination
func (s *SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerms bool) error {
func (s *SimpleImg) ExtractRootfs(opts Options, keepPerms bool) error {
name := opts.ImageName
path := opts.Destination
@@ -180,13 +179,7 @@ func (s *SimpleImg) ExtractRootfs(opts compiler.CompilerBackendOptions, keepPerm
return nil
}
// TODO: Use container-diff (https://github.com/GoogleContainerTools/container-diff) for checking out layer diffs
// Changes uses container-diff (https://github.com/GoogleContainerTools/container-diff) for retrieving out layer diffs
func (i *SimpleImg) Changes(fromImage, toImage compiler.CompilerBackendOptions) ([]compiler.ArtifactLayer, error) {
return GenerateChanges(i, fromImage, toImage)
}
func (*SimpleImg) Push(opts compiler.CompilerBackendOptions) error {
func (*SimpleImg) Push(opts Options) error {
name := opts.ImageName
bus.Manager.Publish(bus.EventImagePrePush, opts)

View File

@@ -13,10 +13,10 @@
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package backend_test
package compiler_test
import (
"github.com/mudler/luet/pkg/compiler"
. "github.com/mudler/luet/pkg/compiler"
. "github.com/mudler/luet/pkg/compiler/backend"
. "github.com/onsi/ginkgo"
@@ -24,7 +24,7 @@ import (
)
var _ = Describe("Docker image diffs", func() {
var b compiler.CompilerBackend
var b CompilerBackend
BeforeEach(func() {
b = NewSimpleDockerBackend()
@@ -32,7 +32,7 @@ var _ = Describe("Docker image diffs", func() {
Context("Generate diffs from docker images", func() {
It("Detect no changes", func() {
opts := compiler.CompilerBackendOptions{
opts := Options{
ImageName: "alpine:latest",
}
err := b.DownloadImage(opts)
@@ -47,18 +47,18 @@ var _ = Describe("Docker image diffs", func() {
})
It("Detects additions and changed files", func() {
err := b.DownloadImage(compiler.CompilerBackendOptions{
err := b.DownloadImage(Options{
ImageName: "quay.io/mocaccino/micro",
})
Expect(err).ToNot(HaveOccurred())
err = b.DownloadImage(compiler.CompilerBackendOptions{
err = b.DownloadImage(Options{
ImageName: "quay.io/mocaccino/extra",
})
Expect(err).ToNot(HaveOccurred())
layers, err := GenerateChanges(b, compiler.CompilerBackendOptions{
layers, err := GenerateChanges(b, Options{
ImageName: "quay.io/mocaccino/micro",
}, compiler.CompilerBackendOptions{
}, Options{
ImageName: "quay.io/mocaccino/extra",
})
Expect(err).ToNot(HaveOccurred())

View File

@@ -19,6 +19,7 @@ import (
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"regexp"
@@ -27,13 +28,14 @@ import (
"time"
bus "github.com/mudler/luet/pkg/bus"
yaml "gopkg.in/yaml.v2"
"github.com/mudler/luet/pkg/compiler/backend"
artifact "github.com/mudler/luet/pkg/compiler/types/artifact"
"github.com/mudler/luet/pkg/compiler/types/options"
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
"github.com/mudler/luet/pkg/helpers"
. "github.com/mudler/luet/pkg/logger"
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/solver"
"github.com/mudler/luet/pkg/tree"
"github.com/pkg/errors"
)
@@ -41,63 +43,55 @@ const BuildFile = "build.yaml"
const DefinitionFile = "definition.yaml"
const CollectionFile = "collection.yaml"
type LuetCompiler struct {
*tree.CompilerRecipe
Backend CompilerBackend
Database pkg.PackageDatabase
PushImageRepository string
PullImageRepository []string
type ArtifactIndex []*artifact.PackageArtifact
PullFirst, KeepImg, Clean bool
Concurrency int
CompressionType CompressionImplementation
Options CompilerOptions
SolverOptions solver.Options
BackedArgs []string
func (i ArtifactIndex) CleanPath() ArtifactIndex {
newIndex := ArtifactIndex{}
for _, art := range i {
// FIXME: This is a dup and makes difficult to add attributes to artifacts
newIndex = append(newIndex, &artifact.PackageArtifact{
Path: path.Base(art.Path),
SourceAssertion: art.SourceAssertion,
CompileSpec: art.CompileSpec,
Dependencies: art.Dependencies,
CompressionType: art.CompressionType,
Checksums: art.Checksums,
Files: art.Files,
})
}
return newIndex
//Update if exists, otherwise just create
}
func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, opt *CompilerOptions, solvopts solver.Options) Compiler {
type LuetCompiler struct {
//*tree.CompilerRecipe
Backend CompilerBackend
Database pkg.PackageDatabase
Options options.Compiler
}
func NewCompiler(p ...options.Option) *LuetCompiler {
c := options.NewDefaultCompiler()
c.Apply(p...)
return &LuetCompiler{Options: *c}
}
func NewLuetCompiler(backend CompilerBackend, db pkg.PackageDatabase, compilerOpts ...options.Option) *LuetCompiler {
// The CompilerRecipe will gives us a tree with only build deps listed.
if len(opt.PullImageRepository) == 0 {
opt.PullImageRepository = []string{opt.PushImageRepository}
}
c := NewCompiler(compilerOpts...)
// c.Options.BackendType
c.Backend = backend
c.Database = db
// c.CompilerRecipe = &tree.CompilerRecipe{
// Recipe: tree.Recipe{Database: db},
// }
return &LuetCompiler{
Backend: backend,
CompilerRecipe: &tree.CompilerRecipe{
Recipe: tree.Recipe{Database: db},
},
Database: db,
PushImageRepository: opt.PushImageRepository,
PullImageRepository: opt.PullImageRepository,
PullFirst: opt.PullFirst,
CompressionType: opt.CompressionType,
KeepImg: opt.KeepImg,
Concurrency: opt.Concurrency,
Options: *opt,
SolverOptions: solvopts,
}
return c
}
// SetBackendArgs sets arbitrary backend arguments.
// Those for example can be commands passed to the docker daemon during build phase,
// as build-args, etc.
func (cs *LuetCompiler) SetBackendArgs(args []string) {
cs.BackedArgs = args
}
// SetConcurrency sets the compiler concurrency
func (cs *LuetCompiler) SetConcurrency(i int) {
cs.Concurrency = i
}
// SetCompressionType sets the compiler compression type for resulting artifacts
func (cs *LuetCompiler) SetCompressionType(t CompressionImplementation) {
cs.CompressionType = t
}
func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan CompilationSpec, a *[]Artifact, m *sync.Mutex, concurrency int, keepPermissions bool, errors chan error) {
func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan *compilerspec.LuetCompilationSpec, a *[]*artifact.PackageArtifact, m *sync.Mutex, concurrency int, keepPermissions bool, errors chan error) {
defer wg.Done()
for s := range cspecs {
@@ -113,17 +107,17 @@ func (cs *LuetCompiler) compilerWorker(i int, wg *sync.WaitGroup, cspecs chan Co
}
// CompileWithReverseDeps compiles the supplied compilationspecs and their reverse dependencies
func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps *compilerspec.LuetCompilationspecs) ([]*artifact.PackageArtifact, []error) {
artifacts, err := cs.CompileParallel(keepPermissions, ps)
if len(err) != 0 {
return artifacts, err
}
Info(":ant: Resolving reverse dependencies")
toCompile := NewLuetCompilationspecs()
toCompile := compilerspec.NewLuetCompilationspecs()
for _, a := range artifacts {
revdeps := a.GetCompileSpec().GetPackage().Revdeps(cs.Database)
revdeps := a.CompileSpec.GetPackage().Revdeps(cs.Database)
for _, r := range revdeps {
spec, asserterr := cs.FromPackage(r)
if err != nil {
@@ -146,15 +140,15 @@ func (cs *LuetCompiler) CompileWithReverseDeps(keepPermissions bool, ps Compilat
// CompileParallel compiles the supplied compilationspecs in parallel
// to note, no specific heuristic is implemented, and the specs are run in parallel as they are.
func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error) {
all := make(chan CompilationSpec)
artifacts := []Artifact{}
func (cs *LuetCompiler) CompileParallel(keepPermissions bool, ps *compilerspec.LuetCompilationspecs) ([]*artifact.PackageArtifact, []error) {
all := make(chan *compilerspec.LuetCompilationSpec)
artifacts := []*artifact.PackageArtifact{}
mutex := &sync.Mutex{}
errors := make(chan error, ps.Len())
var wg = new(sync.WaitGroup)
for i := 0; i < cs.Concurrency; i++ {
for i := 0; i < cs.Options.Concurrency; i++ {
wg.Add(1)
go cs.compilerWorker(i, wg, all, &artifacts, mutex, cs.Concurrency, keepPermissions, errors)
go cs.compilerWorker(i, wg, all, &artifacts, mutex, cs.Options.Concurrency, keepPermissions, errors)
}
for _, p := range ps.All() {
@@ -233,7 +227,7 @@ func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, includ
return nil
}
func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p CompilationSpec, runnerOpts CompilerBackendOptions) (Artifact, error) {
func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec, runnerOpts backend.Options) (*artifact.PackageArtifact, error) {
rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
if err != nil {
@@ -241,7 +235,7 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p Compil
}
defer os.RemoveAll(rootfs) // clean up
err = cs.Backend.ExtractRootfs(CompilerBackendOptions{
err = cs.Backend.ExtractRootfs(backend.Options{
ImageName: runnerOpts.ImageName, Destination: rootfs}, keepPermissions)
if err != nil {
return nil, errors.Wrap(err, "Could not extract rootfs")
@@ -260,18 +254,18 @@ func (cs *LuetCompiler) unpackFs(concurrency int, keepPermissions bool, p Compil
// strip from excludes
cs.stripFromRootfs(p.GetExcludes(), rootfs, false)
}
artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
artifact.SetCompressionType(cs.CompressionType)
a := artifact.NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
a.CompressionType = cs.Options.CompressionType
if err := artifact.Compress(rootfs, concurrency); err != nil {
if err := a.Compress(rootfs, concurrency); err != nil {
return nil, errors.Wrap(err, "Error met while creating package archive")
}
artifact.SetCompileSpec(p)
return artifact, nil
a.CompileSpec = p
return a, nil
}
func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p CompilationSpec, builderOpts, runnerOpts CompilerBackendOptions) (Artifact, error) {
func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec, builderOpts, runnerOpts backend.Options) (*artifact.PackageArtifact, error) {
rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
if err != nil {
@@ -288,30 +282,30 @@ func (cs *LuetCompiler) unpackDelta(concurrency int, keepPermissions bool, p Com
}
Info(pkgTag, ":hammer: Generating delta")
diffs, err := cs.Backend.Changes(builderOpts, runnerOpts)
diffs, err := GenerateChanges(cs.Backend, builderOpts, runnerOpts)
if err != nil {
return nil, errors.Wrap(err, "Could not generate changes from layers")
}
Debug("Extracting image to grab files from delta")
if err := cs.Backend.ExtractRootfs(CompilerBackendOptions{
if err := cs.Backend.ExtractRootfs(backend.Options{
ImageName: runnerOpts.ImageName, Destination: rootfs}, keepPermissions); err != nil {
return nil, errors.Wrap(err, "Could not extract rootfs")
}
artifact, err := ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.CompressionType)
artifact, err := artifact.ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.Options.CompressionType)
if err != nil {
return nil, errors.Wrap(err, "Could not generate deltas")
}
artifact.SetCompileSpec(p)
artifact.CompileSpec = p
return artifact, nil
}
func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,
concurrency int, keepPermissions bool,
p CompilationSpec) (CompilerBackendOptions, CompilerBackendOptions, error) {
p *compilerspec.LuetCompilationSpec) (backend.Options, backend.Options, error) {
var runnerOpts, builderOpts CompilerBackendOptions
var runnerOpts, builderOpts backend.Options
pkgTag := ":package: " + p.GetPackage().HumanReadableString()
@@ -325,7 +319,7 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
fp := p.GetPackage().HashFingerprint(helpers.StripRegistryFromImage(packageImage))
if buildertaggedImage == "" {
buildertaggedImage = cs.PushImageRepository + ":builder-" + fp
buildertaggedImage = cs.Options.PushImageRepository + ":builder-" + fp
Debug(pkgTag, "Creating intermediary image", buildertaggedImage, "from", image)
}
@@ -375,22 +369,22 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
}
builderOpts = CompilerBackendOptions{
builderOpts = backend.Options{
ImageName: buildertaggedImage,
SourcePath: buildDir,
DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile",
Destination: p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
BackendArgs: cs.BackedArgs,
BackendArgs: cs.Options.BackendArgs,
}
runnerOpts = CompilerBackendOptions{
runnerOpts = backend.Options{
ImageName: packageImage,
SourcePath: buildDir,
DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile",
Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
BackendArgs: cs.BackedArgs,
BackendArgs: cs.Options.BackendArgs,
}
buildAndPush := func(opts CompilerBackendOptions) error {
buildAndPush := func(opts backend.Options) error {
buildImage := true
if cs.Options.PullFirst {
err := cs.Backend.DownloadImage(opts)
@@ -430,10 +424,10 @@ func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImag
return builderOpts, runnerOpts, nil
}
func (cs *LuetCompiler) genArtifact(p CompilationSpec, builderOpts, runnerOpts CompilerBackendOptions, concurrency int, keepPermissions bool) (Artifact, error) {
func (cs *LuetCompiler) genArtifact(p *compilerspec.LuetCompilationSpec, builderOpts, runnerOpts backend.Options, concurrency int, keepPermissions bool) (*artifact.PackageArtifact, error) {
// generate Artifact
var artifact Artifact
// generate *artifact.PackageArtifact
var a *artifact.PackageArtifact
var rootfs string
var err error
pkgTag := ":package: " + p.GetPackage().HumanReadableString()
@@ -448,53 +442,53 @@ func (cs *LuetCompiler) genArtifact(p CompilationSpec, builderOpts, runnerOpts C
}
defer os.RemoveAll(rootfs) // clean up
artifact := NewPackageArtifact(fakePackage)
artifact.SetCompressionType(cs.CompressionType)
a := artifact.NewPackageArtifact(fakePackage)
a.CompressionType = cs.Options.CompressionType
if err := artifact.Compress(rootfs, concurrency); err != nil {
if err := a.Compress(rootfs, concurrency); err != nil {
return nil, errors.Wrap(err, "Error met while creating package archive")
}
artifact.SetCompileSpec(p)
artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
a.CompileSpec = p
a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
err = artifact.WriteYaml(p.GetOutputPath())
err = a.WriteYaml(p.GetOutputPath())
if err != nil {
return artifact, errors.Wrap(err, "Failed while writing metadata file")
return a, errors.Wrap(err, "Failed while writing metadata file")
}
Info(pkgTag, " :white_check_mark: done (empty virtual package)")
return artifact, nil
return a, nil
}
if p.UnpackedPackage() {
// Take content of container as a base for our package files
artifact, err = cs.unpackFs(concurrency, keepPermissions, p, runnerOpts)
a, err = cs.unpackFs(concurrency, keepPermissions, p, runnerOpts)
if err != nil {
return nil, errors.Wrap(err, "Error met while extracting image")
}
} else {
// Generate delta between the two images
artifact, err = cs.unpackDelta(concurrency, keepPermissions, p, builderOpts, runnerOpts)
a, err = cs.unpackDelta(concurrency, keepPermissions, p, builderOpts, runnerOpts)
if err != nil {
return nil, errors.Wrap(err, "Error met while generating delta")
}
}
filelist, err := artifact.FileList()
filelist, err := a.FileList()
if err != nil {
return artifact, errors.Wrap(err, "Failed getting package list")
return a, errors.Wrap(err, "Failed getting package list")
}
artifact.SetFiles(filelist)
artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
a.Files = filelist
a.CompileSpec.GetPackage().SetBuildTimestamp(time.Now().String())
err = artifact.WriteYaml(p.GetOutputPath())
err = a.WriteYaml(p.GetOutputPath())
if err != nil {
return artifact, errors.Wrap(err, "Failed while writing metadata file")
return a, errors.Wrap(err, "Failed while writing metadata file")
}
Info(pkgTag, " :white_check_mark: Done")
return artifact, nil
return a, nil
}
func (cs *LuetCompiler) waitForImages(images []string) {
@@ -532,8 +526,8 @@ func oneOfImagesAvailable(images []string, b CompilerBackend) (bool, string) {
func (cs *LuetCompiler) resolveExistingImageHash(imageHash string) string {
var resolvedImage string
toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.PushImageRepository, imageHash)},
genImageList(cs.PullImageRepository, imageHash)...)
toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)},
genImageList(cs.Options.PullImageRepository, imageHash)...)
if exists, which := oneOfImagesExists(toChecklist, cs.Backend); exists {
resolvedImage = which
}
@@ -544,28 +538,43 @@ func (cs *LuetCompiler) resolveExistingImageHash(imageHash string) string {
}
if resolvedImage == "" {
resolvedImage = fmt.Sprintf("%s:%s", cs.PushImageRepository, imageHash)
resolvedImage = fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, imageHash)
}
return resolvedImage
}
func (cs *LuetCompiler) getImageArtifact(hash string, p CompilationSpec) (Artifact, error) {
func LoadArtifactFromYaml(spec *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml"
dat, err := ioutil.ReadFile(spec.Rel(metaFile))
if err != nil {
return nil, errors.Wrap(err, "Error reading file "+metaFile)
}
art, err := artifact.NewPackageArtifactFromYaml(dat)
if err != nil {
return nil, errors.Wrap(err, "Error writing file "+metaFile)
}
// It is relative, set it back to abs
art.Path = spec.Rel(art.Path)
return art, nil
}
func (cs *LuetCompiler) getImageArtifact(hash string, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
// we check if there is an available image with the given hash and
// we return a full artifact if can be loaded locally.
toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.PushImageRepository, hash)},
genImageList(cs.PullImageRepository, hash)...)
toChecklist := append([]string{fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, hash)},
genImageList(cs.Options.PullImageRepository, hash)...)
exists, _ := oneOfImagesExists(toChecklist, cs.Backend)
if art, err := LoadArtifactFromYaml(p); err == nil && exists { // If YAML is correctly loaded, and both images exists, no reason to rebuild.
Debug("Artifact reloaded from YAML. Skipping build")
Debug("Package reloaded from YAML. Skipping build")
return art, nil
}
cs.waitForImages(toChecklist)
available, _ := oneOfImagesAvailable(toChecklist, cs.Backend)
if exists || (cs.Options.PullFirst && available) {
Debug("Image available, returning empty artifact")
return &PackageArtifact{}, nil
return &artifact.PackageArtifact{}, nil
}
return nil, errors.New("artifact not found")
@@ -578,13 +587,13 @@ func (cs *LuetCompiler) getImageArtifact(hash string, p CompilationSpec) (Artifa
func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage string, packageTagHash string,
concurrency int,
keepPermissions, keepImg bool,
p CompilationSpec, generateArtifact bool) (Artifact, error) {
p *compilerspec.LuetCompilationSpec, generateArtifact bool) (*artifact.PackageArtifact, error) {
// If it is a virtual, check if we have to generate an empty artifact or not.
if generateArtifact && p.IsVirtual() {
return cs.genArtifact(p, CompilerBackendOptions{}, CompilerBackendOptions{}, concurrency, keepPermissions)
return cs.genArtifact(p, backend.Options{}, backend.Options{}, concurrency, keepPermissions)
} else if p.IsVirtual() {
return &PackageArtifact{}, nil
return &artifact.PackageArtifact{}, nil
}
if !generateArtifact {
@@ -597,7 +606,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage string, packa
}
// always going to point at the destination from the repo defined
packageImage := fmt.Sprintf("%s:%s", cs.PushImageRepository, packageTagHash)
packageImage := fmt.Sprintf("%s:%s", cs.Options.PushImageRepository, packageTagHash)
builderOpts, runnerOpts, err := cs.buildPackageImage(image, buildertaggedImage, packageImage, concurrency, keepPermissions, p)
if err != nil {
return nil, errors.Wrap(err, "failed building package image")
@@ -616,7 +625,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage string, packa
}
if !generateArtifact {
return &PackageArtifact{}, nil
return &artifact.PackageArtifact{}, nil
}
return cs.genArtifact(p, builderOpts, runnerOpts, concurrency, keepPermissions)
@@ -624,8 +633,8 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage string, packa
// FromDatabase returns all the available compilation specs from a database. If the minimum flag is returned
// it will be computed a minimal subset that will guarantees that all packages are built ( if not targeting a single package explictly )
func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error) {
compilerSpecs := NewLuetCompilationspecs()
func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]*compilerspec.LuetCompilationSpec, error) {
compilerSpecs := compilerspec.NewLuetCompilationspecs()
w := db.World()
@@ -649,11 +658,11 @@ func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst s
}
// ComputeMinimumCompilableSet strips specs that are eventually compiled by leafs
func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error) {
func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...*compilerspec.LuetCompilationSpec) ([]*compilerspec.LuetCompilationSpec, error) {
// Generate a set with all the deps of the provided specs
// we will use that set to remove the deps from the list of provided compilation specs
allDependencies := solver.PackagesAssertions{} // Get all packages that will be in deps
result := []CompilationSpec{}
result := []*compilerspec.LuetCompilationSpec{}
for _, spec := range p {
ass, err := cs.ComputeDepTree(spec)
if err != nil {
@@ -673,9 +682,9 @@ func (cs *LuetCompiler) ComputeMinimumCompilableSet(p ...CompilationSpec) ([]Com
// ComputeDepTree computes the dependency tree of a compilation spec and returns solver assertions
// in order to be able to compile the spec.
func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error) {
func (cs *LuetCompiler) ComputeDepTree(p *compilerspec.LuetCompilationSpec) (solver.PackagesAssertions, error) {
s := solver.NewResolver(cs.SolverOptions, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
s := solver.NewResolver(cs.Options.SolverOptions.Options, pkg.NewInMemoryDatabase(false), cs.Database, pkg.NewInMemoryDatabase(false), cs.Options.SolverOptions.Resolver())
solution, err := s.Install(pkg.Packages{p.GetPackage()})
if err != nil {
@@ -704,13 +713,13 @@ func (cs *LuetCompiler) ComputeDepTree(p CompilationSpec) (solver.PackagesAssert
// Compile is a non-parallel version of CompileParallel. It builds the compilation specs and generates
// an artifact
func (cs *LuetCompiler) Compile(keepPermissions bool, p CompilationSpec) (Artifact, error) {
func (cs *LuetCompiler) Compile(keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
asserts, err := cs.ComputeDepTree(p)
if err != nil {
panic(err)
}
p.SetSourceAssertion(asserts)
return cs.compile(cs.Concurrency, keepPermissions, p)
return cs.compile(cs.Options.Concurrency, keepPermissions, p)
}
func genImageList(refs []string, hash string) []string {
@@ -721,7 +730,7 @@ func genImageList(refs []string, hash string) []string {
return res
}
func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p CompilationSpec) (Artifact, error) {
func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p *compilerspec.LuetCompilationSpec) (*artifact.PackageArtifact, error) {
Info(":package: Compiling", p.GetPackage().HumanReadableString(), ".... :coffee:")
Debug(fmt.Sprintf("%s: has images %t, empty package: %t", p.GetPackage().HumanReadableString(), p.HasImageSource(), p.EmptyPackage()))
@@ -736,17 +745,20 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
targetAssertion := p.GetSourceAssertion().Search(p.GetPackage().GetFingerPrint())
bus.Manager.Publish(bus.EventPackagePreBuild, struct {
CompileSpec CompilationSpec
CompileSpec *compilerspec.LuetCompilationSpec
Assert solver.PackageAssert
}{
CompileSpec: p,
Assert: *targetAssertion,
})
// Update compilespec build options
p.SetBuildOptions(cs.Options)
// - If image is set we just generate a plain dockerfile
// Treat last case (easier) first. The image is provided and we just compute a plain dockerfile with the images listed as above
if p.GetImage() != "" {
return cs.compileWithImage(p.GetImage(), "", targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
return cs.compileWithImage(p.GetImage(), "", targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
}
// - If image is not set, we read a base_image. Then we will build one image from it to kick-off our build based
@@ -755,7 +767,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
// - We later on compute an hash used to identify the image, so each similar deptree keeps the same build image.
dependencies := p.GetSourceAssertion().Drop(p.GetPackage()) // at this point we should have a flattened list of deps to build, including all of them (with all constraints propagated already)
departifacts := []Artifact{} // TODO: Return this somehow
departifacts := []*artifact.PackageArtifact{} // TODO: Return this somehow
var lastHash string
depsN := 0
currentN := 0
@@ -781,7 +793,7 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
Debug(pkgTag, " :arrow_right_hook: :whale: Package image from hash", assertion.Hash.PackageHash)
bus.Manager.Publish(bus.EventPackagePreBuild, struct {
CompileSpec CompilationSpec
CompileSpec *compilerspec.LuetCompilationSpec
Assert solver.PackageAssert
}{
CompileSpec: compileSpec,
@@ -796,30 +808,30 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
if compileSpec.GetImage() != "" {
Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from image")
artifact, err := cs.compileWithImage(compileSpec.GetImage(), resolvedBuildImage, assertion.Hash.PackageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
a, err := cs.compileWithImage(compileSpec.GetImage(), resolvedBuildImage, assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps)
if err != nil {
return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
}
departifacts = append(departifacts, artifact)
departifacts = append(departifacts, a)
Info(pkgTag, ":white_check_mark: Done")
continue
}
Debug(pkgTag, " :wrench: Compiling "+compileSpec.GetPackage().HumanReadableString()+" from tree")
artifact, err := cs.compileWithImage(resolvedBuildImage, "", assertion.Hash.PackageHash, concurrency, keepPermissions, cs.KeepImg, compileSpec, packageDeps)
a, err := cs.compileWithImage(resolvedBuildImage, "", assertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, compileSpec, packageDeps)
if err != nil {
return nil, errors.Wrap(err, "Failed compiling "+compileSpec.GetPackage().HumanReadableString())
}
bus.Manager.Publish(bus.EventPackagePostBuild, struct {
CompileSpec CompilationSpec
Artifact Artifact
CompileSpec *compilerspec.LuetCompilationSpec
Artifact *artifact.PackageArtifact
}{
CompileSpec: compileSpec,
Artifact: artifact,
Artifact: a,
})
departifacts = append(departifacts, artifact)
departifacts = append(departifacts, a)
Info(pkgTag, ":white_check_mark: Done")
}
@@ -831,22 +843,22 @@ func (cs *LuetCompiler) compile(concurrency int, keepPermissions bool, p Compila
resolvedBuildImage := cs.resolveExistingImageHash(lastHash)
Info(":rocket: All dependencies are satisfied, building package requested by the user", p.GetPackage().HumanReadableString())
Info(":package:", p.GetPackage().HumanReadableString(), " Using image: ", resolvedBuildImage)
artifact, err := cs.compileWithImage(resolvedBuildImage, "", targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.KeepImg, p, true)
a, err := cs.compileWithImage(resolvedBuildImage, "", targetAssertion.Hash.PackageHash, concurrency, keepPermissions, cs.Options.KeepImg, p, true)
if err != nil {
return artifact, err
return a, err
}
artifact.SetDependencies(departifacts)
artifact.SetSourceAssertion(p.GetSourceAssertion())
a.Dependencies = departifacts
a.SourceAssertion = p.GetSourceAssertion()
bus.Manager.Publish(bus.EventPackagePostBuild, struct {
CompileSpec CompilationSpec
Artifact Artifact
CompileSpec *compilerspec.LuetCompilationSpec
Artifact *artifact.PackageArtifact
}{
CompileSpec: p,
Artifact: artifact,
Artifact: a,
})
return artifact, err
return a, err
} else {
return departifacts[len(departifacts)-1], nil
}
@@ -857,8 +869,15 @@ type templatedata map[string]interface{}
func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) {
var dataresult []byte
val := pack.Rel(DefinitionFile)
// Update processed build values
dst, err := helpers.UnMarshalValues(cs.Options.BuildValuesFile)
if err != nil {
return nil, errors.Wrap(err, "unmarshalling values")
}
cs.Options.BuildValues = []map[string]interface{}{(map[string]interface{})(dst)}
if _, err := os.Stat(pack.Rel(CollectionFile)); err == nil {
val = pack.Rel(CollectionFile)
@@ -879,11 +898,6 @@ func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) {
raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
dst, err := helpers.UnMarshalValues(cs.Options.BuildValuesFile)
if err != nil {
return nil, errors.Wrap(err, "unmarshalling values")
}
dat, err := helpers.RenderHelm(string(dataBuild), raw, dst)
if err != nil {
return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
@@ -896,12 +910,12 @@ func (cs *LuetCompiler) templatePackage(pack pkg.Package) ([]byte, error) {
}
dataresult = []byte(out)
}
return dataresult, nil
return dataresult, nil
}
// FromPackage returns a compilation spec from a package definition
func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
func (cs *LuetCompiler) FromPackage(p pkg.Package) (*compilerspec.LuetCompilationSpec, error) {
pack, err := cs.Database.FindPackageCandidate(p)
if err != nil {
@@ -913,7 +927,7 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
return nil, errors.Wrap(err, "while rendering package template")
}
return NewLuetCompilationSpec(bytes, pack)
return compilerspec.NewLuetCompilationSpec(bytes, pack)
}
// GetBackend returns the current compilation backend

View File

@@ -21,9 +21,11 @@ import (
. "github.com/mudler/luet/pkg/compiler"
sd "github.com/mudler/luet/pkg/compiler/backend"
"github.com/mudler/luet/pkg/compiler/types/compression"
"github.com/mudler/luet/pkg/compiler/types/options"
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
helpers "github.com/mudler/luet/pkg/helpers"
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/solver"
"github.com/mudler/luet/pkg/tree"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@@ -39,7 +41,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
@@ -54,12 +56,11 @@ var _ = Describe("Compiler", func() {
Expect(spec.GetPreBuildSteps()).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(2)
artifact, err := compiler.Compile(false, spec)
Expect(err).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("test6"))).To(BeTrue())
@@ -83,7 +84,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(1))
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
@@ -98,12 +99,11 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
spec2.SetOutputPath(tmpdir)
compiler.SetConcurrency(2)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
Expect(errs).To(BeNil())
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
})
@@ -118,7 +118,7 @@ var _ = Describe("Compiler", func() {
err = generalRecipe.Load("../../tests/fixtures/templates")
Expect(err).ToNot(HaveOccurred())
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
pkg, err := generalRecipe.GetDatabase().FindPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
@@ -142,7 +142,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
@@ -159,15 +159,14 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
spec2.SetOutputPath(tmpdir)
spec3.SetOutputPath(tmpdir)
compiler.SetConcurrency(2)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2, spec3))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2, spec3))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(3))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("test3"))).To(BeTrue())
@@ -199,7 +198,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(1))
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
@@ -207,23 +206,22 @@ var _ = Describe("Compiler", func() {
Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
spec2.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
artifacts2, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec2))
artifacts2, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec2))
Expect(errs).To(BeNil())
Expect(len(artifacts2)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
for _, artifact := range artifacts2 {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("etc/hosts"))).To(BeTrue())
@@ -241,7 +239,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
@@ -250,15 +248,14 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -276,7 +273,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
@@ -285,15 +282,14 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -312,7 +308,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
@@ -321,15 +317,14 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -348,7 +343,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
@@ -357,14 +352,13 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
@@ -382,7 +376,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
@@ -391,14 +385,13 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("marvin"))).ToNot(BeTrue())
Expect(helpers.Exists(spec.Rel("test5"))).To(BeTrue())
@@ -416,7 +409,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
@@ -425,14 +418,13 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Exists(spec.Rel("var/lib/udhcpd"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("marvin"))).To(BeTrue())
@@ -454,7 +446,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "pkgs-checker", Category: "package", Version: "9999"})
Expect(err).ToNot(HaveOccurred())
@@ -463,15 +455,14 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
@@ -495,7 +486,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
@@ -504,16 +495,15 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
@@ -539,7 +529,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "d", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
@@ -548,16 +538,15 @@ var _ = Describe("Compiler", func() {
// Expect(err).ToNot(HaveOccurred())
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Untar(spec.Rel("c-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
@@ -581,7 +570,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "extra", Category: "layer", Version: "0.1"})
Expect(err).ToNot(HaveOccurred())
@@ -591,13 +580,13 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileWithReverseDeps(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(2))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Untar(spec.Rel("extra-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
@@ -619,7 +608,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(10))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "vhba", Category: "sys-fs-5.4.2", Version: "20190410"})
Expect(err).ToNot(HaveOccurred())
@@ -629,13 +618,13 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].GetDependencies())).To(Equal(6))
Expect(len(artifacts[0].Dependencies)).To(Equal(6))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
Expect(helpers.Untar(spec.Rel("vhba-sys-fs-5.4.2-20190410.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("sabayon-build-portage-layer-0.20191126.package.tar"))).To(BeTrue())
@@ -658,19 +647,19 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(4))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "b", Category: "test", Version: "1.0"})
spec.SetOutputPath(tmpdir)
artifacts, errs := compiler.CompileWithReverseDeps(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileWithReverseDeps(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(4))
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
}
// A deps on B, so A artifacts are here:
@@ -710,7 +699,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(3))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), options.Concurrency(2))
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "c", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
@@ -722,17 +711,16 @@ var _ = Describe("Compiler", func() {
defer os.RemoveAll(tmpdir) // clean up
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(2)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
for _, artifact := range artifacts {
Expect(helpers.Exists(artifact.GetPath())).To(BeTrue())
Expect(helpers.Untar(artifact.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(artifact.Path)).To(BeTrue())
Expect(helpers.Untar(artifact.Path, tmpdir, false)).ToNot(HaveOccurred())
for _, d := range artifact.GetDependencies() {
Expect(helpers.Exists(d.GetPath())).To(BeTrue())
Expect(helpers.Untar(d.GetPath(), tmpdir, false)).ToNot(HaveOccurred())
for _, d := range artifact.Dependencies {
Expect(helpers.Exists(d.Path)).To(BeTrue())
Expect(helpers.Untar(d.Path, tmpdir, false)).ToNot(HaveOccurred())
}
}
@@ -753,7 +741,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
Expect(err).ToNot(HaveOccurred())
@@ -765,12 +753,11 @@ var _ = Describe("Compiler", func() {
defer os.RemoveAll(tmpdir) // clean up
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
Expect(helpers.Untar(spec.Rel("runtime-layer-0.1.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("bin/busybox"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("var"))).ToNot(BeTrue())
@@ -786,7 +773,7 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{
Name: "dironly",
@@ -814,12 +801,10 @@ var _ = Describe("Compiler", func() {
spec.SetOutputPath(tmpdir)
spec2.SetOutputPath(tmpdir2)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec, spec2))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec, spec2))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(2))
Expect(len(artifacts[0].GetDependencies())).To(Equal(0))
Expect(len(artifacts[0].Dependencies)).To(Equal(0))
Expect(helpers.Untar(spec.Rel("dironly-test-1.0.package.tar"), tmpdir, false)).ToNot(HaveOccurred())
Expect(helpers.Exists(spec.Rel("test1"))).To(BeTrue())
@@ -841,11 +826,11 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
Expect(err).ToNot(HaveOccurred())
compiler.SetCompressionType(GZip)
compiler.Options.CompressionType = compression.GZip
Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
tmpdir, err := ioutil.TempDir("", "tree")
@@ -853,12 +838,11 @@ var _ = Describe("Compiler", func() {
defer os.RemoveAll(tmpdir) // clean up
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar.gz"))).To(BeTrue())
Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.package.tar"))).To(BeFalse())
Expect(artifacts[0].Unpack(tmpdir, false)).ToNot(HaveOccurred())
@@ -877,7 +861,7 @@ var _ = Describe("Compiler", func() {
err := generalRecipe.Load("../../tests/fixtures/includeimage")
Expect(err).ToNot(HaveOccurred())
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
specs, err := compiler.FromDatabase(generalRecipe.GetDatabase(), true, "")
Expect(err).ToNot(HaveOccurred())
@@ -896,11 +880,11 @@ var _ = Describe("Compiler", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(2))
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
compiler := NewLuetCompiler(sd.NewSimpleDockerBackend(), generalRecipe.GetDatabase())
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "runtime", Category: "layer", Version: "0.1"})
Expect(err).ToNot(HaveOccurred())
compiler.SetCompressionType(GZip)
compiler.Options.CompressionType = compression.GZip
Expect(spec.GetPackage().GetPath()).ToNot(Equal(""))
tmpdir, err := ioutil.TempDir("", "tree")
@@ -908,20 +892,19 @@ var _ = Describe("Compiler", func() {
defer os.RemoveAll(tmpdir) // clean up
spec.SetOutputPath(tmpdir)
compiler.SetConcurrency(1)
artifacts, errs := compiler.CompileParallel(false, NewLuetCompilationspecs(spec))
artifacts, errs := compiler.CompileParallel(false, compilerspec.NewLuetCompilationspecs(spec))
Expect(errs).To(BeNil())
Expect(len(artifacts)).To(Equal(1))
Expect(len(artifacts[0].GetDependencies())).To(Equal(1))
Expect(artifacts[0].GetFiles()).To(ContainElement("bin/busybox"))
Expect(len(artifacts[0].Dependencies)).To(Equal(1))
Expect(artifacts[0].Files).To(ContainElement("bin/busybox"))
Expect(helpers.Exists(spec.Rel("runtime-layer-0.1.metadata.yaml"))).To(BeTrue())
art, err := LoadArtifactFromYaml(spec)
Expect(err).ToNot(HaveOccurred())
files := art.GetFiles()
files := art.Files
Expect(files).To(ContainElement("bin/busybox"))
})
})

View File

@@ -1,202 +0,0 @@
// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package compiler
import (
"runtime"
"github.com/mudler/luet/pkg/config"
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/solver"
)
type Compiler interface {
Compile(bool, CompilationSpec) (Artifact, error)
CompileParallel(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
CompileWithReverseDeps(keepPermissions bool, ps CompilationSpecs) ([]Artifact, []error)
ComputeDepTree(p CompilationSpec) (solver.PackagesAssertions, error)
ComputeMinimumCompilableSet(p ...CompilationSpec) ([]CompilationSpec, error)
SetConcurrency(i int)
FromPackage(pkg.Package) (CompilationSpec, error)
FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error)
SetBackend(CompilerBackend)
GetBackend() CompilerBackend
SetBackendArgs([]string)
SetCompressionType(t CompressionImplementation)
}
type CompilerBackendOptions struct {
ImageName string
SourcePath string
DockerFileName string
Destination string
Context string
BackendArgs []string
}
type CompilerOptions struct {
PushImageRepository string
PullImageRepository []string
PullFirst, KeepImg, Push bool
Concurrency int
CompressionType CompressionImplementation
Wait bool
OnlyDeps bool
NoDeps bool
SolverOptions config.LuetSolverOptions
BuildValuesFile string
PackageTargetOnly bool
}
func NewDefaultCompilerOptions() *CompilerOptions {
return &CompilerOptions{
PushImageRepository: "luet/cache",
PullFirst: false,
Push: false,
CompressionType: None,
KeepImg: true,
Concurrency: runtime.NumCPU(),
OnlyDeps: false,
NoDeps: false,
}
}
type CompilerBackend interface {
BuildImage(CompilerBackendOptions) error
ExportImage(CompilerBackendOptions) error
RemoveImage(CompilerBackendOptions) error
Changes(fromImage, toImage CompilerBackendOptions) ([]ArtifactLayer, error)
ImageDefinitionToTar(CompilerBackendOptions) error
ExtractRootfs(opts CompilerBackendOptions, keepPerms bool) error
CopyImage(string, string) error
DownloadImage(opts CompilerBackendOptions) error
Push(opts CompilerBackendOptions) error
ImageAvailable(string) bool
ImageExists(string) bool
}
type Artifact interface {
GetPath() string
SetPath(string)
GetDependencies() []Artifact
SetDependencies(d []Artifact)
GetSourceAssertion() solver.PackagesAssertions
SetSourceAssertion(as solver.PackagesAssertions)
SetCompileSpec(as CompilationSpec)
GetCompileSpec() CompilationSpec
WriteYaml(dst string) error
Unpack(dst string, keepPerms bool) error
Compress(src string, concurrency int) error
SetCompressionType(t CompressionImplementation)
FileList() ([]string, error)
Hash() error
Verify() error
SetFiles(f []string)
GetFiles() []string
GetFileName() string
GetChecksums() Checksums
SetChecksums(c Checksums)
GenerateFinalImage(string, CompilerBackend, bool) (CompilerBackendOptions, error)
GetUncompressedName() string
}
type ArtifactNode struct {
Name string `json:"Name"`
Size int `json:"Size"`
}
type ArtifactDiffs struct {
Additions []ArtifactNode `json:"Adds"`
Deletions []ArtifactNode `json:"Dels"`
Changes []ArtifactNode `json:"Mods"`
}
type ArtifactLayer struct {
FromImage string `json:"Image1"`
ToImage string `json:"Image2"`
Diffs ArtifactDiffs `json:"Diff"`
}
type ArtifactLayerSummary struct {
FromImage string `json:"image1"`
ToImage string `json:"image2"`
AddFiles int `json:"add_files"`
AddSizes int64 `json:"add_sizes"`
DelFiles int `json:"del_files"`
DelSizes int64 `json:"del_sizes"`
ChangeFiles int `json:"change_files"`
ChangeSizes int64 `json:"change_sizes"`
}
type ArtifactLayersSummary struct {
Layers []ArtifactLayerSummary `json:"summary"`
}
// CompilationSpec represent a compilation specification derived from a package
type CompilationSpec interface {
ImageUnpack() bool // tells if the definition is just an image
GetIncludes() []string
GetExcludes() []string
RenderBuildImage() (string, error)
WriteBuildImageDefinition(string) error
RenderStepImage(image string) (string, error)
WriteStepImageDefinition(fromimage, path string) error
GetPackage() pkg.Package
BuildSteps() []string
GetSeedImage() string
SetSeedImage(string)
GetImage() string
SetImage(string)
SetOutputPath(string)
GetOutputPath() string
Rel(string) string
GetPreBuildSteps() []string
GetSourceAssertion() solver.PackagesAssertions
SetSourceAssertion(as solver.PackagesAssertions)
GetRetrieve() []string
CopyRetrieves(dest string) error
SetPackageDir(string)
GetPackageDir() string
EmptyPackage() bool
UnpackedPackage() bool
HasImageSource() bool
IsVirtual() bool
}
type CompilationSpecs interface {
Unique() CompilationSpecs
Len() int
All() []CompilationSpec
Add(CompilationSpec)
Remove(s CompilationSpecs) CompilationSpecs
}

View File

@@ -13,7 +13,7 @@
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package compiler
package artifact
import (
"archive/tar"
@@ -36,6 +36,9 @@ import (
"sync"
bus "github.com/mudler/luet/pkg/bus"
backend "github.com/mudler/luet/pkg/compiler/backend"
compression "github.com/mudler/luet/pkg/compiler/types/compression"
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
. "github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/helpers"
. "github.com/mudler/luet/pkg/logger"
@@ -45,54 +48,25 @@ import (
yaml "gopkg.in/yaml.v2"
)
type CompressionImplementation string
const (
None CompressionImplementation = "none" // e.g. tar for standard packages
GZip CompressionImplementation = "gzip"
Zstandard CompressionImplementation = "zstd"
)
type ArtifactIndex []Artifact
func (i ArtifactIndex) CleanPath() ArtifactIndex {
newIndex := ArtifactIndex{}
for _, n := range i {
art := n.(*PackageArtifact)
// FIXME: This is a dup and makes difficult to add attributes to artifacts
newIndex = append(newIndex, &PackageArtifact{
Path: path.Base(n.GetPath()),
SourceAssertion: art.SourceAssertion,
CompileSpec: art.CompileSpec,
Dependencies: art.Dependencies,
CompressionType: art.CompressionType,
Checksums: art.Checksums,
Files: art.Files,
})
}
return newIndex
//Update if exists, otherwise just create
}
// When compiling, we write also a fingerprint.metadata.yaml file with PackageArtifact. In this way we can have another command to create the repository
// which will consist in just of an repository.yaml which is just the repository structure with the list of package artifact.
// In this way a generic client can fetch the packages and, after unpacking the tree, performing queries to install packages.
type PackageArtifact struct {
Path string `json:"path"`
Dependencies []*PackageArtifact `json:"dependencies"`
CompileSpec *LuetCompilationSpec `json:"compilationspec"`
Checksums Checksums `json:"checksums"`
SourceAssertion solver.PackagesAssertions `json:"-"`
CompressionType CompressionImplementation `json:"compressiontype"`
Files []string `json:"files"`
Dependencies []*PackageArtifact `json:"dependencies"`
CompileSpec *compilerspec.LuetCompilationSpec `json:"compilationspec"`
Checksums Checksums `json:"checksums"`
SourceAssertion solver.PackagesAssertions `json:"-"`
CompressionType compression.Implementation `json:"compressiontype"`
Files []string `json:"files"`
}
func NewPackageArtifact(path string) Artifact {
return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: None}
func NewPackageArtifact(path string) *PackageArtifact {
return &PackageArtifact{Path: path, Dependencies: []*PackageArtifact{}, Checksums: Checksums{}, CompressionType: compression.None}
}
func NewPackageArtifactFromYaml(data []byte) (Artifact, error) {
func NewPackageArtifactFromYaml(data []byte) (*PackageArtifact, error) {
p := &PackageArtifact{Checksums: Checksums{}}
err := yaml.Unmarshal(data, &p)
if err != nil {
@@ -102,42 +76,6 @@ func NewPackageArtifactFromYaml(data []byte) (Artifact, error) {
return p, err
}
func LoadArtifactFromYaml(spec CompilationSpec) (Artifact, error) {
metaFile := spec.GetPackage().GetFingerPrint() + ".metadata.yaml"
dat, err := ioutil.ReadFile(spec.Rel(metaFile))
if err != nil {
return nil, errors.Wrap(err, "Error reading file "+metaFile)
}
art, err := NewPackageArtifactFromYaml(dat)
if err != nil {
return nil, errors.Wrap(err, "Error writing file "+metaFile)
}
// It is relative, set it back to abs
art.SetPath(spec.Rel(art.GetPath()))
return art, nil
}
func (a *PackageArtifact) SetCompressionType(t CompressionImplementation) {
a.CompressionType = t
}
func (a *PackageArtifact) GetChecksums() Checksums {
return a.Checksums
}
func (a *PackageArtifact) SetChecksums(c Checksums) {
a.Checksums = c
}
func (a *PackageArtifact) SetFiles(f []string) {
a.Files = f
}
func (a *PackageArtifact) GetFiles() []string {
return a.Files
}
func (a *PackageArtifact) Hash() error {
return a.Checksums.Generate(a)
}
@@ -181,8 +119,8 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
}
//p := a.CompileSpec.GetPackage().GetPath()
mangle.GetCompileSpec().GetPackage().SetPath("")
for _, ass := range mangle.GetCompileSpec().GetSourceAssertion() {
mangle.CompileSpec.GetPackage().SetPath("")
for _, ass := range mangle.CompileSpec.GetSourceAssertion() {
ass.Package.SetPath("")
}
@@ -191,7 +129,7 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
return errors.Wrap(err, "While marshalling for PackageArtifact YAML")
}
err = ioutil.WriteFile(filepath.Join(dst, a.GetCompileSpec().GetPackage().GetFingerPrint()+".metadata.yaml"), data, os.ModePerm)
err = ioutil.WriteFile(filepath.Join(dst, a.CompileSpec.GetPackage().GetFingerPrint()+".metadata.yaml"), data, os.ModePerm)
if err != nil {
return errors.Wrap(err, "While writing PackageArtifact YAML")
}
@@ -201,48 +139,8 @@ func (a *PackageArtifact) WriteYaml(dst string) error {
return nil
}
func (a *PackageArtifact) GetSourceAssertion() solver.PackagesAssertions {
return a.SourceAssertion
}
func (a *PackageArtifact) SetCompileSpec(as CompilationSpec) {
a.CompileSpec = as.(*LuetCompilationSpec)
}
func (a *PackageArtifact) GetCompileSpec() CompilationSpec {
return a.CompileSpec
}
func (a *PackageArtifact) SetSourceAssertion(as solver.PackagesAssertions) {
a.SourceAssertion = as
}
func (a *PackageArtifact) GetDependencies() []Artifact {
ret := []Artifact{}
for _, d := range a.Dependencies {
ret = append(ret, d)
}
return ret
}
func (a *PackageArtifact) SetDependencies(d []Artifact) {
ret := []*PackageArtifact{}
for _, dd := range d {
ret = append(ret, dd.(*PackageArtifact))
}
a.Dependencies = ret
}
func (a *PackageArtifact) GetPath() string {
return a.Path
}
func (a *PackageArtifact) GetFileName() string {
return path.Base(a.GetPath())
}
func (a *PackageArtifact) SetPath(p string) {
a.Path = p
return path.Base(a.Path)
}
func (a *PackageArtifact) genDockerfile() string {
@@ -274,9 +172,13 @@ func CreateArtifactForFile(s string, opts ...func(*PackageArtifact)) (*PackageAr
return a, a.Compress(archive, 1)
}
type ImageBuilder interface {
BuildImage(backend.Options) error
}
// GenerateFinalImage takes an artifact and builds a Docker image with its content
func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend, keepPerms bool) (CompilerBackendOptions, error) {
builderOpts := CompilerBackendOptions{}
func (a *PackageArtifact) GenerateFinalImage(imageName string, b ImageBuilder, keepPerms bool) (backend.Options, error) {
builderOpts := backend.Options{}
archive, err := LuetCfg.GetSystem().TempDir("archive")
if err != nil {
return builderOpts, errors.Wrap(err, "error met while creating tempdir for "+a.Path)
@@ -311,7 +213,7 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend
return builderOpts, errors.Wrap(err, "error met while rendering artifact dockerfile "+a.Path)
}
builderOpts = CompilerBackendOptions{
builderOpts = backend.Options{
ImageName: imageName,
SourcePath: archive,
DockerFileName: dockerFile,
@@ -326,7 +228,7 @@ func (a *PackageArtifact) GenerateFinalImage(imageName string, b CompilerBackend
func (a *PackageArtifact) Compress(src string, concurrency int) error {
switch a.CompressionType {
case Zstandard:
case compression.Zstandard:
err := helpers.Tar(src, a.Path)
if err != nil {
return err
@@ -364,7 +266,7 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
a.Path = zstdFile
return nil
case GZip:
case compression.GZip:
err := helpers.Tar(src, a.Path)
if err != nil {
return err
@@ -409,10 +311,10 @@ func (a *PackageArtifact) Compress(src string, concurrency int) error {
func (a *PackageArtifact) getCompressedName() string {
switch a.CompressionType {
case Zstandard:
case compression.Zstandard:
return a.Path + ".zst"
case GZip:
case compression.GZip:
return a.Path + ".gz"
}
return a.Path
@@ -421,7 +323,7 @@ func (a *PackageArtifact) getCompressedName() string {
// GetUncompressedName returns the artifact path without the extension suffix
func (a *PackageArtifact) GetUncompressedName() string {
switch a.CompressionType {
case Zstandard, GZip:
case compression.Zstandard, compression.GZip:
return strings.TrimSuffix(a.Path, filepath.Ext(a.Path))
}
return a.Path
@@ -509,13 +411,13 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
tarModifier := helpers.NewTarModifierWrapper(dst, tarModifierWrapperFunc)
switch a.CompressionType {
case Zstandard:
case compression.Zstandard:
// Create the uncompressed archive
archive, err := os.Create(a.GetPath() + ".uncompressed")
archive, err := os.Create(a.Path + ".uncompressed")
if err != nil {
return err
}
defer os.RemoveAll(a.GetPath() + ".uncompressed")
defer os.RemoveAll(a.Path + ".uncompressed")
defer archive.Close()
original, err := os.Open(a.Path)
@@ -534,22 +436,22 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
_, err = io.Copy(archive, d)
if err != nil {
return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed")
}
err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
err = helpers.UntarProtect(a.Path+".uncompressed", dst,
LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
if err != nil {
return err
}
return nil
case GZip:
case compression.GZip:
// Create the uncompressed archive
archive, err := os.Create(a.GetPath() + ".uncompressed")
archive, err := os.Create(a.Path + ".uncompressed")
if err != nil {
return err
}
defer os.RemoveAll(a.GetPath() + ".uncompressed")
defer os.RemoveAll(a.Path + ".uncompressed")
defer archive.Close()
original, err := os.Open(a.Path)
@@ -567,10 +469,10 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
_, err = io.Copy(archive, r)
if err != nil {
return errors.Wrap(err, "Cannot copy to "+a.GetPath()+".uncompressed")
return errors.Wrap(err, "Cannot copy to "+a.Path+".uncompressed")
}
err = helpers.UntarProtect(a.GetPath()+".uncompressed", dst,
err = helpers.UntarProtect(a.Path+".uncompressed", dst,
LuetCfg.GetGeneral().SameOwner, protectedFiles, tarModifier)
if err != nil {
return err
@@ -578,7 +480,7 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
return nil
// Defaults to tar only (covers when "none" is supplied)
default:
return helpers.UntarProtect(a.GetPath(), dst, LuetCfg.GetGeneral().SameOwner,
return helpers.UntarProtect(a.Path, dst, LuetCfg.GetGeneral().SameOwner,
protectedFiles, tarModifier)
}
return errors.New("Compression type must be supplied")
@@ -588,12 +490,12 @@ func (a *PackageArtifact) Unpack(dst string, keepPerms bool) error {
func (a *PackageArtifact) FileList() ([]string, error) {
var tr *tar.Reader
switch a.CompressionType {
case Zstandard:
archive, err := os.Create(a.GetPath() + ".uncompressed")
case compression.Zstandard:
archive, err := os.Create(a.Path + ".uncompressed")
if err != nil {
return []string{}, err
}
defer os.RemoveAll(a.GetPath() + ".uncompressed")
defer os.RemoveAll(a.Path + ".uncompressed")
defer archive.Close()
original, err := os.Open(a.Path)
@@ -609,13 +511,13 @@ func (a *PackageArtifact) FileList() ([]string, error) {
}
defer r.Close()
tr = tar.NewReader(r)
case GZip:
case compression.GZip:
// Create the uncompressed archive
archive, err := os.Create(a.GetPath() + ".uncompressed")
archive, err := os.Create(a.Path + ".uncompressed")
if err != nil {
return []string{}, err
}
defer os.RemoveAll(a.GetPath() + ".uncompressed")
defer os.RemoveAll(a.Path + ".uncompressed")
defer archive.Close()
original, err := os.Open(a.Path)
@@ -634,7 +536,7 @@ func (a *PackageArtifact) FileList() ([]string, error) {
// Defaults to tar only (covers when "none" is supplied)
default:
tarFile, err := os.Open(a.GetPath())
tarFile, err := os.Open(a.Path)
if err != nil {
return []string{}, errors.Wrap(err, "Could not open package archive")
}
@@ -729,8 +631,24 @@ func compileRegexes(regexes []string) []*regexp.Regexp {
return result
}
type ArtifactNode struct {
Name string `json:"Name"`
Size int `json:"Size"`
}
type ArtifactDiffs struct {
Additions []ArtifactNode `json:"Adds"`
Deletions []ArtifactNode `json:"Dels"`
Changes []ArtifactNode `json:"Mods"`
}
type ArtifactLayer struct {
FromImage string `json:"Image1"`
ToImage string `json:"Image2"`
Diffs ArtifactDiffs `json:"Diff"`
}
// ExtractArtifactFromDelta extracts deltas from ArtifactLayer from an image in tar format
func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t CompressionImplementation) (Artifact, error) {
func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurrency int, keepPerms bool, includes []string, excludes []string, t compression.Implementation) (*PackageArtifact, error) {
archive, err := LuetCfg.GetSystem().TempDir("archive")
if err != nil {
@@ -852,45 +770,10 @@ func ExtractArtifactFromDelta(src, dst string, layers []ArtifactLayer, concurren
wg.Wait()
a := NewPackageArtifact(dst)
a.SetCompressionType(t)
a.CompressionType = t
err = a.Compress(archive, concurrency)
if err != nil {
return nil, errors.Wrap(err, "Error met while creating package archive")
}
return a, nil
}
func ComputeArtifactLayerSummary(diffs []ArtifactLayer) ArtifactLayersSummary {
ans := ArtifactLayersSummary{
Layers: make([]ArtifactLayerSummary, 0),
}
for _, layer := range diffs {
sum := ArtifactLayerSummary{
FromImage: layer.FromImage,
ToImage: layer.ToImage,
AddFiles: 0,
AddSizes: 0,
DelFiles: 0,
DelSizes: 0,
ChangeFiles: 0,
ChangeSizes: 0,
}
for _, a := range layer.Diffs.Additions {
sum.AddFiles++
sum.AddSizes += int64(a.Size)
}
for _, d := range layer.Diffs.Deletions {
sum.DelFiles++
sum.DelSizes += int64(d.Size)
}
for _, c := range layer.Diffs.Changes {
sum.ChangeFiles++
sum.ChangeSizes += int64(c.Size)
}
ans.Layers = append(ans.Layers, sum)
}
return ans
}

View File

@@ -13,7 +13,7 @@
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package compiler_test
package artifact_test
import (
"io/ioutil"
@@ -22,7 +22,10 @@ import (
"github.com/mudler/luet/pkg/compiler"
. "github.com/mudler/luet/pkg/compiler/backend"
"github.com/mudler/luet/pkg/solver"
backend "github.com/mudler/luet/pkg/compiler/backend"
. "github.com/mudler/luet/pkg/compiler/types/artifact"
compression "github.com/mudler/luet/pkg/compiler/types/compression"
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
. "github.com/mudler/luet/pkg/compiler"
helpers "github.com/mudler/luet/pkg/helpers"
@@ -43,13 +46,10 @@ var _ = Describe("Artifact", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
cc := NewLuetCompiler(nil, generalRecipe.GetDatabase())
lspec, err := cc.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
Expect(err).ToNot(HaveOccurred())
lspec, ok := spec.(*LuetCompilationSpec)
Expect(ok).To(BeTrue())
Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
Expect(lspec.Image).To(Equal("luet/base"))
Expect(lspec.Seed).To(Equal("alpine"))
@@ -81,7 +81,7 @@ ENV PACKAGE_NAME=enman
ENV PACKAGE_VERSION=1.4.0
ENV PACKAGE_CATEGORY=app-admin`))
b := NewSimpleDockerBackend()
opts := CompilerBackendOptions{
opts := backend.Options{
ImageName: "luet/base",
SourcePath: tmpdir,
DockerFileName: "Dockerfile",
@@ -105,7 +105,7 @@ ENV PACKAGE_VERSION=1.4.0
ENV PACKAGE_CATEGORY=app-admin
RUN echo foo > /test
RUN echo bar > /test2`))
opts2 := CompilerBackendOptions{
opts2 := backend.Options{
ImageName: "test",
SourcePath: tmpdir,
DockerFileName: "LuetDockerfile",
@@ -114,7 +114,7 @@ RUN echo bar > /test2`))
Expect(b.BuildImage(opts2)).ToNot(HaveOccurred())
Expect(b.ExportImage(opts2)).ToNot(HaveOccurred())
Expect(helpers.Exists(filepath.Join(tmpdir, "output2.tar"))).To(BeTrue())
diffs, err := b.Changes(opts, opts2)
diffs, err := compiler.GenerateChanges(b, opts, opts2)
Expect(err).ToNot(HaveOccurred())
artifacts := []ArtifactNode{{
@@ -135,13 +135,13 @@ RUN echo bar > /test2`))
Additions: artifacts,
},
}}))
err = b.ExtractRootfs(CompilerBackendOptions{ImageName: "test", Destination: rootfs}, false)
err = b.ExtractRootfs(backend.Options{ImageName: "test", Destination: rootfs}, false)
Expect(err).ToNot(HaveOccurred())
artifact, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, None)
a, err := ExtractArtifactFromDelta(rootfs, filepath.Join(tmpdir, "package.tar"), diffs, 2, false, []string{}, []string{}, compression.None)
Expect(err).ToNot(HaveOccurred())
Expect(helpers.Exists(filepath.Join(tmpdir, "package.tar"))).To(BeTrue())
err = helpers.Untar(artifact.GetPath(), unpacked, false)
err = helpers.Untar(a.Path, unpacked, false)
Expect(err).ToNot(HaveOccurred())
Expect(helpers.Exists(filepath.Join(unpacked, "test"))).To(BeTrue())
Expect(helpers.Exists(filepath.Join(unpacked, "test2"))).To(BeTrue())
@@ -152,13 +152,13 @@ RUN echo bar > /test2`))
Expect(err).ToNot(HaveOccurred())
Expect(content2).To(Equal("bar\n"))
err = artifact.Hash()
err = a.Hash()
Expect(err).ToNot(HaveOccurred())
err = artifact.Verify()
err = a.Verify()
Expect(err).ToNot(HaveOccurred())
Expect(helpers.CopyFile(filepath.Join(tmpdir, "output2.tar"), filepath.Join(tmpdir, "package.tar"))).ToNot(HaveOccurred())
err = artifact.Verify()
err = a.Verify()
Expect(err).To(HaveOccurred())
})
@@ -183,13 +183,13 @@ RUN echo bar > /test2`))
err = ioutil.WriteFile(filepath.Join(tmpdir, "foo", "bar", "test"), testString, 0644)
Expect(err).ToNot(HaveOccurred())
artifact := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
artifact.SetCompileSpec(&LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}})
a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}}
err = artifact.Compress(tmpdir, 1)
err = a.Compress(tmpdir, 1)
Expect(err).ToNot(HaveOccurred())
resultingImage := imageprefix + "foo--1.0"
opts, err := artifact.GenerateFinalImage(resultingImage, b, false)
opts, err := a.GenerateFinalImage(resultingImage, b, false)
Expect(err).ToNot(HaveOccurred())
Expect(opts.ImageName).To(Equal(resultingImage))
@@ -199,7 +199,7 @@ RUN echo bar > /test2`))
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(result) // clean up
err = b.ExtractRootfs(CompilerBackendOptions{ImageName: resultingImage, Destination: result}, false)
err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
Expect(err).ToNot(HaveOccurred())
content, err := ioutil.ReadFile(filepath.Join(result, "test"))
@@ -225,13 +225,13 @@ RUN echo bar > /test2`))
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(tmpWork) // clean up
artifact := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
artifact.SetCompileSpec(&LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}})
a := NewPackageArtifact(filepath.Join(tmpWork, "fake.tar"))
a.CompileSpec = &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Version: "1.0"}}
err = artifact.Compress(tmpdir, 1)
err = a.Compress(tmpdir, 1)
Expect(err).ToNot(HaveOccurred())
resultingImage := imageprefix + "foo--1.0"
opts, err := artifact.GenerateFinalImage(resultingImage, b, false)
opts, err := a.GenerateFinalImage(resultingImage, b, false)
Expect(err).ToNot(HaveOccurred())
Expect(opts.ImageName).To(Equal(resultingImage))
@@ -241,7 +241,7 @@ RUN echo bar > /test2`))
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(result) // clean up
err = b.ExtractRootfs(CompilerBackendOptions{ImageName: resultingImage, Destination: result}, false)
err = b.ExtractRootfs(backend.Options{ImageName: resultingImage, Destination: result}, false)
Expect(err).ToNot(HaveOccurred())
Expect(helpers.DirectoryIsEmpty(result)).To(BeFalse())
@@ -253,15 +253,15 @@ RUN echo bar > /test2`))
It("Retrieves uncompressed name", func() {
a := NewPackageArtifact("foo.tar.gz")
a.SetCompressionType(compiler.GZip)
a.CompressionType = (compression.GZip)
Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
a = NewPackageArtifact("foo.tar.zst")
a.SetCompressionType(compiler.Zstandard)
a.CompressionType = compression.Zstandard
Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
a = NewPackageArtifact("foo.tar")
a.SetCompressionType(compiler.None)
a.CompressionType = compression.None
Expect(a.GetUncompressedName()).To(Equal("foo.tar"))
})
})

View File

@@ -13,7 +13,7 @@
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package compiler
package artifact
import (
@@ -43,7 +43,7 @@ type HashOptions struct {
}
// Generate generates all Checksums supported for the artifact
func (c *Checksums) Generate(a Artifact) error {
func (c *Checksums) Generate(a *PackageArtifact) error {
return c.generateSHA256(a)
}
@@ -56,13 +56,13 @@ func (c Checksums) Compare(d Checksums) error {
return nil
}
func (c *Checksums) generateSHA256(a Artifact) error {
func (c *Checksums) generateSHA256(a *PackageArtifact) error {
return c.generateSum(a, HashOptions{Hasher: sha256.New(), Type: SHA256})
}
func (c *Checksums) generateSum(a Artifact, opts HashOptions) error {
func (c *Checksums) generateSum(a *PackageArtifact, opts HashOptions) error {
f, err := os.Open(a.GetPath())
f, err := os.Open(a.Path)
if err != nil {
return err
}

View File

@@ -13,13 +13,13 @@
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package compiler_test
package artifact_test
import (
"io/ioutil"
"os"
. "github.com/mudler/luet/pkg/compiler"
. "github.com/mudler/luet/pkg/compiler/types/artifact"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

View File

@@ -0,0 +1,9 @@
package compression
type Implementation string
const (
None Implementation = "none" // e.g. tar for standard packages
GZip Implementation = "gzip"
Zstandard Implementation = "zstd"
)

View File

@@ -0,0 +1,191 @@
// Copyright © 2019-2021 Ettore Di Giacinto <mudler@sabayon.org>
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package options
import (
"runtime"
"github.com/mudler/luet/pkg/compiler/types/compression"
"github.com/mudler/luet/pkg/config"
"github.com/mudler/luet/pkg/solver"
)
type Compiler struct {
PushImageRepository string
PullImageRepository []string
PullFirst, KeepImg, Push bool
Concurrency int
CompressionType compression.Implementation
Wait bool
OnlyDeps bool
NoDeps bool
SolverOptions config.LuetSolverOptions
BuildValuesFile []string
BuildValues []map[string]interface{}
PackageTargetOnly bool
BackendArgs []string
BackendType string
}
func NewDefaultCompiler() *Compiler {
return &Compiler{
PushImageRepository: "luet/cache",
PullFirst: false,
Push: false,
CompressionType: compression.None,
KeepImg: true,
Concurrency: runtime.NumCPU(),
OnlyDeps: false,
NoDeps: false,
SolverOptions: config.LuetSolverOptions{Options: solver.Options{Concurrency: 1, Type: solver.SingleCoreSimple}},
}
}
type Option func(cfg *Compiler) error
func (cfg *Compiler) Apply(opts ...Option) error {
for _, opt := range opts {
if opt == nil {
continue
}
if err := opt(cfg); err != nil {
return err
}
}
return nil
}
func WithOptions(opt *Compiler) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg = opt
return nil
}
}
func WithBackendType(r string) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.BackendType = r
return nil
}
}
func WithBuildValues(r []string) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.BuildValuesFile = r
return nil
}
}
func WithPullRepositories(r []string) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.PullImageRepository = r
return nil
}
}
func WithPushRepository(r string) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
if len(cfg.PullImageRepository) == 0 {
cfg.PullImageRepository = []string{cfg.PushImageRepository}
}
cfg.PushImageRepository = r
return nil
}
}
func BackendArgs(r []string) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.BackendArgs = r
return nil
}
}
func PullFirst(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.PullFirst = b
return nil
}
}
func KeepImg(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.KeepImg = b
return nil
}
}
func PushImages(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.Push = b
return nil
}
}
func Wait(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.Wait = b
return nil
}
}
func OnlyDeps(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.OnlyDeps = b
return nil
}
}
func OnlyTarget(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.PackageTargetOnly = b
return nil
}
}
func NoDeps(b bool) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.NoDeps = b
return nil
}
}
func Concurrency(i int) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
if i == 0 {
i = runtime.NumCPU()
}
cfg.Concurrency = i
return nil
}
}
func WithCompressionType(t compression.Implementation) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.CompressionType = t
return nil
}
}
func WithSolverOptions(c config.LuetSolverOptions) func(cfg *Compiler) error {
return func(cfg *Compiler) error {
cfg.SolverOptions = c
return nil
}
}

View File

@@ -13,12 +13,14 @@
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package compiler
package compilerspec
import (
"io/ioutil"
"path/filepath"
options "github.com/mudler/luet/pkg/compiler/types/options"
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/solver"
"github.com/otiai10/copy"
@@ -27,7 +29,7 @@ import (
type LuetCompilationspecs []LuetCompilationSpec
func NewLuetCompilationspecs(s ...CompilationSpec) CompilationSpecs {
func NewLuetCompilationspecs(s ...*LuetCompilationSpec) *LuetCompilationspecs {
all := LuetCompilationspecs{}
for _, spec := range s {
@@ -40,7 +42,7 @@ func (specs LuetCompilationspecs) Len() int {
return len(specs)
}
func (specs *LuetCompilationspecs) Remove(s CompilationSpecs) CompilationSpecs {
func (specs *LuetCompilationspecs) Remove(s *LuetCompilationspecs) *LuetCompilationspecs {
newSpecs := LuetCompilationspecs{}
SPECS:
for _, spec := range specs.All() {
@@ -54,16 +56,12 @@ SPECS:
return &newSpecs
}
func (specs *LuetCompilationspecs) Add(s CompilationSpec) {
c, ok := s.(*LuetCompilationSpec)
if !ok {
panic("LuetCompilationspecs supports only []LuetCompilationSpec")
}
*specs = append(*specs, *c)
func (specs *LuetCompilationspecs) Add(s *LuetCompilationSpec) {
*specs = append(*specs, *s)
}
func (specs *LuetCompilationspecs) All() []CompilationSpec {
var cspecs []CompilationSpec
func (specs *LuetCompilationspecs) All() []*LuetCompilationSpec {
var cspecs []*LuetCompilationSpec
for i, _ := range *specs {
f := (*specs)[i]
cspecs = append(cspecs, &f)
@@ -72,7 +70,7 @@ func (specs *LuetCompilationspecs) All() []CompilationSpec {
return cspecs
}
func (specs *LuetCompilationspecs) Unique() CompilationSpecs {
func (specs *LuetCompilationspecs) Unique() *LuetCompilationspecs {
newSpecs := LuetCompilationspecs{}
seen := map[string]bool{}
@@ -103,9 +101,11 @@ type LuetCompilationSpec struct {
Unpack bool `json:"unpack"`
Includes []string `json:"includes"`
Excludes []string `json:"excludes"`
BuildOptions options.Compiler `json:"build_options"`
}
func NewLuetCompilationSpec(b []byte, p pkg.Package) (CompilationSpec, error) {
func NewLuetCompilationSpec(b []byte, p pkg.Package) (*LuetCompilationSpec, error) {
var spec LuetCompilationSpec
err := yaml.Unmarshal(b, &spec)
if err != nil {
@@ -118,6 +118,10 @@ func (cs *LuetCompilationSpec) GetSourceAssertion() solver.PackagesAssertions {
return cs.SourceAssertion
}
func (cs *LuetCompilationSpec) SetBuildOptions(b options.Compiler) {
cs.BuildOptions = b
}
func (cs *LuetCompilationSpec) SetSourceAssertion(as solver.PackagesAssertions) {
cs.SourceAssertion = as
}

View File

@@ -13,17 +13,18 @@
// You should have received a copy of the GNU General Public License along
// with this program; if not, see <http://www.gnu.org/licenses/>.
package compiler_test
package compilerspec_test
import (
"io/ioutil"
"os"
"path/filepath"
compilerspec "github.com/mudler/luet/pkg/compiler/types/spec"
. "github.com/mudler/luet/pkg/compiler"
helpers "github.com/mudler/luet/pkg/helpers"
pkg "github.com/mudler/luet/pkg/package"
"github.com/mudler/luet/pkg/solver"
"github.com/mudler/luet/pkg/tree"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@@ -32,41 +33,41 @@ import (
var _ = Describe("Spec", func() {
Context("Luet specs", func() {
It("Allows normal operations", func() {
testSpec := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
testSpec2 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "bar", Category: "a", Version: "0"}}
testSpec3 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "baz", Category: "a", Version: "0"}}
testSpec4 := &LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
testSpec := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
testSpec2 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "bar", Category: "a", Version: "0"}}
testSpec3 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "baz", Category: "a", Version: "0"}}
testSpec4 := &compilerspec.LuetCompilationSpec{Package: &pkg.DefaultPackage{Name: "foo", Category: "a", Version: "0"}}
specs := NewLuetCompilationspecs(testSpec, testSpec2)
specs := compilerspec.NewLuetCompilationspecs(testSpec, testSpec2)
Expect(specs.Len()).To(Equal(2))
Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2}))
Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2}))
specs.Add(testSpec3)
Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3}))
Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3}))
specs.Add(testSpec4)
Expect(specs.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3, testSpec4}))
Expect(specs.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3, testSpec4}))
newSpec := specs.Unique()
Expect(newSpec.All()).To(Equal([]CompilationSpec{testSpec, testSpec2, testSpec3}))
Expect(newSpec.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec, testSpec2, testSpec3}))
newSpec2 := specs.Remove(NewLuetCompilationspecs(testSpec, testSpec2))
Expect(newSpec2.All()).To(Equal([]CompilationSpec{testSpec3}))
newSpec2 := specs.Remove(compilerspec.NewLuetCompilationspecs(testSpec, testSpec2))
Expect(newSpec2.All()).To(Equal([]*compilerspec.LuetCompilationSpec{testSpec3}))
})
Context("virtuals", func() {
When("is empty", func() {
It("is virtual", func() {
spec := &LuetCompilationSpec{}
spec := &compilerspec.LuetCompilationSpec{}
Expect(spec.IsVirtual()).To(BeTrue())
})
})
When("has defined steps", func() {
It("is not a virtual", func() {
spec := &LuetCompilationSpec{Steps: []string{"foo"}}
spec := &compilerspec.LuetCompilationSpec{Steps: []string{"foo"}}
Expect(spec.IsVirtual()).To(BeFalse())
})
})
When("has defined image", func() {
It("is not a virtual", func() {
spec := &LuetCompilationSpec{Image: "foo"}
spec := &compilerspec.LuetCompilationSpec{Image: "foo"}
Expect(spec.IsVirtual()).To(BeFalse())
})
})
@@ -82,13 +83,10 @@ var _ = Describe("Spec", func() {
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
lspec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "enman", Category: "app-admin", Version: "1.4.0"})
Expect(err).ToNot(HaveOccurred())
lspec, ok := spec.(*LuetCompilationSpec)
Expect(ok).To(BeTrue())
Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
Expect(lspec.Image).To(Equal("luet/base"))
Expect(lspec.Seed).To(Equal("alpine"))
@@ -137,13 +135,10 @@ RUN echo bar > /test2`))
Expect(len(generalRecipe.GetDatabase().GetPackages())).To(Equal(1))
compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase(), NewDefaultCompilerOptions(), solver.Options{Type: solver.SingleCoreSimple})
spec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
compiler := NewLuetCompiler(nil, generalRecipe.GetDatabase())
lspec, err := compiler.FromPackage(&pkg.DefaultPackage{Name: "a", Category: "test", Version: "1.0"})
Expect(err).ToNot(HaveOccurred())
lspec, ok := spec.(*LuetCompilationSpec)
Expect(ok).To(BeTrue())
Expect(lspec.Steps).To(Equal([]string{"echo foo > /test", "echo bar > /test2"}))
Expect(lspec.Image).To(Equal("luet/base"))
Expect(lspec.Seed).To(Equal("alpine"))