mirror of
https://github.com/mudler/luet.git
synced 2025-09-10 19:49:06 +00:00
Compare commits
7 Commits
Author | SHA1 | Date | |
---|---|---|---|
|
19e6054574 | ||
|
a8624fe451 | ||
|
14c1d6ef24 | ||
|
36c58307e2 | ||
|
665261e526 | ||
|
794c5984a2 | ||
|
a765147c1d |
89
cmd/pack.go
Normal file
89
cmd/pack.go
Normal file
@@ -0,0 +1,89 @@
|
|||||||
|
// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License along
|
||||||
|
// with this program; if not, see <http://www.gnu.org/licenses/>.
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
helpers "github.com/mudler/luet/cmd/helpers"
|
||||||
|
"github.com/mudler/luet/pkg/compiler"
|
||||||
|
. "github.com/mudler/luet/pkg/config"
|
||||||
|
. "github.com/mudler/luet/pkg/logger"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
)
|
||||||
|
|
||||||
|
var packCmd = &cobra.Command{
|
||||||
|
Use: "pack <package name>",
|
||||||
|
Short: "pack a custom package",
|
||||||
|
Long: `pack and creates metadata directly from a source path`,
|
||||||
|
PreRun: func(cmd *cobra.Command, args []string) {
|
||||||
|
viper.BindPFlag("destination", cmd.Flags().Lookup("destination"))
|
||||||
|
viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
|
||||||
|
viper.BindPFlag("source", cmd.Flags().Lookup("source"))
|
||||||
|
},
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
sourcePath := viper.GetString("source")
|
||||||
|
|
||||||
|
dst := viper.GetString("destination")
|
||||||
|
compressionType := viper.GetString("compression")
|
||||||
|
concurrency := LuetCfg.GetGeneral().Concurrency
|
||||||
|
|
||||||
|
if len(args) != 1 {
|
||||||
|
Fatal("You must specify a package name")
|
||||||
|
}
|
||||||
|
|
||||||
|
packageName := args[0]
|
||||||
|
|
||||||
|
p, err := helpers.ParsePackageStr(packageName)
|
||||||
|
if err != nil {
|
||||||
|
Fatal("Invalid package string ", packageName, ": ", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
spec := &compiler.LuetCompilationSpec{Package: p}
|
||||||
|
artifact := compiler.NewPackageArtifact(filepath.Join(dst, p.GetFingerPrint()+".package.tar"))
|
||||||
|
artifact.SetCompressionType(compiler.CompressionImplementation(compressionType))
|
||||||
|
err = artifact.Compress(sourcePath, concurrency)
|
||||||
|
if err != nil {
|
||||||
|
Fatal("failed compressing ", packageName, ": ", err.Error())
|
||||||
|
}
|
||||||
|
artifact.SetCompileSpec(spec)
|
||||||
|
filelist, err := artifact.FileList()
|
||||||
|
if err != nil {
|
||||||
|
Fatal("failed generating file list for ", packageName, ": ", err.Error())
|
||||||
|
}
|
||||||
|
artifact.SetFiles(filelist)
|
||||||
|
artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
|
||||||
|
err = artifact.WriteYaml(dst)
|
||||||
|
if err != nil {
|
||||||
|
Fatal("failed writing metadata yaml file for ", packageName, ": ", err.Error())
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
path, err := os.Getwd()
|
||||||
|
if err != nil {
|
||||||
|
Fatal(err)
|
||||||
|
}
|
||||||
|
packCmd.Flags().String("source", path, "Source folder")
|
||||||
|
packCmd.Flags().String("destination", path, "Destination folder")
|
||||||
|
packCmd.Flags().String("compression", "gzip", "Compression alg: none, gzip")
|
||||||
|
|
||||||
|
RootCmd.AddCommand(packCmd)
|
||||||
|
}
|
@@ -38,7 +38,7 @@ var Verbose bool
|
|||||||
var LockedCommands = []string{"install", "uninstall", "upgrade"}
|
var LockedCommands = []string{"install", "uninstall", "upgrade"}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
LuetCLIVersion = "0.9"
|
LuetCLIVersion = "0.9.2"
|
||||||
LuetEnvPrefix = "LUET"
|
LuetEnvPrefix = "LUET"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@@ -95,7 +95,7 @@ func (*SimpleDocker) ImageExists(imagename string) bool {
|
|||||||
cmd := exec.Command("docker", buildarg...)
|
cmd := exec.Command("docker", buildarg...)
|
||||||
out, err := cmd.CombinedOutput()
|
out, err := cmd.CombinedOutput()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Warning("Image not present")
|
Debug("Image not present")
|
||||||
Debug(string(out))
|
Debug(string(out))
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
@@ -21,8 +21,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"github.com/ghodss/yaml"
|
|
||||||
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -235,7 +233,58 @@ func (cs *LuetCompiler) stripFromRootfs(includes []string, rootfs string, includ
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string, concurrency int, keepPermissions, keepImg bool, p CompilationSpec, generateArtifact bool) (Artifact, error) {
|
func (cs *LuetCompiler) unpackFs(rootfs string, concurrency int, p CompilationSpec) (Artifact, error) {
|
||||||
|
if p.GetPackageDir() != "" {
|
||||||
|
Info(":tophat: Packing from output dir", p.GetPackageDir())
|
||||||
|
rootfs = filepath.Join(rootfs, p.GetPackageDir())
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(p.GetIncludes()) > 0 {
|
||||||
|
// strip from includes
|
||||||
|
cs.stripFromRootfs(p.GetIncludes(), rootfs, true)
|
||||||
|
}
|
||||||
|
if len(p.GetExcludes()) > 0 {
|
||||||
|
// strip from includes
|
||||||
|
cs.stripFromRootfs(p.GetExcludes(), rootfs, false)
|
||||||
|
}
|
||||||
|
artifact := NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
|
||||||
|
artifact.SetCompressionType(cs.CompressionType)
|
||||||
|
|
||||||
|
if err := artifact.Compress(rootfs, concurrency); err != nil {
|
||||||
|
return nil, errors.Wrap(err, "Error met while creating package archive")
|
||||||
|
}
|
||||||
|
|
||||||
|
artifact.SetCompileSpec(p)
|
||||||
|
return artifact, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cs *LuetCompiler) unpackDelta(rootfs string, concurrency int, keepPermissions bool, p CompilationSpec, builderOpts, runnerOpts CompilerBackendOptions) (Artifact, error) {
|
||||||
|
pkgTag := ":package: " + p.GetPackage().HumanReadableString()
|
||||||
|
if err := cs.Backend.ExportImage(builderOpts); err != nil {
|
||||||
|
return nil, errors.Wrap(err, "Could not export image")
|
||||||
|
}
|
||||||
|
if !cs.Options.KeepImageExport {
|
||||||
|
defer os.Remove(builderOpts.Destination)
|
||||||
|
}
|
||||||
|
Info(pkgTag, ":hammer: Generating delta")
|
||||||
|
diffs, err := cs.Backend.Changes(builderOpts.Destination, runnerOpts.Destination)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "Could not generate changes from layers")
|
||||||
|
}
|
||||||
|
artifact, err := ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.CompressionType)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "Could not generate deltas")
|
||||||
|
}
|
||||||
|
|
||||||
|
artifact.SetCompileSpec(p)
|
||||||
|
return artifact, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cs *LuetCompiler) buildPackageImage(image, buildertaggedImage, packageImage string,
|
||||||
|
concurrency int, keepPermissions bool,
|
||||||
|
p CompilationSpec) (CompilerBackendOptions, CompilerBackendOptions, error) {
|
||||||
|
|
||||||
|
var runnerOpts, builderOpts CompilerBackendOptions
|
||||||
|
|
||||||
pkgTag := ":package: " + p.GetPackage().HumanReadableString()
|
pkgTag := ":package: " + p.GetPackage().HumanReadableString()
|
||||||
|
|
||||||
@@ -260,32 +309,23 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
|
|||||||
packageImage = cs.ImageRepository + "-" + fp
|
packageImage = cs.ImageRepository + "-" + fp
|
||||||
}
|
}
|
||||||
|
|
||||||
if !cs.Clean {
|
|
||||||
exists := cs.Backend.ImageExists(buildertaggedImage) && cs.Backend.ImageExists(packageImage)
|
|
||||||
if art, err := LoadArtifactFromYaml(p); err == nil && (cs.Options.SkipIfMetadataExists || exists) {
|
|
||||||
Debug("Artifact reloaded. Skipping build")
|
|
||||||
return art, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
|
p.SetSeedImage(image) // In this case, we ignore the build deps as we suppose that the image has them - otherwise we recompose the tree with a solver,
|
||||||
// and we build all the images first.
|
// and we build all the images first.
|
||||||
|
|
||||||
err := os.MkdirAll(p.Rel("build"), os.ModePerm)
|
err := os.MkdirAll(p.Rel("build"), os.ModePerm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "Error met while creating tempdir for building")
|
return builderOpts, runnerOpts, errors.Wrap(err, "Error met while creating tempdir for building")
|
||||||
}
|
}
|
||||||
buildDir, err := ioutil.TempDir(p.Rel("build"), "pack")
|
buildDir, err := ioutil.TempDir(p.Rel("build"), "pack")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "Error met while creating tempdir for building")
|
return builderOpts, runnerOpts, errors.Wrap(err, "Error met while creating tempdir for building")
|
||||||
}
|
}
|
||||||
defer os.RemoveAll(buildDir) // clean up
|
defer os.RemoveAll(buildDir) // clean up
|
||||||
|
|
||||||
// First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow)
|
// First we copy the source definitions into the output - we create a copy which the builds will need (we need to cache this phase somehow)
|
||||||
err = helpers.CopyDir(p.GetPackage().GetPath(), buildDir)
|
err = helpers.CopyDir(p.GetPackage().GetPath(), buildDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "Could not copy package sources")
|
return builderOpts, runnerOpts, errors.Wrap(err, "Could not copy package sources")
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy file into the build context, the compilespec might have requested to do so.
|
// Copy file into the build context, the compilespec might have requested to do so.
|
||||||
@@ -299,63 +339,75 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
|
|||||||
Info(pkgTag, ":whale: Generating 'builder' image definition from", image)
|
Info(pkgTag, ":whale: Generating 'builder' image definition from", image)
|
||||||
|
|
||||||
// First we create the builder image
|
// First we create the builder image
|
||||||
p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile"))
|
if err := p.WriteBuildImageDefinition(filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+"-builder.dockerfile")); err != nil {
|
||||||
builderOpts := CompilerBackendOptions{
|
return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Then we write the step image, which uses the builder one
|
||||||
|
if err := p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile")); err != nil {
|
||||||
|
return builderOpts, runnerOpts, errors.Wrap(err, "Could not generate image definition")
|
||||||
|
}
|
||||||
|
|
||||||
|
builderOpts = CompilerBackendOptions{
|
||||||
ImageName: buildertaggedImage,
|
ImageName: buildertaggedImage,
|
||||||
SourcePath: buildDir,
|
SourcePath: buildDir,
|
||||||
DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile",
|
DockerFileName: p.GetPackage().GetFingerPrint() + "-builder.dockerfile",
|
||||||
Destination: p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
|
Destination: p.Rel(p.GetPackage().GetFingerPrint() + "-builder.image.tar"),
|
||||||
}
|
}
|
||||||
|
runnerOpts = CompilerBackendOptions{
|
||||||
buildBuilderImage := true
|
|
||||||
if cs.Options.PullFirst {
|
|
||||||
if err := cs.Backend.DownloadImage(builderOpts); err == nil {
|
|
||||||
buildBuilderImage = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if buildBuilderImage {
|
|
||||||
if err = cs.Backend.BuildImage(builderOpts); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "Could not build image: "+image+" "+builderOpts.DockerFileName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = cs.Backend.ExportImage(builderOpts); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "Could not export image")
|
|
||||||
}
|
|
||||||
|
|
||||||
if !cs.Options.KeepImageExport {
|
|
||||||
defer os.Remove(builderOpts.Destination)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cs.Options.Push && buildBuilderImage {
|
|
||||||
if err = cs.Backend.Push(builderOpts); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Then we write the step image, which uses the builder one
|
|
||||||
p.WriteStepImageDefinition(buildertaggedImage, filepath.Join(buildDir, p.GetPackage().GetFingerPrint()+".dockerfile"))
|
|
||||||
runnerOpts := CompilerBackendOptions{
|
|
||||||
ImageName: packageImage,
|
ImageName: packageImage,
|
||||||
SourcePath: buildDir,
|
SourcePath: buildDir,
|
||||||
DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile",
|
DockerFileName: p.GetPackage().GetFingerPrint() + ".dockerfile",
|
||||||
Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
|
Destination: p.Rel(p.GetPackage().GetFingerPrint() + ".image.tar"),
|
||||||
}
|
}
|
||||||
|
|
||||||
buildPackageImage := true
|
buildAndPush := func(opts CompilerBackendOptions) error {
|
||||||
|
buildImage := true
|
||||||
if cs.Options.PullFirst {
|
if cs.Options.PullFirst {
|
||||||
//Best effort pull
|
if err := cs.Backend.DownloadImage(opts); err == nil {
|
||||||
if err := cs.Backend.DownloadImage(runnerOpts); err == nil {
|
buildImage = false
|
||||||
buildPackageImage = false
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if buildImage {
|
||||||
|
if err := cs.Backend.BuildImage(opts); err != nil {
|
||||||
|
return errors.Wrap(err, "Could not build image: "+image+" "+opts.DockerFileName)
|
||||||
|
}
|
||||||
|
if cs.Options.Push {
|
||||||
|
if err = cs.Backend.Push(opts); err != nil {
|
||||||
|
return errors.Wrap(err, "Could not push image: "+image+" "+opts.DockerFileName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
if buildPackageImage {
|
if err := buildAndPush(builderOpts); err != nil {
|
||||||
if err := cs.Backend.BuildImage(runnerOpts); err != nil {
|
return builderOpts, runnerOpts, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
|
||||||
return nil, errors.Wrap(err, "Failed building image for "+runnerOpts.ImageName+" "+runnerOpts.DockerFileName)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := buildAndPush(runnerOpts); err != nil {
|
||||||
|
return builderOpts, runnerOpts, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
|
||||||
|
}
|
||||||
|
|
||||||
|
return builderOpts, runnerOpts, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (cs *LuetCompiler) genArtifact(p CompilationSpec, builderOpts, runnerOpts CompilerBackendOptions, concurrency int, keepPermissions bool) (Artifact, error) {
|
||||||
|
|
||||||
|
// generate Artifact
|
||||||
|
var artifact Artifact
|
||||||
|
var rootfs string
|
||||||
|
var err error
|
||||||
|
unpack := p.ImageUnpack()
|
||||||
|
pkgTag := ":package: " + p.GetPackage().HumanReadableString()
|
||||||
|
|
||||||
|
// If package_dir was specified in the spec, we want to treat the content of the directory
|
||||||
|
// as the root of our archive. ImageUnpack is implied to be true. override it
|
||||||
|
if p.GetPackageDir() != "" {
|
||||||
|
unpack = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// prepare folder content of the image with the package compiled inside
|
||||||
if err := cs.Backend.ExportImage(runnerOpts); err != nil {
|
if err := cs.Backend.ExportImage(runnerOpts); err != nil {
|
||||||
return nil, errors.Wrap(err, "Failed exporting image")
|
return nil, errors.Wrap(err, "Failed exporting image")
|
||||||
}
|
}
|
||||||
@@ -364,23 +416,7 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
|
|||||||
defer os.Remove(runnerOpts.Destination)
|
defer os.Remove(runnerOpts.Destination)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cs.Options.Push && buildPackageImage {
|
rootfs, err = ioutil.TempDir(p.GetOutputPath(), "rootfs")
|
||||||
err = cs.Backend.Push(runnerOpts)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "Could not push image: "+image+" "+builderOpts.DockerFileName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var artifact Artifact
|
|
||||||
unpack := p.ImageUnpack()
|
|
||||||
|
|
||||||
// If package_dir was specified in the spec, we want to treat the content of the directory
|
|
||||||
// as the root of our archive. ImageUnpack is implied to be true. override it
|
|
||||||
if p.GetPackageDir() != "" {
|
|
||||||
unpack = true
|
|
||||||
}
|
|
||||||
|
|
||||||
rootfs, err := ioutil.TempDir(p.GetOutputPath(), "rootfs")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "Could not create tempdir")
|
return nil, errors.Wrap(err, "Could not create tempdir")
|
||||||
}
|
}
|
||||||
@@ -388,66 +424,24 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
|
|||||||
|
|
||||||
// TODO: Compression and such
|
// TODO: Compression and such
|
||||||
err = cs.Backend.ExtractRootfs(CompilerBackendOptions{
|
err = cs.Backend.ExtractRootfs(CompilerBackendOptions{
|
||||||
ImageName: packageImage,
|
ImageName: runnerOpts.ImageName,
|
||||||
SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions)
|
SourcePath: runnerOpts.Destination, Destination: rootfs}, keepPermissions)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "Could not extract rootfs")
|
return nil, errors.Wrap(err, "Could not extract rootfs")
|
||||||
}
|
}
|
||||||
|
|
||||||
if !keepImg {
|
|
||||||
// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
|
|
||||||
// TODO: Handle caching and optionally do not remove things
|
|
||||||
err = cs.Backend.RemoveImage(builderOpts)
|
|
||||||
if err != nil {
|
|
||||||
Warning("Could not remove image ", builderOpts.ImageName)
|
|
||||||
// return nil, errors.Wrap(err, "Could not remove image")
|
|
||||||
}
|
|
||||||
err = cs.Backend.RemoveImage(runnerOpts)
|
|
||||||
if err != nil {
|
|
||||||
Warning("Could not remove image ", builderOpts.ImageName)
|
|
||||||
// return nil, errors.Wrap(err, "Could not remove image")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !generateArtifact {
|
|
||||||
return &PackageArtifact{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if unpack {
|
if unpack {
|
||||||
if p.GetPackageDir() != "" {
|
// Take content of container as a base for our package files
|
||||||
Info(":tophat: Packing from output dir", p.GetPackageDir())
|
artifact, err = cs.unpackFs(rootfs, concurrency, p)
|
||||||
rootfs = filepath.Join(rootfs, p.GetPackageDir())
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(p.GetIncludes()) > 0 {
|
|
||||||
// strip from includes
|
|
||||||
cs.stripFromRootfs(p.GetIncludes(), rootfs, true)
|
|
||||||
}
|
|
||||||
if len(p.GetExcludes()) > 0 {
|
|
||||||
// strip from includes
|
|
||||||
cs.stripFromRootfs(p.GetExcludes(), rootfs, false)
|
|
||||||
}
|
|
||||||
artifact = NewPackageArtifact(p.Rel(p.GetPackage().GetFingerPrint() + ".package.tar"))
|
|
||||||
artifact.SetCompressionType(cs.CompressionType)
|
|
||||||
|
|
||||||
err = artifact.Compress(rootfs, concurrency)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "Error met while creating package archive")
|
return nil, errors.Wrap(err, "Error met while creating package archive")
|
||||||
}
|
}
|
||||||
|
|
||||||
artifact.SetCompileSpec(p)
|
|
||||||
} else {
|
} else {
|
||||||
Info(pkgTag, ":hammer: Generating delta")
|
// Generate delta between the two images
|
||||||
diffs, err := cs.Backend.Changes(p.Rel(p.GetPackage().GetFingerPrint()+"-builder.image.tar"), p.Rel(p.GetPackage().GetFingerPrint()+".image.tar"))
|
artifact, err = cs.unpackDelta(rootfs, concurrency, keepPermissions, p, builderOpts, runnerOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "Could not generate changes from layers")
|
return nil, errors.Wrap(err, "Error met while creating package archive")
|
||||||
}
|
}
|
||||||
artifact, err = ExtractArtifactFromDelta(rootfs, p.Rel(p.GetPackage().GetFingerPrint()+".package.tar"), diffs, concurrency, keepPermissions, p.GetIncludes(), p.GetExcludes(), cs.CompressionType)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "Could not generate deltas")
|
|
||||||
}
|
|
||||||
|
|
||||||
artifact.SetCompileSpec(p)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
filelist, err := artifact.FileList()
|
filelist, err := artifact.FileList()
|
||||||
@@ -456,7 +450,6 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
|
|||||||
}
|
}
|
||||||
|
|
||||||
artifact.SetFiles(filelist)
|
artifact.SetFiles(filelist)
|
||||||
|
|
||||||
artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
|
artifact.GetCompileSpec().GetPackage().SetBuildTimestamp(time.Now().String())
|
||||||
|
|
||||||
err = artifact.WriteYaml(p.GetOutputPath())
|
err = artifact.WriteYaml(p.GetOutputPath())
|
||||||
@@ -468,6 +461,43 @@ func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage
|
|||||||
return artifact, nil
|
return artifact, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (cs *LuetCompiler) compileWithImage(image, buildertaggedImage, packageImage string,
|
||||||
|
concurrency int,
|
||||||
|
keepPermissions, keepImg bool,
|
||||||
|
p CompilationSpec, generateArtifact bool) (Artifact, error) {
|
||||||
|
|
||||||
|
if !cs.Clean {
|
||||||
|
exists := cs.Backend.ImageExists(buildertaggedImage) && cs.Backend.ImageExists(packageImage)
|
||||||
|
if art, err := LoadArtifactFromYaml(p); err == nil && (cs.Options.SkipIfMetadataExists || exists) {
|
||||||
|
Debug("Artifact reloaded. Skipping build")
|
||||||
|
return art, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
builderOpts, runnerOpts, err := cs.buildPackageImage(image, buildertaggedImage, packageImage, concurrency, keepPermissions, p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "failed building package image")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !keepImg {
|
||||||
|
defer func() {
|
||||||
|
// We keep them around, so to not reload them from the tar (which should be the "correct way") and we automatically share the same layers
|
||||||
|
if err := cs.Backend.RemoveImage(builderOpts); err != nil {
|
||||||
|
Warning("Could not remove image ", builderOpts.ImageName)
|
||||||
|
}
|
||||||
|
if err := cs.Backend.RemoveImage(runnerOpts); err != nil {
|
||||||
|
Warning("Could not remove image ", runnerOpts.ImageName)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
if !generateArtifact {
|
||||||
|
return &PackageArtifact{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return cs.genArtifact(p, builderOpts, runnerOpts, concurrency, keepPermissions)
|
||||||
|
}
|
||||||
|
|
||||||
func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error) {
|
func (cs *LuetCompiler) FromDatabase(db pkg.PackageDatabase, minimum bool, dst string) ([]CompilationSpec, error) {
|
||||||
compilerSpecs := NewLuetCompilationspecs()
|
compilerSpecs := NewLuetCompilationspecs()
|
||||||
|
|
||||||
@@ -658,31 +688,11 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
buildFile := pack.Rel(BuildFile)
|
out, err := helpers.RenderFiles(pack.Rel(BuildFile), pack.Rel(DefinitionFile))
|
||||||
if !helpers.Exists(buildFile) {
|
|
||||||
return nil, errors.New("No build file present for " + p.GetFingerPrint())
|
|
||||||
}
|
|
||||||
defFile := pack.Rel(DefinitionFile)
|
|
||||||
if !helpers.Exists(defFile) {
|
|
||||||
return nil, errors.New("No build file present for " + p.GetFingerPrint())
|
|
||||||
}
|
|
||||||
def, err := ioutil.ReadFile(defFile)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
|
||||||
}
|
}
|
||||||
|
|
||||||
build, err := ioutil.ReadFile(buildFile)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var values templatedata
|
|
||||||
if err = yaml.Unmarshal(def, &values); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out, err := helpers.RenderHelm(string(build), values)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return NewLuetCompilationSpec([]byte(out), pack)
|
return NewLuetCompilationSpec([]byte(out), pack)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@@ -1,7 +1,10 @@
|
|||||||
package helpers
|
package helpers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"io/ioutil"
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
"gopkg.in/yaml.v2"
|
||||||
"helm.sh/helm/v3/pkg/chart"
|
"helm.sh/helm/v3/pkg/chart"
|
||||||
"helm.sh/helm/v3/pkg/chartutil"
|
"helm.sh/helm/v3/pkg/chartutil"
|
||||||
"helm.sh/helm/v3/pkg/engine"
|
"helm.sh/helm/v3/pkg/engine"
|
||||||
@@ -31,3 +34,26 @@ func RenderHelm(template string, values map[string]interface{}) (string, error)
|
|||||||
|
|
||||||
return out["templates"], nil
|
return out["templates"], nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type templatedata map[string]interface{}
|
||||||
|
|
||||||
|
func RenderFiles(toTemplate, valuesFile string) (string, error) {
|
||||||
|
raw, err := ioutil.ReadFile(toTemplate)
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.Wrap(err, "reading file "+toTemplate)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !Exists(valuesFile) {
|
||||||
|
return "", errors.Wrap(err, "file not existing "+valuesFile)
|
||||||
|
}
|
||||||
|
def, err := ioutil.ReadFile(valuesFile)
|
||||||
|
if err != nil {
|
||||||
|
return "", errors.Wrap(err, "reading file "+valuesFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
var values templatedata
|
||||||
|
if err = yaml.Unmarshal(def, &values); err != nil {
|
||||||
|
return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
|
||||||
|
}
|
||||||
|
return RenderHelm(string(raw), values)
|
||||||
|
}
|
||||||
|
@@ -30,7 +30,6 @@ import (
|
|||||||
. "github.com/mudler/luet/pkg/logger"
|
. "github.com/mudler/luet/pkg/logger"
|
||||||
pkg "github.com/mudler/luet/pkg/package"
|
pkg "github.com/mudler/luet/pkg/package"
|
||||||
"github.com/mudler/luet/pkg/solver"
|
"github.com/mudler/luet/pkg/solver"
|
||||||
"github.com/mudler/luet/pkg/tree"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
)
|
)
|
||||||
@@ -453,8 +452,7 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
|
|||||||
return errors.Wrap(err, "Failed creating package")
|
return errors.Wrap(err, "Failed creating package")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
executedFinalizer := map[string]bool{}
|
var toFinalize []pkg.Package
|
||||||
|
|
||||||
if !l.Options.NoDeps {
|
if !l.Options.NoDeps {
|
||||||
// TODO: Lower those errors as warning
|
// TODO: Lower those errors as warning
|
||||||
for _, w := range p {
|
for _, w := range p {
|
||||||
@@ -466,36 +464,17 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
|
|||||||
ORDER:
|
ORDER:
|
||||||
for _, ass := range ordered {
|
for _, ass := range ordered {
|
||||||
if ass.Value {
|
if ass.Value {
|
||||||
|
|
||||||
installed, ok := toInstall[ass.Package.GetFingerPrint()]
|
installed, ok := toInstall[ass.Package.GetFingerPrint()]
|
||||||
if !ok {
|
if !ok {
|
||||||
// It was a dep already installed in the system, so we can skip it safely
|
// It was a dep already installed in the system, so we can skip it safely
|
||||||
continue ORDER
|
continue ORDER
|
||||||
}
|
}
|
||||||
|
|
||||||
treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
|
treePackage, err := installed.Repository.GetTree().GetDatabase().FindPackage(ass.Package)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
|
return errors.Wrap(err, "Error getting package "+ass.Package.HumanReadableString())
|
||||||
}
|
}
|
||||||
if helpers.Exists(treePackage.Rel(tree.FinalizerFile)) {
|
|
||||||
finalizerRaw, err := ioutil.ReadFile(treePackage.Rel(tree.FinalizerFile))
|
|
||||||
if err != nil && !l.Options.Force {
|
|
||||||
return errors.Wrap(err, "Error reading file "+treePackage.Rel(tree.FinalizerFile))
|
|
||||||
}
|
|
||||||
if _, exists := executedFinalizer[ass.Package.GetFingerPrint()]; !exists {
|
|
||||||
Info("Executing finalizer for " + ass.Package.HumanReadableString())
|
|
||||||
finalizer, err := NewLuetFinalizerFromYaml(finalizerRaw)
|
|
||||||
if err != nil && !l.Options.Force {
|
|
||||||
return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
|
|
||||||
}
|
|
||||||
err = finalizer.RunInstall(s)
|
|
||||||
if err != nil && !l.Options.Force {
|
|
||||||
return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
|
|
||||||
}
|
|
||||||
executedFinalizer[ass.Package.GetFingerPrint()] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
toFinalize = append(toFinalize, treePackage)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -506,29 +485,11 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp pkg.Packages, s *Sy
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrap(err, "Error getting package "+c.Package.HumanReadableString())
|
return errors.Wrap(err, "Error getting package "+c.Package.HumanReadableString())
|
||||||
}
|
}
|
||||||
if helpers.Exists(treePackage.Rel(tree.FinalizerFile)) {
|
toFinalize = append(toFinalize, treePackage)
|
||||||
finalizerRaw, err := ioutil.ReadFile(treePackage.Rel(tree.FinalizerFile))
|
|
||||||
if err != nil && !l.Options.Force {
|
|
||||||
return errors.Wrap(err, "Error reading file "+treePackage.Rel(tree.FinalizerFile))
|
|
||||||
}
|
|
||||||
if _, exists := executedFinalizer[c.Package.GetFingerPrint()]; !exists {
|
|
||||||
Info(":shell: Executing finalizer for " + c.Package.HumanReadableString())
|
|
||||||
finalizer, err := NewLuetFinalizerFromYaml(finalizerRaw)
|
|
||||||
if err != nil && !l.Options.Force {
|
|
||||||
return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
|
|
||||||
}
|
|
||||||
err = finalizer.RunInstall(s)
|
|
||||||
if err != nil && !l.Options.Force {
|
|
||||||
return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
|
|
||||||
}
|
|
||||||
executedFinalizer[c.Package.GetFingerPrint()] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return s.ExecuteFinalizers(toFinalize, l.Options.Force)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, error) {
|
func (l *LuetInstaller) downloadPackage(a ArtifactMatch) (compiler.Artifact, error) {
|
||||||
|
@@ -1,7 +1,12 @@
|
|||||||
package installer
|
package installer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
. "github.com/mudler/luet/pkg/logger"
|
||||||
|
|
||||||
|
"github.com/mudler/luet/pkg/helpers"
|
||||||
pkg "github.com/mudler/luet/pkg/package"
|
pkg "github.com/mudler/luet/pkg/package"
|
||||||
|
"github.com/mudler/luet/pkg/tree"
|
||||||
|
"github.com/pkg/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
type System struct {
|
type System struct {
|
||||||
@@ -12,3 +17,31 @@ type System struct {
|
|||||||
func (s *System) World() (pkg.Packages, error) {
|
func (s *System) World() (pkg.Packages, error) {
|
||||||
return s.Database.World(), nil
|
return s.Database.World(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type templatedata map[string]interface{}
|
||||||
|
|
||||||
|
func (s *System) ExecuteFinalizers(packs []pkg.Package, force bool) error {
|
||||||
|
executedFinalizer := map[string]bool{}
|
||||||
|
for _, p := range packs {
|
||||||
|
if helpers.Exists(p.Rel(tree.FinalizerFile)) {
|
||||||
|
out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile))
|
||||||
|
if err != nil && !force {
|
||||||
|
return errors.Wrap(err, "reading file "+p.Rel(tree.FinalizerFile))
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, exists := executedFinalizer[p.GetFingerPrint()]; !exists {
|
||||||
|
Info("Executing finalizer for " + p.HumanReadableString())
|
||||||
|
finalizer, err := NewLuetFinalizerFromYaml([]byte(out))
|
||||||
|
if err != nil && !force {
|
||||||
|
return errors.Wrap(err, "Error reading finalizer "+p.Rel(tree.FinalizerFile))
|
||||||
|
}
|
||||||
|
err = finalizer.RunInstall(s)
|
||||||
|
if err != nil && !force {
|
||||||
|
return errors.Wrap(err, "Error executing install finalizer "+p.Rel(tree.FinalizerFile))
|
||||||
|
}
|
||||||
|
executedFinalizer[p.GetFingerPrint()] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
2
tests/fixtures/templatedfinalizers/alpine/build.yaml
vendored
Normal file
2
tests/fixtures/templatedfinalizers/alpine/build.yaml
vendored
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
image: "alpine"
|
||||||
|
unpack: true
|
3
tests/fixtures/templatedfinalizers/alpine/definition.yaml
vendored
Normal file
3
tests/fixtures/templatedfinalizers/alpine/definition.yaml
vendored
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
category: "seed"
|
||||||
|
name: "alpine"
|
||||||
|
version: "1.0"
|
2
tests/fixtures/templatedfinalizers/alpine/finalize.yaml
vendored
Normal file
2
tests/fixtures/templatedfinalizers/alpine/finalize.yaml
vendored
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
install:
|
||||||
|
- echo "{{.Values.name}}" > /tmp/foo
|
77
tests/integration/19_finalizer_templated.sh
Executable file
77
tests/integration/19_finalizer_templated.sh
Executable file
@@ -0,0 +1,77 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
export LUET_NOLOCK=true
|
||||||
|
|
||||||
|
oneTimeSetUp() {
|
||||||
|
export tmpdir="$(mktemp -d)"
|
||||||
|
}
|
||||||
|
|
||||||
|
oneTimeTearDown() {
|
||||||
|
rm -rf "$tmpdir"
|
||||||
|
}
|
||||||
|
|
||||||
|
testBuild() {
|
||||||
|
mkdir $tmpdir/testbuild
|
||||||
|
luet build --tree "$ROOT_DIR/tests/fixtures/templatedfinalizers" --destination $tmpdir/testbuild --compression gzip --all > /dev/null
|
||||||
|
buildst=$?
|
||||||
|
assertEquals 'builds successfully' "$buildst" "0"
|
||||||
|
assertTrue 'create package' "[ -e '$tmpdir/testbuild/alpine-seed-1.0.package.tar.gz' ]"
|
||||||
|
}
|
||||||
|
|
||||||
|
testRepo() {
|
||||||
|
assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
|
||||||
|
luet create-repo --tree "$ROOT_DIR/tests/fixtures/templatedfinalizers" \
|
||||||
|
--output $tmpdir/testbuild \
|
||||||
|
--packages $tmpdir/testbuild \
|
||||||
|
--name "test" \
|
||||||
|
--descr "Test Repo" \
|
||||||
|
--urls $tmpdir/testrootfs \
|
||||||
|
--type disk > /dev/null
|
||||||
|
|
||||||
|
createst=$?
|
||||||
|
assertEquals 'create repo successfully' "$createst" "0"
|
||||||
|
assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
|
||||||
|
}
|
||||||
|
|
||||||
|
testConfig() {
|
||||||
|
mkdir $tmpdir/testrootfs
|
||||||
|
cat <<EOF > $tmpdir/luet.yaml
|
||||||
|
general:
|
||||||
|
debug: true
|
||||||
|
system:
|
||||||
|
rootfs: $tmpdir/testrootfs
|
||||||
|
database_path: "/"
|
||||||
|
database_engine: "boltdb"
|
||||||
|
config_from_host: true
|
||||||
|
repositories:
|
||||||
|
- name: "main"
|
||||||
|
type: "disk"
|
||||||
|
enable: true
|
||||||
|
urls:
|
||||||
|
- "$tmpdir/testbuild"
|
||||||
|
EOF
|
||||||
|
luet config --config $tmpdir/luet.yaml
|
||||||
|
res=$?
|
||||||
|
assertEquals 'config test successfully' "$res" "0"
|
||||||
|
}
|
||||||
|
|
||||||
|
testInstall() {
|
||||||
|
luet install --config $tmpdir/luet.yaml seed/alpine
|
||||||
|
#luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
|
||||||
|
installst=$?
|
||||||
|
assertEquals 'install test successfully' "$installst" "0"
|
||||||
|
assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/bin/busybox' ]"
|
||||||
|
assertTrue 'finalizer runs' "[ -e '$tmpdir/testrootfs/tmp/foo' ]"
|
||||||
|
assertEquals 'finalizer printed used shell' "$(cat $tmpdir/testrootfs/tmp/foo)" 'alpine'
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
testCleanup() {
|
||||||
|
luet cleanup --config $tmpdir/luet.yaml
|
||||||
|
installst=$?
|
||||||
|
assertEquals 'install test successfully' "$installst" "0"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Load shUnit2.
|
||||||
|
. "$ROOT_DIR/tests/integration/shunit2"/shunit2
|
||||||
|
|
Reference in New Issue
Block a user