Compare commits

..

6 Commits
0.9.6 ... 0.9.8

Author SHA1 Message Date
Ettore Di Giacinto
87004c8e78 Tag 0.9.8 2020-11-28 16:29:38 +01:00
Ettore Di Giacinto
0fe30ddcfd Add ability to interpolate during build
Now build takes a --values argument, which is a yaml file that can be
used to interpolate the specs that are going to be compiled.
2020-11-28 15:47:29 +01:00
Ettore Di Giacinto
44d33eceba Set workdir also on step image
Otherwise with DOCKER_SQUASH=true it wouldn't be coherent on where to
find the package files
2020-11-28 12:07:07 +01:00
Ettore Di Giacinto
ca994b07ab Tag 0.9.7 2020-11-28 00:34:46 +01:00
Ettore Di Giacinto
8ce135fe12 Add DOCKER_SQUASH 2020-11-27 23:38:31 +01:00
Ettore Di Giacinto
18d9366bca Minor fixes 2020-11-24 18:27:49 +01:00
22 changed files with 347 additions and 18 deletions

View File

@@ -49,6 +49,7 @@ var buildCmd = &cobra.Command{
viper.BindPFlag("compression", cmd.Flags().Lookup("compression"))
viper.BindPFlag("nodeps", cmd.Flags().Lookup("nodeps"))
viper.BindPFlag("onlydeps", cmd.Flags().Lookup("onlydeps"))
viper.BindPFlag("values", cmd.Flags().Lookup("values"))
viper.BindPFlag("image-repository", cmd.Flags().Lookup("image-repository"))
viper.BindPFlag("push", cmd.Flags().Lookup("push"))
@@ -75,6 +76,8 @@ var buildCmd = &cobra.Command{
databaseType := viper.GetString("database")
compressionType := viper.GetString("compression")
imageRepository := viper.GetString("image-repository")
values := viper.GetString("values")
push := viper.GetBool("push")
pull := viper.GetBool("pull")
keepImages := viper.GetBool("keep-images")
@@ -157,7 +160,7 @@ var buildCmd = &cobra.Command{
opts.KeepImageExport = keepExportedImages
opts.SkipIfMetadataExists = skip
opts.PackageTargetOnly = onlyTarget
opts.BuildValuesFile = values
var solverOpts solver.Options
if concurrent {
solverOpts = solver.Options{Type: solver.ParallelSimple, Concurrency: concurrency}
@@ -291,6 +294,7 @@ func init() {
buildCmd.Flags().Bool("revdeps", false, "Build with revdeps")
buildCmd.Flags().Bool("all", false, "Build all specfiles in the tree")
buildCmd.Flags().Bool("full", false, "Build all packages (optimized)")
buildCmd.Flags().String("values", "", "Build values file to interpolate with each package")
buildCmd.Flags().String("destination", path, "Destination folder")
buildCmd.Flags().String("compression", "none", "Compression alg: none, gzip")

View File

@@ -40,7 +40,7 @@ var Verbose bool
var LockedCommands = []string{"install", "uninstall", "upgrade"}
const (
LuetCLIVersion = "0.9.6"
LuetCLIVersion = "0.9.8"
LuetEnvPrefix = "LUET"
)

View File

@@ -57,7 +57,7 @@ func packageToList(l list.Writer, repo string, p pkg.Package) {
l.AppendItem(p.HumanReadableString())
l.Indent()
l.AppendItem(fmt.Sprintf("Category: %s", p.GetCategory()))
l.AppendItem(fmt.Sprintf("Name: %s", p.GetCategory()))
l.AppendItem(fmt.Sprintf("Name: %s", p.GetName()))
l.AppendItem(fmt.Sprintf("Version: %s", p.GetVersion()))
l.AppendItem(fmt.Sprintf("Description: %s", p.GetDescription()))
l.AppendItem(fmt.Sprintf("Repository: %s ", repo))

1
go.mod
View File

@@ -12,6 +12,7 @@ require (
github.com/docker/docker v17.12.0-ce-rc1.0.20200417035958-130b0bc6032c+incompatible
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
github.com/ecooper/qlearning v0.0.0-20160612200101-3075011a69fd
github.com/fsouza/go-dockerclient v1.6.4
github.com/ghodss/yaml v1.0.0
github.com/hashicorp/go-version v1.2.0
github.com/jedib0t/go-pretty v4.3.0+incompatible

View File

@@ -96,6 +96,7 @@ ENV PACKAGE_CATEGORY=app-admin`))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM luet/base
WORKDIR /luetbuild
ENV PACKAGE_NAME=enman
ENV PACKAGE_VERSION=1.4.0
ENV PACKAGE_CATEGORY=app-admin

View File

@@ -24,6 +24,7 @@ import (
"path/filepath"
"strings"
docker "github.com/fsouza/go-dockerclient"
capi "github.com/mudler/docker-companion/api"
"github.com/mudler/luet/pkg/compiler"
@@ -45,6 +46,7 @@ func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
name := opts.ImageName
path := opts.SourcePath
dockerfileName := opts.DockerFileName
buildarg := []string{"build", "-f", dockerfileName, "-t", name, "."}
Debug(":whale2: Building image " + name)
@@ -56,6 +58,21 @@ func (*SimpleDocker) BuildImage(opts compiler.CompilerBackendOptions) error {
}
Info(":whale: Building image " + name + " done")
if os.Getenv("DOCKER_SQUASH") == "true" {
Info(":whale: Squashing image " + name)
var client *docker.Client
client, err = docker.NewClientFromEnv()
if err != nil {
return errors.Wrap(err, "could not connect to the Docker daemon")
}
err = capi.Squash(client, name, name)
if err != nil {
return errors.Wrap(err, "Failed squashing image")
}
Info(":whale: Squashing image " + name + " done")
}
if config.LuetCfg.GetGeneral().ShowBuildOutput {
Info(string(out))
} else {

View File

@@ -87,6 +87,7 @@ ENV PACKAGE_CATEGORY=app-admin`))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM luet/base
WORKDIR /luetbuild
ENV PACKAGE_NAME=enman
ENV PACKAGE_VERSION=1.4.0
ENV PACKAGE_CATEGORY=app-admin

View File

@@ -27,6 +27,7 @@ import (
"time"
bus "github.com/mudler/luet/pkg/bus"
yaml "gopkg.in/yaml.v2"
"github.com/mudler/luet/pkg/helpers"
. "github.com/mudler/luet/pkg/logger"
@@ -743,13 +744,25 @@ func (cs *LuetCompiler) FromPackage(p pkg.Package) (CompilationSpec, error) {
raw := packsRaw.Find(pack.GetName(), pack.GetCategory(), pack.GetVersion())
dat, err := helpers.RenderHelm(string(dataBuild), raw)
d := map[string]interface{}{}
if len(cs.Options.BuildValuesFile) > 0 {
defBuild, err := ioutil.ReadFile(cs.Options.BuildValuesFile)
if err != nil {
return nil, errors.Wrap(err, "rendering file "+val)
}
err = yaml.Unmarshal(defBuild, &d)
if err != nil {
return nil, errors.Wrap(err, "rendering file "+val)
}
}
dat, err := helpers.RenderHelm(string(dataBuild), raw, d)
if err != nil {
return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
}
dataresult = []byte(dat)
} else {
out, err := helpers.RenderFiles(pack.Rel(BuildFile), val)
out, err := helpers.RenderFiles(pack.Rel(BuildFile), val, cs.Options.BuildValuesFile)
if err != nil {
return nil, errors.Wrap(err, "rendering file "+pack.Rel(BuildFile))
}

View File

@@ -56,6 +56,7 @@ type CompilerOptions struct {
NoDeps bool
SolverOptions config.LuetSolverOptions
SkipIfMetadataExists bool
BuildValuesFile string
PackageTargetOnly bool
}

View File

@@ -242,6 +242,7 @@ RUN ` + s
func (cs *LuetCompilationSpec) RenderStepImage(image string) (string, error) {
spec := `
FROM ` + image + `
WORKDIR /luetbuild
ENV PACKAGE_NAME=` + cs.Package.GetName() + `
ENV PACKAGE_VERSION=` + cs.Package.GetVersion() + `
ENV PACKAGE_CATEGORY=` + cs.Package.GetCategory()

View File

@@ -96,6 +96,7 @@ ENV test=1`))
Expect(err).ToNot(HaveOccurred())
Expect(dockerfile).To(Equal(`
FROM luet/base
WORKDIR /luetbuild
ENV PACKAGE_NAME=enman
ENV PACKAGE_VERSION=1.4.0
ENV PACKAGE_CATEGORY=app-admin
@@ -168,6 +169,7 @@ ENV test=1`))
Expect(dockerfile).To(Equal(`
FROM luet/base
WORKDIR /luetbuild
ENV PACKAGE_NAME=a
ENV PACKAGE_VERSION=1.0
ENV PACKAGE_CATEGORY=test

View File

@@ -11,7 +11,7 @@ import (
)
// RenderHelm renders the template string with helm
func RenderHelm(template string, values map[string]interface{}) (string, error) {
func RenderHelm(template string, values, d map[string]interface{}) (string, error) {
c := &chart.Chart{
Metadata: &chart.Metadata{
Name: "",
@@ -23,7 +23,7 @@ func RenderHelm(template string, values map[string]interface{}) (string, error)
Values: map[string]interface{}{"Values": values},
}
v, err := chartutil.CoalesceValues(c, map[string]interface{}{})
v, err := chartutil.CoalesceValues(c, map[string]interface{}{"Values": d})
if err != nil {
return "", errors.Wrap(err, "while rendering template")
}
@@ -37,7 +37,7 @@ func RenderHelm(template string, values map[string]interface{}) (string, error)
type templatedata map[string]interface{}
func RenderFiles(toTemplate, valuesFile string) (string, error) {
func RenderFiles(toTemplate, valuesFile string, defaultFile string) (string, error) {
raw, err := ioutil.ReadFile(toTemplate)
if err != nil {
return "", errors.Wrap(err, "reading file "+toTemplate)
@@ -46,14 +46,26 @@ func RenderFiles(toTemplate, valuesFile string) (string, error) {
if !Exists(valuesFile) {
return "", errors.Wrap(err, "file not existing "+valuesFile)
}
def, err := ioutil.ReadFile(valuesFile)
val, err := ioutil.ReadFile(valuesFile)
if err != nil {
return "", errors.Wrap(err, "reading file "+valuesFile)
}
var values templatedata
if err = yaml.Unmarshal(def, &values); err != nil {
d := templatedata{}
if len(defaultFile) > 0 {
def, err := ioutil.ReadFile(defaultFile)
if err != nil {
return "", errors.Wrap(err, "reading file "+valuesFile)
}
if err = yaml.Unmarshal(def, &d); err != nil {
return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
}
}
if err = yaml.Unmarshal(val, &values); err != nil {
return "", errors.Wrap(err, "unmarshalling file "+toTemplate)
}
return RenderHelm(string(raw), values)
return RenderHelm(string(raw), values, d)
}

View File

@@ -16,17 +16,132 @@
package helpers_test
import (
"io/ioutil"
"os"
"path/filepath"
. "github.com/mudler/luet/pkg/helpers"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func writeFile(path string, content string) {
err := ioutil.WriteFile(path, []byte(content), 0644)
Expect(err).ToNot(HaveOccurred())
}
var _ = Describe("Helpers", func() {
Context("RenderHelm", func() {
It("Renders templates", func() {
out, err := RenderHelm("{{.Values.Test}}", map[string]interface{}{"Test": "foo"})
out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo"}, map[string]interface{}{"Bar": "bar"})
Expect(err).ToNot(HaveOccurred())
Expect(out).To(Equal("foo"))
Expect(out).To(Equal("foobar"))
})
It("Renders templates with overrides", func() {
out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "baz"}, map[string]interface{}{"Bar": "bar"})
Expect(err).ToNot(HaveOccurred())
Expect(out).To(Equal("foobar"))
})
It("Renders templates", func() {
out, err := RenderHelm("{{.Values.Test}}{{.Values.Bar}}", map[string]interface{}{"Test": "foo", "Bar": "bar"}, map[string]interface{}{})
Expect(err).ToNot(HaveOccurred())
Expect(out).To(Equal("foobar"))
})
It("Render files default overrides", func() {
testDir, err := ioutil.TempDir(os.TempDir(), "test")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(testDir)
toTemplate := filepath.Join(testDir, "totemplate.yaml")
values := filepath.Join(testDir, "values.yaml")
d := filepath.Join(testDir, "default.yaml")
writeFile(toTemplate, `{{.Values.foo}}`)
writeFile(values, `
foo: "bar"
`)
writeFile(d, `
foo: "baz"
`)
Expect(err).ToNot(HaveOccurred())
res, err := RenderFiles(toTemplate, values, d)
Expect(err).ToNot(HaveOccurred())
Expect(res).To(Equal("baz"))
})
It("Render files from values", func() {
testDir, err := ioutil.TempDir(os.TempDir(), "test")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(testDir)
toTemplate := filepath.Join(testDir, "totemplate.yaml")
values := filepath.Join(testDir, "values.yaml")
d := filepath.Join(testDir, "default.yaml")
writeFile(toTemplate, `{{.Values.foo}}`)
writeFile(values, `
foo: "bar"
`)
writeFile(d, `
faa: "baz"
`)
Expect(err).ToNot(HaveOccurred())
res, err := RenderFiles(toTemplate, values, d)
Expect(err).ToNot(HaveOccurred())
Expect(res).To(Equal("bar"))
})
It("Render files from values if no default", func() {
testDir, err := ioutil.TempDir(os.TempDir(), "test")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(testDir)
toTemplate := filepath.Join(testDir, "totemplate.yaml")
values := filepath.Join(testDir, "values.yaml")
writeFile(toTemplate, `{{.Values.foo}}`)
writeFile(values, `
foo: "bar"
`)
Expect(err).ToNot(HaveOccurred())
res, err := RenderFiles(toTemplate, values, "")
Expect(err).ToNot(HaveOccurred())
Expect(res).To(Equal("bar"))
})
It("doesn't interpolate if no one provides the values", func() {
testDir, err := ioutil.TempDir(os.TempDir(), "test")
Expect(err).ToNot(HaveOccurred())
defer os.RemoveAll(testDir)
toTemplate := filepath.Join(testDir, "totemplate.yaml")
values := filepath.Join(testDir, "values.yaml")
d := filepath.Join(testDir, "default.yaml")
writeFile(toTemplate, `{{.Values.foo}}`)
writeFile(values, `
foao: "bar"
`)
writeFile(d, `
faa: "baz"
`)
Expect(err).ToNot(HaveOccurred())
res, err := RenderFiles(toTemplate, values, d)
Expect(err).ToNot(HaveOccurred())
Expect(res).To(Equal(""))
})
})
})

View File

@@ -757,12 +757,11 @@ func (l *LuetInstaller) computeUninstall(p pkg.Package, s *System) (pkg.Packages
}
func (l *LuetInstaller) Uninstall(p pkg.Package, s *System) error {
Spinner(32)
defer SpinnerStop()
toUninstall, err := l.computeUninstall(p, s)
if err != nil {
return errors.Wrap(err, "while computing uninstall")
}
SpinnerStop()
uninstall := func() error {
for _, p := range toUninstall {

View File

@@ -24,7 +24,7 @@ func (s *System) ExecuteFinalizers(packs []pkg.Package, force bool) error {
executedFinalizer := map[string]bool{}
for _, p := range packs {
if helpers.Exists(p.Rel(tree.FinalizerFile)) {
out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile))
out, err := helpers.RenderFiles(p.Rel(tree.FinalizerFile), p.Rel(tree.DefinitionFile), "")
if err != nil && !force {
return errors.Wrap(err, "reading file "+p.Rel(tree.FinalizerFile))
}

View File

@@ -92,7 +92,7 @@ func (r *CompilerRecipe) Load(path string) error {
compileDefPath := pack.Rel(CompilerDefinitionFile)
if helpers.Exists(compileDefPath) {
dat, err := helpers.RenderFiles(compileDefPath, currentpath)
dat, err := helpers.RenderFiles(compileDefPath, currentpath, "")
if err != nil {
return errors.Wrap(err,
"Error templating file "+CompilerDefinitionFile+" from "+
@@ -137,7 +137,7 @@ func (r *CompilerRecipe) Load(path string) error {
if err != nil {
return errors.Wrap(err, "Error reading file "+currentpath)
}
dat, err := helpers.RenderHelm(string(buildyaml), raw)
dat, err := helpers.RenderHelm(string(buildyaml), raw, map[string]interface{}{})
if err != nil {
return errors.Wrap(err,
"Error templating file "+CompilerDefinitionFile+" from "+

View File

@@ -0,0 +1,7 @@
image: quay.io/mocaccino/extra
steps:
- touch /{{.Values.name}}
- touch /build-extra-{{.Values.foo}}
- touch /{{.Values.name}}-{{.Values.bb}}
unpack: true

View File

@@ -0,0 +1,13 @@
packages:
- name: "a"
category: "distro"
version: "0.1"
foo: "baz"
- name: "b"
category: "distro"
version: "0.3"
foo: "f"
- name: "c"
category: "distro"
version: "0.3"
foo: "bar"

View File

@@ -0,0 +1,2 @@
install:
- touch /finalize-{{.Values.name}}

View File

@@ -0,0 +1,4 @@
image: quay.io/mocaccino/extra
steps:
- touch /{{.Values.name}}
- touch /{{.Values.name}}-{{.Values.bb}}

View File

@@ -0,0 +1,3 @@
name: foo
category: test
version: "1.1"

View File

@@ -0,0 +1,133 @@
#!/bin/bash
export LUET_NOLOCK=true
oneTimeSetUp() {
export tmpdir="$(mktemp -d)"
}
oneTimeTearDown() {
rm -rf "$tmpdir"
}
testBuild() {
cat <<EOF > $tmpdir/default.yaml
bb: "ttt"
EOF
mkdir $tmpdir/testbuild
luet build --tree "$ROOT_DIR/tests/fixtures/build_values" --values $tmpdir/default.yaml --destination $tmpdir/testbuild --compression gzip --all
buildst=$?
assertEquals 'builds successfully' "$buildst" "0"
assertTrue 'create package B' "[ -e '$tmpdir/testbuild/b-distro-0.3.package.tar.gz' ]"
assertTrue 'create package A' "[ -e '$tmpdir/testbuild/a-distro-0.1.package.tar.gz' ]"
assertTrue 'create package C' "[ -e '$tmpdir/testbuild/c-distro-0.3.package.tar.gz' ]"
assertTrue 'create package foo' "[ -e '$tmpdir/testbuild/foo-test-1.1.package.tar.gz' ]"
}
testRepo() {
assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
luet create-repo --tree "$ROOT_DIR/tests/fixtures/build_values" \
--output $tmpdir/testbuild \
--packages $tmpdir/testbuild \
--name "test" \
--descr "Test Repo" \
--urls $tmpdir/testrootfs \
--type disk
createst=$?
assertEquals 'create repo successfully' "$createst" "0"
assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
}
testConfig() {
mkdir $tmpdir/testrootfs
cat <<EOF > $tmpdir/luet.yaml
general:
debug: true
system:
rootfs: $tmpdir/testrootfs
database_path: "/"
database_engine: "boltdb"
config_from_host: true
repositories:
- name: "main"
type: "disk"
enable: true
urls:
- "$tmpdir/testbuild"
EOF
luet config --config $tmpdir/luet.yaml
res=$?
assertEquals 'config test successfully' "$res" "0"
}
testInstall() {
luet install -y --config $tmpdir/luet.yaml distro/a
installst=$?
assertEquals 'install test successfully' "$installst" "0"
assertTrue 'package installed A' "[ -e '$tmpdir/testrootfs/a' ]"
# Build time can interpolate on fields which aren't package properties.
assertTrue 'extra field on A' "[ -e '$tmpdir/testrootfs/build-extra-baz' ]"
assertTrue 'package installed A interpolated with values' "[ -e '$tmpdir/testrootfs/a-ttt' ]"
# Finalizers can interpolate only on package field. No extra fields are allowed at this time.
assertTrue 'finalizer executed on A' "[ -e '$tmpdir/testrootfs/finalize-a' ]"
installed=$(luet --config $tmpdir/luet.yaml search --installed .)
searchst=$?
assertEquals 'search exists successfully' "$searchst" "0"
assertContains 'contains distro/a-0.1' "$installed" 'distro/a-0.1'
luet uninstall -y --config $tmpdir/luet.yaml distro/a
installst=$?
assertEquals 'install test successfully' "$installst" "0"
# We do the same check for the others
luet install -y --config $tmpdir/luet.yaml distro/b
installst=$?
assertEquals 'install test successfully' "$installst" "0"
assertTrue 'package installed B' "[ -e '$tmpdir/testrootfs/b' ]"
assertTrue 'package installed B interpolated with values' "[ -e '$tmpdir/testrootfs/b-ttt' ]"
assertTrue 'extra field on B' "[ -e '$tmpdir/testrootfs/build-extra-f' ]"
assertTrue 'finalizer executed on B' "[ -e '$tmpdir/testrootfs/finalize-b' ]"
installed=$(luet --config $tmpdir/luet.yaml search --installed .)
searchst=$?
assertEquals 'search exists successfully' "$searchst" "0"
assertContains 'contains distro/b-0.3' "$installed" 'distro/b-0.3'
luet uninstall -y --config $tmpdir/luet.yaml distro/b
installst=$?
assertEquals 'install test successfully' "$installst" "0"
luet install -y --config $tmpdir/luet.yaml distro/c
installst=$?
assertEquals 'install test successfully' "$installst" "0"
assertTrue 'package installed C' "[ -e '$tmpdir/testrootfs/c' ]"
assertTrue 'extra field on C' "[ -e '$tmpdir/testrootfs/build-extra-bar' ]"
assertTrue 'package installed C interpolated with values' "[ -e '$tmpdir/testrootfs/c-ttt' ]"
assertTrue 'finalizer executed on C' "[ -e '$tmpdir/testrootfs/finalize-c' ]"
installed=$(luet --config $tmpdir/luet.yaml search --installed .)
searchst=$?
assertEquals 'search exists successfully' "$searchst" "0"
assertContains 'contains distro/c-0.3' "$installed" 'distro/c-0.3'
luet uninstall -y --config $tmpdir/luet.yaml distro/c
installst=$?
assertEquals 'install test successfully' "$installst" "0"
luet install -y --config $tmpdir/luet.yaml test/foo
installst=$?
assertEquals 'install test successfully' "$installst" "0"
assertTrue 'package installed foo' "[ -e '$tmpdir/testrootfs/foo' ]"
assertTrue 'package installed foo interpolated with values' "[ -e '$tmpdir/testrootfs/foo-ttt' ]"
}
# Load shUnit2.
. "$ROOT_DIR/tests/integration/shunit2"/shunit2