Merge pull request #76 from mudler/box_finalizers

Add Box finalizers

cmd/exec.go

@@ -0,0 +1,86 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+package cmd
+
+import (
+	"os"
+
+	b64 "encoding/base64"
+
+	"github.com/mudler/luet/pkg/box"
+	. "github.com/mudler/luet/pkg/logger"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var execCmd = &cobra.Command{
+	Use:   "exec --rootfs /path [command]",
+	Short: "Execute a command in the rootfs context",
+	Long:  `Uses unshare technique and pivot root to execute a command inside a folder containing a valid rootfs`,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("stdin", cmd.Flags().Lookup("stdin"))
+		viper.BindPFlag("stdout", cmd.Flags().Lookup("stdout"))
+		viper.BindPFlag("stderr", cmd.Flags().Lookup("stderr"))
+		viper.BindPFlag("rootfs", cmd.Flags().Lookup("rootfs"))
+		viper.BindPFlag("decode", cmd.Flags().Lookup("decode"))
+		viper.BindPFlag("entrypoint", cmd.Flags().Lookup("entrypoint"))
+
+	},
+	// If you change this, look at pkg/box/exec that runs this command and adapt
+	Run: func(cmd *cobra.Command, args []string) {
+
+		stdin := viper.GetBool("stdin")
+		stdout := viper.GetBool("stdout")
+		stderr := viper.GetBool("stderr")
+		rootfs := viper.GetString("rootfs")
+		base := viper.GetBool("decode")
+
+		entrypoint := viper.GetString("entrypoint")
+		if base {
+			var ss []string
+			for _, a := range args {
+				sDec, _ := b64.StdEncoding.DecodeString(a)
+				ss = append(ss, string(sDec))
+			}
+			//If the command to run is complex,using base64 to avoid bad input
+
+			args = ss
+		}
+		Info("Executing", args, "in", rootfs)
+
+		b := box.NewBox(entrypoint, args, rootfs, stdin, stdout, stderr)
+		err := b.Exec()
+		if err != nil {
+			Fatal(err)
+		}
+	},
+}
+
+func init() {
+	path, err := os.Getwd()
+	if err != nil {
+		Fatal(err)
+	}
+	execCmd.Hidden = true
+	execCmd.Flags().String("rootfs", path, "Rootfs path")
+	execCmd.Flags().Bool("stdin", false, "Attach to stdin")
+	execCmd.Flags().Bool("stdout", false, "Attach to stdout")
+	execCmd.Flags().Bool("stderr", false, "Attach to stderr")
+	execCmd.Flags().Bool("decode", false, "Base64 decode")
+
+	execCmd.Flags().String("entrypoint", "/bin/sh", "Entrypoint command (/bin/sh)")
+
+	RootCmd.AddCommand(execCmd)
+}

pkg/box/exec.go

@@ -0,0 +1,157 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package box
+
+import (
+	b64 "encoding/base64"
+	"fmt"
+	"os"
+	"os/exec"
+	"syscall"
+
+	"github.com/pkg/errors"
+
+	helpers "github.com/mudler/luet/pkg/helpers"
+)
+
+type Box interface {
+	Run() error
+	Exec() error
+}
+
+type DefaultBox struct {
+	Name string
+	Root string
+	Env  []string
+	Cmd  string
+	Args []string
+
+	Stdin, Stdout, Stderr bool
+}
+
+func NewBox(cmd string, args []string, rootfs string, stdin, stdout, stderr bool) Box {
+	return &DefaultBox{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+		Cmd:    cmd,
+		Args:   args,
+		Root:   rootfs,
+	}
+}
+
+func (b *DefaultBox) Exec() error {
+
+	if err := mountProc(b.Root); err != nil {
+		return errors.Wrap(err, "Failed mounting proc on rootfs")
+	}
+	if err := mountDev(b.Root); err != nil {
+		return errors.Wrap(err, "Failed mounting dev on rootfs")
+	}
+	if err := PivotRoot(b.Root); err != nil {
+		return errors.Wrap(err, "Failed switching pivot on rootfs")
+	}
+
+	cmd := exec.Command(b.Cmd, b.Args...)
+
+	if b.Stdin {
+		cmd.Stdin = os.Stdin
+	}
+
+	if b.Stderr {
+		cmd.Stderr = os.Stderr
+	}
+
+	if b.Stdout {
+		cmd.Stdout = os.Stdout
+	}
+
+	cmd.Env = b.Env
+
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("Error running the %s command", b.Cmd))
+	}
+	return nil
+}
+
+func (b *DefaultBox) Run() error {
+
+	if !helpers.Exists(b.Root) {
+		return errors.New(b.Root + " does not exist")
+	}
+
+	// This matches with exec CLI command in luet
+	// TODO: Pass by env var as well
+	execCmd := []string{"exec", "--rootfs", b.Root, "--entrypoint", b.Cmd}
+
+	if b.Stdin {
+		execCmd = append(execCmd, "--stdin")
+	}
+
+	if b.Stderr {
+		execCmd = append(execCmd, "--stderr")
+	}
+
+	if b.Stdout {
+		execCmd = append(execCmd, "--stdout")
+	}
+	// Encode the command in base64 to avoid bad input from the args given
+	execCmd = append(execCmd, "--decode")
+
+	for _, a := range b.Args {
+		execCmd = append(execCmd, b64.StdEncoding.EncodeToString([]byte(a)))
+	}
+
+	cmd := exec.Command("/proc/self/exe", execCmd...)
+	if b.Stdin {
+		cmd.Stdin = os.Stdin
+	}
+
+	if b.Stderr {
+		cmd.Stderr = os.Stderr
+	}
+
+	if b.Stdout {
+		cmd.Stdout = os.Stdout
+	}
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Cloneflags: syscall.CLONE_NEWNS |
+			syscall.CLONE_NEWUTS |
+			syscall.CLONE_NEWIPC |
+			syscall.CLONE_NEWPID |
+			syscall.CLONE_NEWNET |
+			syscall.CLONE_NEWUSER,
+		UidMappings: []syscall.SysProcIDMap{
+			{
+				ContainerID: 0,
+				HostID:      os.Getuid(),
+				Size:        1,
+			},
+		},
+		GidMappings: []syscall.SysProcIDMap{
+			{
+				ContainerID: 0,
+				HostID:      os.Getgid(),
+				Size:        1,
+			},
+		},
+	}
+
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, "Failed running Box command")
+	}
+	return nil
+}

pkg/box/rootfs.go

@@ -0,0 +1,91 @@
+// Copyright © 2020 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package box
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+)
+
+func PivotRoot(newroot string) error {
+	putold := filepath.Join(newroot, "/.pivot_root")
+
+	// bind mount newroot to itself - this is a slight hack needed to satisfy the
+	// pivot_root requirement that newroot and putold must not be on the same
+	// filesystem as the current root
+	if err := syscall.Mount(newroot, newroot, "", syscall.MS_BIND|syscall.MS_REC, ""); err != nil {
+		return err
+	}
+
+	// create putold directory
+	if err := os.MkdirAll(putold, 0700); err != nil {
+		return err
+	}
+
+	// call pivot_root
+	if err := syscall.PivotRoot(newroot, putold); err != nil {
+		return err
+	}
+
+	// ensure current working directory is set to new root
+	if err := os.Chdir("/"); err != nil {
+		return err
+	}
+
+	// umount putold, which now lives at /.pivot_root
+	putold = "/.pivot_root"
+	if err := syscall.Unmount(putold, syscall.MNT_DETACH); err != nil {
+		return err
+	}
+
+	// remove putold
+	if err := os.RemoveAll(putold); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func mountProc(newroot string) error {
+	source := "proc"
+	target := filepath.Join(newroot, "/proc")
+	fstype := "proc"
+	flags := 0
+	data := ""
+
+	os.MkdirAll(target, 0755)
+	if err := syscall.Mount(source, target, fstype, uintptr(flags), data); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func mountDev(newroot string) error {
+
+	source := "/dev"
+	target := filepath.Join(newroot, "/dev")
+	fstype := "bind"
+	data := ""
+
+	os.MkdirAll(target, 0755)
+	if err := syscall.Mount(source, target, fstype, syscall.MS_BIND|syscall.MS_REC, data); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/helpers/file.go

@@ -23,6 +23,22 @@ import (
 	copy "github.com/otiai10/copy"
 )
 
+func ListDir(dir string) ([]string, error) {
+	content := []string{}
+
+	err := filepath.Walk(dir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			content = append(content, path)
+
+			return nil
+		})
+
+	return content, err
+}
+
 // Exists reports whether the named file or directory exists.
 func Exists(name string) bool {
 	if _, err := os.Stat(name); err != nil {

pkg/installer/finalizer.go

@@ -0,0 +1,76 @@
+// Copyright © 2019 Ettore Di Giacinto <mudler@gentoo.org>
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package installer
+
+import (
+	"os/exec"
+
+	"github.com/ghodss/yaml"
+	box "github.com/mudler/luet/pkg/box"
+	. "github.com/mudler/luet/pkg/logger"
+
+	"github.com/pkg/errors"
+)
+
+type LuetFinalizer struct {
+	Install   []string `json:"install"`
+	Uninstall []string `json:"uninstall"` // TODO: Where to store?
+}
+
+func (f *LuetFinalizer) RunInstall(s *System) error {
+	for _, c := range f.Install {
+		if s.Target == "/" {
+
+			Info("finalizer on / :", "sh", "-c", c)
+			cmd := exec.Command("sh", "-c", c)
+			stdoutStderr, err := cmd.CombinedOutput()
+			if err != nil {
+				return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
+			}
+			Info(string(stdoutStderr))
+		} else {
+			b := box.NewBox("sh", []string{"-c", c}, s.Target, false, true, true)
+			err := b.Run()
+			if err != nil {
+				return errors.Wrap(err, "Failed running command ")
+			}
+		}
+	}
+	return nil
+}
+
+// TODO: We don't store uninstall finalizers ?!
+func (f *LuetFinalizer) RunUnInstall() error {
+	for _, c := range f.Uninstall {
+		Debug("finalizer:", "sh", "-c", c)
+		cmd := exec.Command("sh", "-c", c)
+		stdoutStderr, err := cmd.CombinedOutput()
+		if err != nil {
+			return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
+		}
+		Info(string(stdoutStderr))
+	}
+	return nil
+}
+
+func NewLuetFinalizerFromYaml(data []byte) (*LuetFinalizer, error) {
+	var p LuetFinalizer
+	err := yaml.Unmarshal(data, &p)
+	if err != nil {
+		return &p, err
+	}
+	return &p, err
+}

pkg/installer/installer.go

@@ -18,13 +18,11 @@ package installer
 import (
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
 
-	"github.com/ghodss/yaml"
 	compiler "github.com/mudler/luet/pkg/compiler"
 	"github.com/mudler/luet/pkg/config"
 	"github.com/mudler/luet/pkg/helpers"
@@ -57,47 +55,6 @@ type ArtifactMatch struct {
 	Repository Repository
 }
 
-type LuetFinalizer struct {
-	Install   []string `json:"install"`
-	Uninstall []string `json:"uninstall"` // TODO: Where to store?
-}
-
-func (f *LuetFinalizer) RunInstall() error {
-	for _, c := range f.Install {
-		Debug("finalizer:", "sh", "-c", c)
-		cmd := exec.Command("sh", "-c", c)
-		stdoutStderr, err := cmd.CombinedOutput()
-		if err != nil {
-			return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
-		}
-		Info(string(stdoutStderr))
-	}
-	return nil
-}
-
-// TODO: We don't store uninstall finalizers ?!
-func (f *LuetFinalizer) RunUnInstall() error {
-	for _, c := range f.Install {
-		Debug("finalizer:", "sh", "-c", c)
-		cmd := exec.Command("sh", "-c", c)
-		stdoutStderr, err := cmd.CombinedOutput()
-		if err != nil {
-			return errors.Wrap(err, "Failed running command: "+string(stdoutStderr))
-		}
-		Info(string(stdoutStderr))
-	}
-	return nil
-}
-
-func NewLuetFinalizerFromYaml(data []byte) (*LuetFinalizer, error) {
-	var p LuetFinalizer
-	err := yaml.Unmarshal(data, &p)
-	if err != nil {
-		return &p, err
-	}
-	return &p, err
-}
-
 func NewLuetInstaller(opts LuetInstallerOptions) Installer {
 	return &LuetInstaller{Options: opts}
 }
@@ -395,13 +352,14 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp []pkg.Package, s *S
 							if err != nil && !l.Options.Force {
 								return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
 							}
-							err = finalizer.RunInstall()
+							err = finalizer.RunInstall(s)
 							if err != nil && !l.Options.Force {
 								return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
 							}
 							executedFinalizer[ass.Package.GetFingerPrint()] = true
 						}
 					}
+
 				}
 			}
 
@@ -423,7 +381,7 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp []pkg.Package, s *S
 					if err != nil && !l.Options.Force {
 						return errors.Wrap(err, "Error reading finalizer "+treePackage.Rel(tree.FinalizerFile))
 					}
-					err = finalizer.RunInstall()
+					err = finalizer.RunInstall(s)
 					if err != nil && !l.Options.Force {
 						return errors.Wrap(err, "Error executing install finalizer "+treePackage.Rel(tree.FinalizerFile))
 					}
@@ -432,6 +390,7 @@ func (l *LuetInstaller) install(syncedRepos Repositories, cp []pkg.Package, s *S
 			}
 		}
 	}
+
 	return nil
 
 }

tests/fixtures/finalizers/alpine/build.yaml

@@ -0,0 +1,2 @@
+image: "alpine"
+unpack: true

tests/fixtures/finalizers/alpine/definition.yaml

@@ -0,0 +1,3 @@
+category: "seed"
+name: "alpine"
+version: "1.0"

tests/fixtures/finalizers/alpine/finalize.yaml

@@ -0,0 +1,2 @@
+install:
+- touch /tmp/foo

tests/integration/07_finalizer.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+export LUET_NOLOCK=true
+
+oneTimeSetUp() {
+export tmpdir="$(mktemp -d)"
+}
+
+oneTimeTearDown() {
+    rm -rf "$tmpdir"
+}
+
+testBuild() {
+    mkdir $tmpdir/testbuild
+    luet build --tree "$ROOT_DIR/tests/fixtures/finalizers" --destination $tmpdir/testbuild --compression gzip --all > /dev/null
+    buildst=$?
+    assertEquals 'builds successfully' "$buildst" "0"
+    assertTrue 'create package' "[ -e '$tmpdir/testbuild/alpine-seed-1.0.package.tar.gz' ]"
+}
+
+testRepo() {
+    assertTrue 'no repository' "[ ! -e '$tmpdir/testbuild/repository.yaml' ]"
+    luet create-repo --tree "$ROOT_DIR/tests/fixtures/finalizers" \
+    --output $tmpdir/testbuild \
+    --packages $tmpdir/testbuild \
+    --name "test" \
+    --descr "Test Repo" \
+    --urls $tmpdir/testrootfs \
+    --type disk > /dev/null
+
+    createst=$?
+    assertEquals 'create repo successfully' "$createst" "0"
+    assertTrue 'create repository' "[ -e '$tmpdir/testbuild/repository.yaml' ]"
+}
+
+testConfig() {
+    mkdir $tmpdir/testrootfs
+    cat <<EOF > $tmpdir/luet.yaml
+general:
+  debug: true
+system:
+  rootfs: $tmpdir/testrootfs
+  database_path: "/"
+  database_engine: "boltdb"
+repositories:
+   - name: "main"
+     type: "disk"
+     enable: true
+     urls:
+       - "$tmpdir/testbuild"
+EOF
+    luet config --config $tmpdir/luet.yaml
+    res=$?
+    assertEquals 'config test successfully' "$res" "0"
+}
+
+testInstall() {
+    luet install --config $tmpdir/luet.yaml seed/alpine
+    #luet install --config $tmpdir/luet.yaml test/c-1.0 > /dev/null
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+    assertTrue 'package installed' "[ -e '$tmpdir/testrootfs/bin/busybox' ]"
+    assertTrue 'finalizer runs' "[ -e '$tmpdir/testrootfs/tmp/foo' ]"
+}
+
+
+testCleanup() {
+    luet cleanup --config $tmpdir/luet.yaml
+    installst=$?
+    assertEquals 'install test successfully' "$installst" "0"
+}
+
+# Load shUnit2.
+. "$ROOT_DIR/tests/integration/shunit2"/shunit2
+