Merge pull request #635 from SUSE/buildah-update

Vendor the latest buildah master
This commit is contained in:
Miloslav Trmač 2019-04-24 07:16:29 +02:00 committed by GitHub
commit 0fa335c149
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 118 additions and 470 deletions

View File

@ -3,7 +3,7 @@ github.com/urfave/cli v1.20.0
github.com/kr/pretty v0.1.0
github.com/kr/text v0.1.0
github.com/containers/image e9c3d17ddb8cb5d48450bc0588525b17b181c049
github.com/containers/buildah 25b7c1164a262934a6526ded1115dfacfc51f8ec
github.com/containers/buildah 810efa340ab43753034e2ed08ec290e4abab7e72
github.com/vbauerster/mpb v3.3.4
github.com/mattn/go-isatty v0.0.4
github.com/VividCortex/ewma v1.1.1

View File

@ -3,6 +3,7 @@
package unshare
import (
"bufio"
"bytes"
"fmt"
"io"
@ -15,7 +16,7 @@ import (
"sync"
"syscall"
"github.com/containers/buildah/util"
"github.com/containers/storage/pkg/idtools"
"github.com/containers/storage/pkg/reexec"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
@ -157,7 +158,7 @@ func (c *Cmd) Start() error {
}
if len(c.UidMappings) == 0 || len(c.GidMappings) == 0 {
uidmap, gidmap, err := util.GetHostIDMappings("")
uidmap, gidmap, err := GetHostIDMappings("")
if err != nil {
fmt.Fprintf(continueWrite, "error reading ID mappings in parent: %v", err)
return errors.Wrapf(err, "error reading ID mappings in parent")
@ -352,7 +353,7 @@ func MaybeReexecUsingUserNamespace(evenForRoot bool) {
// Read the set of ID mappings that we're allowed to use. Each
// range in /etc/subuid and /etc/subgid file is a starting host
// ID and a range size.
uidmap, gidmap, err = util.GetSubIDMappings(me.Username, me.Username)
uidmap, gidmap, err = GetSubIDMappings(me.Username, me.Username)
bailOnError(err, "error reading allowed ID mappings")
if len(uidmap) == 0 {
logrus.Warnf("Found no UID ranges set aside for user %q in /etc/subuid.", me.Username)
@ -384,7 +385,7 @@ func MaybeReexecUsingUserNamespace(evenForRoot bool) {
return
}
// Read the set of ID mappings that we're currently using.
uidmap, gidmap, err = util.GetHostIDMappings("")
uidmap, gidmap, err = GetHostIDMappings("")
bailOnError(err, "error reading current ID mappings")
// Just reuse them.
for i := range uidmap {
@ -404,6 +405,16 @@ func MaybeReexecUsingUserNamespace(evenForRoot bool) {
err = os.Setenv(UsernsEnvName, "1")
bailOnError(err, "error setting %s=1 in environment", UsernsEnvName)
// Set the default isolation type to use the "rootless" method.
if _, present := os.LookupEnv("BUILDAH_ISOLATION"); !present {
if err = os.Setenv("BUILDAH_ISOLATION", "rootless"); err != nil {
if err := os.Setenv("BUILDAH_ISOLATION", "rootless"); err != nil {
logrus.Errorf("error setting BUILDAH_ISOLATION=rootless in environment: %v", err)
os.Exit(1)
}
}
}
// Reuse our stdio.
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
@ -446,3 +457,89 @@ func ExecRunnable(cmd Runnable) {
}
os.Exit(0)
}
// getHostIDMappings reads mappings from the named node under /proc.
func getHostIDMappings(path string) ([]specs.LinuxIDMapping, error) {
var mappings []specs.LinuxIDMapping
f, err := os.Open(path)
if err != nil {
return nil, errors.Wrapf(err, "error reading ID mappings from %q", path)
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
fields := strings.Fields(line)
if len(fields) != 3 {
return nil, errors.Errorf("line %q from %q has %d fields, not 3", line, path, len(fields))
}
cid, err := strconv.ParseUint(fields[0], 10, 32)
if err != nil {
return nil, errors.Wrapf(err, "error parsing container ID value %q from line %q in %q", fields[0], line, path)
}
hid, err := strconv.ParseUint(fields[1], 10, 32)
if err != nil {
return nil, errors.Wrapf(err, "error parsing host ID value %q from line %q in %q", fields[1], line, path)
}
size, err := strconv.ParseUint(fields[2], 10, 32)
if err != nil {
return nil, errors.Wrapf(err, "error parsing size value %q from line %q in %q", fields[2], line, path)
}
mappings = append(mappings, specs.LinuxIDMapping{ContainerID: uint32(cid), HostID: uint32(hid), Size: uint32(size)})
}
return mappings, nil
}
// GetHostIDMappings reads mappings for the specified process (or the current
// process if pid is "self" or an empty string) from the kernel.
func GetHostIDMappings(pid string) ([]specs.LinuxIDMapping, []specs.LinuxIDMapping, error) {
if pid == "" {
pid = "self"
}
uidmap, err := getHostIDMappings(fmt.Sprintf("/proc/%s/uid_map", pid))
if err != nil {
return nil, nil, err
}
gidmap, err := getHostIDMappings(fmt.Sprintf("/proc/%s/gid_map", pid))
if err != nil {
return nil, nil, err
}
return uidmap, gidmap, nil
}
// GetSubIDMappings reads mappings from /etc/subuid and /etc/subgid.
func GetSubIDMappings(user, group string) ([]specs.LinuxIDMapping, []specs.LinuxIDMapping, error) {
mappings, err := idtools.NewIDMappings(user, group)
if err != nil {
return nil, nil, errors.Wrapf(err, "error reading subuid mappings for user %q and subgid mappings for group %q", user, group)
}
var uidmap, gidmap []specs.LinuxIDMapping
for _, m := range mappings.UIDs() {
uidmap = append(uidmap, specs.LinuxIDMapping{
ContainerID: uint32(m.ContainerID),
HostID: uint32(m.HostID),
Size: uint32(m.Size),
})
}
for _, m := range mappings.GIDs() {
gidmap = append(gidmap, specs.LinuxIDMapping{
ContainerID: uint32(m.ContainerID),
HostID: uint32(m.HostID),
Size: uint32(m.Size),
})
}
return uidmap, gidmap, nil
}
// ParseIDMappings parses mapping triples.
func ParseIDMappings(uidmap, gidmap []string) ([]idtools.IDMap, []idtools.IDMap, error) {
uid, err := idtools.ParseIDMap(uidmap, "userns-uid-map")
if err != nil {
return nil, nil, err
}
gid, err := idtools.ParseIDMap(gidmap, "userns-gid-map")
if err != nil {
return nil, nil, err
}
return uid, gid, nil
}

View File

@ -4,6 +4,9 @@ package unshare
import (
"os"
"github.com/containers/storage/pkg/idtools"
"github.com/opencontainers/runtime-spec/specs-go"
)
const (
@ -29,3 +32,14 @@ func RootlessEnv() []string {
// MaybeReexecUsingUserNamespace re-exec the process in a new namespace
func MaybeReexecUsingUserNamespace(evenForRoot bool) {
}
// GetHostIDMappings reads mappings for the specified process (or the current
// process if pid is "self" or an empty string) from the kernel.
func GetHostIDMappings(pid string) ([]specs.LinuxIDMapping, []specs.LinuxIDMapping, error) {
return nil, nil, nil
}
// ParseIDMappings parses mapping triples.
func ParseIDMappings(uidmap, gidmap []string) ([]idtools.IDMap, []idtools.IDMap, error) {
return nil, nil, nil
}

View File

@ -1,35 +0,0 @@
package util
const (
// DefaultRuntime is the default command to use to run the container.
DefaultRuntime = "runc"
// DefaultCNIPluginPath is the default location of CNI plugin helpers.
DefaultCNIPluginPath = "/usr/libexec/cni:/opt/cni/bin"
// DefaultCNIConfigDir is the default location of CNI configuration files.
DefaultCNIConfigDir = "/etc/cni/net.d"
)
var (
// DefaultCapabilities is the list of capabilities which we grant by
// default to containers which are running under UID 0.
DefaultCapabilities = []string{
"CAP_AUDIT_WRITE",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_KILL",
"CAP_MKNOD",
"CAP_NET_BIND_SERVICE",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYS_CHROOT",
}
// DefaultNetworkSysctl is the list of Kernel parameters which we
// grant by default to containers which are running under UID 0.
DefaultNetworkSysctl = map[string]string{
"net.ipv4.ping_group_range": "0 0",
}
)

View File

@ -1,429 +0,0 @@
package util
import (
"bufio"
"fmt"
"io"
"net/url"
"os"
"path"
"strconv"
"strings"
"syscall"
"github.com/containers/image/docker/reference"
"github.com/containers/image/pkg/sysregistriesv2"
"github.com/containers/image/signature"
is "github.com/containers/image/storage"
"github.com/containers/image/transports"
"github.com/containers/image/types"
"github.com/containers/storage"
"github.com/containers/storage/pkg/idtools"
"github.com/docker/distribution/registry/api/errcode"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const (
minimumTruncatedIDLength = 3
// DefaultTransport is a prefix that we apply to an image name if we
// can't find one in the local Store, in order to generate a source
// reference for the image that we can then copy to the local Store.
DefaultTransport = "docker://"
)
var (
// RegistryDefaultPathPrefix contains a per-registry listing of default prefixes
// to prepend to image names that only contain a single path component.
RegistryDefaultPathPrefix = map[string]string{
"index.docker.io": "library",
"docker.io": "library",
}
)
// ResolveName checks if name is a valid image name, and if that name doesn't
// include a domain portion, returns a list of the names which it might
// correspond to in the set of configured registries, the transport used to
// pull the image, and a boolean which is true iff
// 1) the list of search registries was used, and 2) it was empty.
//
// The returned image names never include a transport: prefix, and if transport != "",
// (transport, image) should be a valid input to alltransports.ParseImageName.
// transport == "" indicates that image that already exists in a local storage,
// and the name is valid for store.Image() / storage.Transport.ParseStoreReference().
//
// NOTE: The "list of search registries is empty" check does not count blocked registries,
// and neither the implied "localhost" nor a possible firstRegistry are counted
func ResolveName(name string, firstRegistry string, sc *types.SystemContext, store storage.Store) ([]string, string, bool, error) {
if name == "" {
return nil, "", false, nil
}
// Maybe it's a truncated image ID. Don't prepend a registry name, then.
if len(name) >= minimumTruncatedIDLength {
if img, err := store.Image(name); err == nil && img != nil && strings.HasPrefix(img.ID, name) {
// It's a truncated version of the ID of an image that's present in local storage;
// we need only expand the ID.
return []string{img.ID}, "", false, nil
}
}
// If the image includes a transport's name as a prefix, use it as-is.
if strings.HasPrefix(name, DefaultTransport) {
return []string{strings.TrimPrefix(name, DefaultTransport)}, DefaultTransport, false, nil
}
split := strings.SplitN(name, ":", 2)
if len(split) == 2 {
if trans := transports.Get(split[0]); trans != nil {
return []string{split[1]}, trans.Name(), false, nil
}
}
// If the image name already included a domain component, we're done.
named, err := reference.ParseNormalizedNamed(name)
if err != nil {
return nil, "", false, errors.Wrapf(err, "error parsing image name %q", name)
}
if named.String() == name {
// Parsing produced the same result, so there was a domain name in there to begin with.
return []string{name}, DefaultTransport, false, nil
}
if reference.Domain(named) != "" && RegistryDefaultPathPrefix[reference.Domain(named)] != "" {
// If this domain can cause us to insert something in the middle, check if that happened.
repoPath := reference.Path(named)
domain := reference.Domain(named)
tag := ""
if tagged, ok := named.(reference.Tagged); ok {
tag = ":" + tagged.Tag()
}
digest := ""
if digested, ok := named.(reference.Digested); ok {
digest = "@" + digested.Digest().String()
}
defaultPrefix := RegistryDefaultPathPrefix[reference.Domain(named)] + "/"
if strings.HasPrefix(repoPath, defaultPrefix) && path.Join(domain, repoPath[len(defaultPrefix):])+tag+digest == name {
// Yup, parsing just inserted a bit in the middle, so there was a domain name there to begin with.
return []string{name}, DefaultTransport, false, nil
}
}
// Figure out the list of registries.
var registries []string
searchRegistries, err := sysregistriesv2.FindUnqualifiedSearchRegistries(sc)
if err != nil {
logrus.Debugf("unable to read configured registries to complete %q: %v", name, err)
}
for _, registry := range searchRegistries {
if !registry.Blocked {
registries = append(registries, registry.URL)
}
}
searchRegistriesAreEmpty := len(registries) == 0
// Create all of the combinations. Some registries need an additional component added, so
// use our lookaside map to keep track of them. If there are no configured registries, we'll
// return a name using "localhost" as the registry name.
candidates := []string{}
initRegistries := []string{"localhost"}
if firstRegistry != "" && firstRegistry != "localhost" {
initRegistries = append([]string{firstRegistry}, initRegistries...)
}
for _, registry := range append(initRegistries, registries...) {
if registry == "" {
continue
}
middle := ""
if prefix, ok := RegistryDefaultPathPrefix[registry]; ok && strings.IndexRune(name, '/') == -1 {
middle = prefix
}
candidate := path.Join(registry, middle, name)
candidates = append(candidates, candidate)
}
return candidates, DefaultTransport, searchRegistriesAreEmpty, nil
}
// ExpandNames takes unqualified names, parses them as image names, and returns
// the fully expanded result, including a tag. Names which don't include a registry
// name will be marked for the most-preferred registry (i.e., the first one in our
// configuration).
func ExpandNames(names []string, firstRegistry string, systemContext *types.SystemContext, store storage.Store) ([]string, error) {
expanded := make([]string, 0, len(names))
for _, n := range names {
var name reference.Named
nameList, _, _, err := ResolveName(n, firstRegistry, systemContext, store)
if err != nil {
return nil, errors.Wrapf(err, "error parsing name %q", n)
}
if len(nameList) == 0 {
named, err := reference.ParseNormalizedNamed(n)
if err != nil {
return nil, errors.Wrapf(err, "error parsing name %q", n)
}
name = named
} else {
named, err := reference.ParseNormalizedNamed(nameList[0])
if err != nil {
return nil, errors.Wrapf(err, "error parsing name %q", nameList[0])
}
name = named
}
name = reference.TagNameOnly(name)
expanded = append(expanded, name.String())
}
return expanded, nil
}
// FindImage locates the locally-stored image which corresponds to a given name.
func FindImage(store storage.Store, firstRegistry string, systemContext *types.SystemContext, image string) (types.ImageReference, *storage.Image, error) {
var ref types.ImageReference
var img *storage.Image
var err error
names, _, _, err := ResolveName(image, firstRegistry, systemContext, store)
if err != nil {
return nil, nil, errors.Wrapf(err, "error parsing name %q", image)
}
for _, name := range names {
ref, err = is.Transport.ParseStoreReference(store, name)
if err != nil {
logrus.Debugf("error parsing reference to image %q: %v", name, err)
continue
}
img, err = is.Transport.GetStoreImage(store, ref)
if err != nil {
img2, err2 := store.Image(name)
if err2 != nil {
logrus.Debugf("error locating image %q: %v", name, err2)
continue
}
img = img2
}
break
}
if ref == nil || img == nil {
return nil, nil, errors.Wrapf(err, "error locating image with name %q", image)
}
return ref, img, nil
}
// AddImageNames adds the specified names to the specified image.
func AddImageNames(store storage.Store, firstRegistry string, systemContext *types.SystemContext, image *storage.Image, addNames []string) error {
names, err := ExpandNames(addNames, firstRegistry, systemContext, store)
if err != nil {
return err
}
err = store.SetNames(image.ID, append(image.Names, names...))
if err != nil {
return errors.Wrapf(err, "error adding names (%v) to image %q", names, image.ID)
}
return nil
}
// GetFailureCause checks the type of the error "err" and returns a new
// error message that reflects the reason of the failure.
// In case err type is not a familiar one the error "defaultError" is returned.
func GetFailureCause(err, defaultError error) error {
switch nErr := errors.Cause(err).(type) {
case errcode.Errors:
return err
case errcode.Error, *url.Error:
return nErr
default:
return defaultError
}
}
// WriteError writes `lastError` into `w` if not nil and return the next error `err`
func WriteError(w io.Writer, err error, lastError error) error {
if lastError != nil {
fmt.Fprintln(w, lastError)
}
return err
}
// Runtime is the default command to use to run the container.
func Runtime() string {
runtime := os.Getenv("BUILDAH_RUNTIME")
if runtime != "" {
return runtime
}
return DefaultRuntime
}
// StringInSlice returns a boolean indicating if the exact value s is present
// in the slice slice.
func StringInSlice(s string, slice []string) bool {
for _, v := range slice {
if v == s {
return true
}
}
return false
}
// GetHostIDs uses ID mappings to compute the host-level IDs that will
// correspond to a UID/GID pair in the container.
func GetHostIDs(uidmap, gidmap []specs.LinuxIDMapping, uid, gid uint32) (uint32, uint32, error) {
uidMapped := true
for _, m := range uidmap {
uidMapped = false
if uid >= m.ContainerID && uid < m.ContainerID+m.Size {
uid = (uid - m.ContainerID) + m.HostID
uidMapped = true
break
}
}
if !uidMapped {
return 0, 0, errors.Errorf("container uses ID mappings, but doesn't map UID %d", uid)
}
gidMapped := true
for _, m := range gidmap {
gidMapped = false
if gid >= m.ContainerID && gid < m.ContainerID+m.Size {
gid = (gid - m.ContainerID) + m.HostID
gidMapped = true
break
}
}
if !gidMapped {
return 0, 0, errors.Errorf("container uses ID mappings, but doesn't map GID %d", gid)
}
return uid, gid, nil
}
// GetHostRootIDs uses ID mappings in spec to compute the host-level IDs that will
// correspond to UID/GID 0/0 in the container.
func GetHostRootIDs(spec *specs.Spec) (uint32, uint32, error) {
if spec.Linux == nil {
return 0, 0, nil
}
return GetHostIDs(spec.Linux.UIDMappings, spec.Linux.GIDMappings, 0, 0)
}
// getHostIDMappings reads mappings from the named node under /proc.
func getHostIDMappings(path string) ([]specs.LinuxIDMapping, error) {
var mappings []specs.LinuxIDMapping
f, err := os.Open(path)
if err != nil {
return nil, errors.Wrapf(err, "error reading ID mappings from %q", path)
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
fields := strings.Fields(line)
if len(fields) != 3 {
return nil, errors.Errorf("line %q from %q has %d fields, not 3", line, path, len(fields))
}
cid, err := strconv.ParseUint(fields[0], 10, 32)
if err != nil {
return nil, errors.Wrapf(err, "error parsing container ID value %q from line %q in %q", fields[0], line, path)
}
hid, err := strconv.ParseUint(fields[1], 10, 32)
if err != nil {
return nil, errors.Wrapf(err, "error parsing host ID value %q from line %q in %q", fields[1], line, path)
}
size, err := strconv.ParseUint(fields[2], 10, 32)
if err != nil {
return nil, errors.Wrapf(err, "error parsing size value %q from line %q in %q", fields[2], line, path)
}
mappings = append(mappings, specs.LinuxIDMapping{ContainerID: uint32(cid), HostID: uint32(hid), Size: uint32(size)})
}
return mappings, nil
}
// GetHostIDMappings reads mappings for the specified process (or the current
// process if pid is "self" or an empty string) from the kernel.
func GetHostIDMappings(pid string) ([]specs.LinuxIDMapping, []specs.LinuxIDMapping, error) {
if pid == "" {
pid = "self"
}
uidmap, err := getHostIDMappings(fmt.Sprintf("/proc/%s/uid_map", pid))
if err != nil {
return nil, nil, err
}
gidmap, err := getHostIDMappings(fmt.Sprintf("/proc/%s/gid_map", pid))
if err != nil {
return nil, nil, err
}
return uidmap, gidmap, nil
}
// GetSubIDMappings reads mappings from /etc/subuid and /etc/subgid.
func GetSubIDMappings(user, group string) ([]specs.LinuxIDMapping, []specs.LinuxIDMapping, error) {
mappings, err := idtools.NewIDMappings(user, group)
if err != nil {
return nil, nil, errors.Wrapf(err, "error reading subuid mappings for user %q and subgid mappings for group %q", user, group)
}
var uidmap, gidmap []specs.LinuxIDMapping
for _, m := range mappings.UIDs() {
uidmap = append(uidmap, specs.LinuxIDMapping{
ContainerID: uint32(m.ContainerID),
HostID: uint32(m.HostID),
Size: uint32(m.Size),
})
}
for _, m := range mappings.GIDs() {
gidmap = append(gidmap, specs.LinuxIDMapping{
ContainerID: uint32(m.ContainerID),
HostID: uint32(m.HostID),
Size: uint32(m.Size),
})
}
return uidmap, gidmap, nil
}
// ParseIDMappings parses mapping triples.
func ParseIDMappings(uidmap, gidmap []string) ([]idtools.IDMap, []idtools.IDMap, error) {
uid, err := idtools.ParseIDMap(uidmap, "userns-uid-map")
if err != nil {
return nil, nil, err
}
gid, err := idtools.ParseIDMap(gidmap, "userns-gid-map")
if err != nil {
return nil, nil, err
}
return uid, gid, nil
}
// GetPolicyContext sets up, initializes and returns a new context for the specified policy
func GetPolicyContext(ctx *types.SystemContext) (*signature.PolicyContext, error) {
policy, err := signature.DefaultPolicy(ctx)
if err != nil {
return nil, err
}
policyContext, err := signature.NewPolicyContext(policy)
if err != nil {
return nil, err
}
return policyContext, nil
}
// logIfNotErrno logs the error message unless err is either nil or one of the
// listed syscall.Errno values. It returns true if it logged an error.
func logIfNotErrno(err error, what string, ignores ...syscall.Errno) (logged bool) {
if err == nil {
return false
}
if errno, isErrno := err.(syscall.Errno); isErrno {
for _, ignore := range ignores {
if errno == ignore {
return false
}
}
}
logrus.Error(what)
return true
}
// LogIfNotRetryable logs "what" if err is set and is not an EINTR or EAGAIN
// syscall.Errno. Returns "true" if we can continue.
func LogIfNotRetryable(err error, what string) (retry bool) {
return !logIfNotErrno(err, what, syscall.EINTR, syscall.EAGAIN)
}
// LogIfUnexpectedWhileDraining logs "what" if err is set and is not an EINTR
// or EAGAIN or EIO syscall.Errno.
func LogIfUnexpectedWhileDraining(err error, what string) {
logIfNotErrno(err, what, syscall.EINTR, syscall.EAGAIN, syscall.EIO)
}

View File

@ -8,7 +8,7 @@ github.com/vbauerster/mpb v3.3.4
github.com/mattn/go-isatty v0.0.4
github.com/VividCortex/ewma v1.1.1
github.com/boltdb/bolt v1.3.1
github.com/containers/storage v1.12.1
github.com/containers/storage v1.12.2
github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716
github.com/docker/docker 54dddadc7d5d89fe0be88f76979f6f6ab0dede83
github.com/docker/docker-credential-helpers v0.6.1
@ -65,3 +65,4 @@ github.com/klauspost/cpuid v1.2.0
github.com/onsi/gomega v1.4.3
github.com/spf13/cobra v0.0.3
github.com/spf13/pflag v1.0.3
github.com/ishidawataru/sctp 07191f837fedd2f13d1ec7b5f885f0f3ec54b1cb