github/linuxkit
1
0
Fork 0
mirror of https://github.com/linuxkit/linuxkit.git synced 2026-04-03 05:11:37 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #159 from rn/img-ref

Browse Source 
Restructure Image and image reference code
This commit is contained in:
Justin Cormack
2017-10-09 12:29:13 +01:00
committed by GitHub
parent 9abca80507 1aee0c3d30
commit 70c6aebd26
15 changed files with 1170 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/moby/build.go
View File

@@ -121,7 +121,7 @@ func enforceContentTrust(fullImageName string, config *TrustConfig) bool {
return false
}
func outputImage(image Image, section string, prefix string, m Moby, idMap map[string]uint32, dupMap map[string]string, pull bool, iw *tar.Writer) error {
func outputImage(image *Image, section string, prefix string, m Moby, idMap map[string]uint32, dupMap map[string]string, pull bool, iw *tar.Writer) error {
log.Infof(" Create OCI config for %s", image.Image)
useTrust := enforceContentTrust(image.Image, &m.Trust)
oci, runtime, err := ConfigToOCI(image, useTrust, idMap)
@@ -134,7 +134,7 @@ func outputImage(image Image, section string, prefix string, m Moby, idMap map[s
}
path := path.Join("containers", section, prefix+image.Name)
readonly := oci.Root.Readonly
err = ImageBundle(path, image.Image, config, runtime, iw, useTrust, pull, readonly, dupMap)
err = ImageBundle(path, image.ref, config, runtime, iw, useTrust, pull, readonly, dupMap)
if err != nil {
return fmt.Errorf("Failed to extract root filesystem for %s: %v", image.Image, err)
}
@@ -171,11 +171,11 @@ func Build(m Moby, w io.Writer, pull bool, tp string) error {
// deduplicate containers with the same image
dupMap := map[string]string{}
if m.Kernel.Image != "" {
if m.Kernel.ref != nil {
// get kernel and initrd tarball from container
log.Infof("Extract kernel image: %s", m.Kernel.Image)
log.Infof("Extract kernel image: %s", m.Kernel.ref)
kf := newKernelFilter(iw, m.Kernel.Cmdline, m.Kernel.Binary, m.Kernel.Tar)
err := ImageTar(m.Kernel.Image, "", kf, enforceContentTrust(m.Kernel.Image, &m.Trust), pull, "")
err := ImageTar(m.Kernel.ref, "", kf, enforceContentTrust(m.Kernel.ref.String(), &m.Trust), pull, "")
if err != nil {
return fmt.Errorf("Failed to extract kernel image and tarball: %v", err)
}
@@ -189,9 +189,8 @@ func Build(m Moby, w io.Writer, pull bool, tp string) error {
if len(m.Init) != 0 {
log.Infof("Add init containers:")
}
for _, ii := range m.Init {
log.Infof("Process init image: %s", ii)
err := ImageTar(ii, "", iw, enforceContentTrust(ii, &m.Trust), pull, resolvconfSymlink)
for _, ii := range m.initRefs {
err := ImageTar(ii, "", iw, enforceContentTrust(ii.String(), &m.Trust), pull, resolvconfSymlink)
if err != nil {
return fmt.Errorf("Failed to build init tarball from %s: %v", ii, err)
}
@@ -389,6 +388,8 @@ func tarAppend(iw *tar.Writer, tr *tar.Reader) error {
// this allows inserting metadata into a file in the image
func metadata(m Moby, md string) ([]byte, error) {
// Make sure the Image strings are update to date with the refs
updateImages(&m)
switch md {
case "json":
return json.MarshalIndent(m, "", " ")

92
src/moby/config.go
View File

@@ -9,6 +9,7 @@ import (
"strings"
log "github.com/Sirupsen/logrus"
"github.com/containerd/containerd/reference"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/opencontainers/runtime-spec/specs-go"
@@ -20,11 +21,13 @@ import (
type Moby struct {
Kernel KernelConfig `kernel:"cmdline" json:"kernel,omitempty"`
Init []string `init:"cmdline" json:"init"`
Onboot []Image `yaml:"onboot" json:"onboot"`
Onshutdown []Image `yaml:"onshutdown" json:"onshutdown"`
Services []Image `yaml:"services" json:"services"`
Onboot []*Image `yaml:"onboot" json:"onboot"`
Onshutdown []*Image `yaml:"onshutdown" json:"onshutdown"`
Services []*Image `yaml:"services" json:"services"`
Trust TrustConfig `yaml:"trust" json:"trust,omitempty"`
Files []File `yaml:"files" json:"files"`
initRefs []*reference.Spec
}
// KernelConfig is the type of the config for a kernel
@@ -33,6 +36,8 @@ type KernelConfig struct {
Cmdline string `yaml:"cmdline" json:"cmdline,omitempty"`
Binary string `yaml:"binary" json:"binary,omitempty"`
Tar *string `yaml:"tar" json:"tar,omitempty"`
ref *reference.Spec
}
// TrustConfig is the type of a content trust config
@@ -89,6 +94,8 @@ type Image struct {
UIDMappings *[]specs.LinuxIDMapping `yaml:"uidMappings" json:"uidMappings,omitempty"`
GIDMappings *[]specs.LinuxIDMapping `yaml:"gidMappings" json:"gidMappings,omitempty"`
Runtime *Runtime `yaml:"runtime" json:"runtime,omitempty"`
ref *reference.Spec
}
// Runtime is the type of config processed at runtime, not used to build the OCI spec
@@ -151,6 +158,70 @@ func uniqueServices(m Moby) error {
return nil
}
func extractReferences(m *Moby) error {
if m.Kernel.Image != "" {
r, err := reference.Parse(m.Kernel.Image)
if err != nil {
return fmt.Errorf("extract kernel image reference: %v", err)
}
m.Kernel.ref = &r
}
for _, ii := range m.Init {
r, err := reference.Parse(ii)
if err != nil {
return fmt.Errorf("extract on boot image reference: %v", err)
}
m.initRefs = append(m.initRefs, &r)
}
for _, image := range m.Onboot {
r, err := reference.Parse(image.Image)
if err != nil {
return fmt.Errorf("extract on boot image reference: %v", err)
}
image.ref = &r
}
for _, image := range m.Onshutdown {
r, err := reference.Parse(image.Image)
if err != nil {
return fmt.Errorf("extract on shutdown image reference: %v", err)
}
image.ref = &r
}
for _, image := range m.Services {
r, err := reference.Parse(image.Image)
if err != nil {
return fmt.Errorf("extract service image reference: %v", err)
}
image.ref = &r
}
return nil
}
func updateImages(m *Moby) error {
if m.Kernel.ref != nil {
m.Kernel.Image = m.Kernel.ref.String()
}
for i, ii := range m.initRefs {
m.Init[i] = ii.String()
}
for _, image := range m.Onboot {
if image.ref != nil {
image.Image = image.ref.String()
}
}
for _, image := range m.Onshutdown {
if image.ref != nil {
image.Image = image.ref.String()
}
}
for _, image := range m.Services {
if image.ref != nil {
image.Image = image.ref.String()
}
}
return nil
}
// NewConfig parses a config file
func NewConfig(config []byte) (Moby, error) {
m := Moby{}
@@ -190,6 +261,10 @@ func NewConfig(config []byte) (Moby, error) {
return m, err
}
if err := extractReferences(&m); err != nil {
return m, err
}
return m, nil
}
@@ -208,6 +283,9 @@ func AppendConfig(m0, m1 Moby) (Moby, error) {
if m1.Kernel.Tar != nil {
moby.Kernel.Tar = m1.Kernel.Tar
}
if m1.Kernel.ref != nil {
moby.Kernel.ref = m1.Kernel.ref
}
moby.Init = append(moby.Init, m1.Init...)
moby.Onboot = append(moby.Onboot, m1.Onboot...)
moby.Onshutdown = append(moby.Onshutdown, m1.Onshutdown...)
@@ -215,6 +293,7 @@ func AppendConfig(m0, m1 Moby) (Moby, error) {
moby.Files = append(moby.Files, m1.Files...)
moby.Trust.Image = append(moby.Trust.Image, m1.Trust.Image...)
moby.Trust.Org = append(moby.Trust.Org, m1.Trust.Org...)
moby.initRefs = append(moby.initRefs, m1.initRefs...)
if err := uniqueServices(moby); err != nil {
return moby, err
@@ -290,15 +369,14 @@ func NewImage(config []byte) (Image, error) {
}
// ConfigToOCI converts a config specification to an OCI config file and a runtime config
func ConfigToOCI(image Image, trust bool, idMap map[string]uint32) (specs.Spec, Runtime, error) {
func ConfigToOCI(image *Image, trust bool, idMap map[string]uint32) (specs.Spec, Runtime, error) {
// TODO pass through same docker client to all functions
cli, err := dockerClient()
if err != nil {
return specs.Spec{}, Runtime{}, err
}
inspect, err := dockerInspectImage(cli, image.Image, trust)
inspect, err := dockerInspectImage(cli, image.ref, trust)
if err != nil {
return specs.Spec{}, Runtime{}, err
}
@@ -649,7 +727,7 @@ func idNumeric(v interface{}, idMap map[string]uint32) (uint32, error) {
}
// ConfigInspectToOCI converts a config and the output of image inspect to an OCI config
func ConfigInspectToOCI(yaml Image, inspect types.ImageInspect, idMap map[string]uint32) (specs.Spec, Runtime, error) {
func ConfigInspectToOCI(yaml *Image, inspect types.ImageInspect, idMap map[string]uint32) (specs.Spec, Runtime, error) {
oci := specs.Spec{}
runtime := Runtime{}

6
src/moby/config_test.go
View File

@@ -44,7 +44,7 @@ func TestOverrides(t *testing.T) {
inspect := setupInspect(t, label)
oci, _, err := ConfigInspectToOCI(yaml, inspect, idMap)
oci, _, err := ConfigInspectToOCI(&yaml, inspect, idMap)
if err != nil {
t.Error(err)
}
@@ -72,7 +72,7 @@ func TestInvalidCap(t *testing.T) {
inspect := setupInspect(t, label)
_, _, err := ConfigInspectToOCI(yaml, inspect, idMap)
_, _, err := ConfigInspectToOCI(&yaml, inspect, idMap)
if err == nil {
t.Error("expected error, got valid OCI config")
}
@@ -95,7 +95,7 @@ func TestIdMap(t *testing.T) {
inspect := setupInspect(t, label)
oci, _, err := ConfigInspectToOCI(yaml, inspect, idMap)
oci, _, err := ConfigInspectToOCI(&yaml, inspect, idMap)
if err != nil {
t.Error(err)
}

38
src/moby/docker.go
View File

@@ -14,6 +14,7 @@ import (
"strings"
log "github.com/Sirupsen/logrus"
"github.com/containerd/containerd/reference"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
@@ -97,18 +98,18 @@ func dockerRm(container string) error {
return nil
}
func dockerPull(image string, forcePull, trustedPull bool) error {
log.Debugf("docker pull: %s", image)
func dockerPull(ref *reference.Spec, forcePull, trustedPull bool) error {
log.Debugf("docker pull: %s", ref)
cli, err := dockerClient()
if err != nil {
return errors.New("could not initialize Docker API client")
}
if trustedPull {
log.Debugf("pulling %s with content trust", image)
trustedImg, err := TrustedReference(image)
log.Debugf("pulling %s with content trust", ref)
trustedImg, err := TrustedReference(ref.String())
if err != nil {
return fmt.Errorf("Trusted pull for %s failed: %v", image, err)
return fmt.Errorf("Trusted pull for %s failed: %v", ref, err)
}
// tag the image on a best-effort basis after pulling with content trust,
@@ -117,10 +118,15 @@ func dockerPull(image string, forcePull, trustedPull bool) error {
if err := cli.ImageTag(context.Background(), src, dst); err != nil {
log.Debugf("could not tag trusted image %s to %s", src, dst)
}
}(trustedImg.String(), image)
}(trustedImg.String(), ref.String())
log.Debugf("successfully verified trusted reference %s from notary", trustedImg.String())
image = trustedImg.String()
trustedSpec, err := reference.Parse(trustedImg.String())
if err != nil {
return fmt.Errorf("failed to convert trusted img %s to Spec: %v", trustedImg, err)
}
ref.Locator = trustedSpec.Locator
ref.Object = trustedSpec.Object
imageSearchArg := filters.NewArgs()
imageSearchArg.Add("reference", trustedImg.String())
@@ -130,8 +136,8 @@ func dockerPull(image string, forcePull, trustedPull bool) error {
}
}
log.Infof("Pull image: %s", image)
r, err := cli.ImagePull(context.Background(), image, types.ImagePullOptions{})
log.Infof("Pull image: %s", ref)
r, err := cli.ImagePull(context.Background(), ref.String(), types.ImagePullOptions{})
if err != nil {
return err
}
@@ -140,7 +146,7 @@ func dockerPull(image string, forcePull, trustedPull bool) error {
if err != nil {
return err
}
log.Debugf("docker pull: %s...Done", image)
log.Debugf("docker pull: %s...Done", ref)
return nil
}
@@ -153,17 +159,17 @@ func dockerClient() (*client.Client, error) {
return client.NewEnvClient()
}
func dockerInspectImage(cli *client.Client, image string, trustedPull bool) (types.ImageInspect, error) {
log.Debugf("docker inspect image: %s", image)
func dockerInspectImage(cli *client.Client, ref *reference.Spec, trustedPull bool) (types.ImageInspect, error) {
log.Debugf("docker inspect image: %s", ref)
inspect, _, err := cli.ImageInspectWithRaw(context.Background(), image)
inspect, _, err := cli.ImageInspectWithRaw(context.Background(), ref.String())
if err != nil {
if client.IsErrImageNotFound(err) {
pullErr := dockerPull(image, true, trustedPull)
pullErr := dockerPull(ref, true, trustedPull)
if pullErr != nil {
return types.ImageInspect{}, pullErr
}
inspect, _, err = cli.ImageInspectWithRaw(context.Background(), image)
inspect, _, err = cli.ImageInspectWithRaw(context.Background(), ref.String())
if err != nil {
return types.ImageInspect{}, err
}
@@ -172,7 +178,7 @@ func dockerInspectImage(cli *client.Client, image string, trustedPull bool) (typ
}
}
log.Debugf("docker inspect image: %s...Done", image)
log.Debugf("docker inspect image: %s...Done", ref)
return inspect, nil
}

43
src/moby/image.go
View File

@@ -11,6 +11,7 @@ import (
"strings"
log "github.com/Sirupsen/logrus"
"github.com/containerd/containerd/reference"
"github.com/opencontainers/runtime-spec/specs-go"
)
@@ -81,8 +82,8 @@ func tarPrefix(path string, tw tarWriter) error {
}
// ImageTar takes a Docker image and outputs it to a tar stream
func ImageTar(image, prefix string, tw tarWriter, trust bool, pull bool, resolv string) error {
log.Debugf("image tar: %s %s", image, prefix)
func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull bool, resolv string) error {
log.Debugf("image tar: %s %s", ref, prefix)
if prefix != "" && prefix[len(prefix)-1] != byte('/') {
return fmt.Errorf("prefix does not end with /: %s", prefix)
}
@@ -93,25 +94,25 @@ func ImageTar(image, prefix string, tw tarWriter, trust bool, pull bool, resolv
}
if pull || trust {
err := dockerPull(image, pull, trust)
err := dockerPull(ref, pull, trust)
if err != nil {
return fmt.Errorf("Could not pull image %s: %v", image, err)
return fmt.Errorf("Could not pull image %s: %v", ref, err)
}
}
container, err := dockerCreate(image)
container, err := dockerCreate(ref.String())
if err != nil {
// if the image wasn't found, pull it down. Bail on other errors.
if strings.Contains(err.Error(), "No such image") {
err := dockerPull(image, true, trust)
err := dockerPull(ref, true, trust)
if err != nil {
return fmt.Errorf("Could not pull image %s: %v", image, err)
return fmt.Errorf("Could not pull image %s: %v", ref, err)
}
container, err = dockerCreate(image)
container, err = dockerCreate(ref.String())
if err != nil {
return fmt.Errorf("Failed to docker create image %s: %v", image, err)
return fmt.Errorf("Failed to docker create image %s: %v", ref, err)
}
} else {
return fmt.Errorf("Failed to create docker image %s: %v", image, err)
return fmt.Errorf("Failed to create docker image %s: %v", ref, err)
}
}
contents, err := dockerExport(container)
@@ -137,7 +138,7 @@ func ImageTar(image, prefix string, tw tarWriter, trust bool, pull bool, resolv
return err
}
if exclude[hdr.Name] {
log.Debugf("image tar: %s %s exclude %s", image, prefix, hdr.Name)
log.Debugf("image tar: %s %s exclude %s", ref, prefix, hdr.Name)
_, err = io.Copy(ioutil.Discard, tr)
if err != nil {
return err
@@ -147,7 +148,7 @@ func ImageTar(image, prefix string, tw tarWriter, trust bool, pull bool, resolv
contents := replace[hdr.Name]
hdr.Size = int64(len(contents))
hdr.Name = prefix + hdr.Name
log.Debugf("image tar: %s %s add %s", image, prefix, hdr.Name)
log.Debugf("image tar: %s %s add %s", ref, prefix, hdr.Name)
if err := tw.WriteHeader(hdr); err != nil {
return err
}
@@ -162,7 +163,7 @@ func ImageTar(image, prefix string, tw tarWriter, trust bool, pull bool, resolv
hdr.Size = 0
hdr.Typeflag = tar.TypeSymlink
hdr.Linkname = resolv
log.Debugf("image tar: %s %s add resolv symlink /etc/resolv.conf -> %s", image, prefix, resolv)
log.Debugf("image tar: %s %s add resolv symlink /etc/resolv.conf -> %s", ref, prefix, resolv)
if err := tw.WriteHeader(hdr); err != nil {
return err
}
@@ -172,7 +173,7 @@ func ImageTar(image, prefix string, tw tarWriter, trust bool, pull bool, resolv
return err
}
} else {
log.Debugf("image tar: %s %s add %s", image, prefix, hdr.Name)
log.Debugf("image tar: %s %s add %s", ref, prefix, hdr.Name)
hdr.Name = prefix + hdr.Name
if hdr.Typeflag == tar.TypeLink {
// hard links are referenced by full path so need to be adjusted
@@ -191,7 +192,7 @@ func ImageTar(image, prefix string, tw tarWriter, trust bool, pull bool, resolv
}
// ImageBundle produces an OCI bundle at the given path in a tarball, given an image and a config.json
func ImageBundle(prefix string, image string, config []byte, runtime Runtime, tw tarWriter, trust bool, pull bool, readonly bool, dupMap map[string]string) error {
func ImageBundle(prefix string, ref *reference.Spec, config []byte, runtime Runtime, tw tarWriter, trust bool, pull bool, readonly bool, dupMap map[string]string) error {
// if read only, just unpack in rootfs/ but otherwise set up for overlay
rootExtract := "rootfs"
if !readonly {
@@ -200,17 +201,17 @@ func ImageBundle(prefix string, image string, config []byte, runtime Runtime, tw
// See if we have extracted this image previously
root := path.Join(prefix, rootExtract)
var foundElsewhere = dupMap[image] != ""
var foundElsewhere = dupMap[ref.String()] != ""
if !foundElsewhere {
if err := ImageTar(image, root+"/", tw, trust, pull, ""); err != nil {
if err := ImageTar(ref, root+"/", tw, trust, pull, ""); err != nil {
return err
}
dupMap[image] = root
dupMap[ref.String()] = root
} else {
if err := tarPrefix(prefix+"/", tw); err != nil {
return err
}
root = dupMap[image]
root = dupMap[ref.String()]
}
hdr := &tar.Header{
@@ -274,7 +275,7 @@ func ImageBundle(prefix string, image string, config []byte, runtime Runtime, tw
// write the runtime config
runtimeConfig, err := json.MarshalIndent(runtime, "", " ")
if err != nil {
return fmt.Errorf("Failed to create runtime config for %s: %v", image, err)
return fmt.Errorf("Failed to create runtime config for %s: %v", ref, err)
}
hdr = &tar.Header{
@@ -290,7 +291,7 @@ func ImageBundle(prefix string, image string, config []byte, runtime Runtime, tw
return err
}
log.Debugf("image bundle: %s %s cfg: %s runtime: %s", prefix, image, string(config), string(runtimeConfig))
log.Debugf("image bundle: %s %s cfg: %s runtime: %s", prefix, ref, string(config), string(runtimeConfig))
return nil
}