import cifetch code and add layers command

Signed-off-by: Antonio Murdaca <runcom@redhat.com>
This commit is contained in:
Antonio Murdaca 2016-03-17 11:47:59 +01:00
parent d8fbf24c25
commit 6a00ce47d2
14 changed files with 1370 additions and 21 deletions

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
skopeo
skopeo.1
layers-*

View File

@ -3,7 +3,7 @@ skopeo [![Build Status](https://travis-ci.org/projectatomic/skopeo.svg?branch=ma
_Please be aware `skopeo` is still work in progress_
`skopeo` is a command line utility which is able to _inspect_ a repository on a Docker registry.
`skopeo` is a command line utility which is able to _inspect_ a repository on a Docker registry and fetch images layers.
By _inspect_ I mean it fetches the repository's manifest and it is able to show you a `docker inspect`-like
json output about a whole repository or a tag. This tool, in contrast to `docker inspect`, helps you gather useful information about
a repository or a tag before pulling it (using disk space) - e.g. - which tags are available for the given repository? which labels the image has?
@ -113,6 +113,7 @@ $ make test-integration
```
TODO
-
- update README with `layers` command
- list all images on registry?
- registry v2 search?
- make skopeo docker registry v2 only

550
docker.go Normal file
View File

@ -0,0 +1,550 @@
package main
import (
"crypto/tls"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/homedir"
"github.com/projectatomic/skopeo/docker/reference"
"github.com/projectatomic/skopeo/types"
)
const (
dockerPrefix = "docker://"
dockerHostname = "docker.io"
dockerRegistry = "registry-1.docker.io"
dockerAuthRegistry = "https://index.docker.io/v1/"
dockerCfg = ".docker"
dockerCfgFileName = "config.json"
dockerCfgObsolete = ".dockercfg"
)
var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
type dockerImage struct {
ref reference.Named
tag string
registry string
username string
password string
WWWAuthenticate string
scheme string
rawManifest []byte
}
func (i *dockerImage) RawManifest(version string) ([]byte, error) {
// TODO(runcom): unused version param for now, default to docker v2-1
if err := i.retrieveRawManifest(); err != nil {
return nil, err
}
return i.rawManifest, nil
}
func (i *dockerImage) Manifest(version string) (types.ImageManifest, error) {
// TODO(runcom): port docker/docker implementation under docker/ to just
// use this!!! and do not rely on docker upstream code - will need to support
// v1 fall back also...
return nil, nil
}
func (i *dockerImage) DockerTar() ([]byte, error) {
return nil, nil
}
// will support v1 one day...
type manifest interface {
String() string
GetLayers() []string
}
type manifestSchema1 struct {
Name string
Tag string
FSLayers []struct {
BlobSum string `json:"blobSum"`
} `json:"fsLayers"`
History []struct {
V1Compatibility string `json:"v1Compatibility"`
} `json:"history"`
// TODO(runcom) verify the downloaded manifest
//Signature []byte `json:"signature"`
}
func (m *manifestSchema1) GetLayers() []string {
layers := make([]string, len(m.FSLayers))
for i, layer := range m.FSLayers {
layers[i] = layer.BlobSum
}
return layers
}
func (m *manifestSchema1) String() string {
return fmt.Sprintf("%s-%s", sanitize(m.Name), sanitize(m.Tag))
}
func sanitize(s string) string {
return strings.Replace(s, "/", "-", -1)
}
func (i *dockerImage) makeRequest(method, url string, auth bool, headers map[string]string) (*http.Response, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Docker-Distribution-API-Version", "registry/2.0")
for n, h := range headers {
req.Header.Add(n, h)
}
if auth {
if err := i.setupRequestAuth(req); err != nil {
return nil, err
}
}
// insecure by default for now
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
client := &http.Client{Transport: tr}
res, err := client.Do(req)
if err != nil {
return nil, err
}
return res, nil
}
func (i *dockerImage) setupRequestAuth(req *http.Request) error {
tokens := strings.SplitN(strings.TrimSpace(i.WWWAuthenticate), " ", 2)
if len(tokens) != 2 {
return fmt.Errorf("expected 2 tokens in WWW-Authenticate: %d, %s", len(tokens), i.WWWAuthenticate)
}
switch tokens[0] {
case "Basic":
req.SetBasicAuth(i.username, i.password)
return nil
case "Bearer":
// insecure by default for now
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
client := &http.Client{Transport: tr}
res, err := client.Do(req)
if err != nil {
return err
}
hdr := res.Header.Get("WWW-Authenticate")
if hdr == "" || res.StatusCode != http.StatusUnauthorized {
// no need for bearer? wtf?
return nil
}
tokens = strings.Split(hdr, " ")
tokens = strings.Split(tokens[1], ",")
var realm, service, scope string
for _, token := range tokens {
if strings.HasPrefix(token, "realm") {
realm = strings.Trim(token[len("realm="):], "\"")
}
if strings.HasPrefix(token, "service") {
service = strings.Trim(token[len("service="):], "\"")
}
if strings.HasPrefix(token, "scope") {
scope = strings.Trim(token[len("scope="):], "\"")
}
}
if realm == "" {
return fmt.Errorf("missing realm in bearer auth challenge")
}
if service == "" {
return fmt.Errorf("missing service in bearer auth challenge")
}
// The scope can be empty if we're not getting a token for a specific repo
//if scope == "" && repo != "" {
if scope == "" {
return fmt.Errorf("missing scope in bearer auth challenge")
}
token, err := i.getBearerToken(realm, service, scope)
if err != nil {
return err
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
return nil
}
return fmt.Errorf("no handler for %s authentication", tokens[0])
// support docker bearer with authconfig's Auth string? see docker2aci
}
func (i *dockerImage) getBearerToken(realm, service, scope string) (string, error) {
authReq, err := http.NewRequest("GET", realm, nil)
if err != nil {
return "", err
}
getParams := authReq.URL.Query()
getParams.Add("service", service)
if scope != "" {
getParams.Add("scope", scope)
}
authReq.URL.RawQuery = getParams.Encode()
if i.username != "" && i.password != "" {
authReq.SetBasicAuth(i.username, i.password)
}
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
client := &http.Client{Transport: tr}
res, err := client.Do(authReq)
if err != nil {
return "", err
}
defer res.Body.Close()
switch res.StatusCode {
case http.StatusUnauthorized:
return "", fmt.Errorf("unable to retrieve auth token: 401 unauthorized")
case http.StatusOK:
break
default:
return "", fmt.Errorf("unexpected http code: %d, URL: %s", res.StatusCode, authReq.URL)
}
tokenBlob, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
tokenStruct := struct {
Token string `json:"token"`
}{}
if err := json.Unmarshal(tokenBlob, &tokenStruct); err != nil {
return "", err
}
// TODO(runcom): reuse tokens?
//hostAuthTokens, ok = rb.hostsV2AuthTokens[req.URL.Host]
//if !ok {
//hostAuthTokens = make(map[string]string)
//rb.hostsV2AuthTokens[req.URL.Host] = hostAuthTokens
//}
//hostAuthTokens[repo] = tokenStruct.Token
return tokenStruct.Token, nil
}
func (i *dockerImage) retrieveRawManifest() error {
if i.rawManifest != nil {
return nil
}
pr, err := ping(i.registry)
if err != nil {
return err
}
i.WWWAuthenticate = pr.WWWAuthenticate
i.scheme = pr.scheme
url := i.scheme + "://" + i.registry + "/v2/" + i.ref.RemoteName() + "/manifests/" + i.tag
// TODO(runcom) set manifest version header! schema1 for now - then schema2 etc etc and v1
// TODO(runcom) NO, switch on the resulter manifest like Docker is doing
res, err := i.makeRequest("GET", url, pr.needsAuth(), nil)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
// print body also
return fmt.Errorf("Invalid status code returned when fetching manifest %d", res.StatusCode)
}
manblob, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}
i.rawManifest = manblob
return nil
}
func (i *dockerImage) getSchema1Manifest() (manifest, error) {
if err := i.retrieveRawManifest(); err != nil {
return nil, err
}
mschema1 := &manifestSchema1{}
if err := json.Unmarshal(i.rawManifest, mschema1); err != nil {
return nil, err
}
if err := fixManifestLayers(mschema1); err != nil {
return nil, err
}
return mschema1, nil
}
func (i *dockerImage) Layers(layers ...string) error {
m, err := i.getSchema1Manifest()
if err != nil {
return err
}
tmpDir, err := ioutil.TempDir(".", "layers-"+m.String()+"-")
if err != nil {
return err
}
data, err := json.Marshal(m)
if err != nil {
return err
}
if err := ioutil.WriteFile(path.Join(tmpDir, "manifest.json"), data, 0644); err != nil {
return err
}
url := i.scheme + "://" + i.registry + "/v2/" + i.ref.RemoteName() + "/blobs/"
if len(layers) == 0 {
layers = m.GetLayers()
}
for _, l := range layers {
if !strings.HasPrefix(l, "sha256:") {
l = "sha256:" + l
}
if err := i.getLayer(l, url, tmpDir); err != nil {
return err
}
}
return nil
}
func (i *dockerImage) getLayer(l, url, tmpDir string) error {
lurl := url + l
logrus.Infof("Downloading %s", lurl)
res, err := i.makeRequest("GET", lurl, i.WWWAuthenticate != "", nil)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
// print url also
return fmt.Errorf("Invalid status code returned when fetching blob %d", res.StatusCode)
}
layerPath := path.Join(tmpDir, strings.Replace(l, "sha256:", "", -1)+".tar")
layerFile, err := os.Create(layerPath)
if err != nil {
return err
}
if _, err := io.Copy(layerFile, res.Body); err != nil {
return err
}
if err := layerFile.Sync(); err != nil {
return err
}
return nil
}
func parseDockerImage(img string) (types.Image, error) {
ref, err := reference.ParseNamed(img)
if err != nil {
return nil, err
}
if reference.IsNameOnly(ref) {
ref = reference.WithDefaultTag(ref)
}
var tag string
switch x := ref.(type) {
case reference.Canonical:
tag = x.Digest().String()
case reference.NamedTagged:
tag = x.Tag()
}
var registry string
hostname := ref.Hostname()
if hostname == dockerHostname {
registry = dockerRegistry
} else {
registry = hostname
}
username, password, err := getAuth(ref.Hostname())
if err != nil {
return nil, err
}
return &dockerImage{
ref: ref,
tag: tag,
registry: registry,
username: username,
password: password,
}, nil
}
func getDefaultConfigDir(confPath string) string {
return filepath.Join(homedir.Get(), confPath)
}
type DockerAuthConfigObsolete struct {
Auth string `json:"auth"`
}
type DockerAuthConfig struct {
Auth string `json:"auth,omitempty"`
}
type DockerConfigFile struct {
AuthConfigs map[string]DockerAuthConfig `json:"auths"`
}
func decodeDockerAuth(s string) (string, string, error) {
decoded, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return "", "", err
}
parts := strings.SplitN(string(decoded), ":", 2)
if len(parts) != 2 {
return "", "", fmt.Errorf("invalid auth configuration file")
}
user := parts[0]
password := strings.Trim(parts[1], "\x00")
return user, password, nil
}
func getAuth(hostname string) (string, string, error) {
if hostname == dockerHostname {
hostname = dockerAuthRegistry
}
dockerCfgPath := filepath.Join(getDefaultConfigDir(".docker"), dockerCfgFileName)
if _, err := os.Stat(dockerCfgPath); err == nil {
j, err := ioutil.ReadFile(dockerCfgPath)
if err != nil {
return "", "", err
}
var dockerAuth DockerConfigFile
if err := json.Unmarshal(j, &dockerAuth); err != nil {
return "", "", err
}
// try the normal case
if c, ok := dockerAuth.AuthConfigs[hostname]; ok {
return decodeDockerAuth(c.Auth)
}
} else if os.IsNotExist(err) {
oldDockerCfgPath := filepath.Join(getDefaultConfigDir(dockerCfgObsolete))
if _, err := os.Stat(oldDockerCfgPath); err != nil {
return "", "", nil //missing file is not an error
}
j, err := ioutil.ReadFile(oldDockerCfgPath)
if err != nil {
return "", "", err
}
var dockerAuthOld map[string]DockerAuthConfigObsolete
if err := json.Unmarshal(j, &dockerAuthOld); err != nil {
return "", "", err
}
if c, ok := dockerAuthOld[hostname]; ok {
return decodeDockerAuth(c.Auth)
}
} else {
// if file is there but we can't stat it for any reason other
// than it doesn't exist then stop
return "", "", fmt.Errorf("%s - %v", dockerCfgPath, err)
}
return "", "", nil
}
type APIErr struct {
Code string
Message string
Detail interface{}
}
type pingResponse struct {
WWWAuthenticate string
APIVersion string
scheme string
errors []APIErr
}
func (pr *pingResponse) needsAuth() bool {
return pr.WWWAuthenticate != ""
}
func ping(registry string) (*pingResponse, error) {
// insecure by default for now
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
client := &http.Client{Transport: tr}
ping := func(scheme string) (*pingResponse, error) {
resp, err := client.Get(scheme + "://" + registry + "/v2/")
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusUnauthorized {
return nil, fmt.Errorf("error pinging repository, response code %d", resp.StatusCode)
}
pr := &pingResponse{}
pr.WWWAuthenticate = resp.Header.Get("WWW-Authenticate")
pr.APIVersion = resp.Header.Get("Docker-Distribution-Api-Version")
pr.scheme = scheme
if resp.StatusCode == http.StatusUnauthorized {
type APIErrors struct {
Errors []APIErr
}
errs := &APIErrors{}
if err := json.NewDecoder(resp.Body).Decode(errs); err != nil {
return nil, err
}
pr.errors = errs.Errors
}
return pr, nil
}
scheme := "https"
pr, err := ping(scheme)
if err != nil {
scheme = "http"
pr, err = ping(scheme)
if err == nil {
return pr, nil
}
}
return pr, err
}
func fixManifestLayers(manifest *manifestSchema1) error {
type imageV1 struct {
ID string
Parent string
}
imgs := make([]*imageV1, len(manifest.FSLayers))
for i := range manifest.FSLayers {
img := &imageV1{}
if err := json.Unmarshal([]byte(manifest.History[i].V1Compatibility), img); err != nil {
return err
}
imgs[i] = img
if err := validateV1ID(img.ID); err != nil {
return err
}
}
if imgs[len(imgs)-1].Parent != "" {
return errors.New("Invalid parent ID in the base layer of the image.")
}
// check general duplicates to error instead of a deadlock
idmap := make(map[string]struct{})
var lastID string
for _, img := range imgs {
// skip IDs that appear after each other, we handle those later
if _, exists := idmap[img.ID]; img.ID != lastID && exists {
return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
}
lastID = img.ID
idmap[lastID] = struct{}{}
}
// backwards loop so that we keep the remaining indexes after removing items
for i := len(imgs) - 2; i >= 0; i-- {
if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue
manifest.FSLayers = append(manifest.FSLayers[:i], manifest.FSLayers[i+1:]...)
manifest.History = append(manifest.History[:i], manifest.History[i+1:]...)
} else if imgs[i].Parent != imgs[i+1].ID {
return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", imgs[i+1].ID, imgs[i].Parent)
}
}
return nil
}
func validateV1ID(id string) error {
if ok := validHex.MatchString(id); !ok {
return fmt.Errorf("image ID %q is invalid", id)
}
return nil
}

5
docker/README Normal file
View File

@ -0,0 +1,5 @@
TODO
Eventually we want to get rid of inspect pkg which uses docker upstream code
and use, instead, docker.go which calls the api directly
Be aware that docker pkg do not fall back to v1! this must be implemented soon

550
docker/docker.go Normal file
View File

@ -0,0 +1,550 @@
package docker
import (
"crypto/tls"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/homedir"
"github.com/projectatomic/skopeo/docker/reference"
"github.com/projectatomic/skopeo/types"
)
const (
dockerPrefix = "docker://"
dockerHostname = "docker.io"
dockerRegistry = "registry-1.docker.io"
dockerAuthRegistry = "https://index.docker.io/v1/"
dockerCfg = ".docker"
dockerCfgFileName = "config.json"
dockerCfgObsolete = ".dockercfg"
)
var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
type dockerImage struct {
ref reference.Named
tag string
registry string
username string
password string
WWWAuthenticate string
scheme string
rawManifest []byte
}
func (i *dockerImage) RawManifest(version string) ([]byte, error) {
// TODO(runcom): unused version param for now, default to docker v2-1
if err := i.retrieveRawManifest(); err != nil {
return nil, err
}
return i.rawManifest, nil
}
func (i *dockerImage) Manifest(version string) (types.ImageManifest, error) {
// TODO(runcom): port docker/docker implementation under docker/ to just
// use this!!! and do not rely on docker upstream code - will need to support
// v1 fall back also...
return nil, nil
}
func (i *dockerImage) DockerTar() ([]byte, error) {
return nil, nil
}
// will support v1 one day...
type manifest interface {
String() string
GetLayers() []string
}
type manifestSchema1 struct {
Name string
Tag string
FSLayers []struct {
BlobSum string `json:"blobSum"`
} `json:"fsLayers"`
History []struct {
V1Compatibility string `json:"v1Compatibility"`
} `json:"history"`
// TODO(runcom) verify the downloaded manifest
//Signature []byte `json:"signature"`
}
func (m *manifestSchema1) GetLayers() []string {
layers := make([]string, len(m.FSLayers))
for i, layer := range m.FSLayers {
layers[i] = layer.BlobSum
}
return layers
}
func (m *manifestSchema1) String() string {
return fmt.Sprintf("%s-%s", sanitize(m.Name), sanitize(m.Tag))
}
func sanitize(s string) string {
return strings.Replace(s, "/", "-", -1)
}
func (i *dockerImage) makeRequest(method, url string, auth bool, headers map[string]string) (*http.Response, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Docker-Distribution-API-Version", "registry/2.0")
for n, h := range headers {
req.Header.Add(n, h)
}
if auth {
if err := i.setupRequestAuth(req); err != nil {
return nil, err
}
}
// insecure by default for now
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
client := &http.Client{Transport: tr}
res, err := client.Do(req)
if err != nil {
return nil, err
}
return res, nil
}
func (i *dockerImage) setupRequestAuth(req *http.Request) error {
tokens := strings.SplitN(strings.TrimSpace(i.WWWAuthenticate), " ", 2)
if len(tokens) != 2 {
return fmt.Errorf("expected 2 tokens in WWW-Authenticate: %d, %s", len(tokens), i.WWWAuthenticate)
}
switch tokens[0] {
case "Basic":
req.SetBasicAuth(i.username, i.password)
return nil
case "Bearer":
// insecure by default for now
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
client := &http.Client{Transport: tr}
res, err := client.Do(req)
if err != nil {
return err
}
hdr := res.Header.Get("WWW-Authenticate")
if hdr == "" || res.StatusCode != http.StatusUnauthorized {
// no need for bearer? wtf?
return nil
}
tokens = strings.Split(hdr, " ")
tokens = strings.Split(tokens[1], ",")
var realm, service, scope string
for _, token := range tokens {
if strings.HasPrefix(token, "realm") {
realm = strings.Trim(token[len("realm="):], "\"")
}
if strings.HasPrefix(token, "service") {
service = strings.Trim(token[len("service="):], "\"")
}
if strings.HasPrefix(token, "scope") {
scope = strings.Trim(token[len("scope="):], "\"")
}
}
if realm == "" {
return fmt.Errorf("missing realm in bearer auth challenge")
}
if service == "" {
return fmt.Errorf("missing service in bearer auth challenge")
}
// The scope can be empty if we're not getting a token for a specific repo
//if scope == "" && repo != "" {
if scope == "" {
return fmt.Errorf("missing scope in bearer auth challenge")
}
token, err := i.getBearerToken(realm, service, scope)
if err != nil {
return err
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
return nil
}
return fmt.Errorf("no handler for %s authentication", tokens[0])
// support docker bearer with authconfig's Auth string? see docker2aci
}
func (i *dockerImage) getBearerToken(realm, service, scope string) (string, error) {
authReq, err := http.NewRequest("GET", realm, nil)
if err != nil {
return "", err
}
getParams := authReq.URL.Query()
getParams.Add("service", service)
if scope != "" {
getParams.Add("scope", scope)
}
authReq.URL.RawQuery = getParams.Encode()
if i.username != "" && i.password != "" {
authReq.SetBasicAuth(i.username, i.password)
}
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
client := &http.Client{Transport: tr}
res, err := client.Do(authReq)
if err != nil {
return "", err
}
defer res.Body.Close()
switch res.StatusCode {
case http.StatusUnauthorized:
return "", fmt.Errorf("unable to retrieve auth token: 401 unauthorized")
case http.StatusOK:
break
default:
return "", fmt.Errorf("unexpected http code: %d, URL: %s", res.StatusCode, authReq.URL)
}
tokenBlob, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
tokenStruct := struct {
Token string `json:"token"`
}{}
if err := json.Unmarshal(tokenBlob, &tokenStruct); err != nil {
return "", err
}
// TODO(runcom): reuse tokens?
//hostAuthTokens, ok = rb.hostsV2AuthTokens[req.URL.Host]
//if !ok {
//hostAuthTokens = make(map[string]string)
//rb.hostsV2AuthTokens[req.URL.Host] = hostAuthTokens
//}
//hostAuthTokens[repo] = tokenStruct.Token
return tokenStruct.Token, nil
}
func (i *dockerImage) retrieveRawManifest() error {
if i.rawManifest != nil {
return nil
}
pr, err := ping(i.registry)
if err != nil {
return err
}
i.WWWAuthenticate = pr.WWWAuthenticate
i.scheme = pr.scheme
url := i.scheme + "://" + i.registry + "/v2/" + i.ref.RemoteName() + "/manifests/" + i.tag
// TODO(runcom) set manifest version header! schema1 for now - then schema2 etc etc and v1
// TODO(runcom) NO, switch on the resulter manifest like Docker is doing
res, err := i.makeRequest("GET", url, pr.needsAuth(), nil)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
// print body also
return fmt.Errorf("Invalid status code returned when fetching manifest %d", res.StatusCode)
}
manblob, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}
i.rawManifest = manblob
return nil
}
func (i *dockerImage) getSchema1Manifest() (manifest, error) {
if err := i.retrieveRawManifest(); err != nil {
return nil, err
}
mschema1 := &manifestSchema1{}
if err := json.Unmarshal(i.rawManifest, mschema1); err != nil {
return nil, err
}
if err := fixManifestLayers(mschema1); err != nil {
return nil, err
}
return mschema1, nil
}
func (i *dockerImage) Layers(layers ...string) error {
m, err := i.getSchema1Manifest()
if err != nil {
return err
}
tmpDir, err := ioutil.TempDir(".", "layers-"+m.String()+"-")
if err != nil {
return err
}
data, err := json.Marshal(m)
if err != nil {
return err
}
if err := ioutil.WriteFile(path.Join(tmpDir, "manifest.json"), data, 0644); err != nil {
return err
}
url := i.scheme + "://" + i.registry + "/v2/" + i.ref.RemoteName() + "/blobs/"
if len(layers) == 0 {
layers = m.GetLayers()
}
for _, l := range layers {
if !strings.HasPrefix(l, "sha256:") {
l = "sha256:" + l
}
if err := i.getLayer(l, url, tmpDir); err != nil {
return err
}
}
return nil
}
func (i *dockerImage) getLayer(l, url, tmpDir string) error {
lurl := url + l
logrus.Infof("Downloading %s", lurl)
res, err := i.makeRequest("GET", lurl, i.WWWAuthenticate != "", nil)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
// print url also
return fmt.Errorf("Invalid status code returned when fetching blob %d", res.StatusCode)
}
layerPath := path.Join(tmpDir, strings.Replace(l, "sha256:", "", -1)+".tar")
layerFile, err := os.Create(layerPath)
if err != nil {
return err
}
if _, err := io.Copy(layerFile, res.Body); err != nil {
return err
}
if err := layerFile.Sync(); err != nil {
return err
}
return nil
}
func parseDockerImage(img string) (types.Image, error) {
ref, err := reference.ParseNamed(img)
if err != nil {
return nil, err
}
if reference.IsNameOnly(ref) {
ref = reference.WithDefaultTag(ref)
}
var tag string
switch x := ref.(type) {
case reference.Canonical:
tag = x.Digest().String()
case reference.NamedTagged:
tag = x.Tag()
}
var registry string
hostname := ref.Hostname()
if hostname == dockerHostname {
registry = dockerRegistry
} else {
registry = hostname
}
username, password, err := getAuth(ref.Hostname())
if err != nil {
return nil, err
}
return &dockerImage{
ref: ref,
tag: tag,
registry: registry,
username: username,
password: password,
}, nil
}
func getDefaultConfigDir(confPath string) string {
return filepath.Join(homedir.Get(), confPath)
}
type DockerAuthConfigObsolete struct {
Auth string `json:"auth"`
}
type DockerAuthConfig struct {
Auth string `json:"auth,omitempty"`
}
type DockerConfigFile struct {
AuthConfigs map[string]DockerAuthConfig `json:"auths"`
}
func decodeDockerAuth(s string) (string, string, error) {
decoded, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return "", "", err
}
parts := strings.SplitN(string(decoded), ":", 2)
if len(parts) != 2 {
return "", "", fmt.Errorf("invalid auth configuration file")
}
user := parts[0]
password := strings.Trim(parts[1], "\x00")
return user, password, nil
}
func getAuth(hostname string) (string, string, error) {
if hostname == dockerHostname {
hostname = dockerAuthRegistry
}
dockerCfgPath := filepath.Join(getDefaultConfigDir(".docker"), dockerCfgFileName)
if _, err := os.Stat(dockerCfgPath); err == nil {
j, err := ioutil.ReadFile(dockerCfgPath)
if err != nil {
return "", "", err
}
var dockerAuth DockerConfigFile
if err := json.Unmarshal(j, &dockerAuth); err != nil {
return "", "", err
}
// try the normal case
if c, ok := dockerAuth.AuthConfigs[hostname]; ok {
return decodeDockerAuth(c.Auth)
}
} else if os.IsNotExist(err) {
oldDockerCfgPath := filepath.Join(getDefaultConfigDir(dockerCfgObsolete))
if _, err := os.Stat(oldDockerCfgPath); err != nil {
return "", "", nil //missing file is not an error
}
j, err := ioutil.ReadFile(oldDockerCfgPath)
if err != nil {
return "", "", err
}
var dockerAuthOld map[string]DockerAuthConfigObsolete
if err := json.Unmarshal(j, &dockerAuthOld); err != nil {
return "", "", err
}
if c, ok := dockerAuthOld[hostname]; ok {
return decodeDockerAuth(c.Auth)
}
} else {
// if file is there but we can't stat it for any reason other
// than it doesn't exist then stop
return "", "", fmt.Errorf("%s - %v", dockerCfgPath, err)
}
return "", "", nil
}
type APIErr struct {
Code string
Message string
Detail interface{}
}
type pingResponse struct {
WWWAuthenticate string
APIVersion string
scheme string
errors []APIErr
}
func (pr *pingResponse) needsAuth() bool {
return pr.WWWAuthenticate != ""
}
func ping(registry string) (*pingResponse, error) {
// insecure by default for now
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
client := &http.Client{Transport: tr}
ping := func(scheme string) (*pingResponse, error) {
resp, err := client.Get(scheme + "://" + registry + "/v2/")
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusUnauthorized {
return nil, fmt.Errorf("error pinging repository, response code %d", resp.StatusCode)
}
pr := &pingResponse{}
pr.WWWAuthenticate = resp.Header.Get("WWW-Authenticate")
pr.APIVersion = resp.Header.Get("Docker-Distribution-Api-Version")
pr.scheme = scheme
if resp.StatusCode == http.StatusUnauthorized {
type APIErrors struct {
Errors []APIErr
}
errs := &APIErrors{}
if err := json.NewDecoder(resp.Body).Decode(errs); err != nil {
return nil, err
}
pr.errors = errs.Errors
}
return pr, nil
}
scheme := "https"
pr, err := ping(scheme)
if err != nil {
scheme = "http"
pr, err = ping(scheme)
if err == nil {
return pr, nil
}
}
return pr, err
}
func fixManifestLayers(manifest *manifestSchema1) error {
type imageV1 struct {
ID string
Parent string
}
imgs := make([]*imageV1, len(manifest.FSLayers))
for i := range manifest.FSLayers {
img := &imageV1{}
if err := json.Unmarshal([]byte(manifest.History[i].V1Compatibility), img); err != nil {
return err
}
imgs[i] = img
if err := validateV1ID(img.ID); err != nil {
return err
}
}
if imgs[len(imgs)-1].Parent != "" {
return errors.New("Invalid parent ID in the base layer of the image.")
}
// check general duplicates to error instead of a deadlock
idmap := make(map[string]struct{})
var lastID string
for _, img := range imgs {
// skip IDs that appear after each other, we handle those later
if _, exists := idmap[img.ID]; img.ID != lastID && exists {
return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID)
}
lastID = img.ID
idmap[lastID] = struct{}{}
}
// backwards loop so that we keep the remaining indexes after removing items
for i := len(imgs) - 2; i >= 0; i-- {
if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue
manifest.FSLayers = append(manifest.FSLayers[:i], manifest.FSLayers[i+1:]...)
manifest.History = append(manifest.History[:i], manifest.History[i+1:]...)
} else if imgs[i].Parent != imgs[i+1].ID {
return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", imgs[i+1].ID, imgs[i].Parent)
}
}
return nil
}
func validateV1ID(id string) error {
if ok := validHex.MatchString(id); !ok {
return fmt.Errorf("image ID %q is invalid", id)
}
return nil
}

View File

@ -1,4 +1,4 @@
package docker
package inspect
import (
"encoding/json"
@ -47,7 +47,7 @@ func (f fallbackError) Error() string {
}
type manifestFetcher interface {
Fetch(ctx context.Context, ref reference.Named) (*types.ImageManifest, error)
Fetch(ctx context.Context, ref reference.Named) (types.ImageManifest, error)
}
func validateName(name string) error {
@ -62,7 +62,7 @@ func validateName(name string) error {
return nil
}
func GetData(c *cli.Context, name string) (*types.ImageManifest, error) {
func GetData(c *cli.Context, name string) (types.ImageManifest, error) {
if err := validateName(name); err != nil {
return nil, err
}
@ -101,7 +101,7 @@ func GetData(c *cli.Context, name string) (*types.ImageManifest, error) {
ctx = context.Background()
lastErr error
discardNoSupportErrors bool
imgInspect *types.ImageManifest
imgInspect types.ImageManifest
confirmedV2 bool
confirmedTLSRegistries = make(map[string]struct{})
)
@ -248,12 +248,12 @@ func validateRepoName(name string) error {
return nil
}
func makeImageManifest(img *image.Image, tag string, dgst digest.Digest, tagList []string) *types.ImageManifest {
func makeImageManifest(img *image.Image, tag string, dgst digest.Digest, tagList []string) types.ImageManifest {
var digest string
if err := dgst.Validate(); err == nil {
digest = dgst.String()
}
return &types.ImageManifest{
return &types.DockerImageManifest{
Tag: tag,
Digest: digest,
RepoTags: tagList,

View File

@ -1,4 +1,4 @@
package docker
package inspect
import (
"encoding/json"
@ -31,9 +31,9 @@ type v1ManifestFetcher struct {
session *registry.Session
}
func (mf *v1ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (*types.ImageManifest, error) {
func (mf *v1ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (types.ImageManifest, error) {
var (
imgInspect *types.ImageManifest
imgInspect types.ImageManifest
)
if _, isCanonical := ref.(reference.Canonical); isCanonical {
// Allowing fallback, because HTTPS v1 is before HTTP v2
@ -68,7 +68,7 @@ func (mf *v1ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (*t
return imgInspect, nil
}
func (mf *v1ManifestFetcher) fetchWithSession(ctx context.Context, ref reference.Named) (*types.ImageManifest, error) {
func (mf *v1ManifestFetcher) fetchWithSession(ctx context.Context, ref reference.Named) (types.ImageManifest, error) {
repoData, err := mf.session.GetRepositoryData(mf.repoInfo)
if err != nil {
if strings.Contains(err.Error(), "HTTP code: 404") {

View File

@ -1,4 +1,4 @@
package docker
package inspect
import (
"encoding/json"
@ -34,9 +34,9 @@ type v2ManifestFetcher struct {
service *registry.Service
}
func (mf *v2ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (*types.ImageManifest, error) {
func (mf *v2ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (types.ImageManifest, error) {
var (
imgInspect *types.ImageManifest
imgInspect types.ImageManifest
err error
)
@ -60,7 +60,7 @@ func (mf *v2ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (*t
return imgInspect, err
}
func (mf *v2ManifestFetcher) fetchWithRepository(ctx context.Context, ref reference.Named) (*types.ImageManifest, error) {
func (mf *v2ManifestFetcher) fetchWithRepository(ctx context.Context, ref reference.Named) (types.ImageManifest, error) {
var (
manifest distribution.Manifest
tagOrDigest string // Used for logging/progress only

View File

@ -0,0 +1,211 @@
// COPY FROM DOCKER/DOCKER
package reference
import (
"errors"
"fmt"
"regexp"
"strings"
"github.com/docker/distribution/digest"
distreference "github.com/docker/distribution/reference"
)
const (
// DefaultTag defines the default tag used when performing images related actions and no tag or digest is specified
DefaultTag = "latest"
// DefaultHostname is the default built-in hostname
DefaultHostname = "docker.io"
// LegacyDefaultHostname is automatically converted to DefaultHostname
LegacyDefaultHostname = "index.docker.io"
// DefaultRepoPrefix is the prefix used for default repositories in default host
DefaultRepoPrefix = "library/"
)
// Named is an object with a full name
type Named interface {
// Name returns normalized repository name, like "ubuntu".
Name() string
// String returns full reference, like "ubuntu@sha256:abcdef..."
String() string
// FullName returns full repository name with hostname, like "docker.io/library/ubuntu"
FullName() string
// Hostname returns hostname for the reference, like "docker.io"
Hostname() string
// RemoteName returns the repository component of the full name, like "library/ubuntu"
RemoteName() string
}
// NamedTagged is an object including a name and tag.
type NamedTagged interface {
Named
Tag() string
}
// Canonical reference is an object with a fully unique
// name including a name with hostname and digest
type Canonical interface {
Named
Digest() digest.Digest
}
// ParseNamed parses s and returns a syntactically valid reference implementing
// the Named interface. The reference must have a name, otherwise an error is
// returned.
// If an error was encountered it is returned, along with a nil Reference.
func ParseNamed(s string) (Named, error) {
named, err := distreference.ParseNamed(s)
if err != nil {
return nil, fmt.Errorf("Error parsing reference: %q is not a valid repository/tag", s)
}
r, err := WithName(named.Name())
if err != nil {
return nil, err
}
if canonical, isCanonical := named.(distreference.Canonical); isCanonical {
return WithDigest(r, canonical.Digest())
}
if tagged, isTagged := named.(distreference.NamedTagged); isTagged {
return WithTag(r, tagged.Tag())
}
return r, nil
}
// WithName returns a named object representing the given string. If the input
// is invalid ErrReferenceInvalidFormat will be returned.
func WithName(name string) (Named, error) {
name, err := normalize(name)
if err != nil {
return nil, err
}
if err := validateName(name); err != nil {
return nil, err
}
r, err := distreference.WithName(name)
if err != nil {
return nil, err
}
return &namedRef{r}, nil
}
// WithTag combines the name from "name" and the tag from "tag" to form a
// reference incorporating both the name and the tag.
func WithTag(name Named, tag string) (NamedTagged, error) {
r, err := distreference.WithTag(name, tag)
if err != nil {
return nil, err
}
return &taggedRef{namedRef{r}}, nil
}
// WithDigest combines the name from "name" and the digest from "digest" to form
// a reference incorporating both the name and the digest.
func WithDigest(name Named, digest digest.Digest) (Canonical, error) {
r, err := distreference.WithDigest(name, digest)
if err != nil {
return nil, err
}
return &canonicalRef{namedRef{r}}, nil
}
type namedRef struct {
distreference.Named
}
type taggedRef struct {
namedRef
}
type canonicalRef struct {
namedRef
}
func (r *namedRef) FullName() string {
hostname, remoteName := splitHostname(r.Name())
return hostname + "/" + remoteName
}
func (r *namedRef) Hostname() string {
hostname, _ := splitHostname(r.Name())
return hostname
}
func (r *namedRef) RemoteName() string {
_, remoteName := splitHostname(r.Name())
return remoteName
}
func (r *taggedRef) Tag() string {
return r.namedRef.Named.(distreference.NamedTagged).Tag()
}
func (r *canonicalRef) Digest() digest.Digest {
return r.namedRef.Named.(distreference.Canonical).Digest()
}
// WithDefaultTag adds a default tag to a reference if it only has a repo name.
func WithDefaultTag(ref Named) Named {
if IsNameOnly(ref) {
ref, _ = WithTag(ref, DefaultTag)
}
return ref
}
// IsNameOnly returns true if reference only contains a repo name.
func IsNameOnly(ref Named) bool {
if _, ok := ref.(NamedTagged); ok {
return false
}
if _, ok := ref.(Canonical); ok {
return false
}
return true
}
// splitHostname splits a repository name to hostname and remotename string.
// If no valid hostname is found, the default hostname is used. Repository name
// needs to be already validated before.
func splitHostname(name string) (hostname, remoteName string) {
i := strings.IndexRune(name, '/')
if i == -1 || (!strings.ContainsAny(name[:i], ".:") && name[:i] != "localhost") {
hostname, remoteName = DefaultHostname, name
} else {
hostname, remoteName = name[:i], name[i+1:]
}
if hostname == LegacyDefaultHostname {
hostname = DefaultHostname
}
if hostname == DefaultHostname && !strings.ContainsRune(remoteName, '/') {
remoteName = DefaultRepoPrefix + remoteName
}
return
}
// normalize returns a repository name in its normalized form, meaning it
// will not contain default hostname nor library/ prefix for official images.
func normalize(name string) (string, error) {
host, remoteName := splitHostname(name)
if strings.ToLower(remoteName) != remoteName {
return "", errors.New("invalid reference format: repository name must be lowercase")
}
if host == DefaultHostname {
if strings.HasPrefix(remoteName, DefaultRepoPrefix) {
return strings.TrimPrefix(remoteName, DefaultRepoPrefix), nil
}
return remoteName, nil
}
return name, nil
}
// EDIT FROM DOCKER/DOCKER TO NOT IMPORT IMAGE.V1
func validateName(name string) error {
if err := ValidateIDV1(name); err == nil {
return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name)
}
return nil
}
var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
// ValidateIDV1 checks whether an ID string is a valid image ID.
func ValidateIDV1(id string) error {
if ok := validHex.MatchString(id); !ok {
return fmt.Errorf("image ID %q is invalid", id)
}
return nil
}

View File

@ -7,7 +7,7 @@ import (
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
"github.com/projectatomic/skopeo/docker"
pkgInspect "github.com/projectatomic/skopeo/docker/inspect"
"github.com/projectatomic/skopeo/types"
)
@ -29,16 +29,16 @@ var inspectCmd = cli.Command{
},
}
func inspect(c *cli.Context) (*types.ImageManifest, error) {
func inspect(c *cli.Context) (types.ImageManifest, error) {
var (
imgInspect *types.ImageManifest
imgInspect types.ImageManifest
err error
name = c.Args().First()
)
switch {
case strings.HasPrefix(name, types.DockerPrefix):
imgInspect, err = docker.GetData(c, strings.Replace(name, "docker://", "", -1))
imgInspect, err = pkgInspect.GetData(c, strings.Replace(name, "docker://", "", -1))
if err != nil {
return nil, err
}

View File

@ -1 +1,22 @@
package main
import (
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
)
// TODO(runcom): document args and usage
var layersCmd = cli.Command{
Name: "layers",
Usage: "get images layers",
ArgsUsage: ``,
Action: func(context *cli.Context) {
img, err := parseImage(context.Args().First())
if err != nil {
logrus.Fatal(err)
}
if err := img.Layers(context.Args().Tail()...); err != nil {
logrus.Fatal(err)
}
},
}

View File

@ -49,6 +49,7 @@ func main() {
}
app.Commands = []cli.Command{
inspectCmd,
layersCmd,
}
if err := app.Run(os.Args); err != nil {
logrus.Fatal(err)

View File

@ -27,7 +27,12 @@ type Image interface {
DockerTar() ([]byte, error) // ??? also, configure output directory
}
type ImageManifest struct {
// TODO(runcom)
type ImageManifest interface {
Labels() map[string]string
}
type DockerImageManifest struct {
Tag string
Digest string
RepoTags []string
@ -41,3 +46,7 @@ type ImageManifest struct {
Os string
Layers []string // ???
}
func (m *DockerImageManifest) Labels() map[string]string {
return m.Config.Labels
}

View File

@ -10,7 +10,7 @@ import (
func parseImage(img string) (types.Image, error) {
switch {
case strings.HasPrefix(img, types.DockerPrefix):
//return parseDockerImage(strings.TrimPrefix(img, dockerPrefix))
return parseDockerImage(strings.TrimPrefix(img, dockerPrefix))
//case strings.HasPrefix(img, appcPrefix):
//
}