diff --git a/.gitignore b/.gitignore index da177d95..772132c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ skopeo skopeo.1 +layers-* diff --git a/README.md b/README.md index 78ed413c..1275509c 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ skopeo [ - e.g. - which tags are available for the given repository? which labels the image has? @@ -113,6 +113,7 @@ $ make test-integration ``` TODO - +- update README with `layers` command - list all images on registry? - registry v2 search? - make skopeo docker registry v2 only diff --git a/docker.go b/docker.go new file mode 100644 index 00000000..67ba44ed --- /dev/null +++ b/docker.go @@ -0,0 +1,550 @@ +package main + +import ( + "crypto/tls" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "path/filepath" + "regexp" + "strings" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/homedir" + "github.com/projectatomic/skopeo/docker/reference" + "github.com/projectatomic/skopeo/types" +) + +const ( + dockerPrefix = "docker://" + dockerHostname = "docker.io" + dockerRegistry = "registry-1.docker.io" + dockerAuthRegistry = "https://index.docker.io/v1/" + + dockerCfg = ".docker" + dockerCfgFileName = "config.json" + dockerCfgObsolete = ".dockercfg" +) + +var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) + +type dockerImage struct { + ref reference.Named + tag string + registry string + username string + password string + WWWAuthenticate string + scheme string + rawManifest []byte +} + +func (i *dockerImage) RawManifest(version string) ([]byte, error) { + // TODO(runcom): unused version param for now, default to docker v2-1 + if err := i.retrieveRawManifest(); err != nil { + return nil, err + } + return i.rawManifest, nil +} + +func (i *dockerImage) Manifest(version string) (types.ImageManifest, error) { + // TODO(runcom): port docker/docker implementation under docker/ to just + // use this!!! and do not rely on docker upstream code - will need to support + // v1 fall back also... + return nil, nil +} + +func (i *dockerImage) DockerTar() ([]byte, error) { + return nil, nil +} + +// will support v1 one day... +type manifest interface { + String() string + GetLayers() []string +} + +type manifestSchema1 struct { + Name string + Tag string + FSLayers []struct { + BlobSum string `json:"blobSum"` + } `json:"fsLayers"` + History []struct { + V1Compatibility string `json:"v1Compatibility"` + } `json:"history"` + // TODO(runcom) verify the downloaded manifest + //Signature []byte `json:"signature"` +} + +func (m *manifestSchema1) GetLayers() []string { + layers := make([]string, len(m.FSLayers)) + for i, layer := range m.FSLayers { + layers[i] = layer.BlobSum + } + return layers +} + +func (m *manifestSchema1) String() string { + return fmt.Sprintf("%s-%s", sanitize(m.Name), sanitize(m.Tag)) +} + +func sanitize(s string) string { + return strings.Replace(s, "/", "-", -1) +} + +func (i *dockerImage) makeRequest(method, url string, auth bool, headers map[string]string) (*http.Response, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Docker-Distribution-API-Version", "registry/2.0") + for n, h := range headers { + req.Header.Add(n, h) + } + if auth { + if err := i.setupRequestAuth(req); err != nil { + return nil, err + } + } + // insecure by default for now + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + client := &http.Client{Transport: tr} + res, err := client.Do(req) + if err != nil { + return nil, err + } + return res, nil +} + +func (i *dockerImage) setupRequestAuth(req *http.Request) error { + tokens := strings.SplitN(strings.TrimSpace(i.WWWAuthenticate), " ", 2) + if len(tokens) != 2 { + return fmt.Errorf("expected 2 tokens in WWW-Authenticate: %d, %s", len(tokens), i.WWWAuthenticate) + } + switch tokens[0] { + case "Basic": + req.SetBasicAuth(i.username, i.password) + return nil + case "Bearer": + // insecure by default for now + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + client := &http.Client{Transport: tr} + res, err := client.Do(req) + if err != nil { + return err + } + hdr := res.Header.Get("WWW-Authenticate") + if hdr == "" || res.StatusCode != http.StatusUnauthorized { + // no need for bearer? wtf? + return nil + } + tokens = strings.Split(hdr, " ") + tokens = strings.Split(tokens[1], ",") + var realm, service, scope string + for _, token := range tokens { + if strings.HasPrefix(token, "realm") { + realm = strings.Trim(token[len("realm="):], "\"") + } + if strings.HasPrefix(token, "service") { + service = strings.Trim(token[len("service="):], "\"") + } + if strings.HasPrefix(token, "scope") { + scope = strings.Trim(token[len("scope="):], "\"") + } + } + + if realm == "" { + return fmt.Errorf("missing realm in bearer auth challenge") + } + if service == "" { + return fmt.Errorf("missing service in bearer auth challenge") + } + // The scope can be empty if we're not getting a token for a specific repo + //if scope == "" && repo != "" { + if scope == "" { + return fmt.Errorf("missing scope in bearer auth challenge") + } + token, err := i.getBearerToken(realm, service, scope) + if err != nil { + return err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + return nil + } + return fmt.Errorf("no handler for %s authentication", tokens[0]) + // support docker bearer with authconfig's Auth string? see docker2aci +} + +func (i *dockerImage) getBearerToken(realm, service, scope string) (string, error) { + authReq, err := http.NewRequest("GET", realm, nil) + if err != nil { + return "", err + } + getParams := authReq.URL.Query() + getParams.Add("service", service) + if scope != "" { + getParams.Add("scope", scope) + } + authReq.URL.RawQuery = getParams.Encode() + if i.username != "" && i.password != "" { + authReq.SetBasicAuth(i.username, i.password) + } + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + client := &http.Client{Transport: tr} + res, err := client.Do(authReq) + if err != nil { + return "", err + } + defer res.Body.Close() + switch res.StatusCode { + case http.StatusUnauthorized: + return "", fmt.Errorf("unable to retrieve auth token: 401 unauthorized") + case http.StatusOK: + break + default: + return "", fmt.Errorf("unexpected http code: %d, URL: %s", res.StatusCode, authReq.URL) + } + tokenBlob, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + tokenStruct := struct { + Token string `json:"token"` + }{} + if err := json.Unmarshal(tokenBlob, &tokenStruct); err != nil { + return "", err + } + // TODO(runcom): reuse tokens? + //hostAuthTokens, ok = rb.hostsV2AuthTokens[req.URL.Host] + //if !ok { + //hostAuthTokens = make(map[string]string) + //rb.hostsV2AuthTokens[req.URL.Host] = hostAuthTokens + //} + //hostAuthTokens[repo] = tokenStruct.Token + return tokenStruct.Token, nil +} + +func (i *dockerImage) retrieveRawManifest() error { + if i.rawManifest != nil { + return nil + } + pr, err := ping(i.registry) + if err != nil { + return err + } + i.WWWAuthenticate = pr.WWWAuthenticate + i.scheme = pr.scheme + url := i.scheme + "://" + i.registry + "/v2/" + i.ref.RemoteName() + "/manifests/" + i.tag + // TODO(runcom) set manifest version header! schema1 for now - then schema2 etc etc and v1 + // TODO(runcom) NO, switch on the resulter manifest like Docker is doing + res, err := i.makeRequest("GET", url, pr.needsAuth(), nil) + if err != nil { + return err + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + // print body also + return fmt.Errorf("Invalid status code returned when fetching manifest %d", res.StatusCode) + } + manblob, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + i.rawManifest = manblob + return nil +} + +func (i *dockerImage) getSchema1Manifest() (manifest, error) { + if err := i.retrieveRawManifest(); err != nil { + return nil, err + } + mschema1 := &manifestSchema1{} + if err := json.Unmarshal(i.rawManifest, mschema1); err != nil { + return nil, err + } + if err := fixManifestLayers(mschema1); err != nil { + return nil, err + } + return mschema1, nil +} + +func (i *dockerImage) Layers(layers ...string) error { + m, err := i.getSchema1Manifest() + if err != nil { + return err + } + tmpDir, err := ioutil.TempDir(".", "layers-"+m.String()+"-") + if err != nil { + return err + } + data, err := json.Marshal(m) + if err != nil { + return err + } + if err := ioutil.WriteFile(path.Join(tmpDir, "manifest.json"), data, 0644); err != nil { + return err + } + url := i.scheme + "://" + i.registry + "/v2/" + i.ref.RemoteName() + "/blobs/" + if len(layers) == 0 { + layers = m.GetLayers() + } + for _, l := range layers { + if !strings.HasPrefix(l, "sha256:") { + l = "sha256:" + l + } + if err := i.getLayer(l, url, tmpDir); err != nil { + return err + } + } + return nil +} + +func (i *dockerImage) getLayer(l, url, tmpDir string) error { + lurl := url + l + logrus.Infof("Downloading %s", lurl) + res, err := i.makeRequest("GET", lurl, i.WWWAuthenticate != "", nil) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + // print url also + return fmt.Errorf("Invalid status code returned when fetching blob %d", res.StatusCode) + } + layerPath := path.Join(tmpDir, strings.Replace(l, "sha256:", "", -1)+".tar") + layerFile, err := os.Create(layerPath) + if err != nil { + return err + } + if _, err := io.Copy(layerFile, res.Body); err != nil { + return err + } + if err := layerFile.Sync(); err != nil { + return err + } + return nil +} + +func parseDockerImage(img string) (types.Image, error) { + ref, err := reference.ParseNamed(img) + if err != nil { + return nil, err + } + if reference.IsNameOnly(ref) { + ref = reference.WithDefaultTag(ref) + } + var tag string + switch x := ref.(type) { + case reference.Canonical: + tag = x.Digest().String() + case reference.NamedTagged: + tag = x.Tag() + } + var registry string + hostname := ref.Hostname() + if hostname == dockerHostname { + registry = dockerRegistry + } else { + registry = hostname + } + username, password, err := getAuth(ref.Hostname()) + if err != nil { + return nil, err + } + return &dockerImage{ + ref: ref, + tag: tag, + registry: registry, + username: username, + password: password, + }, nil +} + +func getDefaultConfigDir(confPath string) string { + return filepath.Join(homedir.Get(), confPath) +} + +type DockerAuthConfigObsolete struct { + Auth string `json:"auth"` +} + +type DockerAuthConfig struct { + Auth string `json:"auth,omitempty"` +} + +type DockerConfigFile struct { + AuthConfigs map[string]DockerAuthConfig `json:"auths"` +} + +func decodeDockerAuth(s string) (string, string, error) { + decoded, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return "", "", err + } + parts := strings.SplitN(string(decoded), ":", 2) + if len(parts) != 2 { + return "", "", fmt.Errorf("invalid auth configuration file") + } + user := parts[0] + password := strings.Trim(parts[1], "\x00") + return user, password, nil +} + +func getAuth(hostname string) (string, string, error) { + if hostname == dockerHostname { + hostname = dockerAuthRegistry + } + dockerCfgPath := filepath.Join(getDefaultConfigDir(".docker"), dockerCfgFileName) + if _, err := os.Stat(dockerCfgPath); err == nil { + j, err := ioutil.ReadFile(dockerCfgPath) + if err != nil { + return "", "", err + } + var dockerAuth DockerConfigFile + if err := json.Unmarshal(j, &dockerAuth); err != nil { + return "", "", err + } + // try the normal case + if c, ok := dockerAuth.AuthConfigs[hostname]; ok { + return decodeDockerAuth(c.Auth) + } + } else if os.IsNotExist(err) { + oldDockerCfgPath := filepath.Join(getDefaultConfigDir(dockerCfgObsolete)) + if _, err := os.Stat(oldDockerCfgPath); err != nil { + return "", "", nil //missing file is not an error + } + j, err := ioutil.ReadFile(oldDockerCfgPath) + if err != nil { + return "", "", err + } + var dockerAuthOld map[string]DockerAuthConfigObsolete + if err := json.Unmarshal(j, &dockerAuthOld); err != nil { + return "", "", err + } + if c, ok := dockerAuthOld[hostname]; ok { + return decodeDockerAuth(c.Auth) + } + } else { + // if file is there but we can't stat it for any reason other + // than it doesn't exist then stop + return "", "", fmt.Errorf("%s - %v", dockerCfgPath, err) + } + return "", "", nil +} + +type APIErr struct { + Code string + Message string + Detail interface{} +} + +type pingResponse struct { + WWWAuthenticate string + APIVersion string + scheme string + errors []APIErr +} + +func (pr *pingResponse) needsAuth() bool { + return pr.WWWAuthenticate != "" +} + +func ping(registry string) (*pingResponse, error) { + // insecure by default for now + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + client := &http.Client{Transport: tr} + ping := func(scheme string) (*pingResponse, error) { + resp, err := client.Get(scheme + "://" + registry + "/v2/") + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusUnauthorized { + return nil, fmt.Errorf("error pinging repository, response code %d", resp.StatusCode) + } + pr := &pingResponse{} + pr.WWWAuthenticate = resp.Header.Get("WWW-Authenticate") + pr.APIVersion = resp.Header.Get("Docker-Distribution-Api-Version") + pr.scheme = scheme + if resp.StatusCode == http.StatusUnauthorized { + type APIErrors struct { + Errors []APIErr + } + errs := &APIErrors{} + if err := json.NewDecoder(resp.Body).Decode(errs); err != nil { + return nil, err + } + pr.errors = errs.Errors + } + return pr, nil + } + scheme := "https" + pr, err := ping(scheme) + if err != nil { + scheme = "http" + pr, err = ping(scheme) + if err == nil { + return pr, nil + } + } + return pr, err +} + +func fixManifestLayers(manifest *manifestSchema1) error { + type imageV1 struct { + ID string + Parent string + } + imgs := make([]*imageV1, len(manifest.FSLayers)) + for i := range manifest.FSLayers { + img := &imageV1{} + + if err := json.Unmarshal([]byte(manifest.History[i].V1Compatibility), img); err != nil { + return err + } + + imgs[i] = img + if err := validateV1ID(img.ID); err != nil { + return err + } + } + if imgs[len(imgs)-1].Parent != "" { + return errors.New("Invalid parent ID in the base layer of the image.") + } + // check general duplicates to error instead of a deadlock + idmap := make(map[string]struct{}) + var lastID string + for _, img := range imgs { + // skip IDs that appear after each other, we handle those later + if _, exists := idmap[img.ID]; img.ID != lastID && exists { + return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID) + } + lastID = img.ID + idmap[lastID] = struct{}{} + } + // backwards loop so that we keep the remaining indexes after removing items + for i := len(imgs) - 2; i >= 0; i-- { + if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue + manifest.FSLayers = append(manifest.FSLayers[:i], manifest.FSLayers[i+1:]...) + manifest.History = append(manifest.History[:i], manifest.History[i+1:]...) + } else if imgs[i].Parent != imgs[i+1].ID { + return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", imgs[i+1].ID, imgs[i].Parent) + } + } + return nil +} + +func validateV1ID(id string) error { + if ok := validHex.MatchString(id); !ok { + return fmt.Errorf("image ID %q is invalid", id) + } + return nil +} diff --git a/docker/README b/docker/README new file mode 100644 index 00000000..993e7ccc --- /dev/null +++ b/docker/README @@ -0,0 +1,5 @@ +TODO + +Eventually we want to get rid of inspect pkg which uses docker upstream code +and use, instead, docker.go which calls the api directly +Be aware that docker pkg do not fall back to v1! this must be implemented soon diff --git a/docker/docker.go b/docker/docker.go new file mode 100644 index 00000000..609d15d4 --- /dev/null +++ b/docker/docker.go @@ -0,0 +1,550 @@ +package docker + +import ( + "crypto/tls" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "path/filepath" + "regexp" + "strings" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/homedir" + "github.com/projectatomic/skopeo/docker/reference" + "github.com/projectatomic/skopeo/types" +) + +const ( + dockerPrefix = "docker://" + dockerHostname = "docker.io" + dockerRegistry = "registry-1.docker.io" + dockerAuthRegistry = "https://index.docker.io/v1/" + + dockerCfg = ".docker" + dockerCfgFileName = "config.json" + dockerCfgObsolete = ".dockercfg" +) + +var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) + +type dockerImage struct { + ref reference.Named + tag string + registry string + username string + password string + WWWAuthenticate string + scheme string + rawManifest []byte +} + +func (i *dockerImage) RawManifest(version string) ([]byte, error) { + // TODO(runcom): unused version param for now, default to docker v2-1 + if err := i.retrieveRawManifest(); err != nil { + return nil, err + } + return i.rawManifest, nil +} + +func (i *dockerImage) Manifest(version string) (types.ImageManifest, error) { + // TODO(runcom): port docker/docker implementation under docker/ to just + // use this!!! and do not rely on docker upstream code - will need to support + // v1 fall back also... + return nil, nil +} + +func (i *dockerImage) DockerTar() ([]byte, error) { + return nil, nil +} + +// will support v1 one day... +type manifest interface { + String() string + GetLayers() []string +} + +type manifestSchema1 struct { + Name string + Tag string + FSLayers []struct { + BlobSum string `json:"blobSum"` + } `json:"fsLayers"` + History []struct { + V1Compatibility string `json:"v1Compatibility"` + } `json:"history"` + // TODO(runcom) verify the downloaded manifest + //Signature []byte `json:"signature"` +} + +func (m *manifestSchema1) GetLayers() []string { + layers := make([]string, len(m.FSLayers)) + for i, layer := range m.FSLayers { + layers[i] = layer.BlobSum + } + return layers +} + +func (m *manifestSchema1) String() string { + return fmt.Sprintf("%s-%s", sanitize(m.Name), sanitize(m.Tag)) +} + +func sanitize(s string) string { + return strings.Replace(s, "/", "-", -1) +} + +func (i *dockerImage) makeRequest(method, url string, auth bool, headers map[string]string) (*http.Response, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Docker-Distribution-API-Version", "registry/2.0") + for n, h := range headers { + req.Header.Add(n, h) + } + if auth { + if err := i.setupRequestAuth(req); err != nil { + return nil, err + } + } + // insecure by default for now + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + client := &http.Client{Transport: tr} + res, err := client.Do(req) + if err != nil { + return nil, err + } + return res, nil +} + +func (i *dockerImage) setupRequestAuth(req *http.Request) error { + tokens := strings.SplitN(strings.TrimSpace(i.WWWAuthenticate), " ", 2) + if len(tokens) != 2 { + return fmt.Errorf("expected 2 tokens in WWW-Authenticate: %d, %s", len(tokens), i.WWWAuthenticate) + } + switch tokens[0] { + case "Basic": + req.SetBasicAuth(i.username, i.password) + return nil + case "Bearer": + // insecure by default for now + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + client := &http.Client{Transport: tr} + res, err := client.Do(req) + if err != nil { + return err + } + hdr := res.Header.Get("WWW-Authenticate") + if hdr == "" || res.StatusCode != http.StatusUnauthorized { + // no need for bearer? wtf? + return nil + } + tokens = strings.Split(hdr, " ") + tokens = strings.Split(tokens[1], ",") + var realm, service, scope string + for _, token := range tokens { + if strings.HasPrefix(token, "realm") { + realm = strings.Trim(token[len("realm="):], "\"") + } + if strings.HasPrefix(token, "service") { + service = strings.Trim(token[len("service="):], "\"") + } + if strings.HasPrefix(token, "scope") { + scope = strings.Trim(token[len("scope="):], "\"") + } + } + + if realm == "" { + return fmt.Errorf("missing realm in bearer auth challenge") + } + if service == "" { + return fmt.Errorf("missing service in bearer auth challenge") + } + // The scope can be empty if we're not getting a token for a specific repo + //if scope == "" && repo != "" { + if scope == "" { + return fmt.Errorf("missing scope in bearer auth challenge") + } + token, err := i.getBearerToken(realm, service, scope) + if err != nil { + return err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + return nil + } + return fmt.Errorf("no handler for %s authentication", tokens[0]) + // support docker bearer with authconfig's Auth string? see docker2aci +} + +func (i *dockerImage) getBearerToken(realm, service, scope string) (string, error) { + authReq, err := http.NewRequest("GET", realm, nil) + if err != nil { + return "", err + } + getParams := authReq.URL.Query() + getParams.Add("service", service) + if scope != "" { + getParams.Add("scope", scope) + } + authReq.URL.RawQuery = getParams.Encode() + if i.username != "" && i.password != "" { + authReq.SetBasicAuth(i.username, i.password) + } + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + client := &http.Client{Transport: tr} + res, err := client.Do(authReq) + if err != nil { + return "", err + } + defer res.Body.Close() + switch res.StatusCode { + case http.StatusUnauthorized: + return "", fmt.Errorf("unable to retrieve auth token: 401 unauthorized") + case http.StatusOK: + break + default: + return "", fmt.Errorf("unexpected http code: %d, URL: %s", res.StatusCode, authReq.URL) + } + tokenBlob, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", err + } + tokenStruct := struct { + Token string `json:"token"` + }{} + if err := json.Unmarshal(tokenBlob, &tokenStruct); err != nil { + return "", err + } + // TODO(runcom): reuse tokens? + //hostAuthTokens, ok = rb.hostsV2AuthTokens[req.URL.Host] + //if !ok { + //hostAuthTokens = make(map[string]string) + //rb.hostsV2AuthTokens[req.URL.Host] = hostAuthTokens + //} + //hostAuthTokens[repo] = tokenStruct.Token + return tokenStruct.Token, nil +} + +func (i *dockerImage) retrieveRawManifest() error { + if i.rawManifest != nil { + return nil + } + pr, err := ping(i.registry) + if err != nil { + return err + } + i.WWWAuthenticate = pr.WWWAuthenticate + i.scheme = pr.scheme + url := i.scheme + "://" + i.registry + "/v2/" + i.ref.RemoteName() + "/manifests/" + i.tag + // TODO(runcom) set manifest version header! schema1 for now - then schema2 etc etc and v1 + // TODO(runcom) NO, switch on the resulter manifest like Docker is doing + res, err := i.makeRequest("GET", url, pr.needsAuth(), nil) + if err != nil { + return err + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + // print body also + return fmt.Errorf("Invalid status code returned when fetching manifest %d", res.StatusCode) + } + manblob, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + i.rawManifest = manblob + return nil +} + +func (i *dockerImage) getSchema1Manifest() (manifest, error) { + if err := i.retrieveRawManifest(); err != nil { + return nil, err + } + mschema1 := &manifestSchema1{} + if err := json.Unmarshal(i.rawManifest, mschema1); err != nil { + return nil, err + } + if err := fixManifestLayers(mschema1); err != nil { + return nil, err + } + return mschema1, nil +} + +func (i *dockerImage) Layers(layers ...string) error { + m, err := i.getSchema1Manifest() + if err != nil { + return err + } + tmpDir, err := ioutil.TempDir(".", "layers-"+m.String()+"-") + if err != nil { + return err + } + data, err := json.Marshal(m) + if err != nil { + return err + } + if err := ioutil.WriteFile(path.Join(tmpDir, "manifest.json"), data, 0644); err != nil { + return err + } + url := i.scheme + "://" + i.registry + "/v2/" + i.ref.RemoteName() + "/blobs/" + if len(layers) == 0 { + layers = m.GetLayers() + } + for _, l := range layers { + if !strings.HasPrefix(l, "sha256:") { + l = "sha256:" + l + } + if err := i.getLayer(l, url, tmpDir); err != nil { + return err + } + } + return nil +} + +func (i *dockerImage) getLayer(l, url, tmpDir string) error { + lurl := url + l + logrus.Infof("Downloading %s", lurl) + res, err := i.makeRequest("GET", lurl, i.WWWAuthenticate != "", nil) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + // print url also + return fmt.Errorf("Invalid status code returned when fetching blob %d", res.StatusCode) + } + layerPath := path.Join(tmpDir, strings.Replace(l, "sha256:", "", -1)+".tar") + layerFile, err := os.Create(layerPath) + if err != nil { + return err + } + if _, err := io.Copy(layerFile, res.Body); err != nil { + return err + } + if err := layerFile.Sync(); err != nil { + return err + } + return nil +} + +func parseDockerImage(img string) (types.Image, error) { + ref, err := reference.ParseNamed(img) + if err != nil { + return nil, err + } + if reference.IsNameOnly(ref) { + ref = reference.WithDefaultTag(ref) + } + var tag string + switch x := ref.(type) { + case reference.Canonical: + tag = x.Digest().String() + case reference.NamedTagged: + tag = x.Tag() + } + var registry string + hostname := ref.Hostname() + if hostname == dockerHostname { + registry = dockerRegistry + } else { + registry = hostname + } + username, password, err := getAuth(ref.Hostname()) + if err != nil { + return nil, err + } + return &dockerImage{ + ref: ref, + tag: tag, + registry: registry, + username: username, + password: password, + }, nil +} + +func getDefaultConfigDir(confPath string) string { + return filepath.Join(homedir.Get(), confPath) +} + +type DockerAuthConfigObsolete struct { + Auth string `json:"auth"` +} + +type DockerAuthConfig struct { + Auth string `json:"auth,omitempty"` +} + +type DockerConfigFile struct { + AuthConfigs map[string]DockerAuthConfig `json:"auths"` +} + +func decodeDockerAuth(s string) (string, string, error) { + decoded, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return "", "", err + } + parts := strings.SplitN(string(decoded), ":", 2) + if len(parts) != 2 { + return "", "", fmt.Errorf("invalid auth configuration file") + } + user := parts[0] + password := strings.Trim(parts[1], "\x00") + return user, password, nil +} + +func getAuth(hostname string) (string, string, error) { + if hostname == dockerHostname { + hostname = dockerAuthRegistry + } + dockerCfgPath := filepath.Join(getDefaultConfigDir(".docker"), dockerCfgFileName) + if _, err := os.Stat(dockerCfgPath); err == nil { + j, err := ioutil.ReadFile(dockerCfgPath) + if err != nil { + return "", "", err + } + var dockerAuth DockerConfigFile + if err := json.Unmarshal(j, &dockerAuth); err != nil { + return "", "", err + } + // try the normal case + if c, ok := dockerAuth.AuthConfigs[hostname]; ok { + return decodeDockerAuth(c.Auth) + } + } else if os.IsNotExist(err) { + oldDockerCfgPath := filepath.Join(getDefaultConfigDir(dockerCfgObsolete)) + if _, err := os.Stat(oldDockerCfgPath); err != nil { + return "", "", nil //missing file is not an error + } + j, err := ioutil.ReadFile(oldDockerCfgPath) + if err != nil { + return "", "", err + } + var dockerAuthOld map[string]DockerAuthConfigObsolete + if err := json.Unmarshal(j, &dockerAuthOld); err != nil { + return "", "", err + } + if c, ok := dockerAuthOld[hostname]; ok { + return decodeDockerAuth(c.Auth) + } + } else { + // if file is there but we can't stat it for any reason other + // than it doesn't exist then stop + return "", "", fmt.Errorf("%s - %v", dockerCfgPath, err) + } + return "", "", nil +} + +type APIErr struct { + Code string + Message string + Detail interface{} +} + +type pingResponse struct { + WWWAuthenticate string + APIVersion string + scheme string + errors []APIErr +} + +func (pr *pingResponse) needsAuth() bool { + return pr.WWWAuthenticate != "" +} + +func ping(registry string) (*pingResponse, error) { + // insecure by default for now + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + client := &http.Client{Transport: tr} + ping := func(scheme string) (*pingResponse, error) { + resp, err := client.Get(scheme + "://" + registry + "/v2/") + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusUnauthorized { + return nil, fmt.Errorf("error pinging repository, response code %d", resp.StatusCode) + } + pr := &pingResponse{} + pr.WWWAuthenticate = resp.Header.Get("WWW-Authenticate") + pr.APIVersion = resp.Header.Get("Docker-Distribution-Api-Version") + pr.scheme = scheme + if resp.StatusCode == http.StatusUnauthorized { + type APIErrors struct { + Errors []APIErr + } + errs := &APIErrors{} + if err := json.NewDecoder(resp.Body).Decode(errs); err != nil { + return nil, err + } + pr.errors = errs.Errors + } + return pr, nil + } + scheme := "https" + pr, err := ping(scheme) + if err != nil { + scheme = "http" + pr, err = ping(scheme) + if err == nil { + return pr, nil + } + } + return pr, err +} + +func fixManifestLayers(manifest *manifestSchema1) error { + type imageV1 struct { + ID string + Parent string + } + imgs := make([]*imageV1, len(manifest.FSLayers)) + for i := range manifest.FSLayers { + img := &imageV1{} + + if err := json.Unmarshal([]byte(manifest.History[i].V1Compatibility), img); err != nil { + return err + } + + imgs[i] = img + if err := validateV1ID(img.ID); err != nil { + return err + } + } + if imgs[len(imgs)-1].Parent != "" { + return errors.New("Invalid parent ID in the base layer of the image.") + } + // check general duplicates to error instead of a deadlock + idmap := make(map[string]struct{}) + var lastID string + for _, img := range imgs { + // skip IDs that appear after each other, we handle those later + if _, exists := idmap[img.ID]; img.ID != lastID && exists { + return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID) + } + lastID = img.ID + idmap[lastID] = struct{}{} + } + // backwards loop so that we keep the remaining indexes after removing items + for i := len(imgs) - 2; i >= 0; i-- { + if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue + manifest.FSLayers = append(manifest.FSLayers[:i], manifest.FSLayers[i+1:]...) + manifest.History = append(manifest.History[:i], manifest.History[i+1:]...) + } else if imgs[i].Parent != imgs[i+1].ID { + return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", imgs[i+1].ID, imgs[i].Parent) + } + } + return nil +} + +func validateV1ID(id string) error { + if ok := validHex.MatchString(id); !ok { + return fmt.Errorf("image ID %q is invalid", id) + } + return nil +} diff --git a/docker/inspect.go b/docker/inspect/inspect.go similarity index 96% rename from docker/inspect.go rename to docker/inspect/inspect.go index c350feb6..7f89362d 100644 --- a/docker/inspect.go +++ b/docker/inspect/inspect.go @@ -1,4 +1,4 @@ -package docker +package inspect import ( "encoding/json" @@ -47,7 +47,7 @@ func (f fallbackError) Error() string { } type manifestFetcher interface { - Fetch(ctx context.Context, ref reference.Named) (*types.ImageManifest, error) + Fetch(ctx context.Context, ref reference.Named) (types.ImageManifest, error) } func validateName(name string) error { @@ -62,7 +62,7 @@ func validateName(name string) error { return nil } -func GetData(c *cli.Context, name string) (*types.ImageManifest, error) { +func GetData(c *cli.Context, name string) (types.ImageManifest, error) { if err := validateName(name); err != nil { return nil, err } @@ -101,7 +101,7 @@ func GetData(c *cli.Context, name string) (*types.ImageManifest, error) { ctx = context.Background() lastErr error discardNoSupportErrors bool - imgInspect *types.ImageManifest + imgInspect types.ImageManifest confirmedV2 bool confirmedTLSRegistries = make(map[string]struct{}) ) @@ -248,12 +248,12 @@ func validateRepoName(name string) error { return nil } -func makeImageManifest(img *image.Image, tag string, dgst digest.Digest, tagList []string) *types.ImageManifest { +func makeImageManifest(img *image.Image, tag string, dgst digest.Digest, tagList []string) types.ImageManifest { var digest string if err := dgst.Validate(); err == nil { digest = dgst.String() } - return &types.ImageManifest{ + return &types.DockerImageManifest{ Tag: tag, Digest: digest, RepoTags: tagList, diff --git a/docker/inspect_v1.go b/docker/inspect/inspect_v1.go similarity index 97% rename from docker/inspect_v1.go rename to docker/inspect/inspect_v1.go index 93b18a0f..8fcd24cb 100644 --- a/docker/inspect_v1.go +++ b/docker/inspect/inspect_v1.go @@ -1,4 +1,4 @@ -package docker +package inspect import ( "encoding/json" @@ -31,9 +31,9 @@ type v1ManifestFetcher struct { session *registry.Session } -func (mf *v1ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (*types.ImageManifest, error) { +func (mf *v1ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (types.ImageManifest, error) { var ( - imgInspect *types.ImageManifest + imgInspect types.ImageManifest ) if _, isCanonical := ref.(reference.Canonical); isCanonical { // Allowing fallback, because HTTPS v1 is before HTTP v2 @@ -68,7 +68,7 @@ func (mf *v1ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (*t return imgInspect, nil } -func (mf *v1ManifestFetcher) fetchWithSession(ctx context.Context, ref reference.Named) (*types.ImageManifest, error) { +func (mf *v1ManifestFetcher) fetchWithSession(ctx context.Context, ref reference.Named) (types.ImageManifest, error) { repoData, err := mf.session.GetRepositoryData(mf.repoInfo) if err != nil { if strings.Contains(err.Error(), "HTTP code: 404") { diff --git a/docker/inspect_v2.go b/docker/inspect/inspect_v2.go similarity index 98% rename from docker/inspect_v2.go rename to docker/inspect/inspect_v2.go index 719d61e3..eb9469e2 100644 --- a/docker/inspect_v2.go +++ b/docker/inspect/inspect_v2.go @@ -1,4 +1,4 @@ -package docker +package inspect import ( "encoding/json" @@ -34,9 +34,9 @@ type v2ManifestFetcher struct { service *registry.Service } -func (mf *v2ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (*types.ImageManifest, error) { +func (mf *v2ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (types.ImageManifest, error) { var ( - imgInspect *types.ImageManifest + imgInspect types.ImageManifest err error ) @@ -60,7 +60,7 @@ func (mf *v2ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (*t return imgInspect, err } -func (mf *v2ManifestFetcher) fetchWithRepository(ctx context.Context, ref reference.Named) (*types.ImageManifest, error) { +func (mf *v2ManifestFetcher) fetchWithRepository(ctx context.Context, ref reference.Named) (types.ImageManifest, error) { var ( manifest distribution.Manifest tagOrDigest string // Used for logging/progress only diff --git a/docker/reference/reference.go b/docker/reference/reference.go new file mode 100644 index 00000000..ba695b97 --- /dev/null +++ b/docker/reference/reference.go @@ -0,0 +1,211 @@ +// COPY FROM DOCKER/DOCKER +package reference + +import ( + "errors" + "fmt" + "regexp" + "strings" + + "github.com/docker/distribution/digest" + distreference "github.com/docker/distribution/reference" +) + +const ( + // DefaultTag defines the default tag used when performing images related actions and no tag or digest is specified + DefaultTag = "latest" + // DefaultHostname is the default built-in hostname + DefaultHostname = "docker.io" + // LegacyDefaultHostname is automatically converted to DefaultHostname + LegacyDefaultHostname = "index.docker.io" + // DefaultRepoPrefix is the prefix used for default repositories in default host + DefaultRepoPrefix = "library/" +) + +// Named is an object with a full name +type Named interface { + // Name returns normalized repository name, like "ubuntu". + Name() string + // String returns full reference, like "ubuntu@sha256:abcdef..." + String() string + // FullName returns full repository name with hostname, like "docker.io/library/ubuntu" + FullName() string + // Hostname returns hostname for the reference, like "docker.io" + Hostname() string + // RemoteName returns the repository component of the full name, like "library/ubuntu" + RemoteName() string +} + +// NamedTagged is an object including a name and tag. +type NamedTagged interface { + Named + Tag() string +} + +// Canonical reference is an object with a fully unique +// name including a name with hostname and digest +type Canonical interface { + Named + Digest() digest.Digest +} + +// ParseNamed parses s and returns a syntactically valid reference implementing +// the Named interface. The reference must have a name, otherwise an error is +// returned. +// If an error was encountered it is returned, along with a nil Reference. +func ParseNamed(s string) (Named, error) { + named, err := distreference.ParseNamed(s) + if err != nil { + return nil, fmt.Errorf("Error parsing reference: %q is not a valid repository/tag", s) + } + r, err := WithName(named.Name()) + if err != nil { + return nil, err + } + if canonical, isCanonical := named.(distreference.Canonical); isCanonical { + return WithDigest(r, canonical.Digest()) + } + if tagged, isTagged := named.(distreference.NamedTagged); isTagged { + return WithTag(r, tagged.Tag()) + } + return r, nil +} + +// WithName returns a named object representing the given string. If the input +// is invalid ErrReferenceInvalidFormat will be returned. +func WithName(name string) (Named, error) { + name, err := normalize(name) + if err != nil { + return nil, err + } + if err := validateName(name); err != nil { + return nil, err + } + r, err := distreference.WithName(name) + if err != nil { + return nil, err + } + return &namedRef{r}, nil +} + +// WithTag combines the name from "name" and the tag from "tag" to form a +// reference incorporating both the name and the tag. +func WithTag(name Named, tag string) (NamedTagged, error) { + r, err := distreference.WithTag(name, tag) + if err != nil { + return nil, err + } + return &taggedRef{namedRef{r}}, nil +} + +// WithDigest combines the name from "name" and the digest from "digest" to form +// a reference incorporating both the name and the digest. +func WithDigest(name Named, digest digest.Digest) (Canonical, error) { + r, err := distreference.WithDigest(name, digest) + if err != nil { + return nil, err + } + return &canonicalRef{namedRef{r}}, nil +} + +type namedRef struct { + distreference.Named +} +type taggedRef struct { + namedRef +} +type canonicalRef struct { + namedRef +} + +func (r *namedRef) FullName() string { + hostname, remoteName := splitHostname(r.Name()) + return hostname + "/" + remoteName +} +func (r *namedRef) Hostname() string { + hostname, _ := splitHostname(r.Name()) + return hostname +} +func (r *namedRef) RemoteName() string { + _, remoteName := splitHostname(r.Name()) + return remoteName +} +func (r *taggedRef) Tag() string { + return r.namedRef.Named.(distreference.NamedTagged).Tag() +} +func (r *canonicalRef) Digest() digest.Digest { + return r.namedRef.Named.(distreference.Canonical).Digest() +} + +// WithDefaultTag adds a default tag to a reference if it only has a repo name. +func WithDefaultTag(ref Named) Named { + if IsNameOnly(ref) { + ref, _ = WithTag(ref, DefaultTag) + } + return ref +} + +// IsNameOnly returns true if reference only contains a repo name. +func IsNameOnly(ref Named) bool { + if _, ok := ref.(NamedTagged); ok { + return false + } + if _, ok := ref.(Canonical); ok { + return false + } + return true +} + +// splitHostname splits a repository name to hostname and remotename string. +// If no valid hostname is found, the default hostname is used. Repository name +// needs to be already validated before. +func splitHostname(name string) (hostname, remoteName string) { + i := strings.IndexRune(name, '/') + if i == -1 || (!strings.ContainsAny(name[:i], ".:") && name[:i] != "localhost") { + hostname, remoteName = DefaultHostname, name + } else { + hostname, remoteName = name[:i], name[i+1:] + } + if hostname == LegacyDefaultHostname { + hostname = DefaultHostname + } + if hostname == DefaultHostname && !strings.ContainsRune(remoteName, '/') { + remoteName = DefaultRepoPrefix + remoteName + } + return +} + +// normalize returns a repository name in its normalized form, meaning it +// will not contain default hostname nor library/ prefix for official images. +func normalize(name string) (string, error) { + host, remoteName := splitHostname(name) + if strings.ToLower(remoteName) != remoteName { + return "", errors.New("invalid reference format: repository name must be lowercase") + } + if host == DefaultHostname { + if strings.HasPrefix(remoteName, DefaultRepoPrefix) { + return strings.TrimPrefix(remoteName, DefaultRepoPrefix), nil + } + return remoteName, nil + } + return name, nil +} + +// EDIT FROM DOCKER/DOCKER TO NOT IMPORT IMAGE.V1 + +func validateName(name string) error { + if err := ValidateIDV1(name); err == nil { + return fmt.Errorf("Invalid repository name (%s), cannot specify 64-byte hexadecimal strings", name) + } + return nil +} + +var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) + +// ValidateIDV1 checks whether an ID string is a valid image ID. +func ValidateIDV1(id string) error { + if ok := validHex.MatchString(id); !ok { + return fmt.Errorf("image ID %q is invalid", id) + } + return nil +} diff --git a/inspect.go b/inspect.go index 76b4cd64..08e5add4 100644 --- a/inspect.go +++ b/inspect.go @@ -7,7 +7,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/codegangsta/cli" - "github.com/projectatomic/skopeo/docker" + pkgInspect "github.com/projectatomic/skopeo/docker/inspect" "github.com/projectatomic/skopeo/types" ) @@ -29,16 +29,16 @@ var inspectCmd = cli.Command{ }, } -func inspect(c *cli.Context) (*types.ImageManifest, error) { +func inspect(c *cli.Context) (types.ImageManifest, error) { var ( - imgInspect *types.ImageManifest + imgInspect types.ImageManifest err error name = c.Args().First() ) switch { case strings.HasPrefix(name, types.DockerPrefix): - imgInspect, err = docker.GetData(c, strings.Replace(name, "docker://", "", -1)) + imgInspect, err = pkgInspect.GetData(c, strings.Replace(name, "docker://", "", -1)) if err != nil { return nil, err } diff --git a/layers.go b/layers.go index 06ab7d0f..7dbab1bb 100644 --- a/layers.go +++ b/layers.go @@ -1 +1,22 @@ package main + +import ( + "github.com/Sirupsen/logrus" + "github.com/codegangsta/cli" +) + +// TODO(runcom): document args and usage +var layersCmd = cli.Command{ + Name: "layers", + Usage: "get images layers", + ArgsUsage: ``, + Action: func(context *cli.Context) { + img, err := parseImage(context.Args().First()) + if err != nil { + logrus.Fatal(err) + } + if err := img.Layers(context.Args().Tail()...); err != nil { + logrus.Fatal(err) + } + }, +} diff --git a/main.go b/main.go index 963cca72..3bcc03a1 100644 --- a/main.go +++ b/main.go @@ -49,6 +49,7 @@ func main() { } app.Commands = []cli.Command{ inspectCmd, + layersCmd, } if err := app.Run(os.Args); err != nil { logrus.Fatal(err) diff --git a/types/types.go b/types/types.go index e77ffae5..0e9000ee 100644 --- a/types/types.go +++ b/types/types.go @@ -27,7 +27,12 @@ type Image interface { DockerTar() ([]byte, error) // ??? also, configure output directory } -type ImageManifest struct { +// TODO(runcom) +type ImageManifest interface { + Labels() map[string]string +} + +type DockerImageManifest struct { Tag string Digest string RepoTags []string @@ -41,3 +46,7 @@ type ImageManifest struct { Os string Layers []string // ??? } + +func (m *DockerImageManifest) Labels() map[string]string { + return m.Config.Labels +} diff --git a/utils.go b/utils.go index 25ed6e08..966c2417 100644 --- a/utils.go +++ b/utils.go @@ -10,7 +10,7 @@ import ( func parseImage(img string) (types.Image, error) { switch { case strings.HasPrefix(img, types.DockerPrefix): - //return parseDockerImage(strings.TrimPrefix(img, dockerPrefix)) + return parseDockerImage(strings.TrimPrefix(img, dockerPrefix)) //case strings.HasPrefix(img, appcPrefix): // }