diff --git a/README.md b/README.md index 957cdbd0..6e76bfd5 100644 --- a/README.md +++ b/README.md @@ -118,8 +118,7 @@ TODO - list all images on registry? - registry v2 search? - make skopeo docker registry v2 only -- output raw manifest -- download layers and support docker load tar(s) +- download layers in parallel and support docker load tar(s) - get rid of docker/docker code (?) - show repo tags via flag or when reference isn't tagged or digested - add tests (integration with deployed registries in container - Docker-like) diff --git a/docker.go b/docker.go index 4ba4afe2..297afaf3 100644 --- a/docker.go +++ b/docker.go @@ -17,7 +17,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/homedir" - "github.com/projectatomic/skopeo/docker/reference" + "github.com/projectatomic/skopeo/reference" "github.com/projectatomic/skopeo/types" ) diff --git a/docker/docker.go b/docker/docker.go deleted file mode 100644 index 609d15d4..00000000 --- a/docker/docker.go +++ /dev/null @@ -1,550 +0,0 @@ -package docker - -import ( - "crypto/tls" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - "path" - "path/filepath" - "regexp" - "strings" - - "github.com/Sirupsen/logrus" - "github.com/docker/docker/pkg/homedir" - "github.com/projectatomic/skopeo/docker/reference" - "github.com/projectatomic/skopeo/types" -) - -const ( - dockerPrefix = "docker://" - dockerHostname = "docker.io" - dockerRegistry = "registry-1.docker.io" - dockerAuthRegistry = "https://index.docker.io/v1/" - - dockerCfg = ".docker" - dockerCfgFileName = "config.json" - dockerCfgObsolete = ".dockercfg" -) - -var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) - -type dockerImage struct { - ref reference.Named - tag string - registry string - username string - password string - WWWAuthenticate string - scheme string - rawManifest []byte -} - -func (i *dockerImage) RawManifest(version string) ([]byte, error) { - // TODO(runcom): unused version param for now, default to docker v2-1 - if err := i.retrieveRawManifest(); err != nil { - return nil, err - } - return i.rawManifest, nil -} - -func (i *dockerImage) Manifest(version string) (types.ImageManifest, error) { - // TODO(runcom): port docker/docker implementation under docker/ to just - // use this!!! and do not rely on docker upstream code - will need to support - // v1 fall back also... - return nil, nil -} - -func (i *dockerImage) DockerTar() ([]byte, error) { - return nil, nil -} - -// will support v1 one day... -type manifest interface { - String() string - GetLayers() []string -} - -type manifestSchema1 struct { - Name string - Tag string - FSLayers []struct { - BlobSum string `json:"blobSum"` - } `json:"fsLayers"` - History []struct { - V1Compatibility string `json:"v1Compatibility"` - } `json:"history"` - // TODO(runcom) verify the downloaded manifest - //Signature []byte `json:"signature"` -} - -func (m *manifestSchema1) GetLayers() []string { - layers := make([]string, len(m.FSLayers)) - for i, layer := range m.FSLayers { - layers[i] = layer.BlobSum - } - return layers -} - -func (m *manifestSchema1) String() string { - return fmt.Sprintf("%s-%s", sanitize(m.Name), sanitize(m.Tag)) -} - -func sanitize(s string) string { - return strings.Replace(s, "/", "-", -1) -} - -func (i *dockerImage) makeRequest(method, url string, auth bool, headers map[string]string) (*http.Response, error) { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - req.Header.Set("Docker-Distribution-API-Version", "registry/2.0") - for n, h := range headers { - req.Header.Add(n, h) - } - if auth { - if err := i.setupRequestAuth(req); err != nil { - return nil, err - } - } - // insecure by default for now - tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} - client := &http.Client{Transport: tr} - res, err := client.Do(req) - if err != nil { - return nil, err - } - return res, nil -} - -func (i *dockerImage) setupRequestAuth(req *http.Request) error { - tokens := strings.SplitN(strings.TrimSpace(i.WWWAuthenticate), " ", 2) - if len(tokens) != 2 { - return fmt.Errorf("expected 2 tokens in WWW-Authenticate: %d, %s", len(tokens), i.WWWAuthenticate) - } - switch tokens[0] { - case "Basic": - req.SetBasicAuth(i.username, i.password) - return nil - case "Bearer": - // insecure by default for now - tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} - client := &http.Client{Transport: tr} - res, err := client.Do(req) - if err != nil { - return err - } - hdr := res.Header.Get("WWW-Authenticate") - if hdr == "" || res.StatusCode != http.StatusUnauthorized { - // no need for bearer? wtf? - return nil - } - tokens = strings.Split(hdr, " ") - tokens = strings.Split(tokens[1], ",") - var realm, service, scope string - for _, token := range tokens { - if strings.HasPrefix(token, "realm") { - realm = strings.Trim(token[len("realm="):], "\"") - } - if strings.HasPrefix(token, "service") { - service = strings.Trim(token[len("service="):], "\"") - } - if strings.HasPrefix(token, "scope") { - scope = strings.Trim(token[len("scope="):], "\"") - } - } - - if realm == "" { - return fmt.Errorf("missing realm in bearer auth challenge") - } - if service == "" { - return fmt.Errorf("missing service in bearer auth challenge") - } - // The scope can be empty if we're not getting a token for a specific repo - //if scope == "" && repo != "" { - if scope == "" { - return fmt.Errorf("missing scope in bearer auth challenge") - } - token, err := i.getBearerToken(realm, service, scope) - if err != nil { - return err - } - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) - return nil - } - return fmt.Errorf("no handler for %s authentication", tokens[0]) - // support docker bearer with authconfig's Auth string? see docker2aci -} - -func (i *dockerImage) getBearerToken(realm, service, scope string) (string, error) { - authReq, err := http.NewRequest("GET", realm, nil) - if err != nil { - return "", err - } - getParams := authReq.URL.Query() - getParams.Add("service", service) - if scope != "" { - getParams.Add("scope", scope) - } - authReq.URL.RawQuery = getParams.Encode() - if i.username != "" && i.password != "" { - authReq.SetBasicAuth(i.username, i.password) - } - tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} - client := &http.Client{Transport: tr} - res, err := client.Do(authReq) - if err != nil { - return "", err - } - defer res.Body.Close() - switch res.StatusCode { - case http.StatusUnauthorized: - return "", fmt.Errorf("unable to retrieve auth token: 401 unauthorized") - case http.StatusOK: - break - default: - return "", fmt.Errorf("unexpected http code: %d, URL: %s", res.StatusCode, authReq.URL) - } - tokenBlob, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", err - } - tokenStruct := struct { - Token string `json:"token"` - }{} - if err := json.Unmarshal(tokenBlob, &tokenStruct); err != nil { - return "", err - } - // TODO(runcom): reuse tokens? - //hostAuthTokens, ok = rb.hostsV2AuthTokens[req.URL.Host] - //if !ok { - //hostAuthTokens = make(map[string]string) - //rb.hostsV2AuthTokens[req.URL.Host] = hostAuthTokens - //} - //hostAuthTokens[repo] = tokenStruct.Token - return tokenStruct.Token, nil -} - -func (i *dockerImage) retrieveRawManifest() error { - if i.rawManifest != nil { - return nil - } - pr, err := ping(i.registry) - if err != nil { - return err - } - i.WWWAuthenticate = pr.WWWAuthenticate - i.scheme = pr.scheme - url := i.scheme + "://" + i.registry + "/v2/" + i.ref.RemoteName() + "/manifests/" + i.tag - // TODO(runcom) set manifest version header! schema1 for now - then schema2 etc etc and v1 - // TODO(runcom) NO, switch on the resulter manifest like Docker is doing - res, err := i.makeRequest("GET", url, pr.needsAuth(), nil) - if err != nil { - return err - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - // print body also - return fmt.Errorf("Invalid status code returned when fetching manifest %d", res.StatusCode) - } - manblob, err := ioutil.ReadAll(res.Body) - if err != nil { - return err - } - i.rawManifest = manblob - return nil -} - -func (i *dockerImage) getSchema1Manifest() (manifest, error) { - if err := i.retrieveRawManifest(); err != nil { - return nil, err - } - mschema1 := &manifestSchema1{} - if err := json.Unmarshal(i.rawManifest, mschema1); err != nil { - return nil, err - } - if err := fixManifestLayers(mschema1); err != nil { - return nil, err - } - return mschema1, nil -} - -func (i *dockerImage) Layers(layers ...string) error { - m, err := i.getSchema1Manifest() - if err != nil { - return err - } - tmpDir, err := ioutil.TempDir(".", "layers-"+m.String()+"-") - if err != nil { - return err - } - data, err := json.Marshal(m) - if err != nil { - return err - } - if err := ioutil.WriteFile(path.Join(tmpDir, "manifest.json"), data, 0644); err != nil { - return err - } - url := i.scheme + "://" + i.registry + "/v2/" + i.ref.RemoteName() + "/blobs/" - if len(layers) == 0 { - layers = m.GetLayers() - } - for _, l := range layers { - if !strings.HasPrefix(l, "sha256:") { - l = "sha256:" + l - } - if err := i.getLayer(l, url, tmpDir); err != nil { - return err - } - } - return nil -} - -func (i *dockerImage) getLayer(l, url, tmpDir string) error { - lurl := url + l - logrus.Infof("Downloading %s", lurl) - res, err := i.makeRequest("GET", lurl, i.WWWAuthenticate != "", nil) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - // print url also - return fmt.Errorf("Invalid status code returned when fetching blob %d", res.StatusCode) - } - layerPath := path.Join(tmpDir, strings.Replace(l, "sha256:", "", -1)+".tar") - layerFile, err := os.Create(layerPath) - if err != nil { - return err - } - if _, err := io.Copy(layerFile, res.Body); err != nil { - return err - } - if err := layerFile.Sync(); err != nil { - return err - } - return nil -} - -func parseDockerImage(img string) (types.Image, error) { - ref, err := reference.ParseNamed(img) - if err != nil { - return nil, err - } - if reference.IsNameOnly(ref) { - ref = reference.WithDefaultTag(ref) - } - var tag string - switch x := ref.(type) { - case reference.Canonical: - tag = x.Digest().String() - case reference.NamedTagged: - tag = x.Tag() - } - var registry string - hostname := ref.Hostname() - if hostname == dockerHostname { - registry = dockerRegistry - } else { - registry = hostname - } - username, password, err := getAuth(ref.Hostname()) - if err != nil { - return nil, err - } - return &dockerImage{ - ref: ref, - tag: tag, - registry: registry, - username: username, - password: password, - }, nil -} - -func getDefaultConfigDir(confPath string) string { - return filepath.Join(homedir.Get(), confPath) -} - -type DockerAuthConfigObsolete struct { - Auth string `json:"auth"` -} - -type DockerAuthConfig struct { - Auth string `json:"auth,omitempty"` -} - -type DockerConfigFile struct { - AuthConfigs map[string]DockerAuthConfig `json:"auths"` -} - -func decodeDockerAuth(s string) (string, string, error) { - decoded, err := base64.StdEncoding.DecodeString(s) - if err != nil { - return "", "", err - } - parts := strings.SplitN(string(decoded), ":", 2) - if len(parts) != 2 { - return "", "", fmt.Errorf("invalid auth configuration file") - } - user := parts[0] - password := strings.Trim(parts[1], "\x00") - return user, password, nil -} - -func getAuth(hostname string) (string, string, error) { - if hostname == dockerHostname { - hostname = dockerAuthRegistry - } - dockerCfgPath := filepath.Join(getDefaultConfigDir(".docker"), dockerCfgFileName) - if _, err := os.Stat(dockerCfgPath); err == nil { - j, err := ioutil.ReadFile(dockerCfgPath) - if err != nil { - return "", "", err - } - var dockerAuth DockerConfigFile - if err := json.Unmarshal(j, &dockerAuth); err != nil { - return "", "", err - } - // try the normal case - if c, ok := dockerAuth.AuthConfigs[hostname]; ok { - return decodeDockerAuth(c.Auth) - } - } else if os.IsNotExist(err) { - oldDockerCfgPath := filepath.Join(getDefaultConfigDir(dockerCfgObsolete)) - if _, err := os.Stat(oldDockerCfgPath); err != nil { - return "", "", nil //missing file is not an error - } - j, err := ioutil.ReadFile(oldDockerCfgPath) - if err != nil { - return "", "", err - } - var dockerAuthOld map[string]DockerAuthConfigObsolete - if err := json.Unmarshal(j, &dockerAuthOld); err != nil { - return "", "", err - } - if c, ok := dockerAuthOld[hostname]; ok { - return decodeDockerAuth(c.Auth) - } - } else { - // if file is there but we can't stat it for any reason other - // than it doesn't exist then stop - return "", "", fmt.Errorf("%s - %v", dockerCfgPath, err) - } - return "", "", nil -} - -type APIErr struct { - Code string - Message string - Detail interface{} -} - -type pingResponse struct { - WWWAuthenticate string - APIVersion string - scheme string - errors []APIErr -} - -func (pr *pingResponse) needsAuth() bool { - return pr.WWWAuthenticate != "" -} - -func ping(registry string) (*pingResponse, error) { - // insecure by default for now - tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} - client := &http.Client{Transport: tr} - ping := func(scheme string) (*pingResponse, error) { - resp, err := client.Get(scheme + "://" + registry + "/v2/") - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusUnauthorized { - return nil, fmt.Errorf("error pinging repository, response code %d", resp.StatusCode) - } - pr := &pingResponse{} - pr.WWWAuthenticate = resp.Header.Get("WWW-Authenticate") - pr.APIVersion = resp.Header.Get("Docker-Distribution-Api-Version") - pr.scheme = scheme - if resp.StatusCode == http.StatusUnauthorized { - type APIErrors struct { - Errors []APIErr - } - errs := &APIErrors{} - if err := json.NewDecoder(resp.Body).Decode(errs); err != nil { - return nil, err - } - pr.errors = errs.Errors - } - return pr, nil - } - scheme := "https" - pr, err := ping(scheme) - if err != nil { - scheme = "http" - pr, err = ping(scheme) - if err == nil { - return pr, nil - } - } - return pr, err -} - -func fixManifestLayers(manifest *manifestSchema1) error { - type imageV1 struct { - ID string - Parent string - } - imgs := make([]*imageV1, len(manifest.FSLayers)) - for i := range manifest.FSLayers { - img := &imageV1{} - - if err := json.Unmarshal([]byte(manifest.History[i].V1Compatibility), img); err != nil { - return err - } - - imgs[i] = img - if err := validateV1ID(img.ID); err != nil { - return err - } - } - if imgs[len(imgs)-1].Parent != "" { - return errors.New("Invalid parent ID in the base layer of the image.") - } - // check general duplicates to error instead of a deadlock - idmap := make(map[string]struct{}) - var lastID string - for _, img := range imgs { - // skip IDs that appear after each other, we handle those later - if _, exists := idmap[img.ID]; img.ID != lastID && exists { - return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID) - } - lastID = img.ID - idmap[lastID] = struct{}{} - } - // backwards loop so that we keep the remaining indexes after removing items - for i := len(imgs) - 2; i >= 0; i-- { - if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue - manifest.FSLayers = append(manifest.FSLayers[:i], manifest.FSLayers[i+1:]...) - manifest.History = append(manifest.History[:i], manifest.History[i+1:]...) - } else if imgs[i].Parent != imgs[i+1].ID { - return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", imgs[i+1].ID, imgs[i].Parent) - } - } - return nil -} - -func validateV1ID(id string) error { - if ok := validHex.MatchString(id); !ok { - return fmt.Errorf("image ID %q is invalid", id) - } - return nil -} diff --git a/docker/reference/reference.go b/reference/reference.go similarity index 99% rename from docker/reference/reference.go rename to reference/reference.go index ba695b97..3c0a80d0 100644 --- a/docker/reference/reference.go +++ b/reference/reference.go @@ -1,5 +1,4 @@ -// COPY FROM DOCKER/DOCKER -package reference +package reference // COPY WITH EDITS FROM DOCKER/DOCKER import ( "errors"