diff --git a/cmd/skopeo/inspect.go b/cmd/skopeo/inspect.go index 203ed481..a1d3e987 100644 --- a/cmd/skopeo/inspect.go +++ b/cmd/skopeo/inspect.go @@ -3,13 +3,10 @@ package main import ( "encoding/json" "fmt" - "strings" "github.com/Sirupsen/logrus" "github.com/codegangsta/cli" "github.com/projectatomic/skopeo" - pkgInspect "github.com/projectatomic/skopeo/docker/inspect" - "github.com/projectatomic/skopeo/types" ) var inspectCmd = cli.Command{ @@ -47,22 +44,3 @@ var inspectCmd = cli.Command{ fmt.Println(string(out)) }, } - -func inspect(c *cli.Context) (types.ImageManifest, error) { - var ( - imgInspect types.ImageManifest - err error - name = c.Args().First() - ) - - switch { - case strings.HasPrefix(name, types.DockerPrefix): - imgInspect, err = pkgInspect.GetData(c, strings.Replace(name, "docker://", "", -1)) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("%s image is invalid, please use 'docker://'", name) - } - return imgInspect, nil -} diff --git a/docker.go b/docker.go index 7ee829dd..119c121c 100644 --- a/docker.go +++ b/docker.go @@ -404,16 +404,16 @@ func getDefaultConfigDir(confPath string) string { return filepath.Join(homedir.Get(), confPath) } -type DockerAuthConfigObsolete struct { +type dockerAuthConfigObsolete struct { Auth string `json:"auth"` } -type DockerAuthConfig struct { +type dockerAuthConfig struct { Auth string `json:"auth,omitempty"` } -type DockerConfigFile struct { - AuthConfigs map[string]DockerAuthConfig `json:"auths"` +type dockerConfigFile struct { + AuthConfigs map[string]dockerAuthConfig `json:"auths"` } func decodeDockerAuth(s string) (string, string, error) { @@ -440,7 +440,7 @@ func getAuth(hostname string) (string, string, error) { if err != nil { return "", "", err } - var dockerAuth DockerConfigFile + var dockerAuth dockerConfigFile if err := json.Unmarshal(j, &dockerAuth); err != nil { return "", "", err } @@ -457,7 +457,7 @@ func getAuth(hostname string) (string, string, error) { if err != nil { return "", "", err } - var dockerAuthOld map[string]DockerAuthConfigObsolete + var dockerAuthOld map[string]dockerAuthConfigObsolete if err := json.Unmarshal(j, &dockerAuthOld); err != nil { return "", "", err } @@ -472,7 +472,7 @@ func getAuth(hostname string) (string, string, error) { return "", "", nil } -type APIErr struct { +type apiErr struct { Code string Message string Detail interface{} @@ -482,7 +482,7 @@ type pingResponse struct { WWWAuthenticate string APIVersion string scheme string - errors []APIErr + errors []apiErr } func (pr *pingResponse) needsAuth() bool { @@ -508,7 +508,7 @@ func ping(registry string) (*pingResponse, error) { pr.scheme = scheme if resp.StatusCode == http.StatusUnauthorized { type APIErrors struct { - Errors []APIErr + Errors []apiErr } errs := &APIErrors{} if err := json.NewDecoder(resp.Body).Decode(errs); err != nil { diff --git a/docker/README b/docker/README deleted file mode 100644 index 993e7ccc..00000000 --- a/docker/README +++ /dev/null @@ -1,5 +0,0 @@ -TODO - -Eventually we want to get rid of inspect pkg which uses docker upstream code -and use, instead, docker.go which calls the api directly -Be aware that docker pkg do not fall back to v1! this must be implemented soon diff --git a/docker/inspect/inspect.go b/docker/inspect/inspect.go deleted file mode 100644 index 7f89362d..00000000 --- a/docker/inspect/inspect.go +++ /dev/null @@ -1,345 +0,0 @@ -package inspect - -import ( - "encoding/json" - "fmt" - "strings" - "syscall" - "time" - - "github.com/Sirupsen/logrus" - "github.com/codegangsta/cli" - "github.com/docker/distribution/digest" - distreference "github.com/docker/distribution/reference" - "github.com/docker/distribution/registry/api/errcode" - "github.com/docker/distribution/registry/api/v2" - "github.com/docker/distribution/registry/client" - "github.com/docker/docker/api" - "github.com/docker/docker/cliconfig" - "github.com/docker/docker/distribution" - "github.com/docker/docker/dockerversion" - "github.com/docker/docker/image" - "github.com/docker/docker/opts" - versionPkg "github.com/docker/docker/pkg/version" - "github.com/docker/docker/reference" - "github.com/docker/docker/registry" - engineTypes "github.com/docker/engine-api/types" - registryTypes "github.com/docker/engine-api/types/registry" - "github.com/projectatomic/skopeo/types" - "golang.org/x/net/context" -) - -// fallbackError wraps an error that can possibly allow fallback to a different -// endpoint. -type fallbackError struct { - // err is the error being wrapped. - err error - // confirmedV2 is set to true if it was confirmed that the registry - // supports the v2 protocol. This is used to limit fallbacks to the v1 - // protocol. - confirmedV2 bool - transportOK bool -} - -// Error renders the FallbackError as a string. -func (f fallbackError) Error() string { - return f.err.Error() -} - -type manifestFetcher interface { - Fetch(ctx context.Context, ref reference.Named) (types.ImageManifest, error) -} - -func validateName(name string) error { - distref, err := distreference.ParseNamed(name) - if err != nil { - return err - } - hostname, _ := distreference.SplitHostname(distref) - if hostname == "" { - return fmt.Errorf("Please use a fully qualified repository name") - } - return nil -} - -func GetData(c *cli.Context, name string) (types.ImageManifest, error) { - if err := validateName(name); err != nil { - return nil, err - } - ref, err := reference.ParseNamed(name) - if err != nil { - return nil, err - } - repoInfo, err := registry.ParseRepositoryInfo(ref) - if err != nil { - return nil, err - } - authConfig, err := getAuthConfig(c, repoInfo.Index) - if err != nil { - return nil, err - } - if err := validateRepoName(repoInfo.Name()); err != nil { - return nil, err - } - options := ®istry.Options{} - options.Mirrors = opts.NewListOpts(nil) - options.InsecureRegistries = opts.NewListOpts(nil) - options.InsecureRegistries.Set("0.0.0.0/0") - registryService := registry.NewService(options) - // TODO(runcom): hacky, provide a way of passing tls cert (flag?) to be used to lookup - for _, ic := range registryService.Config.IndexConfigs { - ic.Secure = false - } - - endpoints, err := registryService.LookupPullEndpoints(repoInfo.Hostname()) - if err != nil { - return nil, err - } - logrus.Debugf("endpoints: %v", endpoints) - - var ( - ctx = context.Background() - lastErr error - discardNoSupportErrors bool - imgInspect types.ImageManifest - confirmedV2 bool - confirmedTLSRegistries = make(map[string]struct{}) - ) - - for _, endpoint := range endpoints { - // make sure I can reach the registry, same as docker pull does - v1endpoint, err := endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(), nil) - if err != nil { - return nil, err - } - if _, err := v1endpoint.Ping(); err != nil { - if strings.Contains(err.Error(), "timeout") { - return nil, err - } - continue - } - - if confirmedV2 && endpoint.Version == registry.APIVersion1 { - logrus.Debugf("Skipping v1 endpoint %s because v2 registry was detected", endpoint.URL) - continue - } - - if endpoint.URL.Scheme != "https" { - if _, confirmedTLS := confirmedTLSRegistries[endpoint.URL.Host]; confirmedTLS { - logrus.Debugf("Skipping non-TLS endpoint %s for host/port that appears to use TLS", endpoint.URL) - continue - } - } - - logrus.Debugf("Trying to fetch image manifest of %s repository from %s %s", repoInfo.Name(), endpoint.URL, endpoint.Version) - - //fetcher, err := newManifestFetcher(endpoint, repoInfo, config) - fetcher, err := newManifestFetcher(endpoint, repoInfo, authConfig, registryService) - if err != nil { - lastErr = err - continue - } - - if imgInspect, err = fetcher.Fetch(ctx, ref); err != nil { - // Was this fetch cancelled? If so, don't try to fall back. - fallback := false - select { - case <-ctx.Done(): - default: - if fallbackErr, ok := err.(fallbackError); ok { - fallback = true - confirmedV2 = confirmedV2 || fallbackErr.confirmedV2 - if fallbackErr.transportOK && endpoint.URL.Scheme == "https" { - confirmedTLSRegistries[endpoint.URL.Host] = struct{}{} - } - err = fallbackErr.err - } - } - if fallback { - if _, ok := err.(distribution.ErrNoSupport); !ok { - // Because we found an error that's not ErrNoSupport, discard all subsequent ErrNoSupport errors. - discardNoSupportErrors = true - // save the current error - lastErr = err - } else if !discardNoSupportErrors { - // Save the ErrNoSupport error, because it's either the first error or all encountered errors - // were also ErrNoSupport errors. - lastErr = err - } - continue - } - logrus.Errorf("Not continuing with pull after error: %v", err) - return nil, err - } - - return imgInspect, nil - } - - if lastErr == nil { - lastErr = fmt.Errorf("no endpoints found for %s", ref.String()) - } - - return nil, lastErr -} - -func newManifestFetcher(endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo, authConfig engineTypes.AuthConfig, registryService *registry.Service) (manifestFetcher, error) { - switch endpoint.Version { - case registry.APIVersion2: - return &v2ManifestFetcher{ - endpoint: endpoint, - authConfig: authConfig, - service: registryService, - repoInfo: repoInfo, - }, nil - case registry.APIVersion1: - return &v1ManifestFetcher{ - endpoint: endpoint, - authConfig: authConfig, - service: registryService, - repoInfo: repoInfo, - }, nil - } - return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL) -} - -func getAuthConfig(c *cli.Context, index *registryTypes.IndexInfo) (engineTypes.AuthConfig, error) { - var ( - username = c.GlobalString("username") - password = c.GlobalString("password") - cfg = c.GlobalString("docker-cfg") - defAuthConfig = engineTypes.AuthConfig{ - Username: c.GlobalString("username"), - Password: c.GlobalString("password"), - Email: "stub@example.com", - } - ) - - // - // FINAL TODO(runcom): avoid returning empty config! just fallthrough and return - // the first useful authconfig - // - - // TODO(runcom): ??? atomic needs this - // TODO(runcom): implement this to opt-in for docker-cfg, no need to make this - // work by default with docker's conf - //useDockerConf := c.GlobalString("use-docker-cfg") - - if username != "" && password != "" { - return defAuthConfig, nil - } - - confFile, err := cliconfig.Load(cfg) - if err != nil { - return engineTypes.AuthConfig{}, err - } - authConfig := registry.ResolveAuthConfig(confFile.AuthConfigs, index) - logrus.Debugf("authConfig for %s: %v", index.Name, authConfig) - - return authConfig, nil -} - -func validateRepoName(name string) error { - if name == "" { - return fmt.Errorf("Repository name can't be empty") - } - if name == api.NoBaseImageSpecifier { - return fmt.Errorf("'%s' is a reserved name", api.NoBaseImageSpecifier) - } - return nil -} - -func makeImageManifest(img *image.Image, tag string, dgst digest.Digest, tagList []string) types.ImageManifest { - var digest string - if err := dgst.Validate(); err == nil { - digest = dgst.String() - } - return &types.DockerImageManifest{ - Tag: tag, - Digest: digest, - RepoTags: tagList, - Comment: img.Comment, - Created: img.Created.Format(time.RFC3339Nano), - ContainerConfig: &img.ContainerConfig, - DockerVersion: img.DockerVersion, - Author: img.Author, - Config: img.Config, - Architecture: img.Architecture, - Os: img.OS, - } -} - -func makeRawConfigFromV1Config(imageJSON []byte, rootfs *image.RootFS, history []image.History) (map[string]*json.RawMessage, error) { - var dver struct { - DockerVersion string `json:"docker_version"` - } - - if err := json.Unmarshal(imageJSON, &dver); err != nil { - return nil, err - } - - useFallback := versionPkg.Version(dver.DockerVersion).LessThan("1.8.3") - - if useFallback { - var v1Image image.V1Image - err := json.Unmarshal(imageJSON, &v1Image) - if err != nil { - return nil, err - } - imageJSON, err = json.Marshal(v1Image) - if err != nil { - return nil, err - } - } - - var c map[string]*json.RawMessage - if err := json.Unmarshal(imageJSON, &c); err != nil { - return nil, err - } - - c["rootfs"] = rawJSON(rootfs) - c["history"] = rawJSON(history) - - return c, nil -} - -func rawJSON(value interface{}) *json.RawMessage { - jsonval, err := json.Marshal(value) - if err != nil { - return nil - } - return (*json.RawMessage)(&jsonval) -} - -func continueOnError(err error) bool { - switch v := err.(type) { - case errcode.Errors: - if len(v) == 0 { - return true - } - return continueOnError(v[0]) - case distribution.ErrNoSupport: - return continueOnError(v.Err) - case errcode.Error: - return shouldV2Fallback(v) - case *client.UnexpectedHTTPResponseError: - return true - case ImageConfigPullError: - return false - case error: - return !strings.Contains(err.Error(), strings.ToLower(syscall.ENOSPC.Error())) - } - // let's be nice and fallback if the error is a completely - // unexpected one. - // If new errors have to be handled in some way, please - // add them to the switch above. - return true -} - -// shouldV2Fallback returns true if this error is a reason to fall back to v1. -func shouldV2Fallback(err errcode.Error) bool { - switch err.Code { - case errcode.ErrorCodeUnauthorized, v2.ErrorCodeManifestUnknown, v2.ErrorCodeNameUnknown: - return true - } - return false -} diff --git a/docker/inspect/inspect_v1.go b/docker/inspect/inspect_v1.go deleted file mode 100644 index 8fcd24cb..00000000 --- a/docker/inspect/inspect_v1.go +++ /dev/null @@ -1,170 +0,0 @@ -package inspect - -import ( - "encoding/json" - "errors" - "fmt" - "strings" - - "github.com/Sirupsen/logrus" - "github.com/docker/distribution" - "github.com/docker/distribution/registry/client/transport" - dockerdistribution "github.com/docker/docker/distribution" - "github.com/docker/docker/dockerversion" - "github.com/docker/docker/image" - "github.com/docker/docker/image/v1" - "github.com/docker/docker/reference" - "github.com/docker/docker/registry" - engineTypes "github.com/docker/engine-api/types" - "github.com/projectatomic/skopeo/types" - "golang.org/x/net/context" -) - -type v1ManifestFetcher struct { - endpoint registry.APIEndpoint - repoInfo *registry.RepositoryInfo - repo distribution.Repository - confirmedV2 bool - // wrap in a config? - authConfig engineTypes.AuthConfig - service *registry.Service - session *registry.Session -} - -func (mf *v1ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (types.ImageManifest, error) { - var ( - imgInspect types.ImageManifest - ) - if _, isCanonical := ref.(reference.Canonical); isCanonical { - // Allowing fallback, because HTTPS v1 is before HTTP v2 - return nil, fallbackError{err: dockerdistribution.ErrNoSupport{errors.New("Cannot pull by digest with v1 registry")}} - } - tlsConfig, err := mf.service.TLSConfig(mf.repoInfo.Index.Name) - if err != nil { - return nil, err - } - // Adds Docker-specific headers as well as user-specified headers (metaHeaders) - tr := transport.NewTransport( - registry.NewTransport(tlsConfig), - //registry.DockerHeaders(mf.config.MetaHeaders)..., - registry.DockerHeaders(dockerversion.DockerUserAgent(), nil)..., - ) - client := registry.HTTPClient(tr) - //v1Endpoint, err := mf.endpoint.ToV1Endpoint(mf.config.MetaHeaders) - v1Endpoint, err := mf.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(), nil) - if err != nil { - logrus.Debugf("Could not get v1 endpoint: %v", err) - return nil, fallbackError{err: err} - } - mf.session, err = registry.NewSession(client, &mf.authConfig, v1Endpoint) - if err != nil { - logrus.Debugf("Fallback from error: %s", err) - return nil, fallbackError{err: err} - } - imgInspect, err = mf.fetchWithSession(ctx, ref) - if err != nil { - return nil, err - } - return imgInspect, nil -} - -func (mf *v1ManifestFetcher) fetchWithSession(ctx context.Context, ref reference.Named) (types.ImageManifest, error) { - repoData, err := mf.session.GetRepositoryData(mf.repoInfo) - if err != nil { - if strings.Contains(err.Error(), "HTTP code: 404") { - return nil, fmt.Errorf("Error: image %s not found", mf.repoInfo.RemoteName()) - } - // Unexpected HTTP error - return nil, err - } - - var tagsList map[string]string - tagsList, err = mf.session.GetRemoteTags(repoData.Endpoints, mf.repoInfo) - if err != nil { - logrus.Errorf("unable to get remote tags: %s", err) - return nil, err - } - - logrus.Debugf("Retrieving the tag list") - tagged, isTagged := ref.(reference.NamedTagged) - var tagID, tag string - if isTagged { - tag = tagged.Tag() - tagsList[tagged.Tag()] = tagID - } else { - ref, err = reference.WithTag(ref, reference.DefaultTag) - if err != nil { - return nil, err - } - tagged, _ := ref.(reference.NamedTagged) - tag = tagged.Tag() - tagsList[tagged.Tag()] = tagID - } - tagID, err = mf.session.GetRemoteTag(repoData.Endpoints, mf.repoInfo, tag) - if err == registry.ErrRepoNotFound { - return nil, fmt.Errorf("Tag %s not found in repository %s", tag, mf.repoInfo.FullName()) - } - if err != nil { - logrus.Errorf("unable to get remote tags: %s", err) - return nil, err - } - - tagList := []string{} - for tag := range tagsList { - tagList = append(tagList, tag) - } - - img := repoData.ImgList[tagID] - - var pulledImg *image.Image - for _, ep := range mf.repoInfo.Index.Mirrors { - if pulledImg, err = mf.pullImageJSON(img.ID, ep, repoData.Tokens); err != nil { - // Don't report errors when pulling from mirrors. - logrus.Debugf("Error pulling image json of %s:%s, mirror: %s, %s", mf.repoInfo.FullName(), img.Tag, ep, err) - continue - } - break - } - if pulledImg == nil { - for _, ep := range repoData.Endpoints { - if pulledImg, err = mf.pullImageJSON(img.ID, ep, repoData.Tokens); err != nil { - // It's not ideal that only the last error is returned, it would be better to concatenate the errors. - logrus.Infof("Error pulling image json of %s:%s, endpoint: %s, %v", mf.repoInfo.FullName(), img.Tag, ep, err) - continue - } - break - } - } - if err != nil { - return nil, fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, mf.repoInfo.FullName(), err) - } - if pulledImg == nil { - return nil, fmt.Errorf("No such image %s:%s", mf.repoInfo.FullName(), tag) - } - - return makeImageManifest(pulledImg, tag, "", tagList), nil -} - -func (mf *v1ManifestFetcher) pullImageJSON(imgID, endpoint string, token []string) (*image.Image, error) { - imgJSON, _, err := mf.session.GetRemoteImageJSON(imgID, endpoint) - if err != nil { - return nil, err - } - h, err := v1.HistoryFromConfig(imgJSON, false) - if err != nil { - return nil, err - } - configRaw, err := makeRawConfigFromV1Config(imgJSON, image.NewRootFS(), []image.History{h}) - if err != nil { - return nil, err - } - config, err := json.Marshal(configRaw) - if err != nil { - return nil, err - } - img, err := image.NewFromJSON(config) - if err != nil { - return nil, err - } - return img, nil -} diff --git a/docker/inspect/inspect_v2.go b/docker/inspect/inspect_v2.go deleted file mode 100644 index eb9469e2..00000000 --- a/docker/inspect/inspect_v2.go +++ /dev/null @@ -1,486 +0,0 @@ -package inspect - -import ( - "encoding/json" - "errors" - "fmt" - "runtime" - - "github.com/Sirupsen/logrus" - "github.com/docker/distribution" - "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest/manifestlist" - "github.com/docker/distribution/manifest/schema1" - "github.com/docker/distribution/manifest/schema2" - "github.com/docker/distribution/registry/api/errcode" - "github.com/docker/distribution/registry/client" - dockerdistribution "github.com/docker/docker/distribution" - "github.com/docker/docker/image" - "github.com/docker/docker/image/v1" - "github.com/docker/docker/reference" - "github.com/docker/docker/registry" - engineTypes "github.com/docker/engine-api/types" - "github.com/projectatomic/skopeo/types" - "golang.org/x/net/context" -) - -type v2ManifestFetcher struct { - endpoint registry.APIEndpoint - repoInfo *registry.RepositoryInfo - repo distribution.Repository - confirmedV2 bool - // wrap in a config? - authConfig engineTypes.AuthConfig - service *registry.Service -} - -func (mf *v2ManifestFetcher) Fetch(ctx context.Context, ref reference.Named) (types.ImageManifest, error) { - var ( - imgInspect types.ImageManifest - err error - ) - - //mf.repo, mf.confirmedV2, err = distribution.NewV2Repository(ctx, mf.repoInfo, mf.endpoint, mf.config.MetaHeaders, mf.config.AuthConfig, "pull") - mf.repo, mf.confirmedV2, err = dockerdistribution.NewV2Repository(ctx, mf.repoInfo, mf.endpoint, nil, &mf.authConfig, "pull") - if err != nil { - logrus.Debugf("Error getting v2 registry: %v", err) - return nil, err - } - - imgInspect, err = mf.fetchWithRepository(ctx, ref) - if err != nil { - if _, ok := err.(fallbackError); ok { - return nil, err - } - if continueOnError(err) { - logrus.Errorf("Error trying v2 registry: %v", err) - return nil, fallbackError{err: err, confirmedV2: mf.confirmedV2, transportOK: true} - } - } - return imgInspect, err -} - -func (mf *v2ManifestFetcher) fetchWithRepository(ctx context.Context, ref reference.Named) (types.ImageManifest, error) { - var ( - manifest distribution.Manifest - tagOrDigest string // Used for logging/progress only - tagList = []string{} - ) - - manSvc, err := mf.repo.Manifests(ctx) - if err != nil { - return nil, err - } - - if _, isTagged := ref.(reference.NamedTagged); !isTagged { - ref, err = reference.WithTag(ref, reference.DefaultTag) - if err != nil { - return nil, err - } - } - - if tagged, isTagged := ref.(reference.NamedTagged); isTagged { - // NOTE: not using TagService.Get, since it uses HEAD requests - // against the manifests endpoint, which are not supported by - // all registry versions. - manifest, err = manSvc.Get(ctx, "", client.WithTag(tagged.Tag())) - if err != nil { - return nil, allowV1Fallback(err) - } - tagOrDigest = tagged.Tag() - } else if digested, isDigested := ref.(reference.Canonical); isDigested { - manifest, err = manSvc.Get(ctx, digested.Digest()) - if err != nil { - return nil, err - } - tagOrDigest = digested.Digest().String() - } else { - return nil, fmt.Errorf("internal error: reference has neither a tag nor a digest: %s", ref.String()) - } - - if manifest == nil { - return nil, fmt.Errorf("image manifest does not exist for tag or digest %q", tagOrDigest) - } - - // If manSvc.Get succeeded, we can be confident that the registry on - // the other side speaks the v2 protocol. - mf.confirmedV2 = true - - tagList, err = mf.repo.Tags(ctx).All(ctx) - if err != nil { - // If this repository doesn't exist on V2, we should - // permit a fallback to V1. - return nil, allowV1Fallback(err) - } - - var ( - image *image.Image - manifestDigest digest.Digest - ) - - switch v := manifest.(type) { - case *schema1.SignedManifest: - image, manifestDigest, err = mf.pullSchema1(ctx, ref, v) - if err != nil { - return nil, err - } - case *schema2.DeserializedManifest: - image, manifestDigest, err = mf.pullSchema2(ctx, ref, v) - if err != nil { - return nil, err - } - case *manifestlist.DeserializedManifestList: - image, manifestDigest, err = mf.pullManifestList(ctx, ref, v) - if err != nil { - return nil, err - } - default: - return nil, errors.New("unsupported manifest format") - } - - // TODO(runcom) - //var showTags bool - //if reference.IsNameOnly(ref) { - //showTags = true - //logrus.Debug("Using default tag: latest") - //ref = reference.WithDefaultTag(ref) - //} - //_ = showTags - return makeImageManifest(image, tagOrDigest, manifestDigest, tagList), nil -} - -func (mf *v2ManifestFetcher) pullSchema1(ctx context.Context, ref reference.Named, unverifiedManifest *schema1.SignedManifest) (img *image.Image, manifestDigest digest.Digest, err error) { - var verifiedManifest *schema1.Manifest - verifiedManifest, err = verifySchema1Manifest(unverifiedManifest, ref) - if err != nil { - return nil, "", err - } - - // remove duplicate layers and check parent chain validity - err = fixManifestLayers(verifiedManifest) - if err != nil { - return nil, "", err - } - - // Image history converted to the new format - var history []image.History - - // Note that the order of this loop is in the direction of bottom-most - // to top-most, so that the downloads slice gets ordered correctly. - for i := len(verifiedManifest.FSLayers) - 1; i >= 0; i-- { - var throwAway struct { - ThrowAway bool `json:"throwaway,omitempty"` - } - if err := json.Unmarshal([]byte(verifiedManifest.History[i].V1Compatibility), &throwAway); err != nil { - return nil, "", err - } - - h, err := v1.HistoryFromConfig([]byte(verifiedManifest.History[i].V1Compatibility), throwAway.ThrowAway) - if err != nil { - return nil, "", err - } - history = append(history, h) - } - - rootFS := image.NewRootFS() - configRaw, err := makeRawConfigFromV1Config([]byte(verifiedManifest.History[0].V1Compatibility), rootFS, history) - - config, err := json.Marshal(configRaw) - if err != nil { - return nil, "", err - } - - img, err = image.NewFromJSON(config) - if err != nil { - return nil, "", err - } - - manifestDigest = digest.FromBytes(unverifiedManifest.Canonical) - - return img, manifestDigest, nil -} - -func verifySchema1Manifest(signedManifest *schema1.SignedManifest, ref reference.Named) (m *schema1.Manifest, err error) { - // If pull by digest, then verify the manifest digest. NOTE: It is - // important to do this first, before any other content validation. If the - // digest cannot be verified, don't even bother with those other things. - if digested, isCanonical := ref.(reference.Canonical); isCanonical { - verifier, err := digest.NewDigestVerifier(digested.Digest()) - if err != nil { - return nil, err - } - if _, err := verifier.Write(signedManifest.Canonical); err != nil { - return nil, err - } - if !verifier.Verified() { - err := fmt.Errorf("image verification failed for digest %s", digested.Digest()) - logrus.Error(err) - return nil, err - } - } - m = &signedManifest.Manifest - - if m.SchemaVersion != 1 { - return nil, fmt.Errorf("unsupported schema version %d for %q", m.SchemaVersion, ref.String()) - } - if len(m.FSLayers) != len(m.History) { - return nil, fmt.Errorf("length of history not equal to number of layers for %q", ref.String()) - } - if len(m.FSLayers) == 0 { - return nil, fmt.Errorf("no FSLayers in manifest for %q", ref.String()) - } - return m, nil -} - -func fixManifestLayers(m *schema1.Manifest) error { - imgs := make([]*image.V1Image, len(m.FSLayers)) - for i := range m.FSLayers { - img := &image.V1Image{} - - if err := json.Unmarshal([]byte(m.History[i].V1Compatibility), img); err != nil { - return err - } - - imgs[i] = img - if err := v1.ValidateID(img.ID); err != nil { - return err - } - } - - if imgs[len(imgs)-1].Parent != "" && runtime.GOOS != "windows" { - // Windows base layer can point to a base layer parent that is not in manifest. - return errors.New("Invalid parent ID in the base layer of the image.") - } - - // check general duplicates to error instead of a deadlock - idmap := make(map[string]struct{}) - - var lastID string - for _, img := range imgs { - // skip IDs that appear after each other, we handle those later - if _, exists := idmap[img.ID]; img.ID != lastID && exists { - return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID) - } - lastID = img.ID - idmap[lastID] = struct{}{} - } - - // backwards loop so that we keep the remaining indexes after removing items - for i := len(imgs) - 2; i >= 0; i-- { - if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue - m.FSLayers = append(m.FSLayers[:i], m.FSLayers[i+1:]...) - m.History = append(m.History[:i], m.History[i+1:]...) - } else if imgs[i].Parent != imgs[i+1].ID { - return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", imgs[i+1].ID, imgs[i].Parent) - } - } - - return nil -} - -func (mf *v2ManifestFetcher) pullSchema2(ctx context.Context, ref reference.Named, mfst *schema2.DeserializedManifest) (img *image.Image, manifestDigest digest.Digest, err error) { - manifestDigest, err = schema2ManifestDigest(ref, mfst) - if err != nil { - return nil, "", err - } - - target := mfst.Target() - - configChan := make(chan []byte, 1) - errChan := make(chan error, 1) - var cancel func() - ctx, cancel = context.WithCancel(ctx) - - // Pull the image config - go func() { - configJSON, err := mf.pullSchema2ImageConfig(ctx, target.Digest) - if err != nil { - errChan <- ImageConfigPullError{Err: err} - cancel() - return - } - configChan <- configJSON - }() - - var ( - configJSON []byte // raw serialized image config - unmarshalledConfig image.Image // deserialized image config - ) - if runtime.GOOS == "windows" { - configJSON, unmarshalledConfig, err = receiveConfig(configChan, errChan) - if err != nil { - return nil, "", err - } - if unmarshalledConfig.RootFS == nil { - return nil, "", errors.New("image config has no rootfs section") - } - } - - if configJSON == nil { - configJSON, unmarshalledConfig, err = receiveConfig(configChan, errChan) - if err != nil { - return nil, "", err - } - } - - img, err = image.NewFromJSON(configJSON) - if err != nil { - return nil, "", err - } - - return img, manifestDigest, nil -} - -func (mf *v2ManifestFetcher) pullSchema2ImageConfig(ctx context.Context, dgst digest.Digest) (configJSON []byte, err error) { - blobs := mf.repo.Blobs(ctx) - configJSON, err = blobs.Get(ctx, dgst) - if err != nil { - return nil, err - } - - // Verify image config digest - verifier, err := digest.NewDigestVerifier(dgst) - if err != nil { - return nil, err - } - if _, err := verifier.Write(configJSON); err != nil { - return nil, err - } - if !verifier.Verified() { - err := fmt.Errorf("image config verification failed for digest %s", dgst) - logrus.Error(err) - return nil, err - } - - return configJSON, nil -} - -func receiveConfig(configChan <-chan []byte, errChan <-chan error) ([]byte, image.Image, error) { - select { - case configJSON := <-configChan: - var unmarshalledConfig image.Image - if err := json.Unmarshal(configJSON, &unmarshalledConfig); err != nil { - return nil, image.Image{}, err - } - return configJSON, unmarshalledConfig, nil - case err := <-errChan: - return nil, image.Image{}, err - // Don't need a case for ctx.Done in the select because cancellation - // will trigger an error in p.pullSchema2ImageConfig. - } -} - -// ImageConfigPullError is an error pulling the image config blob -// (only applies to schema2). -type ImageConfigPullError struct { - Err error -} - -// Error returns the error string for ImageConfigPullError. -func (e ImageConfigPullError) Error() string { - return "error pulling image configuration: " + e.Err.Error() -} - -// allowV1Fallback checks if the error is a possible reason to fallback to v1 -// (even if confirmedV2 has been set already), and if so, wraps the error in -// a fallbackError with confirmedV2 set to false. Otherwise, it returns the -// error unmodified. -func allowV1Fallback(err error) error { - switch v := err.(type) { - case errcode.Errors: - if len(v) != 0 { - if v0, ok := v[0].(errcode.Error); ok && shouldV2Fallback(v0) { - return fallbackError{err: err, confirmedV2: false, transportOK: true} - } - } - case errcode.Error: - if shouldV2Fallback(v) { - return fallbackError{err: err, confirmedV2: false, transportOK: true} - } - } - return err -} - -// schema2ManifestDigest computes the manifest digest, and, if pulling by -// digest, ensures that it matches the requested digest. -func schema2ManifestDigest(ref reference.Named, mfst distribution.Manifest) (digest.Digest, error) { - _, canonical, err := mfst.Payload() - if err != nil { - return "", err - } - - // If pull by digest, then verify the manifest digest. - if digested, isDigested := ref.(reference.Canonical); isDigested { - verifier, err := digest.NewDigestVerifier(digested.Digest()) - if err != nil { - return "", err - } - if _, err := verifier.Write(canonical); err != nil { - return "", err - } - if !verifier.Verified() { - err := fmt.Errorf("manifest verification failed for digest %s", digested.Digest()) - logrus.Error(err) - return "", err - } - return digested.Digest(), nil - } - - return digest.FromBytes(canonical), nil -} - -// pullManifestList handles "manifest lists" which point to various -// platform-specifc manifests. -func (mf *v2ManifestFetcher) pullManifestList(ctx context.Context, ref reference.Named, mfstList *manifestlist.DeserializedManifestList) (img *image.Image, manifestListDigest digest.Digest, err error) { - manifestListDigest, err = schema2ManifestDigest(ref, mfstList) - if err != nil { - return nil, "", err - } - - var manifestDigest digest.Digest - for _, manifestDescriptor := range mfstList.Manifests { - // TODO(aaronl): The manifest list spec supports optional - // "features" and "variant" fields. These are not yet used. - // Once they are, their values should be interpreted here. - if manifestDescriptor.Platform.Architecture == runtime.GOARCH && manifestDescriptor.Platform.OS == runtime.GOOS { - manifestDigest = manifestDescriptor.Digest - break - } - } - - if manifestDigest == "" { - return nil, "", errors.New("no supported platform found in manifest list") - } - - manSvc, err := mf.repo.Manifests(ctx) - if err != nil { - return nil, "", err - } - - manifest, err := manSvc.Get(ctx, manifestDigest) - if err != nil { - return nil, "", err - } - - manifestRef, err := reference.WithDigest(ref, manifestDigest) - if err != nil { - return nil, "", err - } - - switch v := manifest.(type) { - case *schema1.SignedManifest: - img, _, err = mf.pullSchema1(ctx, manifestRef, v) - if err != nil { - return nil, "", err - } - case *schema2.DeserializedManifest: - img, _, err = mf.pullSchema2(ctx, manifestRef, v) - if err != nil { - return nil, "", err - } - default: - return nil, "", errors.New("unsupported manifest format") - } - - return img, manifestListDigest, err -} diff --git a/integration/check_test.go b/integration/check_test.go index efb4ddac..ae05a5db 100644 --- a/integration/check_test.go +++ b/integration/check_test.go @@ -84,7 +84,7 @@ func (s *SkopeoSuite) TestVersion(c *check.C) { func (s *SkopeoSuite) TestCanAuthToPrivateRegistryV2WithoutDockerCfg(c *check.C) { out, err := exec.Command(skopeoBinary, "--docker-cfg=''", "--username="+s.regV2WithAuth.username, "--password="+s.regV2WithAuth.password, "inspect", fmt.Sprintf("docker://%s/busybox:latest", s.regV2WithAuth.url)).CombinedOutput() c.Assert(err, check.NotNil, check.Commentf(string(out))) - wanted := "Error: image busybox not found" + wanted := "Invalid status code returned when fetching manifest 401" if !strings.Contains(string(out), wanted) { c.Fatalf("wanted %s, got %s", wanted, string(out)) } @@ -93,7 +93,7 @@ func (s *SkopeoSuite) TestCanAuthToPrivateRegistryV2WithoutDockerCfg(c *check.C) func (s *SkopeoSuite) TestNeedAuthToPrivateRegistryV2WithoutDockerCfg(c *check.C) { out, err := exec.Command(skopeoBinary, "--docker-cfg=''", "inspect", fmt.Sprintf("docker://%s/busybox:latest", s.regV2WithAuth.url)).CombinedOutput() c.Assert(err, check.NotNil, check.Commentf(string(out))) - wanted := "no basic auth credentials" + wanted := "Invalid status code returned when fetching manifest 401" if !strings.Contains(string(out), wanted) { c.Fatalf("wanted %s, got %s", wanted, string(out)) } @@ -104,11 +104,11 @@ func (s *SkopeoSuite) TestNeedAuthToPrivateRegistryV2WithoutDockerCfg(c *check.C func (s *SkopeoSuite) TestNoNeedAuthToPrivateRegistryV2ImageNotFound(c *check.C) { out, err := exec.Command(skopeoBinary, "inspect", fmt.Sprintf("docker://%s/busybox:latest", s.regV2.url)).CombinedOutput() c.Assert(err, check.NotNil, check.Commentf(string(out))) - wanted := "Error: image busybox not found" + wanted := "Invalid status code returned when fetching manifest 404" if !strings.Contains(string(out), wanted) { c.Fatalf("wanted %s, got %s", wanted, string(out)) } - wanted = "no basic auth credentials" + wanted = "Invalid status code returned when fetching manifest 401" if strings.Contains(string(out), wanted) { c.Fatalf("not wanted %s, got %s", wanted, string(out)) } diff --git a/types/types.go b/types/types.go index ac56ea0e..3b36070b 100644 --- a/types/types.go +++ b/types/types.go @@ -5,20 +5,24 @@ import ( ) const ( + // DockerPrefix is the URL-like schema prefix used for Docker image references. DockerPrefix = "docker://" ) +// Registry is a service providing repositories. type Registry interface { Repositories() []Repository Repository(ref string) Repository Lookup(term string) []Image // docker registry v1 only AFAICT, v2 can be built hacking with Images() } +// Repository is a set of images. type Repository interface { Images() []Image Image(ref string) Image // ref == image name w/o registry part } +// Image is a Docker image in a repository. type Image interface { // ref to repository? Layers(layers ...string) error // configure download directory? Call it DownloadLayers? @@ -27,11 +31,14 @@ type Image interface { DockerTar() ([]byte, error) // ??? also, configure output directory } +// ImageManifest is the interesting subset of metadata about an Image. // TODO(runcom) type ImageManifest interface { Labels() map[string]string } +// DockerImageManifest is a set of metadata describing Docker images and their manifest.json files. +// Note that this is not exactly manifest.json, e.g. some fields have been added. type DockerImageManifest struct { Tag string Digest string @@ -47,6 +54,7 @@ type DockerImageManifest struct { Layers []string // ??? } +// Labels returns labels attached to this image. func (m *DockerImageManifest) Labels() map[string]string { if m.Config == nil { return nil diff --git a/utils.go b/utils.go index 43bf6679..fc90ec32 100644 --- a/utils.go +++ b/utils.go @@ -7,6 +7,7 @@ import ( "github.com/projectatomic/skopeo/types" ) +// ParseImage converts image URL-like string to an initialized handler for that image. func ParseImage(img string) (types.Image, error) { switch { case strings.HasPrefix(img, types.DockerPrefix): diff --git a/version.go b/version.go index 93cd1cc1..0ddd894b 100644 --- a/version.go +++ b/version.go @@ -1,5 +1,7 @@ package skopeo +// Version is a version of thils build. const Version = "0.1.12-dev" +// GitCommit is a git commit hash of this build. It is ordinarily overriden by LDFLAGS in Makefile. var GitCommit = ""