From 67d82c7d596379ada0b4549c38ff051c1fa850e5 Mon Sep 17 00:00:00 2001 From: Roman Vynar Date: Tue, 18 Feb 2020 23:31:56 +0200 Subject: [PATCH] Amend tag info page, change logging. * Amend representation of the tag info page * Change logging library, add "-log-level" argument and put most of the logging into DEBUG mode --- CHANGELOG.md | 8 ++- README.md | 15 ++++- events/event_listener.go | 4 +- go.mod | 4 +- go.sum | 15 +++-- main.go | 79 ++++++++++++++----------- registry/client.go | 121 +++++++++++++++++++++++---------------- registry/common.go | 32 ++++++----- registry/common_test.go | 4 +- registry/tasks.go | 4 +- templates/tag_info.html | 47 ++++++++------- 11 files changed, 199 insertions(+), 134 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 988b039..f9e6b7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,17 @@ ## Changelog -### 0.9.0 (2020-02-17) - unreleased +### 0.9.0 (2020-02-19) - unreleased +* Upgrade Go version to 1.13.7, alpine to 3.11 and other dependencies. * Support Manifest List v2. This enables the proper display of multi-arch images, such as those generated by Docker BuildX or manually (thanks to Christoph Honal @StarGate01). -* Upgrade Go version to 1.13.7, alpine to 3.11 and other dependencies. + So now we support the following formats: Manifest v2 schema 1, Manifest v2 schema 2, Manifest List v2 schema 2 + and all their confusing combinations. +* Change logging library, add "-log-level" argument and put most of the logging into DEBUG mode. * You can define timezone when running container by adding `TZ` env var, e.g. "-e TZ=America/Los_Angeles" (thanks to @gminog). * Fix initial ownership of /opt/data dir in Dockerfile. +* Amend representation of the tag info page. ### 0.8.2 (2019-07-30) diff --git a/README.md b/README.md index a4b4d7a..b11a75b 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ * Web UI for Docker Registry 2.6+ * Browse repositories and tags -* Display Docker image details by layers including both manifests v1 and v2 +* Display image details by layers +* Support Manifest v2 schema 1, Manifest v2 schema 2, Manifest List v2 schema 2 and their confusing combinations * Fast and small, written on Go * Automatically discover an authentication method (basic auth, token service etc.) * Caching the list of repositories, tag counts and refreshing in background @@ -43,7 +44,7 @@ you need to ensure the sqlite db can be written, i.e. mount a folder as listed a To run with a custom TZ: - -e TZ=America/Los_Angeles + -e TZ=America/Los_Angeles ## Configure event listener on Docker Registry @@ -110,6 +111,16 @@ Note, the cron schedule format includes seconds! See https://godoc.org/github.co To increase http request verbosity, run container with `-e GOREQUEST_DEBUG=1`. +### About Docker image formats... + +Docker image formats and their confusing combinations as supported by this UI: + +* Manifest v2 schema 1 only: older format, e.g. created with Docker 1.9. +* Manifest v2 schema 1 + Manifest v2 schema 2: current format of a single image, the image history are coming from schema 1, should be referenced by repo:tag name. +* Manifest v2 schema 1 + Manifest List v2 schema 2: multi-arch image format containing digests of sub-images, the image history are coming from schema 1 (no idea from what sub-image it was picked up when created), should be referenced by repo:tag name. +* Manifest v2 schema 2: current image format referenced by its digest sha256, no image history. +* Manifest List v2 schema 2: multi-arch image format referenced by its digest sha256, no image history. + ### Screenshots ![image](screenshots/1.png) diff --git a/events/event_listener.go b/events/event_listener.go index 561ea35..6fe7cfe 100644 --- a/events/event_listener.go +++ b/events/event_listener.go @@ -8,8 +8,8 @@ import ( "os" "strings" - "github.com/hhkbp2/go-logging" "github.com/quiq/docker-registry-ui/registry" + "github.com/sirupsen/logrus" // 🐒 patching of "database/sql". _ "github.com/go-sql-driver/mysql" @@ -37,7 +37,7 @@ type EventListener struct { databaseLocation string retention int eventDeletion bool - logger logging.Logger + logger *logrus.Entry } type eventData struct { diff --git a/go.mod b/go.mod index df20dac..1748347 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,12 @@ require ( github.com/CloudyKit/jet v2.1.2+incompatible github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect github.com/go-sql-driver/mysql v1.5.0 - github.com/hhkbp2/go-logging v0.3.0 - github.com/hhkbp2/go-strftime v0.0.0-20150709091403-d82166ec6782 // indirect - github.com/hhkbp2/testify v0.0.0-20150512090439-112845ebc045 // indirect github.com/labstack/echo/v4 v4.1.14 github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/parnurzeal/gorequest v0.2.16 github.com/pkg/errors v0.9.1 // indirect github.com/robfig/cron v1.2.0 + github.com/sirupsen/logrus v1.4.2 github.com/smartystreets/goconvey v1.6.4 github.com/tidwall/gjson v1.5.0 gopkg.in/yaml.v2 v2.2.8 diff --git a/go.sum b/go.sum index e560299..3df453e 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/CloudyKit/jet v2.1.2+incompatible h1:ybZoYzMBdoijK6I+Ke3vg9GZsmlKo/Zh github.com/CloudyKit/jet v2.1.2+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 h1:pEtiCjIXx3RvGjlUJuCNxNOw0MNblyR9Wi+vJGBFh+8= @@ -14,14 +16,10 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/hhkbp2/go-logging v0.3.0 h1:P8WoxS+VgYa0Nfu9HSKgQZZmz7wrGzpuy33mBeAX4+I= -github.com/hhkbp2/go-logging v0.3.0/go.mod h1:zAp/KbVJna4DHUdeSPYGsRNn9c62x569NIr9ssBuZ/I= -github.com/hhkbp2/go-strftime v0.0.0-20150709091403-d82166ec6782 h1:Evl9i7wBY3bjJ3NqHs0ldhnKOdQL4Kaum9ve1JAmiCE= -github.com/hhkbp2/go-strftime v0.0.0-20150709091403-d82166ec6782/go.mod h1:x8/IOQ5qQ4DKfiTmD9wBhQ40edg5wh7gMRwdLg07mMw= -github.com/hhkbp2/testify v0.0.0-20150512090439-112845ebc045 h1:MmQwR3zANTXzs2yZexVBDY6qcH2vJXOl/2dZFkWVM7w= -github.com/hhkbp2/testify v0.0.0-20150512090439-112845ebc045/go.mod h1:8DUHF4igllRoOCbQKJsylsDqROcRtPTdb+SQUfjCYLo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/labstack/echo/v4 v4.1.14 h1:h8XP66UfB3tUm+L3QPw7tmwAu3pJaA/nyfHPCcz46ic= github.com/labstack/echo/v4 v4.1.14/go.mod h1:Q5KZ1vD3V5FEzjM79hjwVrC3ABr7F5IdM23bXQMRDGg= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= @@ -44,11 +42,15 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/gjson v1.5.0 h1:QCssIUI7J0RStkzIcI4A7O6P8rDA5wi5IPf70uqKSxg= @@ -75,6 +77,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8 h1:JA8d3MPx/IToSyXZG/RhwYEtfrKO1Fxrqe8KrkiLXKM= diff --git a/main.go b/main.go index d4dbc38..07a766b 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,7 @@ import ( "github.com/quiq/docker-registry-ui/events" "github.com/quiq/docker-registry-ui/registry" "github.com/robfig/cron" + "github.com/sirupsen/logrus" "github.com/tidwall/gjson" "gopkg.in/yaml.v2" ) @@ -53,16 +54,23 @@ type apiClient struct { func main() { var ( - a apiClient - configFile string - purgeTags bool - purgeDryRun bool + a apiClient + + configFile, loggingLevel string + purgeTags, purgeDryRun bool ) flag.StringVar(&configFile, "config-file", "config.yml", "path to the config file") + flag.StringVar(&loggingLevel, "log-level", "info", "logging level") flag.BoolVar(&purgeTags, "purge-tags", false, "purge old tags instead of running a web server") flag.BoolVar(&purgeDryRun, "dry-run", false, "dry-run for purging task, does not delete anything") flag.Parse() + if loggingLevel != "info" { + if level, err := logrus.ParseLevel(loggingLevel); err == nil { + logrus.SetLevel(level) + } + } + // Read config file. if _, err := os.Stat(configFile); os.IsNotExist(err) { panic(err) @@ -209,17 +217,35 @@ func (a *apiClient) viewTagInfo(c echo.Context) error { repoPath = fmt.Sprintf("%s/%s", namespace, repo) } + // Retrieve full image info from various versions of manifests sha256, infoV1, infoV2 := a.client.TagInfo(repoPath, tag, false) - manifests := a.client.Manifests(repoPath, tag) + sha256list, manifests := a.client.ManifestList(repoPath, tag) if (infoV1 == "" || infoV2 == "") && len(manifests) == 0 { return c.Redirect(http.StatusSeeOther, fmt.Sprintf("%s/%s/%s", a.config.BasePath, namespace, repo)) } - isListOnly := (infoV1 == "" && infoV2 == "") - newRepoPath := gjson.Get(infoV1, "name").String() - if newRepoPath != "" { - repoPath = newRepoPath + + created := gjson.Get(gjson.Get(infoV1, "history.0.v1Compatibility").String(), "created").String() + isDigest := strings.HasPrefix(tag, "sha256:") + if len(manifests) > 0 { + sha256 = sha256list } + // Gather layers v2 + var layersV2 []map[string]gjson.Result + for _, s := range gjson.Get(infoV2, "layers").Array() { + layersV2 = append(layersV2, s.Map()) + } + + // Gather layers v1 + var layersV1 []map[string]interface{} + for _, s := range gjson.Get(infoV1, "history.#.v1Compatibility").Array() { + m, _ := gjson.Parse(s.String()).Value().(map[string]interface{}) + // Sort key in the map to show the ordered on UI. + m["ordered_keys"] = registry.SortedMapKeys(m) + layersV1 = append(layersV1, m) + } + + // Count image size var imageSize int64 if gjson.Get(infoV2, "layers").Exists() { for _, s := range gjson.Get(infoV2, "layers.#.size").Array() { @@ -231,56 +257,45 @@ func (a *apiClient) viewTagInfo(c echo.Context) error { } } - var layersV2 []map[string]gjson.Result - for _, s := range gjson.Get(infoV2, "layers").Array() { - layersV2 = append(layersV2, s.Map()) - } - - var layersV1 []map[string]interface{} - for _, s := range gjson.Get(infoV1, "history.#.v1Compatibility").Array() { - m, _ := gjson.Parse(s.String()).Value().(map[string]interface{}) - // Sort key in the map to show the ordered on UI. - m["ordered_keys"] = registry.SortedMapKeys(m) - layersV1 = append(layersV1, m) - } - + // Count layers layersCount := len(layersV2) if layersCount == 0 { layersCount = len(gjson.Get(infoV1, "fsLayers").Array()) } - isDigest := strings.HasPrefix(tag, "sha256:") - var digests []map[string]interface{} + // Gather sub-image info of multi-arch image + var digestList []map[string]interface{} for _, s := range manifests { r, _ := gjson.Parse(s.String()).Value().(map[string]interface{}) if s.Get("mediaType").String() == "application/vnd.docker.distribution.manifest.v2+json" { - _, _, dInfo := a.client.TagInfo(repoPath, s.Get("digest").String(), false) + _, dInfoV1, _ := a.client.TagInfo(repoPath, s.Get("digest").String(), true) var dSize int64 - for _, d := range gjson.Get(dInfo, "layers.#.size").Array() { + for _, d := range gjson.Get(dInfoV1, "layers.#.size").Array() { dSize = dSize + d.Int() } r["size"] = dSize } else { r["size"] = s.Get("size").Int() } + delete(r, "mediaType") r["ordered_keys"] = registry.SortedMapKeys(r) - digests = append(digests, r) + digestList = append(digestList, r) } + // Populate template vars data := jet.VarMap{} data.Set("namespace", namespace) data.Set("repo", repo) - data.Set("sha256", sha256) - data.Set("imageSize", imageSize) data.Set("tag", tag) data.Set("repoPath", repoPath) - data.Set("created", gjson.Get(gjson.Get(infoV1, "history.0.v1Compatibility").String(), "created").String()) + data.Set("sha256", sha256) + data.Set("imageSize", imageSize) + data.Set("created", created) data.Set("layersCount", layersCount) data.Set("layersV2", layersV2) data.Set("layersV1", layersV1) data.Set("isDigest", isDigest) - data.Set("isListOnly", isListOnly) - data.Set("digests", digests) + data.Set("digestList", digestList) return c.Render(http.StatusOK, "tag_info.html", data) } diff --git a/registry/client.go b/registry/client.go index 184c0ef..d88292c 100644 --- a/registry/client.go +++ b/registry/client.go @@ -10,11 +10,13 @@ import ( "sync" "time" - "github.com/hhkbp2/go-logging" "github.com/parnurzeal/gorequest" + "github.com/sirupsen/logrus" "github.com/tidwall/gjson" ) +const userAgent = "docker-registry-ui" + // Client main class. type Client struct { url string @@ -22,7 +24,7 @@ type Client struct { username string password string request *gorequest.SuperAgent - logger logging.Logger + logger *logrus.Entry mux sync.Mutex tokens map[string]string repos map[string][]string @@ -44,7 +46,8 @@ func NewClient(url string, verifyTLS bool, username, password string) *Client { repos: map[string][]string{}, tagCounts: map[string]int{}, } - resp, _, errs := c.request.Get(c.url+"/v2/").Set("User-Agent", "docker-registry-ui").End() + resp, _, errs := c.request.Get(c.url+"/v2/"). + Set("User-Agent", userAgent).End() if len(errs) > 0 { c.logger.Error(errs[0]) return nil @@ -82,14 +85,18 @@ func NewClient(url string, verifyTLS bool, username, password string) *Client { func (c *Client) getToken(scope string) string { // Check if we have already a token and it's not expired. if token, ok := c.tokens[scope]; ok { - resp, _, _ := c.request.Get(c.url+"/v2/").Set("Authorization", fmt.Sprintf("Bearer %s", token)).Set("User-Agent", "docker-registry-ui").End() + resp, _, _ := c.request.Get(c.url+"/v2/"). + Set("Authorization", fmt.Sprintf("Bearer %s", token)). + Set("User-Agent", userAgent).End() if resp != nil && resp.StatusCode == 200 { return token } } request := gorequest.New().TLSClientConfig(&tls.Config{InsecureSkipVerify: !c.verifyTLS}) - resp, data, errs := request.Get(fmt.Sprintf("%s&scope=%s", c.authURL, scope)).SetBasicAuth(c.username, c.password).Set("User-Agent", "docker-registry-ui").End() + resp, data, errs := request.Get(fmt.Sprintf("%s&scope=%s", c.authURL, scope)). + SetBasicAuth(c.username, c.password). + Set("User-Agent", userAgent).End() if len(errs) > 0 { c.logger.Error(errs[0]) return "" @@ -105,53 +112,38 @@ func (c *Client) getToken(scope string) string { return c.tokens[scope] } -// callRegistry make an HTTP request to Docker registry. -func (c *Client) callRegistry(uri, scope string, manifest uint, delete bool, list bool) (string, gorequest.Response) { - endpoint := "manifest" - if list { - endpoint = "manifest.list" - } - acceptHeader := fmt.Sprintf("application/vnd.docker.distribution.%s.v%d+json", endpoint, manifest) +// callRegistry make an HTTP request to retrieve data from Docker registry. +func (c *Client) callRegistry(uri, scope, manifestFormat string) (string, gorequest.Response) { + acceptHeader := fmt.Sprintf("application/vnd.docker.distribution.%s+json", manifestFormat) authHeader := "" if c.authURL != "" { authHeader = fmt.Sprintf("Bearer %s", c.getToken(scope)) } - resp, data, errs := c.request.Get(c.url+uri).Set("Accept", acceptHeader).Set("Authorization", authHeader).Set("User-Agent", "docker-registry-ui").End() + resp, data, errs := c.request.Get(c.url+uri). + Set("Accept", acceptHeader). + Set("Authorization", authHeader). + Set("User-Agent", userAgent).End() if len(errs) > 0 { c.logger.Error(errs[0]) return "", resp } - c.logger.Info("GET ", uri, " ", resp.Status) + c.logger.Debugf("GET %s %s", uri, resp.Status) // Returns 404 when no tags in the repo. if resp.StatusCode != 200 { return "", resp } + // Ensure Docker-Content-Digest header is present as we use it in various places. + // The header is probably in AWS ECR case. digest := resp.Header.Get("Docker-Content-Digest") if digest == "" { - // Try to get digest from body instead, should be equal to what would be presented - // in Docker-Content-Digest + // Try to get digest from body instead, should be equal to what would be presented in Docker-Content-Digest. h := crypto.SHA256.New() h.Write([]byte(data)) resp.Header.Set("Docker-Content-Digest", fmt.Sprintf("sha256:%x", h.Sum(nil))) } - - if delete { - // Delete by manifest digest reference. - parts := strings.Split(uri, "/manifests/") - uri = parts[0] + "/manifests/" + digest - resp, _, errs := c.request.Delete(c.url+uri).Set("Accept", acceptHeader).Set("Authorization", authHeader).Set("User-Agent", "docker-registry-ui").End() - if len(errs) > 0 { - c.logger.Error(errs[0]) - } else { - // Returns 202 on success. - c.logger.Info("DELETE ", uri, " (", parts[1], ") ", resp.Status) - } - return "", resp - } - return data, resp } @@ -183,7 +175,7 @@ func (c *Client) Repositories(useCache bool) map[string][]string { uri := "/v2/_catalog" c.repos = map[string][]string{} for { - data, resp := c.callRegistry(uri, scope, 2, false, false) + data, resp := c.callRegistry(uri, scope, "manifest.v2") if data == "" { return c.repos } @@ -216,7 +208,7 @@ func (c *Client) Repositories(useCache bool) map[string][]string { // Tags get tags for the repo. func (c *Client) Tags(repo string) []string { scope := fmt.Sprintf("repository:%s:*", repo) - data, _ := c.callRegistry(fmt.Sprintf("/v2/%s/tags/list", repo), scope, 2, false, false) + data, _ := c.callRegistry(fmt.Sprintf("/v2/%s/tags/list", repo), scope, "manifest.v2") var tags []string for _, t := range gjson.Get(data, "tags").Array() { tags = append(tags, t.String()) @@ -224,32 +216,45 @@ func (c *Client) Tags(repo string) []string { return tags } -// Manifests gets manifest list entries for a tag for the repo. -func (c *Client) Manifests(repo string, tag string) []gjson.Result { +// ManifestList gets manifest list entries for a tag for the repo. +func (c *Client) ManifestList(repo, tag string) (string, []gjson.Result) { scope := fmt.Sprintf("repository:%s:*", repo) - data, _ := c.callRegistry(fmt.Sprintf("/v2/%s/manifests/%s", repo, tag), scope, 2, false, true) - return gjson.Get(data, "manifests").Array() + uri := fmt.Sprintf("/v2/%s/manifests/%s", repo, tag) + // If manifest.list.v2 does not exist because it's a normal image, + // the registry returns manifest.v1 or manifest.v2 if requested by sha256. + info, resp := c.callRegistry(uri, scope, "manifest.list.v2") + digest := resp.Header.Get("Docker-Content-Digest") + sha256 := "" + if digest != "" { + sha256 = digest[7:] + } + c.logger.Debugf(`Received manifest.list.v2 with sha256 "%s" from %s: %s`, sha256, uri, info) + return sha256, gjson.Get(info, "manifests").Array() } -// TagInfo get image info for the repo tag. -func (c *Client) TagInfo(repo, tag string, v1only bool) (rsha256, rinfoV1, rinfoV2 string) { +// TagInfo get image info for the repo tag or digest sha256. +func (c *Client) TagInfo(repo, tag string, v1only bool) (string, string, string) { scope := fmt.Sprintf("repository:%s:*", repo) - infoV1, _ := c.callRegistry(fmt.Sprintf("/v2/%s/manifests/%s", repo, tag), scope, 1, false, false) - if infoV1 == "" { - return "", "", "" - } - - if v1only { + uri := fmt.Sprintf("/v2/%s/manifests/%s", repo, tag) + // Note, if manifest.v1 does not exist because the image is requested by sha256, + // the registry returns manifest.v2 instead or manifest.list.v2 if it's the manifest list! + infoV1, _ := c.callRegistry(uri, scope, "manifest.v1") + c.logger.Debugf("Received manifest.v1 from %s: %s", uri, infoV1) + if infoV1 == "" || v1only { return "", infoV1, "" } - infoV2, resp := c.callRegistry(fmt.Sprintf("/v2/%s/manifests/%s", repo, tag), scope, 2, false, false) + // Note, if manifest.v2 does not exist because the image is in the older format (Docker 1.9), + // the registry returns manifest.v1 instead or manifest.list.v2 if it's the manifest list requested by sha256! + infoV2, resp := c.callRegistry(uri, scope, "manifest.v2") + c.logger.Debugf("Received manifest.v2 from %s: %s", uri, infoV2) digest := resp.Header.Get("Docker-Content-Digest") if infoV2 == "" || digest == "" { return "", "", "" } sha256 := digest[7:] + c.logger.Debugf("sha256 for %s/%s is %s", repo, tag, sha256) return sha256, infoV1, infoV2 } @@ -261,7 +266,8 @@ func (c *Client) TagCounts() map[string]int { // CountTags count repository tags in background regularly. func (c *Client) CountTags(interval uint8) { for { - c.logger.Info("Calculating tags in background...") + start := time.Now() + c.logger.Info("[CountTags] Calculating image tags...") catalog := c.Repositories(false) for n, repos := range catalog { for _, r := range repos { @@ -272,7 +278,7 @@ func (c *Client) CountTags(interval uint8) { c.tagCounts[fmt.Sprintf("%s/%s", n, r)] = len(c.Tags(repoPath)) } } - c.logger.Info("Tags calculation complete.") + c.logger.Infof("[CountTags] Job complete (%v).", time.Now().Sub(start)) time.Sleep(time.Duration(interval) * time.Minute) } } @@ -280,5 +286,22 @@ func (c *Client) CountTags(interval uint8) { // DeleteTag delete image tag. func (c *Client) DeleteTag(repo, tag string) { scope := fmt.Sprintf("repository:%s:*", repo) - c.callRegistry(fmt.Sprintf("/v2/%s/manifests/%s", repo, tag), scope, 2, true, false) + // Get sha256 digest for tag. + _, resp := c.callRegistry(fmt.Sprintf("/v2/%s/manifests/%s", repo, tag), scope, "manifest.v2") + + // Delete by manifest digest reference. + authHeader := "" + if c.authURL != "" { + authHeader = fmt.Sprintf("Bearer %s", c.getToken(scope)) + } + uri := fmt.Sprintf("/v2/%s/manifests/%s", repo, resp.Header.Get("Docker-Content-Digest")) + resp, _, errs := c.request.Delete(c.url+uri). + Set("Authorization", authHeader). + Set("User-Agent", userAgent).End() + if len(errs) > 0 { + c.logger.Error(errs[0]) + } else { + // Returns 202 on success. + c.logger.Infof("DELETE %s (tag:%s) %s", uri, tag, resp.Status) + } } diff --git a/registry/common.go b/registry/common.go index 1e5a2b8..c60919e 100644 --- a/registry/common.go +++ b/registry/common.go @@ -2,23 +2,24 @@ package registry import ( "fmt" + "os" "reflect" "sort" + "time" - "github.com/hhkbp2/go-logging" + "github.com/sirupsen/logrus" ) -// SetupLogging configure logging. -func SetupLogging(name string) logging.Logger { - logger := logging.GetLogger(name) - handler := logging.NewStdoutHandler() - format := "%(asctime)s - %(name)s - %(levelname)s - %(message)s" - dateFormat := "%Y-%m-%d %H:%M:%S" - formatter := logging.NewStandardFormatter(format, dateFormat) - handler.SetFormatter(formatter) - logger.SetLevel(logging.LevelInfo) - logger.AddHandler(handler) - return logger +// SetupLogging setup logger +func SetupLogging(name string) *logrus.Entry { + logrus.SetFormatter(&logrus.TextFormatter{ + TimestampFormat: time.RFC3339, + FullTimestamp: true, + }) + // Output to stdout instead of the default stderr. + logrus.SetOutput(os.Stdout) + + return logrus.WithFields(logrus.Fields{"logger": name}) } // SortedMapKeys sort keys of the map where values can be of any type. @@ -40,7 +41,12 @@ func PrettySize(size float64) string { size = size / 1024 i = i + 1 } - return fmt.Sprintf("%.*f %s", 0, size, units[i]) + // Format decimals as follow: 0 B, 0 KB, 0.0 MB, 0.00 GB + decimals := i - 1 + if decimals < 0 { + decimals = 0 + } + return fmt.Sprintf("%.*f %s", decimals, size, units[i]) } // ItemInSlice check if item is an element of slice diff --git a/registry/common_test.go b/registry/common_test.go index 6e1b93f..eb6ee2b 100644 --- a/registry/common_test.go +++ b/registry/common_test.go @@ -37,8 +37,8 @@ func TestPrettySize(t *testing.T) { 123: "123 B", 23123: "23 KB", 23923: "23 KB", - 723425120: "690 MB", - 8534241213: "8 GB", + 723425120: "689.9 MB", + 8534241213: "7.95 GB", } for key, val := range input { convey.So(PrettySize(key), convey.ShouldEqual, val) diff --git a/registry/tasks.go b/registry/tasks.go index 908d692..c11cefe 100644 --- a/registry/tasks.go +++ b/registry/tasks.go @@ -5,7 +5,7 @@ import ( "sort" "time" - "github.com/hhkbp2/go-logging" + "github.com/sirupsen/logrus" "github.com/tidwall/gjson" ) @@ -36,7 +36,7 @@ func (p timeSlice) Swap(i, j int) { func PurgeOldTags(client *Client, purgeDryRun bool, purgeTagsKeepDays, purgeTagsKeepCount int) { logger := SetupLogging("registry.tasks.PurgeOldTags") // Reduce client logging. - client.logger.SetLevel(logging.LevelError) + client.logger.Logger.SetLevel(logrus.ErrorLevel) dryRunText := "" if purgeDryRun { diff --git a/templates/tag_info.html b/templates/tag_info.html index 716f427..51faca8 100644 --- a/templates/tag_info.html +++ b/templates/tag_info.html @@ -11,36 +11,45 @@
  • {{ repo|url_decode }}
  • {{ tag }}
    • + +

    Image Details

    - + - + - {{if not isListOnly}} - + {{if not isDigest}} - + + + {{end}} + {{if not digestList}} + + + + + {{end}} - + + - - - - {{end}}
    Image DetailsSummary
    Image{{ registryHost }}/{{ repoPath }}:{{ tag }}Image URL{{ registryHost }}/{{ repoPath }}{{if isDigest}}@{{else}}:{{end}}{{ tag }}
    sha256{{ sha256 }}Digestsha256:{{ sha256 }}
    Created On{{ created|pretty_time }}Created On{{ created|pretty_time }}
    Image Size{{ imageSize|pretty_size }}
    Layer Count{{ layersCount }}
    Image Size{{ imageSize|pretty_size }}Manifest Formats{{if not isDigest}}Manifest v2 schema 1{{else}}Manifest v2 schema 1{{end}} | + {{if not digestList && layersV2}}Manifest v2 schema 2{{else}}Manifest v2 schema 2{{end}} | + {{if digestList}}Manifest List v2 schema 2{{else}}Manifest List v2 schema 2{{end}} +
    Layer Count{{ layersCount }}
    -{{if digests}} -

    Manifest List v2

    -{{range index, manifest := digests}} +{{if digestList}} +

    Multi-arch Sub-images

    +{{range index, manifest := digestList}} @@ -60,11 +69,7 @@ {{else if key == "size"}} {{else if key == "digest"}} - {{if not isListOnly}} - - {{else}} - - {{end}} + {{else}} {{end}} @@ -73,7 +78,7 @@
    {{ manifest[key]|pretty_size }}{{ manifest["digest"] }}{{ manifest["digest"] }}{{ manifest["digest"] }}{{ manifest[key] }}
    {{end}} {{else if layersV2}} -

    Manifest v2

    +

    Blobs

    @@ -92,8 +97,8 @@
    {{end}} -{{if not isListOnly && not isDigest}} -

    Manifest v1

    +{{if not isDigest}} +

    Image History

    {{range index, layer := layersV1}}