From ae0e8f570dc1d01222b411af6539f27f83bb63e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kupai=20J=C3=B3zsef?= Date: Sun, 22 Mar 2020 13:35:11 +0000 Subject: [PATCH] Added support for nested namespaces --- .gitignore | 1 + Dockerfile | 5 +- main.go | 112 ++++++++++++++++------------- registry/client.go | 49 +++---------- registry/common.go | 9 +++ registry/tasks.go | 39 ++++------ templates/{tags.html => list.html} | 39 +++++++--- templates/repositories.html | 68 ------------------ templates/tag_info.html | 7 +- 9 files changed, 132 insertions(+), 197 deletions(-) rename templates/{tags.html => list.html} (63%) delete mode 100644 templates/repositories.html diff --git a/.gitignore b/.gitignore index 7d21d57..59cebc7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ config-dev.yml data/registry_events.db vendor/ +*.swp diff --git a/Dockerfile b/Dockerfile index 01841a9..da1026c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,11 @@ RUN apk update && \ WORKDIR /opt/src ADD events events ADD registry registry -ADD *.go go.mod go.sum ./ +ADD go.mod go.sum ./ +RUN go mod download + +ADD *.go ./ RUN go test -v ./registry && \ go build -o /opt/docker-registry-ui *.go diff --git a/main.go b/main.go index c76183b..c4f55bb 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "os" + "regexp" "strings" "github.com/CloudyKit/jet" @@ -52,6 +53,21 @@ type apiClient struct { config configData } +type breadCrumb struct { + segment string + path string +} + +func getBreadCrumbs(path string) []breadCrumb { + ret := []breadCrumb{} + segments := strings.Split(path, "/") + for i := 0; i < len(segments); i++ { + e := breadCrumb{segment: segments[i], path: strings.Join(segments[0:i+1], "/")} + ret = append(ret, e) + } + return ret +} + func main() { var ( a apiClient @@ -149,13 +165,10 @@ func main() { e.File("/favicon.ico", "static/favicon.ico") e.Static(a.config.BasePath+"/static", "static") if a.config.BasePath != "" { - e.GET(a.config.BasePath, a.viewRepositories) + e.GET(a.config.BasePath, a.dispatchRequest) } - e.GET(a.config.BasePath+"/", a.viewRepositories) - e.GET(a.config.BasePath+"/:namespace", a.viewRepositories) - e.GET(a.config.BasePath+"/:namespace/:repo", a.viewTags) - e.GET(a.config.BasePath+"/:namespace/:repo/:tag", a.viewTagInfo) - e.GET(a.config.BasePath+"/:namespace/:repo/:tag/delete", a.deleteTag) + e.GET(a.config.BasePath+"/", a.dispatchRequest) + e.GET(a.config.BasePath+"/*", a.dispatchRequest) e.GET(a.config.BasePath+"/events", a.viewLog) // Protected event listener. @@ -170,58 +183,65 @@ func main() { e.Logger.Fatal(e.Start(a.config.ListenAddr)) } -func (a *apiClient) viewRepositories(c echo.Context) error { - namespace := c.Param("namespace") - if namespace == "" { - namespace = "library" +func (a *apiClient) dispatchRequest(c echo.Context) error { + path := c.Request().URL.Path + if strings.HasPrefix(path, a.config.BasePath) { + path = path[len(a.config.BasePath):] + } + if strings.HasPrefix(path, "/") { + path = path[1:] } - repos, _ := a.client.Repositories(true)[namespace] - data := jet.VarMap{} - data.Set("namespace", namespace) - data.Set("namespaces", a.client.Namespaces()) - data.Set("repos", repos) - data.Set("tagCounts", a.client.TagCounts()) - - return c.Render(http.StatusOK, "repositories.html", data) + segments := strings.Split(path, "/") + manifestRequest, _ := regexp.MatchString(".*/manifests/[^/]*$", path) + manifestDeleteRequest, _ := regexp.MatchString(".*/manifests/[^/]*/delete$", path) + if manifestRequest { + repoPath := strings.Join(segments[0:len(segments)-2], "/") + tagName := segments[len(segments)-1] + return a.viewTagInfo(c, repoPath, tagName) + } else if manifestDeleteRequest { + repoPath := strings.Join(segments[0:len(segments)-3], "/") + tagName := segments[len(segments)-2] + return a.deleteTag(c, repoPath, tagName) + } else { + return a.listRepo(c, path) + } } -func (a *apiClient) viewTags(c echo.Context) error { - namespace := c.Param("namespace") - repo := c.Param("repo") - repoPath := repo - if namespace != "library" { - repoPath = fmt.Sprintf("%s/%s", namespace, repo) - } - +func (a *apiClient) listRepo(c echo.Context, repoPath string) error { tags := a.client.Tags(repoPath) + repos := a.client.RepositoriesList(true) + + filterExpression := repoPath + if !strings.HasSuffix(repoPath, "/") && repoPath != "" { + filterExpression += "/" + } + matching_repos := registry.FilterStringSlice(repos, func(s string) bool { + return strings.HasPrefix(s, filterExpression) && len(s) > len(repoPath) + }) deleteAllowed := a.checkDeletePermission(c.Request().Header.Get("X-WEBAUTH-USER")) data := jet.VarMap{} - data.Set("namespace", namespace) - data.Set("repo", repo) + data.Set("repoPath", repoPath) + data.Set("breadCrumbs", getBreadCrumbs(repoPath)) data.Set("tags", tags) + data.Set("repos", matching_repos) data.Set("deleteAllowed", deleteAllowed) + data.Set("tagCounts", a.client.TagCounts()) repoPath, _ = url.PathUnescape(repoPath) data.Set("events", a.eventListener.GetEvents(repoPath)) - return c.Render(http.StatusOK, "tags.html", data) + return c.Render(http.StatusOK, "list.html", data) } -func (a *apiClient) viewTagInfo(c echo.Context) error { - namespace := c.Param("namespace") - repo := c.Param("repo") - tag := c.Param("tag") - repoPath := repo - if namespace != "library" { - repoPath = fmt.Sprintf("%s/%s", namespace, repo) - } +func (a *apiClient) viewTagInfo(c echo.Context, repoPath string, tag string) error { + repo := repoPath // Retrieve full image info from various versions of manifests sha256, infoV1, infoV2 := a.client.TagInfo(repoPath, tag, false) sha256list, manifests := a.client.ManifestList(repoPath, tag) if (infoV1 == "" || infoV2 == "") && len(manifests) == 0 { - return c.Redirect(http.StatusSeeOther, fmt.Sprintf("%s/%s/%s", a.config.BasePath, namespace, repo)) + return c.Redirect(http.StatusSeeOther, fmt.Sprintf("%s/%s", a.config.BasePath, repo)) } created := gjson.Get(gjson.Get(infoV1, "history.0.v1Compatibility").String(), "created").String() @@ -277,7 +297,7 @@ func (a *apiClient) viewTagInfo(c echo.Context) error { r["size"] = dSize // Create link here because there is a bug with jet template when referencing a value by map key in the "if" condition under "range". if r["mediaType"] == "application/vnd.docker.distribution.manifest.v2+json" { - r["digest"] = fmt.Sprintf(`%s`, a.config.BasePath, namespace, repo, r["digest"], r["digest"]) + r["digest"] = fmt.Sprintf(`%s`, a.config.BasePath, repo, r["digest"], r["digest"]) } } else { // Sub-image of the cache type. @@ -289,10 +309,10 @@ func (a *apiClient) viewTagInfo(c echo.Context) error { // Populate template vars data := jet.VarMap{} - data.Set("namespace", namespace) data.Set("repo", repo) data.Set("tag", tag) data.Set("repoPath", repoPath) + data.Set("breadCrumbs", getBreadCrumbs(repoPath)) data.Set("sha256", sha256) data.Set("imageSize", imageSize) data.Set("created", created) @@ -305,20 +325,12 @@ func (a *apiClient) viewTagInfo(c echo.Context) error { return c.Render(http.StatusOK, "tag_info.html", data) } -func (a *apiClient) deleteTag(c echo.Context) error { - namespace := c.Param("namespace") - repo := c.Param("repo") - tag := c.Param("tag") - repoPath := repo - if namespace != "library" { - repoPath = fmt.Sprintf("%s/%s", namespace, repo) - } - +func (a *apiClient) deleteTag(c echo.Context, repoPath string, tag string) error { if a.checkDeletePermission(c.Request().Header.Get("X-WEBAUTH-USER")) { a.client.DeleteTag(repoPath, tag) } - return c.Redirect(http.StatusSeeOther, fmt.Sprintf("%s/%s/%s", a.config.BasePath, namespace, repo)) + return c.Redirect(http.StatusSeeOther, fmt.Sprintf("%s/%s", a.config.BasePath, repoPath)) } // checkDeletePermission check if tag deletion is allowed whether by anyone or permitted users. diff --git a/registry/client.go b/registry/client.go index eba7049..bee36f2 100644 --- a/registry/client.go +++ b/registry/client.go @@ -5,7 +5,6 @@ import ( "crypto/tls" "fmt" "regexp" - "sort" "strings" "sync" "time" @@ -27,7 +26,7 @@ type Client struct { logger *logrus.Entry mux sync.Mutex tokens map[string]string - repos map[string][]string + repos []string tagCounts map[string]int authURL string } @@ -43,7 +42,7 @@ func NewClient(url string, verifyTLS bool, username, password string) *Client { request: gorequest.New().TLSClientConfig(&tls.Config{InsecureSkipVerify: !verifyTLS}), logger: SetupLogging("registry.client"), tokens: map[string]string{}, - repos: map[string][]string{}, + repos: []string{}, tagCounts: map[string]int{}, } resp, _, errs := c.request.Get(c.url+"/v2/"). @@ -153,21 +152,7 @@ func (c *Client) callRegistry(uri, scope, manifestFormat string) (string, gorequ return data, resp } -// Namespaces list repo namespaces. -func (c *Client) Namespaces() []string { - namespaces := make([]string, 0, len(c.repos)) - for k := range c.repos { - namespaces = append(namespaces, k) - } - if !ItemInSlice("library", namespaces) { - namespaces = append(namespaces, "library") - } - sort.Strings(namespaces) - return namespaces -} - -// Repositories list repos by namespaces where 'library' is the default one. -func (c *Client) Repositories(useCache bool) map[string][]string { +func (c *Client) RepositoriesList(useCache bool) []string { // Return from cache if available. if len(c.repos) > 0 && useCache { return c.repos @@ -179,7 +164,7 @@ func (c *Client) Repositories(useCache bool) map[string][]string { linkRegexp := regexp.MustCompile("^<(.*?)>;.*$") scope := "registry:catalog:*" uri := "/v2/_catalog" - c.repos = map[string][]string{} + c.repos = []string{} for { data, resp := c.callRegistry(uri, scope, "manifest.v2") if data == "" { @@ -187,14 +172,7 @@ func (c *Client) Repositories(useCache bool) map[string][]string { } for _, r := range gjson.Get(data, "repositories").Array() { - namespace := "library" - repo := r.String() - if strings.Contains(repo, "/") { - f := strings.SplitN(repo, "/", 2) - namespace = f[0] - repo = f[1] - } - c.repos[namespace] = append(c.repos[namespace], repo) + c.repos = append(c.repos, r.String()) } // pagination @@ -274,15 +252,9 @@ func (c *Client) CountTags(interval uint8) { for { start := time.Now() c.logger.Info("[CountTags] Calculating image tags...") - catalog := c.Repositories(false) - for n, repos := range catalog { - for _, r := range repos { - repoPath := r - if n != "library" { - repoPath = fmt.Sprintf("%s/%s", n, r) - } - c.tagCounts[fmt.Sprintf("%s/%s", n, r)] = len(c.Tags(repoPath)) - } + catalog := c.RepositoriesList(false) + for _, repoPath := range catalog { + c.tagCounts[repoPath] = len(c.Tags(repoPath)) } c.logger.Infof("[CountTags] Job complete (%v).", time.Now().Sub(start)) time.Sleep(time.Duration(interval) * time.Minute) @@ -307,10 +279,7 @@ func (c *Client) DeleteTag(repo, tag string) { if len(errs) > 0 { c.logger.Error(errs[0]) } else { - // Returns 202 on success. - if !strings.Contains(repo, "/") { - c.tagCounts["library/"+repo]-- - } else { + if c.tagCounts[repo] > 0 { c.tagCounts[repo]-- } c.logger.Infof("DELETE %s (tag:%s) %s", uri, tag, resp.Status) diff --git a/registry/common.go b/registry/common.go index c60919e..87385bc 100644 --- a/registry/common.go +++ b/registry/common.go @@ -58,3 +58,12 @@ func ItemInSlice(item string, slice []string) bool { } return false } + +func FilterStringSlice(ss []string, test func(string) bool) (ret []string) { + for _, s := range ss { + if test(s) { + ret = append(ret, s) + } + } + return +} diff --git a/registry/tasks.go b/registry/tasks.go index c11cefe..6364410 100644 --- a/registry/tasks.go +++ b/registry/tasks.go @@ -44,40 +44,31 @@ func PurgeOldTags(client *Client, purgeDryRun bool, purgeTagsKeepDays, purgeTags dryRunText = "skipped" } logger.Info("Scanning registry for repositories, tags and their creation dates...") - catalog := client.Repositories(true) - // catalog := map[string][]string{"library": []string{""}} + catalog := client.RepositoriesList(true) now := time.Now().UTC() repos := map[string]timeSlice{} - count := 0 - for namespace := range catalog { - count = count + len(catalog[namespace]) - for _, repo := range catalog[namespace] { - if namespace != "library" { - repo = fmt.Sprintf("%s/%s", namespace, repo) - } - - tags := client.Tags(repo) - logger.Infof("[%s] scanning %d tags...", repo, len(tags)) - if len(tags) == 0 { + for _, repo := range catalog { + tags := client.Tags(repo) + logger.Infof("[%s] scanning %d tags...", repo, len(tags)) + if len(tags) == 0 { + continue + } + for _, tag := range tags { + _, infoV1, _ := client.TagInfo(repo, tag, true) + if infoV1 == "" { + logger.Errorf("[%s] missing manifest v1 for tag %s", repo, tag) continue } - for _, tag := range tags { - _, infoV1, _ := client.TagInfo(repo, tag, true) - if infoV1 == "" { - logger.Errorf("[%s] missing manifest v1 for tag %s", repo, tag) - continue - } - created := gjson.Get(gjson.Get(infoV1, "history.0.v1Compatibility").String(), "created").Time() - repos[repo] = append(repos[repo], tagData{name: tag, created: created}) - } + created := gjson.Get(gjson.Get(infoV1, "history.0.v1Compatibility").String(), "created").Time() + repos[repo] = append(repos[repo], tagData{name: tag, created: created}) } } - logger.Infof("Scanned %d repositories.", count) + logger.Infof("Scanned %d repositories.", len(catalog)) logger.Info("Filtering out tags for purging...") purgeTags := map[string][]string{} keepTags := map[string][]string{} - count = 0 + count := 0 for _, repo := range SortedMapKeys(repos) { // Sort tags by "created" from newest to oldest. sortedTags := make(timeSlice, 0, len(repos[repo])) diff --git a/templates/tags.html b/templates/list.html similarity index 63% rename from templates/tags.html rename to templates/list.html index 8704a03..ae02397 100644 --- a/templates/tags.html +++ b/templates/list.html @@ -6,14 +6,18 @@ {{end}} @@ -33,27 +36,43 @@ {{block body()}}
  1. {{ registryHost }}
    2. - {{if namespace != "library"}} -
  2. {{ namespace }}
    3. + {{range crumb := breadCrumbs}} +
  3. {{ crumb.segment }}
    4. {{end}} -
  4. {{ repo|url_decode }}
- + + + {{range tag := tags}} + + + + {{end}} + {{range repo := repos}} + + + + {{end}} diff --git a/templates/repositories.html b/templates/repositories.html deleted file mode 100644 index fd7ac5d..0000000 --- a/templates/repositories.html +++ /dev/null @@ -1,68 +0,0 @@ -{{extends "base.html"}} - -{{block head()}} - -{{end}} - -{{block body()}} -
- -
-
-
    -
  1. Namespace
    2. -
-
- -
    -
  1. {{ registryHost }}
    2. - {{if namespace != "library"}} -
  2. {{ namespace }}
    3. - {{end}} -
- -
Tag NameNameTag countType
- {{ tag }} + {{ tag }} {{if deleteAllowed}} - Delete + Delete {{end}} n/aTags
+ {{noprefix_repo := repo[len(repoPath):]}} + {{if noprefix_repo[0:1] == "/"}} + {{noprefix_repo = noprefix_repo[1:]}} + {{end}} + {{ noprefix_repo }} + {{ tagCounts[repo] }}Repositories
- - - - - - - - {{range repo := repos}} - {{if !isset(tagCounts[namespace+"/"+repo]) || (isset(tagCounts[namespace+"/"+repo]) && tagCounts[namespace+"/"+repo] > 0)}} - - - - - {{end}} - {{end}} - -
RepositoryTags
{{ repo }}{{ tagCounts[namespace+"/"+repo] }}
-{{end}} diff --git a/templates/tag_info.html b/templates/tag_info.html index 718f3d9..9a7f258 100644 --- a/templates/tag_info.html +++ b/templates/tag_info.html @@ -5,11 +5,10 @@ {{block body()}}
  1. {{ registryHost }}
    2. - {{if namespace != "library"}} -
  2. {{ namespace }}
    3. + {{range crumb := breadCrumbs}} +
  3. {{ crumb.segment }}
    4. {{end}} -
  4. {{ repo|url_decode }}
    5. -
  5. {{ tag }}
    6. +
  6. {{ tag }}

Image Details