From 17480452857b5d532f7e5f6a08a17200ca41ef8c Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Sun, 22 Jun 2025 19:22:51 +0800 Subject: [PATCH] Refactor packages (#34777) --- modules/web/router_path.go | 36 ++- modules/web/router_test.go | 82 ++++--- routers/api/packages/api.go | 232 +++----------------- routers/api/packages/conda/conda.go | 26 ++- routers/api/packages/container/blob.go | 8 +- routers/api/packages/container/container.go | 2 +- routers/api/packages/container/manifest.go | 8 + services/packages/container/cleanup.go | 2 +- 8 files changed, 149 insertions(+), 247 deletions(-) diff --git a/modules/web/router_path.go b/modules/web/router_path.go index 1531ccd01c..ce041eedab 100644 --- a/modules/web/router_path.go +++ b/modules/web/router_path.go @@ -6,6 +6,7 @@ package web import ( "net/http" "regexp" + "slices" "strings" "code.gitea.io/gitea/modules/container" @@ -36,11 +37,21 @@ func (g *RouterPathGroup) ServeHTTP(resp http.ResponseWriter, req *http.Request) g.r.chiRouter.NotFoundHandler().ServeHTTP(resp, req) } +type RouterPathGroupPattern struct { + re *regexp.Regexp + params []routerPathParam + middlewares []any +} + // MatchPath matches the request method, and uses regexp to match the path. -// The pattern uses "<...>" to define path parameters, for example: "/" (different from chi router) -// It is only designed to resolve some special cases which chi router can't handle. +// The pattern uses "<...>" to define path parameters, for example, "/" (different from chi router) +// It is only designed to resolve some special cases that chi router can't handle. // For most cases, it shouldn't be used because it needs to iterate all rules to find the matched one (inefficient). func (g *RouterPathGroup) MatchPath(methods, pattern string, h ...any) { + g.MatchPattern(methods, g.PatternRegexp(pattern), h...) +} + +func (g *RouterPathGroup) MatchPattern(methods string, pattern *RouterPathGroupPattern, h ...any) { g.matchers = append(g.matchers, newRouterPathMatcher(methods, pattern, h...)) } @@ -96,8 +107,8 @@ func isValidMethod(name string) bool { return false } -func newRouterPathMatcher(methods, pattern string, h ...any) *routerPathMatcher { - middlewares, handlerFunc := wrapMiddlewareAndHandler(nil, h) +func newRouterPathMatcher(methods string, patternRegexp *RouterPathGroupPattern, h ...any) *routerPathMatcher { + middlewares, handlerFunc := wrapMiddlewareAndHandler(patternRegexp.middlewares, h) p := &routerPathMatcher{methods: make(container.Set[string]), middlewares: middlewares, handlerFunc: handlerFunc} for method := range strings.SplitSeq(methods, ",") { method = strings.TrimSpace(method) @@ -106,19 +117,25 @@ func newRouterPathMatcher(methods, pattern string, h ...any) *routerPathMatcher } p.methods.Add(method) } + p.re, p.params = patternRegexp.re, patternRegexp.params + return p +} + +func patternRegexp(pattern string, h ...any) *RouterPathGroupPattern { + p := &RouterPathGroupPattern{middlewares: slices.Clone(h)} re := []byte{'^'} lastEnd := 0 for lastEnd < len(pattern) { start := strings.IndexByte(pattern[lastEnd:], '<') if start == -1 { - re = append(re, pattern[lastEnd:]...) + re = append(re, regexp.QuoteMeta(pattern[lastEnd:])...) break } end := strings.IndexByte(pattern[lastEnd+start:], '>') if end == -1 { panic("invalid pattern: " + pattern) } - re = append(re, pattern[lastEnd:lastEnd+start]...) + re = append(re, regexp.QuoteMeta(pattern[lastEnd:lastEnd+start])...) partName, partExp, _ := strings.Cut(pattern[lastEnd+start+1:lastEnd+start+end], ":") lastEnd += start + end + 1 @@ -140,7 +157,10 @@ func newRouterPathMatcher(methods, pattern string, h ...any) *routerPathMatcher p.params = append(p.params, param) } re = append(re, '$') - reStr := string(re) - p.re = regexp.MustCompile(reStr) + p.re = regexp.MustCompile(string(re)) return p } + +func (g *RouterPathGroup) PatternRegexp(pattern string, h ...any) *RouterPathGroupPattern { + return patternRegexp(pattern, h...) +} diff --git a/modules/web/router_test.go b/modules/web/router_test.go index 21619012ea..1cee2b879b 100644 --- a/modules/web/router_test.go +++ b/modules/web/router_test.go @@ -34,7 +34,7 @@ func TestPathProcessor(t *testing.T) { testProcess := func(pattern, uri string, expectedPathParams map[string]string) { chiCtx := chi.NewRouteContext() chiCtx.RouteMethod = "GET" - p := newRouterPathMatcher("GET", pattern, http.NotFound) + p := newRouterPathMatcher("GET", patternRegexp(pattern), http.NotFound) assert.True(t, p.matchPath(chiCtx, uri), "use pattern %s to process uri %s", pattern, uri) assert.Equal(t, expectedPathParams, chiURLParamsToMap(chiCtx), "use pattern %s to process uri %s", pattern, uri) } @@ -56,18 +56,20 @@ func TestRouter(t *testing.T) { recorder.Body = buff type resultStruct struct { - method string - pathParams map[string]string - handlerMark string + method string + pathParams map[string]string + handlerMarks []string } - var res resultStruct + var res resultStruct h := func(optMark ...string) func(resp http.ResponseWriter, req *http.Request) { mark := util.OptionalArg(optMark, "") return func(resp http.ResponseWriter, req *http.Request) { res.method = req.Method res.pathParams = chiURLParamsToMap(chi.RouteContext(req.Context())) - res.handlerMark = mark + if mark != "" { + res.handlerMarks = append(res.handlerMarks, mark) + } } } @@ -77,6 +79,8 @@ func TestRouter(t *testing.T) { if stop := req.FormValue("stop"); stop != "" && (mark == "" || mark == stop) { h(stop)(resp, req) resp.WriteHeader(http.StatusOK) + } else if mark != "" { + res.handlerMarks = append(res.handlerMarks, mark) } } } @@ -108,7 +112,7 @@ func TestRouter(t *testing.T) { m.Delete("", h()) }) m.PathGroup("/*", func(g *RouterPathGroup) { - g.MatchPath("GET", `//`, stopMark("s2"), h("match-path")) + g.MatchPattern("GET", g.PatternRegexp(`//`, stopMark("s2")), stopMark("s3"), h("match-path")) }, stopMark("s1")) }) }) @@ -126,31 +130,31 @@ func TestRouter(t *testing.T) { } t.Run("RootRouter", func(t *testing.T) { - testRoute(t, "GET /the-user/the-repo/other", resultStruct{method: "GET", handlerMark: "not-found:/"}) + testRoute(t, "GET /the-user/the-repo/other", resultStruct{method: "GET", handlerMarks: []string{"not-found:/"}}) testRoute(t, "GET /the-user/the-repo/pulls", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "pulls"}, - handlerMark: "list-issues-b", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "pulls"}, + handlerMarks: []string{"list-issues-b"}, }) testRoute(t, "GET /the-user/the-repo/issues/123", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, - handlerMark: "view-issue", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, + handlerMarks: []string{"view-issue"}, }) testRoute(t, "GET /the-user/the-repo/issues/123?stop=hijack", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, - handlerMark: "hijack", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "type": "issues", "index": "123"}, + handlerMarks: []string{"hijack"}, }) testRoute(t, "POST /the-user/the-repo/issues/123/update", resultStruct{ - method: "POST", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "index": "123"}, - handlerMark: "update-issue", + method: "POST", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "index": "123"}, + handlerMarks: []string{"update-issue"}, }) }) t.Run("Sub Router", func(t *testing.T) { - testRoute(t, "GET /api/v1/other", resultStruct{method: "GET", handlerMark: "not-found:/api/v1"}) + testRoute(t, "GET /api/v1/other", resultStruct{method: "GET", handlerMarks: []string{"not-found:/api/v1"}}) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches", resultStruct{ method: "GET", pathParams: map[string]string{"username": "the-user", "reponame": "the-repo"}, @@ -179,31 +183,37 @@ func TestRouter(t *testing.T) { t.Run("MatchPath", func(t *testing.T) { testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, - handlerMark: "match-path", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, + handlerMarks: []string{"s1", "s2", "s3", "match-path"}, }) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1%2fd2/fn", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1%2fd2/fn", "dir": "d1%2fd2", "file": "fn"}, - handlerMark: "match-path", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1%2fd2/fn", "dir": "d1%2fd2", "file": "fn"}, + handlerMarks: []string{"s1", "s2", "s3", "match-path"}, }) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/000", resultStruct{ - method: "GET", - pathParams: map[string]string{"reponame": "the-repo", "username": "the-user", "*": "d1/d2/000"}, - handlerMark: "not-found:/api/v1", + method: "GET", + pathParams: map[string]string{"reponame": "the-repo", "username": "the-user", "*": "d1/d2/000"}, + handlerMarks: []string{"s1", "not-found:/api/v1"}, }) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn?stop=s1", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn"}, - handlerMark: "s1", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn"}, + handlerMarks: []string{"s1"}, }) testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn?stop=s2", resultStruct{ - method: "GET", - pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, - handlerMark: "s2", + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, + handlerMarks: []string{"s1", "s2"}, + }) + + testRoute(t, "GET /api/v1/repos/the-user/the-repo/branches/d1/d2/fn?stop=s3", resultStruct{ + method: "GET", + pathParams: map[string]string{"username": "the-user", "reponame": "the-repo", "*": "d1/d2/fn", "dir": "d1/d2", "file": "fn"}, + handlerMarks: []string{"s1", "s2", "s3"}, }) }) } diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index df5897e45e..f65c4b99ff 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -5,8 +5,6 @@ package packages import ( "net/http" - "regexp" - "strings" auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/perm" @@ -282,42 +280,10 @@ func CommonRoutes() *web.Router { }) }) }, reqPackageAccess(perm.AccessModeRead)) - r.Group("/conda", func() { - var ( - downloadPattern = regexp.MustCompile(`\A(.+/)?(.+)/((?:[^/]+(?:\.tar\.bz2|\.conda))|(?:current_)?repodata\.json(?:\.bz2)?)\z`) - uploadPattern = regexp.MustCompile(`\A(.+/)?([^/]+(?:\.tar\.bz2|\.conda))\z`) - ) - - r.Get("/*", func(ctx *context.Context) { - m := downloadPattern.FindStringSubmatch(ctx.PathParam("*")) - if len(m) == 0 { - ctx.Status(http.StatusNotFound) - return - } - - ctx.SetPathParam("channel", strings.TrimSuffix(m[1], "/")) - ctx.SetPathParam("architecture", m[2]) - ctx.SetPathParam("filename", m[3]) - - switch m[3] { - case "repodata.json", "repodata.json.bz2", "current_repodata.json", "current_repodata.json.bz2": - conda.EnumeratePackages(ctx) - default: - conda.DownloadPackageFile(ctx) - } - }) - r.Put("/*", reqPackageAccess(perm.AccessModeWrite), func(ctx *context.Context) { - m := uploadPattern.FindStringSubmatch(ctx.PathParam("*")) - if len(m) == 0 { - ctx.Status(http.StatusNotFound) - return - } - - ctx.SetPathParam("channel", strings.TrimSuffix(m[1], "/")) - ctx.SetPathParam("filename", m[2]) - - conda.UploadPackageFile(ctx) - }) + r.PathGroup("/conda/*", func(g *web.RouterPathGroup) { + g.MatchPath("GET", "//", conda.ListOrGetPackages) + g.MatchPath("GET", "///", conda.ListOrGetPackages) + g.MatchPath("PUT", "//", reqPackageAccess(perm.AccessModeWrite), conda.UploadPackageFile) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/cran", func() { r.Group("/src", func() { @@ -358,60 +324,15 @@ func CommonRoutes() *web.Router { }, reqPackageAccess(perm.AccessModeRead)) r.Group("/go", func() { r.Put("/upload", reqPackageAccess(perm.AccessModeWrite), goproxy.UploadPackage) - r.Get("/sumdb/sum.golang.org/supported", func(ctx *context.Context) { - ctx.Status(http.StatusNotFound) - }) + r.Get("/sumdb/sum.golang.org/supported", http.NotFound) - // Manual mapping of routes because the package name contains slashes which chi does not support // https://go.dev/ref/mod#goproxy-protocol - r.Get("/*", func(ctx *context.Context) { - path := ctx.PathParam("*") - - if strings.HasSuffix(path, "/@latest") { - ctx.SetPathParam("name", path[:len(path)-len("/@latest")]) - ctx.SetPathParam("version", "latest") - - goproxy.PackageVersionMetadata(ctx) - return - } - - parts := strings.SplitN(path, "/@v/", 2) - if len(parts) != 2 { - ctx.Status(http.StatusNotFound) - return - } - - ctx.SetPathParam("name", parts[0]) - - // /@v/list - if parts[1] == "list" { - goproxy.EnumeratePackageVersions(ctx) - return - } - - // /@v/.zip - if strings.HasSuffix(parts[1], ".zip") { - ctx.SetPathParam("version", parts[1][:len(parts[1])-len(".zip")]) - - goproxy.DownloadPackageFile(ctx) - return - } - // /@v/.info - if strings.HasSuffix(parts[1], ".info") { - ctx.SetPathParam("version", parts[1][:len(parts[1])-len(".info")]) - - goproxy.PackageVersionMetadata(ctx) - return - } - // /@v/.mod - if strings.HasSuffix(parts[1], ".mod") { - ctx.SetPathParam("version", parts[1][:len(parts[1])-len(".mod")]) - - goproxy.PackageVersionGoModContent(ctx) - return - } - - ctx.Status(http.StatusNotFound) + r.PathGroup("/*", func(g *web.RouterPathGroup) { + g.MatchPath("GET", "//@", goproxy.PackageVersionMetadata) + g.MatchPath("GET", "//@v/list", goproxy.EnumeratePackageVersions) + g.MatchPath("GET", "//@v/.zip", goproxy.DownloadPackageFile) + g.MatchPath("GET", "//@v/.info", goproxy.PackageVersionMetadata) + g.MatchPath("GET", "//@v/.mod", goproxy.PackageVersionGoModContent) }) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/generic", func() { @@ -532,82 +453,24 @@ func CommonRoutes() *web.Router { }) }) }, reqPackageAccess(perm.AccessModeRead)) + r.Group("/pypi", func() { r.Post("/", reqPackageAccess(perm.AccessModeWrite), pypi.UploadPackageFile) r.Get("/files/{id}/{version}/{filename}", pypi.DownloadPackageFile) r.Get("/simple/{id}", pypi.PackageMetadata) }, reqPackageAccess(perm.AccessModeRead)) - r.Group("/rpm", func() { - r.Group("/repository.key", func() { - r.Head("", rpm.GetRepositoryKey) - r.Get("", rpm.GetRepositoryKey) - }) - var ( - repoPattern = regexp.MustCompile(`\A(.*?)\.repo\z`) - uploadPattern = regexp.MustCompile(`\A(.*?)/upload\z`) - filePattern = regexp.MustCompile(`\A(.*?)/package/([^/]+)/([^/]+)/([^/]+)(?:/([^/]+\.rpm)|)\z`) - repoFilePattern = regexp.MustCompile(`\A(.*?)/repodata/([^/]+)\z`) - ) - - r.Methods("HEAD,GET,PUT,DELETE", "*", func(ctx *context.Context) { - path := ctx.PathParam("*") - isHead := ctx.Req.Method == http.MethodHead - isGetHead := ctx.Req.Method == http.MethodHead || ctx.Req.Method == http.MethodGet - isPut := ctx.Req.Method == http.MethodPut - isDelete := ctx.Req.Method == http.MethodDelete - - m := repoPattern.FindStringSubmatch(path) - if len(m) == 2 && isGetHead { - ctx.SetPathParam("group", strings.Trim(m[1], "/")) - rpm.GetRepositoryConfig(ctx) - return - } - - m = repoFilePattern.FindStringSubmatch(path) - if len(m) == 3 && isGetHead { - ctx.SetPathParam("group", strings.Trim(m[1], "/")) - ctx.SetPathParam("filename", m[2]) - if isHead { - rpm.CheckRepositoryFileExistence(ctx) - } else { - rpm.GetRepositoryFile(ctx) - } - return - } - - m = uploadPattern.FindStringSubmatch(path) - if len(m) == 2 && isPut { - reqPackageAccess(perm.AccessModeWrite)(ctx) - if ctx.Written() { - return - } - ctx.SetPathParam("group", strings.Trim(m[1], "/")) - rpm.UploadPackageFile(ctx) - return - } - - m = filePattern.FindStringSubmatch(path) - if len(m) == 6 && (isGetHead || isDelete) { - ctx.SetPathParam("group", strings.Trim(m[1], "/")) - ctx.SetPathParam("name", m[2]) - ctx.SetPathParam("version", m[3]) - ctx.SetPathParam("architecture", m[4]) - if isGetHead { - rpm.DownloadPackageFile(ctx) - } else { - reqPackageAccess(perm.AccessModeWrite)(ctx) - if ctx.Written() { - return - } - rpm.DeletePackageFile(ctx) - } - return - } - - ctx.Status(http.StatusNotFound) - }) + r.Methods("HEAD,GET", "/rpm.repo", reqPackageAccess(perm.AccessModeRead), rpm.GetRepositoryConfig) + r.PathGroup("/rpm/*", func(g *web.RouterPathGroup) { + g.MatchPath("HEAD,GET", "/repository.key", rpm.GetRepositoryKey) + g.MatchPath("HEAD,GET", "/.repo", rpm.GetRepositoryConfig) + g.MatchPath("HEAD", "//repodata/", rpm.CheckRepositoryFileExistence) + g.MatchPath("GET", "//repodata/", rpm.GetRepositoryFile) + g.MatchPath("PUT", "//upload", reqPackageAccess(perm.AccessModeWrite), rpm.UploadPackageFile) + g.MatchPath("HEAD,GET", "//package///", rpm.DownloadPackageFile) + g.MatchPath("DELETE", "//package///", reqPackageAccess(perm.AccessModeWrite), rpm.DeletePackageFile) }, reqPackageAccess(perm.AccessModeRead)) + r.Group("/rubygems", func() { r.Get("/specs.4.8.gz", rubygems.EnumeratePackages) r.Get("/latest_specs.4.8.gz", rubygems.EnumeratePackagesLatest) @@ -621,6 +484,7 @@ func CommonRoutes() *web.Router { r.Delete("/yank", rubygems.DeletePackage) }, reqPackageAccess(perm.AccessModeWrite)) }, reqPackageAccess(perm.AccessModeRead)) + r.Group("/swift", func() { r.Group("", func() { // Needs to be unauthenticated. r.Post("", swift.CheckAuthenticate) @@ -632,31 +496,12 @@ func CommonRoutes() *web.Router { r.Get("", swift.EnumeratePackageVersions) r.Get(".json", swift.EnumeratePackageVersions) }, swift.CheckAcceptMediaType(swift.AcceptJSON)) - r.Group("/{version}", func() { - r.Get("/Package.swift", swift.CheckAcceptMediaType(swift.AcceptSwift), swift.DownloadManifest) - r.Put("", reqPackageAccess(perm.AccessModeWrite), swift.CheckAcceptMediaType(swift.AcceptJSON), swift.UploadPackageFile) - r.Get("", func(ctx *context.Context) { - // Can't use normal routes here: https://github.com/go-chi/chi/issues/781 - - version := ctx.PathParam("version") - if strings.HasSuffix(version, ".zip") { - swift.CheckAcceptMediaType(swift.AcceptZip)(ctx) - if ctx.Written() { - return - } - ctx.SetPathParam("version", version[:len(version)-4]) - swift.DownloadPackageFile(ctx) - } else { - swift.CheckAcceptMediaType(swift.AcceptJSON)(ctx) - if ctx.Written() { - return - } - if strings.HasSuffix(version, ".json") { - ctx.SetPathParam("version", version[:len(version)-5]) - } - swift.PackageVersionMetadata(ctx) - } - }) + r.PathGroup("/*", func(g *web.RouterPathGroup) { + g.MatchPath("GET", "/.json", swift.CheckAcceptMediaType(swift.AcceptJSON), swift.PackageVersionMetadata) + g.MatchPath("GET", "/.zip", swift.CheckAcceptMediaType(swift.AcceptZip), swift.DownloadPackageFile) + g.MatchPath("GET", "//Package.swift", swift.CheckAcceptMediaType(swift.AcceptSwift), swift.DownloadManifest) + g.MatchPath("GET", "/", swift.CheckAcceptMediaType(swift.AcceptJSON), swift.PackageVersionMetadata) + g.MatchPath("PUT", "/", reqPackageAccess(perm.AccessModeWrite), swift.CheckAcceptMediaType(swift.AcceptJSON), swift.UploadPackageFile) }) }) r.Get("/identifiers", swift.CheckAcceptMediaType(swift.AcceptJSON), swift.LookupPackageIdentifiers) @@ -705,18 +550,13 @@ func ContainerRoutes() *web.Router { r.PathGroup("/*", func(g *web.RouterPathGroup) { g.MatchPath("POST", "//blobs/uploads", reqPackageAccess(perm.AccessModeWrite), container.VerifyImageName, container.PostBlobsUploads) g.MatchPath("GET", "//tags/list", container.VerifyImageName, container.GetTagsList) - g.MatchPath("GET,PATCH,PUT,DELETE", `//blobs/uploads/`, reqPackageAccess(perm.AccessModeWrite), container.VerifyImageName, func(ctx *context.Context) { - switch ctx.Req.Method { - case http.MethodGet: - container.GetBlobsUpload(ctx) - case http.MethodPatch: - container.PatchBlobsUpload(ctx) - case http.MethodPut: - container.PutBlobsUpload(ctx) - default: /* DELETE */ - container.DeleteBlobsUpload(ctx) - } - }) + + patternBlobsUploadsUUID := g.PatternRegexp(`//blobs/uploads/`, reqPackageAccess(perm.AccessModeWrite), container.VerifyImageName) + g.MatchPattern("GET", patternBlobsUploadsUUID, container.GetBlobsUpload) + g.MatchPattern("PATCH", patternBlobsUploadsUUID, container.PatchBlobsUpload) + g.MatchPattern("PUT", patternBlobsUploadsUUID, container.PutBlobsUpload) + g.MatchPattern("DELETE", patternBlobsUploadsUUID, container.DeleteBlobsUpload) + g.MatchPath("HEAD", `//blobs/`, container.VerifyImageName, container.HeadBlob) g.MatchPath("GET", `//blobs/`, container.VerifyImageName, container.GetBlob) g.MatchPath("DELETE", `//blobs/`, container.VerifyImageName, reqPackageAccess(perm.AccessModeWrite), container.DeleteBlob) diff --git a/routers/api/packages/conda/conda.go b/routers/api/packages/conda/conda.go index fe7542dd18..cfe069d6db 100644 --- a/routers/api/packages/conda/conda.go +++ b/routers/api/packages/conda/conda.go @@ -36,6 +36,24 @@ func apiError(ctx *context.Context, status int, obj any) { }) } +func isCondaPackageFileName(filename string) bool { + return strings.HasSuffix(filename, ".tar.bz2") || strings.HasSuffix(filename, ".conda") +} + +func ListOrGetPackages(ctx *context.Context) { + filename := ctx.PathParam("filename") + switch filename { + case "repodata.json", "repodata.json.bz2", "current_repodata.json", "current_repodata.json.bz2": + EnumeratePackages(ctx) + return + } + if isCondaPackageFileName(filename) { + DownloadPackageFile(ctx) + return + } + ctx.NotFound(nil) +} + func EnumeratePackages(ctx *context.Context) { type Info struct { Subdir string `json:"subdir"` @@ -174,6 +192,12 @@ func EnumeratePackages(ctx *context.Context) { } func UploadPackageFile(ctx *context.Context) { + filename := ctx.PathParam("filename") + if !isCondaPackageFileName(filename) { + apiError(ctx, http.StatusBadRequest, nil) + return + } + upload, needToClose, err := ctx.UploadStream() if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -191,7 +215,7 @@ func UploadPackageFile(ctx *context.Context) { defer buf.Close() var pck *conda_module.Package - if strings.HasSuffix(strings.ToLower(ctx.PathParam("filename")), ".tar.bz2") { + if strings.HasSuffix(filename, ".tar.bz2") { pck, err = conda_module.ParsePackageBZ2(buf) } else { pck, err = conda_module.ParsePackageConda(buf, buf.Size()) diff --git a/routers/api/packages/container/blob.go b/routers/api/packages/container/blob.go index 2ea9b3839c..abfc21f95a 100644 --- a/routers/api/packages/container/blob.go +++ b/routers/api/packages/container/blob.go @@ -90,14 +90,14 @@ func mountBlob(ctx context.Context, pi *packages_service.PackageInfo, pb *packag }) } -func containerPkgName(piOwnerID int64, piName string) string { - return fmt.Sprintf("pkg_%d_container_%s", piOwnerID, strings.ToLower(piName)) +func containerGlobalLockKey(piOwnerID int64, piName, usage string) string { + return fmt.Sprintf("pkg_%d_container_%s_%s", piOwnerID, strings.ToLower(piName), usage) } func getOrCreateUploadVersion(ctx context.Context, pi *packages_service.PackageInfo) (*packages_model.PackageVersion, error) { var uploadVersion *packages_model.PackageVersion - releaser, err := globallock.Lock(ctx, containerPkgName(pi.Owner.ID, pi.Name)) + releaser, err := globallock.Lock(ctx, containerGlobalLockKey(pi.Owner.ID, pi.Name, "package")) if err != nil { return nil, err } @@ -178,7 +178,7 @@ func createFileForBlob(ctx context.Context, pv *packages_model.PackageVersion, p } func deleteBlob(ctx context.Context, ownerID int64, image string, digest digest.Digest) error { - releaser, err := globallock.Lock(ctx, containerPkgName(ownerID, image)) + releaser, err := globallock.Lock(ctx, containerGlobalLockKey(ownerID, image, "blob")) if err != nil { return err } diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go index d1b80daccf..aeec16be4b 100644 --- a/routers/api/packages/container/container.go +++ b/routers/api/packages/container/container.go @@ -32,7 +32,7 @@ import ( packages_service "code.gitea.io/gitea/services/packages" container_service "code.gitea.io/gitea/services/packages/container" - digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/go-digest" ) // maximum size of a container manifest diff --git a/routers/api/packages/container/manifest.go b/routers/api/packages/container/manifest.go index b69b7af3f7..22ea11c8ce 100644 --- a/routers/api/packages/container/manifest.go +++ b/routers/api/packages/container/manifest.go @@ -16,6 +16,7 @@ import ( packages_model "code.gitea.io/gitea/models/packages" container_model "code.gitea.io/gitea/models/packages/container" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/globallock" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" @@ -61,6 +62,13 @@ func processManifest(ctx context.Context, mci *manifestCreationInfo, buf *packag } } + // .../container/manifest.go:453:createManifestBlob() [E] Error inserting package blob: Error 1062 (23000): Duplicate entry '..........' for key 'package_blob.UQE_package_blob_md5' + releaser, err := globallock.Lock(ctx, containerGlobalLockKey(mci.Owner.ID, mci.Image, "manifest")) + if err != nil { + return "", err + } + defer releaser() + if container_module.IsMediaTypeImageManifest(mci.MediaType) { return processOciImageManifest(ctx, mci, buf) } else if container_module.IsMediaTypeImageIndex(mci.MediaType) { diff --git a/services/packages/container/cleanup.go b/services/packages/container/cleanup.go index d15d6b6c84..263562a396 100644 --- a/services/packages/container/cleanup.go +++ b/services/packages/container/cleanup.go @@ -13,7 +13,7 @@ import ( container_module "code.gitea.io/gitea/modules/packages/container" packages_service "code.gitea.io/gitea/services/packages" - digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/go-digest" ) // Cleanup removes expired container data