From 534917d57670d82703567131e2b33fd945e6f8cb Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 26 Jan 2024 14:18:19 +0800 Subject: [PATCH 1/5] Don't remove all mirror repository's releases when mirroring (#28817) Fix #22066 # Purpose This PR fix the releases will be deleted when mirror repository sync the tags. # The problem In the previous implementation of #19125. All releases record in databases of one mirror repository will be deleted before sync. Ref: https://github.com/go-gitea/gitea/pull/19125/files#diff-2aa04998a791c30e5a02b49a97c07fcd93d50e8b31640ce2ddb1afeebf605d02R481 # The Pros This PR introduced a new method which will load all releases from databases and all tags on git data into memory. And detect which tags needs to be inserted, which tags need to be updated or deleted. Only tags releases(IsTag=true) which are not included in git data will be deleted, only tags which sha1 changed will be updated. So it will not delete any real releases include drafts. # The Cons The drawback is the memory usage will be higher than before if there are many tags on this repository. This PR defined a special release struct to reduce columns loaded from database to memory. --- modules/repository/repo.go | 76 ++++++++++++++++++++++++++++++--- modules/repository/repo_test.go | 76 +++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 6 deletions(-) create mode 100644 modules/repository/repo_test.go diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 5af69763d15..5352da0378d 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -508,6 +508,18 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re return nil } +// shortRelease to reduce load memory, this struct can replace repo_model.Release +type shortRelease struct { + ID int64 + TagName string + Sha1 string + IsTag bool +} + +func (shortRelease) TableName() string { + return "release" +} + // pullMirrorReleaseSync is a pull-mirror specific tag<->release table // synchronization which overwrites all Releases from the repository tags. This // can be relied on since a pull-mirror is always identical to its @@ -521,16 +533,20 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) } err = db.WithTx(ctx, func(ctx context.Context) error { - // - // clear out existing releases - // - if _, err := db.DeleteByBean(ctx, &repo_model.Release{RepoID: repo.ID}); err != nil { - return fmt.Errorf("unable to clear releases for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) + dbReleases, err := db.Find[shortRelease](ctx, repo_model.FindReleasesOptions{ + RepoID: repo.ID, + IncludeDrafts: true, + IncludeTags: true, + }) + if err != nil { + return fmt.Errorf("unable to FindReleases in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) } + + inserts, deletes, updates := calcSync(tags, dbReleases) // // make release set identical to upstream tags // - for _, tag := range tags { + for _, tag := range inserts { release := repo_model.Release{ RepoID: repo.ID, TagName: tag.Name, @@ -547,6 +563,25 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git return fmt.Errorf("unable insert tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err) } } + + // only delete tags releases + if len(deletes) > 0 { + if _, err := db.GetEngine(ctx).Where("repo_id=?", repo.ID). + In("id", deletes). + Delete(&repo_model.Release{}); err != nil { + return fmt.Errorf("unable to delete tags for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) + } + } + + for _, tag := range updates { + if _, err := db.GetEngine(ctx).Where("repo_id = ? AND lower_tag_name = ?", repo.ID, strings.ToLower(tag.Name)). + Cols("sha1"). + Update(&repo_model.Release{ + Sha1: tag.Object.String(), + }); err != nil { + return fmt.Errorf("unable to update tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err) + } + } return nil }) if err != nil { @@ -556,3 +591,32 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git log.Trace("pullMirrorReleaseSync: done rebuilding %d releases", numTags) return nil } + +func calcSync(destTags []*git.Tag, dbTags []*shortRelease) ([]*git.Tag, []int64, []*git.Tag) { + destTagMap := make(map[string]*git.Tag) + for _, tag := range destTags { + destTagMap[tag.Name] = tag + } + dbTagMap := make(map[string]*shortRelease) + for _, rel := range dbTags { + dbTagMap[rel.TagName] = rel + } + + inserted := make([]*git.Tag, 0, 10) + updated := make([]*git.Tag, 0, 10) + for _, tag := range destTags { + rel := dbTagMap[tag.Name] + if rel == nil { + inserted = append(inserted, tag) + } else if rel.Sha1 != tag.Object.String() { + updated = append(updated, tag) + } + } + deleted := make([]int64, 0, 10) + for _, tag := range dbTags { + if destTagMap[tag.TagName] == nil && tag.IsTag { + deleted = append(deleted, tag.ID) + } + } + return inserted, deleted, updated +} diff --git a/modules/repository/repo_test.go b/modules/repository/repo_test.go new file mode 100644 index 00000000000..68980f92f94 --- /dev/null +++ b/modules/repository/repo_test.go @@ -0,0 +1,76 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "testing" + + "code.gitea.io/gitea/modules/git" + + "github.com/stretchr/testify/assert" +) + +func Test_calcSync(t *testing.T) { + gitTags := []*git.Tag{ + /*{ + Name: "v0.1.0-beta", //deleted tag + Object: git.MustIDFromString(""), + }, + { + Name: "v0.1.1-beta", //deleted tag but release should not be deleted because it's a release + Object: git.MustIDFromString(""), + }, + */ + { + Name: "v1.0.0", // keep as before + Object: git.MustIDFromString("1006e6e13c73ad3d9e2d5682ad266b5016523485"), + }, + { + Name: "v1.1.0", // retagged with new commit id + Object: git.MustIDFromString("bbdb7df30248e7d4a26a909c8d2598a152e13868"), + }, + { + Name: "v1.2.0", // new tag + Object: git.MustIDFromString("a5147145e2f24d89fd6d2a87826384cc1d253267"), + }, + } + + dbReleases := []*shortRelease{ + { + ID: 1, + TagName: "v0.1.0-beta", + Sha1: "244758d7da8dd1d9e0727e8cb7704ed4ba9a17c3", + IsTag: true, + }, + { + ID: 2, + TagName: "v0.1.1-beta", + Sha1: "244758d7da8dd1d9e0727e8cb7704ed4ba9a17c3", + IsTag: false, + }, + { + ID: 3, + TagName: "v1.0.0", + Sha1: "1006e6e13c73ad3d9e2d5682ad266b5016523485", + }, + { + ID: 4, + TagName: "v1.1.0", + Sha1: "53ab18dcecf4152b58328d1f47429510eb414d50", + }, + } + + inserts, deletes, updates := calcSync(gitTags, dbReleases) + if assert.EqualValues(t, 1, len(inserts), "inserts") { + assert.EqualValues(t, *gitTags[2], *inserts[0], "inserts equal") + } + + if assert.EqualValues(t, 1, len(deletes), "deletes") { + assert.EqualValues(t, 1, deletes[0], "deletes equal") + } + + if assert.EqualValues(t, 1, len(updates), "updates") { + assert.EqualValues(t, *gitTags[1], *updates[0], "updates equal") + } +} From a240d5dfa7e261f2fb703cf24b1ba4dc6aa47bfd Mon Sep 17 00:00:00 2001 From: wackbyte Date: Fri, 26 Jan 2024 09:15:57 -0500 Subject: [PATCH 2/5] Fix non-alphabetic sorting of repo topics (#28938) --- models/repo/topic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/repo/topic.go b/models/repo/topic.go index b71f43bc880..79b13e320da 100644 --- a/models/repo/topic.go +++ b/models/repo/topic.go @@ -366,7 +366,7 @@ func syncTopicsInRepository(sess db.Engine, repoID int64) error { topicNames := make([]string, 0, 25) if err := sess.Table("topic").Cols("name"). Join("INNER", "repo_topic", "repo_topic.topic_id = topic.id"). - Where("repo_topic.repo_id = ?", repoID).Desc("topic.repo_count").Find(&topicNames); err != nil { + Where("repo_topic.repo_id = ?", repoID).Asc("topic.name").Find(&topicNames); err != nil { return err } From 01acd1eea38f25d2b21f56ec15dd162ca6005fbf Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 27 Jan 2024 04:36:01 +0100 Subject: [PATCH 3/5] Strip `/` from relative links (#28932) Fixes #28915 Restores the old behaviour: https://github.com/go-gitea/gitea/pull/26745/files#diff-d78a9d361b1fddc12218e4dd42f42d39d6be1fda184041e06bb6fb30f0d94c59L96 --- modules/markup/markdown/goldmark.go | 10 +++++---- modules/markup/markdown/markdown_test.go | 26 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/modules/markup/markdown/goldmark.go b/modules/markup/markdown/goldmark.go index 1db3cbad7e1..3dc5530e008 100644 --- a/modules/markup/markdown/goldmark.go +++ b/modules/markup/markdown/goldmark.go @@ -85,9 +85,11 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa // 2. If they're not wrapped with a link they need a link wrapper // Check if the destination is a real link - link := v.Destination - if len(link) > 0 && !markup.IsLink(link) { - v.Destination = []byte(giteautil.URLJoin(ctx.Links.ResolveMediaLink(ctx.IsWiki), string(link))) + if len(v.Destination) > 0 && !markup.IsLink(v.Destination) { + v.Destination = []byte(giteautil.URLJoin( + ctx.Links.ResolveMediaLink(ctx.IsWiki), + strings.TrimLeft(string(v.Destination), "/"), + )) } parent := n.Parent() @@ -103,7 +105,7 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa // Duplicate the current image node image := ast.NewImage(ast.NewLink()) - image.Destination = link + image.Destination = v.Destination image.Title = v.Title for _, attr := range v.Attributes() { image.SetAttribute(attr.Name, attr.Value) diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index 1edfb3e6644..bdf4011fa24 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -580,6 +580,8 @@ https://example.com/file.bin [[local link|file.bin]] [[remote link|https://example.com]] ![local image](image.jpg) +![local image](path/file) +![local image](/path/file) ![remote image](https://example.com/image.jpg) [[local image|image.jpg]] [[remote link|https://example.com/image.jpg]] @@ -609,6 +611,8 @@ mail@domain.com local link
remote link
local image
+local image
+local image
remote image
local image
remote link
@@ -634,6 +638,8 @@ space

local link
remote link
local image
+local image
+local image
remote image
local image
remote link
@@ -661,6 +667,8 @@ space

local link
remote link
local image
+local image
+local image
remote image
local image
remote link
@@ -688,6 +696,8 @@ space

local link
remote link
local image
+local image
+local image
remote image
local image
remote link
@@ -715,6 +725,8 @@ space

local link
remote link
local image
+local image
+local image
remote image
local image
remote link
@@ -742,6 +754,8 @@ space

local link
remote link
local image
+local image
+local image
remote image
local image
remote link
@@ -770,6 +784,8 @@ space

local link
remote link
local image
+local image
+local image
remote image
local image
remote link
@@ -798,6 +814,8 @@ space

local link
remote link
local image
+local image
+local image
remote image
local image
remote link
@@ -826,6 +844,8 @@ space

local link
remote link
local image
+local image
+local image
remote image
local image
remote link
@@ -854,6 +874,8 @@ space

local link
remote link
local image
+local image
+local image
remote image
local image
remote link
@@ -883,6 +905,8 @@ space

local link
remote link
local image
+local image
+local image
remote image
local image
remote link
@@ -912,6 +936,8 @@ space

local link
remote link
local image
+local image
+local image
remote image
local image
remote link
From fc1bae00a4cc07d832eb9405ec8fd8f4e52c0197 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 27 Jan 2024 10:27:34 +0100 Subject: [PATCH 4/5] Fix SSPI user creation (#28948) Fixes #28945 Setting the avatar is wrong and creating a random password is equal to leave it empty. --- services/auth/sspi.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/services/auth/sspi.go b/services/auth/sspi.go index 57ba0462c57..0e974fde8f3 100644 --- a/services/auth/sspi.go +++ b/services/auth/sspi.go @@ -11,7 +11,6 @@ import ( "sync" "code.gitea.io/gitea/models/auth" - "code.gitea.io/gitea/models/avatars" "code.gitea.io/gitea/models/db" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" @@ -167,12 +166,9 @@ func (s *SSPI) shouldAuthenticate(req *http.Request) (shouldAuth bool) { func (s *SSPI) newUser(ctx context.Context, username string, cfg *sspi.Source) (*user_model.User, error) { email := gouuid.New().String() + "@localhost.localdomain" user := &user_model.User{ - Name: username, - Email: email, - Passwd: gouuid.New().String(), - Language: cfg.DefaultLanguage, - UseCustomAvatar: true, - Avatar: avatars.DefaultAvatarLink(), + Name: username, + Email: email, + Language: cfg.DefaultLanguage, } emailNotificationPreference := user_model.EmailNotificationsDisabled overwriteDefault := &user_model.CreateUserOverwriteOptions{ From 0e650dca3076bbf8e1a4d1a80cef3275a51af658 Mon Sep 17 00:00:00 2001 From: Yarden Shoham Date: Sat, 27 Jan 2024 14:27:37 +0200 Subject: [PATCH 5/5] Make loading animation less aggressive (#28955) The current animation loops in a very fast manner, causing a slight feeling of uncomfortableness. This change slows it a bit for a smoother experience. # Before ![before](https://github.com/go-gitea/gitea/assets/20454870/215a722d-feb4-4643-819d-c37a620c5e48) # After ![after](https://github.com/go-gitea/gitea/assets/20454870/7acb1fab-9157-4f4d-8cc7-45fea0234b47) Signed-off-by: Yarden Shoham --- web_src/css/modules/animations.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/css/modules/animations.css b/web_src/css/modules/animations.css index cac824d87d6..87eb6a75cf0 100644 --- a/web_src/css/modules/animations.css +++ b/web_src/css/modules/animations.css @@ -22,7 +22,7 @@ height: min(4em, 66.6%); aspect-ratio: 1; transform: translate(-50%, -50%); - animation: isloadingspin 500ms infinite linear; + animation: isloadingspin 1000ms infinite linear; border-width: 4px; border-style: solid; border-color: var(--color-secondary) var(--color-secondary) var(--color-secondary-dark-8) var(--color-secondary-dark-8);