Make configFetcher fallback work for gitea (#357)

#299 who closed #133 did not take into account, that that gitea (and eventually) other forges do return 200 and empty string if file was not found - this make configFetcher more resilient
This commit is contained in:
6543 2021-09-26 01:23:17 +02:00 committed by GitHub
parent 4b6188d761
commit 5a05a7fe6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 121 additions and 70 deletions

View File

@ -1,6 +1,7 @@
package shared package shared
import ( import (
"fmt"
"strings" "strings"
"time" "time"
@ -31,13 +32,13 @@ func (cf *configFetcher) Fetch() (files []*remote.FileMeta, err error) {
for i := 0; i < 5; i++ { for i := 0; i < 5; i++ {
select { select {
case <-time.After(time.Second * time.Duration(i)): case <-time.After(time.Second * time.Duration(i)):
if len(cf.repo.Config) > 0 { if len(config) > 0 {
// either a file // either a file
if !strings.HasSuffix(config, "/") { if !strings.HasSuffix(config, "/") {
file, err = cf.remote_.File(cf.user, cf.repo, cf.build, config) file, err = cf.remote_.File(cf.user, cf.repo, cf.build, config)
if err == nil { if err == nil && len(file) != 0 {
return []*remote.FileMeta{{ return []*remote.FileMeta{{
Name: cf.repo.Config, Name: config,
Data: file, Data: file,
}}, nil }}, nil
} }
@ -54,12 +55,13 @@ func (cf *configFetcher) Fetch() (files []*remote.FileMeta, err error) {
// test .woodpecker/ folder // test .woodpecker/ folder
// if folder is not supported we will get a "Not implemented" error and continue // if folder is not supported we will get a "Not implemented" error and continue
files, err = cf.remote_.Dir(cf.user, cf.repo, cf.build, ".woodpecker") files, err = cf.remote_.Dir(cf.user, cf.repo, cf.build, ".woodpecker")
if err == nil { files = filterPipelineFiles(files)
return filterPipelineFiles(files), nil if err == nil && len(files) != 0 {
return files, nil
} }
file, err = cf.remote_.File(cf.user, cf.repo, cf.build, ".woodpecker.yml") file, err = cf.remote_.File(cf.user, cf.repo, cf.build, ".woodpecker.yml")
if err == nil { if err == nil && len(file) != 0 {
return []*remote.FileMeta{{ return []*remote.FileMeta{{
Name: ".woodpecker.yml", Name: ".woodpecker.yml",
Data: file, Data: file,
@ -67,14 +69,19 @@ func (cf *configFetcher) Fetch() (files []*remote.FileMeta, err error) {
} }
file, err = cf.remote_.File(cf.user, cf.repo, cf.build, ".drone.yml") file, err = cf.remote_.File(cf.user, cf.repo, cf.build, ".drone.yml")
if err == nil { if err == nil && len(file) != 0 {
return []*remote.FileMeta{{ return []*remote.FileMeta{{
Name: ".drone.yml", Name: ".drone.yml",
Data: file, Data: file,
}}, nil }}, nil
} }
if err == nil && len(files) == 0 {
return nil, fmt.Errorf("ConfigFetcher: Fallback did not found config")
}
} }
// TODO: retry loop is inactive and could maybe be fixed/deleted
return nil, err return nil, err
} }
} }

View File

@ -5,31 +5,45 @@ import (
"path/filepath" "path/filepath"
"testing" "testing"
"github.com/stretchr/testify/mock"
"github.com/woodpecker-ci/woodpecker/model" "github.com/woodpecker-ci/woodpecker/model"
"github.com/woodpecker-ci/woodpecker/server/remote" "github.com/woodpecker-ci/woodpecker/server/remote"
"github.com/woodpecker-ci/woodpecker/server/remote/mocks" "github.com/woodpecker-ci/woodpecker/server/remote/mocks"
"github.com/woodpecker-ci/woodpecker/server/shared" "github.com/woodpecker-ci/woodpecker/server/shared"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
) )
func TestFetch(t *testing.T) { func TestFetch(t *testing.T) {
t.Parallel() t.Parallel()
type file struct {
name string
data []byte
}
dummyData := []byte("TEST")
testTable := []struct { testTable := []struct {
name string name string
repoConfig string repoConfig string
files []string files []file
expectedFileNames []string expectedFileNames []string
expectedError bool expectedError bool
}{ }{
{ {
name: "Default config - .woodpecker/", name: "Default config - .woodpecker/",
repoConfig: "", repoConfig: "",
files: []string{ files: []file{{
".woodpecker/text.txt", name: ".woodpecker/text.txt",
".woodpecker/release.yml", data: dummyData,
".woodpecker/image.png", }, {
}, name: ".woodpecker/release.yml",
data: dummyData,
}, {
name: ".woodpecker/image.png",
data: dummyData,
}},
expectedFileNames: []string{ expectedFileNames: []string{
".woodpecker/release.yml", ".woodpecker/release.yml",
}, },
@ -38,9 +52,10 @@ func TestFetch(t *testing.T) {
{ {
name: "Default config - .woodpecker.yml", name: "Default config - .woodpecker.yml",
repoConfig: "", repoConfig: "",
files: []string{ files: []file{{
".woodpecker.yml", name: ".woodpecker.yml",
}, data: dummyData,
}},
expectedFileNames: []string{ expectedFileNames: []string{
".woodpecker.yml", ".woodpecker.yml",
}, },
@ -49,9 +64,10 @@ func TestFetch(t *testing.T) {
{ {
name: "Default config - .drone.yml", name: "Default config - .drone.yml",
repoConfig: "", repoConfig: "",
files: []string{ files: []file{{
".drone.yml", name: ".drone.yml",
}, data: dummyData,
}},
expectedFileNames: []string{ expectedFileNames: []string{
".drone.yml", ".drone.yml",
}, },
@ -60,17 +76,20 @@ func TestFetch(t *testing.T) {
{ {
name: "Default config - Empty repo", name: "Default config - Empty repo",
repoConfig: "", repoConfig: "",
files: []string{}, files: []file{},
expectedFileNames: []string{}, expectedFileNames: []string{},
expectedError: true, expectedError: true,
}, },
{ {
name: "Default config - Additional sub-folders", name: "Default config - Additional sub-folders",
repoConfig: "", repoConfig: "",
files: []string{ files: []file{{
".woodpecker/test.yml", name: ".woodpecker/test.yml",
".woodpecker/sub-folder/config.yml", data: dummyData,
}, }, {
name: ".woodpecker/sub-folder/config.yml",
data: dummyData,
}},
expectedFileNames: []string{ expectedFileNames: []string{
".woodpecker/test.yml", ".woodpecker/test.yml",
}, },
@ -79,25 +98,55 @@ func TestFetch(t *testing.T) {
{ {
name: "Default config - Additional none .yml files", name: "Default config - Additional none .yml files",
repoConfig: "", repoConfig: "",
files: []string{ files: []file{{
".woodpecker/notes.txt", name: ".woodpecker/notes.txt",
".woodpecker/image.png", data: dummyData,
".woodpecker/test.yml", }, {
}, name: ".woodpecker/image.png",
data: dummyData,
}, {
name: ".woodpecker/test.yml",
data: dummyData,
}},
expectedFileNames: []string{ expectedFileNames: []string{
".woodpecker/test.yml", ".woodpecker/test.yml",
}, },
expectedError: false, expectedError: false,
}, },
{
name: "Default config - Empty Folder",
repoConfig: " ",
files: []file{{
name: ".woodpecker/.keep",
data: dummyData,
}, {
name: ".woodpecker.yml",
data: nil,
}, {
name: ".drone.yml",
data: dummyData,
}},
expectedFileNames: []string{
".drone.yml",
},
expectedError: false,
},
{ {
name: "Special config - folder (ignoring default files)", name: "Special config - folder (ignoring default files)",
repoConfig: ".my-ci-folder/", repoConfig: ".my-ci-folder/",
files: []string{ files: []file{{
".woodpecker/test.yml", name: ".woodpecker/test.yml",
".woodpecker.yml", data: dummyData,
".drone.yml", }, {
".my-ci-folder/test.yml", name: ".woodpecker.yml",
}, data: dummyData,
}, {
name: ".drone.yml",
data: dummyData,
}, {
name: ".my-ci-folder/test.yml",
data: dummyData,
}},
expectedFileNames: []string{ expectedFileNames: []string{
".my-ci-folder/test.yml", ".my-ci-folder/test.yml",
}, },
@ -106,9 +155,10 @@ func TestFetch(t *testing.T) {
{ {
name: "Special config - folder", name: "Special config - folder",
repoConfig: ".my-ci-folder/", repoConfig: ".my-ci-folder/",
files: []string{ files: []file{{
".my-ci-folder/test.yml", name: ".my-ci-folder/test.yml",
}, data: dummyData,
}},
expectedFileNames: []string{ expectedFileNames: []string{
".my-ci-folder/test.yml", ".my-ci-folder/test.yml",
}, },
@ -117,9 +167,10 @@ func TestFetch(t *testing.T) {
{ {
name: "Special config - subfolder", name: "Special config - subfolder",
repoConfig: ".my-ci-folder/my-config/", repoConfig: ".my-ci-folder/my-config/",
files: []string{ files: []file{{
".my-ci-folder/my-config/test.yml", name: ".my-ci-folder/my-config/test.yml",
}, data: dummyData,
}},
expectedFileNames: []string{ expectedFileNames: []string{
".my-ci-folder/my-config/test.yml", ".my-ci-folder/my-config/test.yml",
}, },
@ -128,9 +179,10 @@ func TestFetch(t *testing.T) {
{ {
name: "Special config - file", name: "Special config - file",
repoConfig: ".config.yml", repoConfig: ".config.yml",
files: []string{ files: []file{{
".config.yml", name: ".config.yml",
}, data: dummyData,
}},
expectedFileNames: []string{ expectedFileNames: []string{
".config.yml", ".config.yml",
}, },
@ -139,9 +191,10 @@ func TestFetch(t *testing.T) {
{ {
name: "Special config - file inside subfolder", name: "Special config - file inside subfolder",
repoConfig: ".my-ci-folder/sub-folder/config.yml", repoConfig: ".my-ci-folder/sub-folder/config.yml",
files: []string{ files: []file{{
".my-ci-folder/sub-folder/config.yml", name: ".my-ci-folder/sub-folder/config.yml",
}, data: dummyData,
}},
expectedFileNames: []string{ expectedFileNames: []string{
".my-ci-folder/sub-folder/config.yml", ".my-ci-folder/sub-folder/config.yml",
}, },
@ -150,7 +203,7 @@ func TestFetch(t *testing.T) {
{ {
name: "Special config - empty repo", name: "Special config - empty repo",
repoConfig: ".config.yml", repoConfig: ".config.yml",
files: []string{}, files: []file{},
expectedFileNames: []string{}, expectedFileNames: []string{},
expectedError: true, expectedError: true,
}, },
@ -163,13 +216,15 @@ func TestFetch(t *testing.T) {
r := new(mocks.Remote) r := new(mocks.Remote)
dirs := map[string][]*remote.FileMeta{} dirs := map[string][]*remote.FileMeta{}
for _, file := range tt.files { for _, file := range tt.files {
r.On("File", mock.Anything, mock.Anything, mock.Anything, file).Return([]byte{}, nil) r.On("File", mock.Anything, mock.Anything, mock.Anything, file.name).Return(file.data, nil)
path := filepath.Dir(file) path := filepath.Dir(file.name)
if path != "." {
dirs[path] = append(dirs[path], &remote.FileMeta{ dirs[path] = append(dirs[path], &remote.FileMeta{
Name: file, Name: file.name,
Data: []byte{}, Data: file.data,
}) })
} }
}
for path, files := range dirs { for path, files := range dirs {
r.On("Dir", mock.Anything, mock.Anything, mock.Anything, path).Return(files, nil) r.On("Dir", mock.Anything, mock.Anything, mock.Anything, path).Return(files, nil)
@ -192,22 +247,11 @@ func TestFetch(t *testing.T) {
t.Fatal("error fetching config:", err) t.Fatal("error fetching config:", err)
} }
matchingFiles := 0 matchingFiles := make([]string, len(files))
for _, expectedFileName := range tt.expectedFileNames { for i := range files {
for _, file := range files { matchingFiles[i] = files[i].Name
if file.Name == expectedFileName {
matchingFiles += 1
}
}
}
if matchingFiles != len(tt.expectedFileNames) {
var receivedFileNames []string
for _, file := range files {
receivedFileNames = append(receivedFileNames, file.Name)
}
t.Fatal("expected some other pipeline files", tt.expectedFileNames, receivedFileNames)
} }
assert.ElementsMatch(t, tt.expectedFileNames, matchingFiles, "expected some other pipeline files")
}) })
} }
} }