Replay pipeline using cli exec by downloading metadata (#4103)

Co-authored-by: Anbraten <6918444+anbraten@users.noreply.github.com>
This commit is contained in:
6543
2024-09-25 07:20:51 +02:00
committed by GitHub
parent 1a6c8dfec6
commit fcc57dfc38
21 changed files with 927 additions and 194 deletions

View File

@@ -30,8 +30,10 @@ import (
"go.woodpecker-ci.org/woodpecker/v2/server"
"go.woodpecker-ci.org/woodpecker/v2/server/model"
"go.woodpecker-ci.org/woodpecker/v2/server/pipeline"
"go.woodpecker-ci.org/woodpecker/v2/server/pipeline/stepbuilder"
"go.woodpecker-ci.org/woodpecker/v2/server/router/middleware/session"
"go.woodpecker-ci.org/woodpecker/v2/server/store"
"go.woodpecker-ci.org/woodpecker/v2/server/store/types"
)
// CreatePipeline
@@ -392,6 +394,47 @@ func GetPipelineConfig(c *gin.Context) {
c.JSON(http.StatusOK, configs)
}
// GetPipelineMetadata
//
// @Summary Get metadata for a pipeline or a specific workflow, including previous pipeline info
// @Router /repos/{repo_id}/pipelines/{number}/metadata [get]
// @Produce json
// @Success 200 {object} metadata.Metadata
// @Tags Pipelines
// @Param Authorization header string true "Insert your personal access token" default(Bearer <personal access token>)
// @Param repo_id path int true "the repository id"
// @Param number path int true "the number of the pipeline"
func GetPipelineMetadata(c *gin.Context) {
repo := session.Repo(c)
num, err := strconv.ParseInt(c.Param("number"), 10, 64)
if err != nil {
_ = c.AbortWithError(http.StatusBadRequest, err)
return
}
_store := store.FromContext(c)
currentPipeline, err := _store.GetPipelineNumber(repo, num)
if err != nil {
handleDBError(c, err)
return
}
forge, err := server.Config.Services.Manager.ForgeFromRepo(repo)
if err != nil {
c.AbortWithStatus(http.StatusInternalServerError)
return
}
prevPipeline, err := _store.GetPipelineLastBefore(repo, currentPipeline.Branch, currentPipeline.ID)
if err != nil && !errors.Is(err, types.RecordNotExist) {
handleDBError(c, err)
return
}
metadata := stepbuilder.MetadataFromStruct(forge, repo, currentPipeline, prevPipeline, nil, server.Config.Server.Host)
c.JSON(http.StatusOK, metadata)
}
// CancelPipeline
//
// @Summary Cancel a pipeline

View File

@@ -1,129 +1,217 @@
// Copyright 2024 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/franela/goblin"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
"go.woodpecker-ci.org/woodpecker/v2/server"
forge_mocks "go.woodpecker-ci.org/woodpecker/v2/server/forge/mocks"
"go.woodpecker-ci.org/woodpecker/v2/server/model"
"go.woodpecker-ci.org/woodpecker/v2/server/store/mocks"
mocks_manager "go.woodpecker-ci.org/woodpecker/v2/server/services/mocks"
store_mocks "go.woodpecker-ci.org/woodpecker/v2/server/store/mocks"
"go.woodpecker-ci.org/woodpecker/v2/server/store/types"
)
var fakePipeline = &model.Pipeline{
ID: 2,
Number: 2,
Status: model.StatusSuccess,
}
func TestGetPipelines(t *testing.T) {
gin.SetMode(gin.TestMode)
g := goblin.Goblin(t)
g.Describe("Pipeline", func() {
g.It("should get pipelines", func() {
pipelines := []*model.Pipeline{fakePipeline}
t.Run("should get pipelines", func(t *testing.T) {
pipelines := []*model.Pipeline{fakePipeline}
mockStore := mocks.NewStore(t)
mockStore.On("GetPipelineList", mock.Anything, mock.Anything, mock.Anything).Return(pipelines, nil)
mockStore := store_mocks.NewStore(t)
mockStore.On("GetPipelineList", mock.Anything, mock.Anything, mock.Anything).Return(pipelines, nil)
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
c.Set("store", mockStore)
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
c.Set("store", mockStore)
GetPipelines(c)
GetPipelines(c)
mockStore.AssertCalled(t, "GetPipelineList", mock.Anything, mock.Anything, mock.Anything)
assert.Equal(t, http.StatusOK, c.Writer.Status())
})
mockStore.AssertCalled(t, "GetPipelineList", mock.Anything, mock.Anything, mock.Anything)
assert.Equal(t, http.StatusOK, c.Writer.Status())
})
g.It("should not parse pipeline filter", func() {
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Request, _ = http.NewRequest(http.MethodDelete, "/?before=2023-01-16&after=2023-01-15", nil)
t.Run("should not parse pipeline filter", func(t *testing.T) {
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Request, _ = http.NewRequest(http.MethodDelete, "/?before=2023-01-16&after=2023-01-15", nil)
GetPipelines(c)
GetPipelines(c)
assert.Equal(t, http.StatusBadRequest, c.Writer.Status())
})
assert.Equal(t, http.StatusBadRequest, c.Writer.Status())
})
g.It("should parse pipeline filter", func() {
pipelines := []*model.Pipeline{fakePipeline}
t.Run("should parse pipeline filter", func(t *testing.T) {
pipelines := []*model.Pipeline{fakePipeline}
mockStore := mocks.NewStore(t)
mockStore.On("GetPipelineList", mock.Anything, mock.Anything, mock.Anything).Return(pipelines, nil)
mockStore := store_mocks.NewStore(t)
mockStore.On("GetPipelineList", mock.Anything, mock.Anything, mock.Anything).Return(pipelines, nil)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Set("store", mockStore)
c.Request, _ = http.NewRequest(http.MethodDelete, "/?2023-01-16T15:00:00Z&after=2023-01-15T15:00:00Z", nil)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Set("store", mockStore)
c.Request, _ = http.NewRequest(http.MethodDelete, "/?2023-01-16T15:00:00Z&after=2023-01-15T15:00:00Z", nil)
GetPipelines(c)
GetPipelines(c)
assert.Equal(t, http.StatusOK, c.Writer.Status())
})
assert.Equal(t, http.StatusOK, c.Writer.Status())
})
g.It("should parse pipeline filter with tz offset", func() {
pipelines := []*model.Pipeline{fakePipeline}
t.Run("should parse pipeline filter with tz offset", func(t *testing.T) {
pipelines := []*model.Pipeline{fakePipeline}
mockStore := mocks.NewStore(t)
mockStore.On("GetPipelineList", mock.Anything, mock.Anything, mock.Anything).Return(pipelines, nil)
mockStore := store_mocks.NewStore(t)
mockStore.On("GetPipelineList", mock.Anything, mock.Anything, mock.Anything).Return(pipelines, nil)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Set("store", mockStore)
c.Request, _ = http.NewRequest(http.MethodDelete, "/?before=2023-01-16T15:00:00%2B01:00&after=2023-01-15T15:00:00%2B01:00", nil)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Set("store", mockStore)
c.Request, _ = http.NewRequest(http.MethodDelete, "/?before=2023-01-16T15:00:00%2B01:00&after=2023-01-15T15:00:00%2B01:00", nil)
GetPipelines(c)
GetPipelines(c)
assert.Equal(t, http.StatusOK, c.Writer.Status())
})
assert.Equal(t, http.StatusOK, c.Writer.Status())
})
}
func TestDeletePipeline(t *testing.T) {
gin.SetMode(gin.TestMode)
g := goblin.Goblin(t)
g.Describe("Pipeline", func() {
g.It("should delete pipeline", func() {
mockStore := mocks.NewStore(t)
mockStore.On("GetPipelineNumber", mock.Anything, mock.Anything).Return(fakePipeline, nil)
mockStore.On("DeletePipeline", mock.Anything).Return(nil)
t.Run("should delete pipeline", func(t *testing.T) {
mockStore := store_mocks.NewStore(t)
mockStore.On("GetPipelineNumber", mock.Anything, mock.Anything).Return(fakePipeline, nil)
mockStore.On("DeletePipeline", mock.Anything).Return(nil)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Set("store", mockStore)
c.Params = gin.Params{{Key: "number", Value: "2"}}
DeletePipeline(c)
mockStore.AssertCalled(t, "GetPipelineNumber", mock.Anything, mock.Anything)
mockStore.AssertCalled(t, "DeletePipeline", mock.Anything)
assert.Equal(t, http.StatusNoContent, c.Writer.Status())
})
t.Run("should not delete without pipeline number", func(t *testing.T) {
c, _ := gin.CreateTestContext(httptest.NewRecorder())
DeletePipeline(c)
assert.Equal(t, http.StatusBadRequest, c.Writer.Status())
})
t.Run("should not delete pending", func(t *testing.T) {
fakePipeline := *fakePipeline
fakePipeline.Status = model.StatusPending
mockStore := store_mocks.NewStore(t)
mockStore.On("GetPipelineNumber", mock.Anything, mock.Anything).Return(&fakePipeline, nil)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Set("store", mockStore)
c.Params = gin.Params{{Key: "number", Value: "2"}}
DeletePipeline(c)
mockStore.AssertCalled(t, "GetPipelineNumber", mock.Anything, mock.Anything)
mockStore.AssertNotCalled(t, "DeletePipeline", mock.Anything)
assert.Equal(t, http.StatusUnprocessableEntity, c.Writer.Status())
})
}
func TestGetPipelineMetadata(t *testing.T) {
gin.SetMode(gin.TestMode)
prevPipeline := &model.Pipeline{
ID: 1,
Number: 1,
Status: model.StatusFailure,
}
fakeRepo := &model.Repo{ID: 1}
mockForge := forge_mocks.NewForge(t)
mockForge.On("Name").Return("mock")
mockForge.On("URL").Return("https://codeberg.org")
mockManager := mocks_manager.NewManager(t)
mockManager.On("ForgeFromRepo", fakeRepo).Return(mockForge, nil)
server.Config.Services.Manager = mockManager
mockStore := store_mocks.NewStore(t)
mockStore.On("GetPipelineNumber", mock.Anything, int64(2)).Return(fakePipeline, nil)
mockStore.On("GetPipelineLastBefore", mock.Anything, mock.Anything, int64(2)).Return(prevPipeline, nil)
t.Run("PipelineMetadata", func(t *testing.T) {
t.Run("should get pipeline metadata", func(t *testing.T) {
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
c.Params = gin.Params{{Key: "number", Value: "2"}}
c.Set("store", mockStore)
c.Params = gin.Params{{Key: "number", Value: "1"}}
c.Set("forge", mockForge)
c.Set("repo", fakeRepo)
DeletePipeline(c)
GetPipelineMetadata(c)
mockStore.AssertCalled(t, "GetPipelineNumber", mock.Anything, mock.Anything)
mockStore.AssertCalled(t, "DeletePipeline", mock.Anything)
assert.Equal(t, http.StatusNoContent, c.Writer.Status())
assert.Equal(t, http.StatusOK, w.Code)
var response metadata.Metadata
err := json.Unmarshal(w.Body.Bytes(), &response)
assert.NoError(t, err)
assert.Equal(t, int64(1), response.Repo.ID)
assert.Equal(t, int64(2), response.Curr.Number)
assert.Equal(t, int64(1), response.Prev.Number)
})
g.It("should not delete without pipeline number", func() {
c, _ := gin.CreateTestContext(httptest.NewRecorder())
t.Run("should return bad request for invalid pipeline number", func(t *testing.T) {
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
c.Params = gin.Params{{Key: "number", Value: "invalid"}}
DeletePipeline(c)
GetPipelineMetadata(c)
assert.Equal(t, http.StatusBadRequest, c.Writer.Status())
assert.Equal(t, http.StatusBadRequest, w.Code)
})
g.It("should not delete pending", func() {
fakePipeline.Status = model.StatusPending
t.Run("should return not found for non-existent pipeline", func(t *testing.T) {
mockStore := store_mocks.NewStore(t)
mockStore.On("GetPipelineNumber", mock.Anything, int64(3)).Return((*model.Pipeline)(nil), types.RecordNotExist)
mockStore := mocks.NewStore(t)
mockStore.On("GetPipelineNumber", mock.Anything, mock.Anything).Return(fakePipeline, nil)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
c.Params = gin.Params{{Key: "number", Value: "3"}}
c.Set("store", mockStore)
c.Params = gin.Params{{Key: "number", Value: "1"}}
c.Set("repo", fakeRepo)
DeletePipeline(c)
GetPipelineMetadata(c)
mockStore.AssertCalled(t, "GetPipelineNumber", mock.Anything, mock.Anything)
mockStore.AssertNotCalled(t, "DeletePipeline", mock.Anything)
assert.Equal(t, http.StatusUnprocessableEntity, c.Writer.Status())
assert.Equal(t, http.StatusNotFound, w.Code)
})
})
}

View File

@@ -38,7 +38,7 @@ func parsePipeline(forge forge.Forge, store store.Store, currentPipeline *model.
}
// get the previous pipeline so that we can send status change notifications
last, err := store.GetPipelineLastBefore(repo, currentPipeline.Branch, currentPipeline.ID)
prev, err := store.GetPipelineLastBefore(repo, currentPipeline.Branch, currentPipeline.ID)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
log.Error().Err(err).Str("repo", repo.FullName).Msgf("error getting last pipeline before pipeline number '%d'", currentPipeline.Number)
}
@@ -74,7 +74,7 @@ func parsePipeline(forge forge.Forge, store store.Store, currentPipeline *model.
b := stepbuilder.StepBuilder{
Repo: repo,
Curr: currentPipeline,
Last: last,
Prev: prev,
Netrc: netrc,
Secs: secs,
Regs: regs,

View File

@@ -25,7 +25,7 @@ import (
)
// MetadataFromStruct return the metadata from a pipeline will run with.
func MetadataFromStruct(forge metadata.ServerForge, repo *model.Repo, pipeline, last *model.Pipeline, workflow *model.Workflow, sysURL string) metadata.Metadata {
func MetadataFromStruct(forge metadata.ServerForge, repo *model.Repo, pipeline, prev *model.Pipeline, workflow *model.Workflow, sysURL string) metadata.Metadata {
host := sysURL
uri, err := url.Parse(sysURL)
if err == nil {
@@ -78,7 +78,7 @@ func MetadataFromStruct(forge metadata.ServerForge, repo *model.Repo, pipeline,
return metadata.Metadata{
Repo: fRepo,
Curr: metadataPipelineFromModelPipeline(pipeline, true),
Prev: metadataPipelineFromModelPipeline(last, false),
Prev: metadataPipelineFromModelPipeline(prev, false),
Workflow: fWorkflow,
Step: metadata.Step{},
Sys: metadata.System{

View File

@@ -33,7 +33,7 @@ func TestMetadataFromStruct(t *testing.T) {
name string
forge metadata.ServerForge
repo *model.Repo
pipeline, last *model.Pipeline
pipeline, prev *model.Pipeline
workflow *model.Workflow
sysURL string
expectedMetadata metadata.Metadata
@@ -63,7 +63,7 @@ func TestMetadataFromStruct(t *testing.T) {
forge: forge,
repo: &model.Repo{FullName: "testUser/testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", Clone: "https://gitea.com/testUser/testRepo.git", CloneSSH: "git@gitea.com:testUser/testRepo.git", Branch: "main", IsSCMPrivate: true, SCMKind: "git"},
pipeline: &model.Pipeline{Number: 3, ChangedFiles: []string{"test.go", "markdown file.md"}},
last: &model.Pipeline{Number: 2},
prev: &model.Pipeline{Number: 2},
workflow: &model.Workflow{Name: "hello"},
sysURL: "https://example.com",
expectedMetadata: metadata.Metadata{
@@ -98,7 +98,7 @@ func TestMetadataFromStruct(t *testing.T) {
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
result := MetadataFromStruct(testCase.forge, testCase.repo, testCase.pipeline, testCase.last, testCase.workflow, testCase.sysURL)
result := MetadataFromStruct(testCase.forge, testCase.repo, testCase.pipeline, testCase.prev, testCase.workflow, testCase.sysURL)
assert.EqualValues(t, testCase.expectedMetadata, result)
assert.EqualValues(t, testCase.expectedEnviron, result.Environ())
})

View File

@@ -42,7 +42,7 @@ import (
type StepBuilder struct {
Repo *model.Repo
Curr *model.Pipeline
Last *model.Pipeline
Prev *model.Pipeline
Netrc *model.Netrc
Secs []*model.Secret
Regs []*model.Registry
@@ -115,7 +115,7 @@ func (b *StepBuilder) Build() (items []*Item, errorsAndWarnings error) {
}
func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.Axis, data string) (item *Item, errorsAndWarnings error) {
workflowMetadata := MetadataFromStruct(b.Forge, b.Repo, b.Curr, b.Last, workflow, b.Host)
workflowMetadata := MetadataFromStruct(b.Forge, b.Repo, b.Curr, b.Prev, workflow, b.Host)
environ := b.environmentVariables(workflowMetadata, axis)
// add global environment variables for substituting

View File

@@ -42,7 +42,7 @@ func TestGlobalEnvsubst(t *testing.T) {
Message: "aaa",
Event: model.EventPush,
},
Last: &model.Pipeline{},
Prev: &model.Pipeline{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},
@@ -81,7 +81,7 @@ func TestMissingGlobalEnvsubst(t *testing.T) {
Message: "aaa",
Event: model.EventPush,
},
Last: &model.Pipeline{},
Prev: &model.Pipeline{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},
@@ -116,7 +116,7 @@ func TestMultilineEnvsubst(t *testing.T) {
Message: `aaa
bbb`,
},
Last: &model.Pipeline{},
Prev: &model.Pipeline{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},
@@ -159,7 +159,7 @@ func TestMultiPipeline(t *testing.T) {
Curr: &model.Pipeline{
Event: model.EventPush,
},
Last: &model.Pipeline{},
Prev: &model.Pipeline{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},
@@ -200,7 +200,7 @@ func TestDependsOn(t *testing.T) {
Curr: &model.Pipeline{
Event: model.EventPush,
},
Last: &model.Pipeline{},
Prev: &model.Pipeline{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},
@@ -255,7 +255,7 @@ func TestRunsOn(t *testing.T) {
Curr: &model.Pipeline{
Event: model.EventPush,
},
Last: &model.Pipeline{},
Prev: &model.Pipeline{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},
@@ -296,7 +296,7 @@ func TestPipelineName(t *testing.T) {
Curr: &model.Pipeline{
Event: model.EventPush,
},
Last: &model.Pipeline{},
Prev: &model.Pipeline{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},
@@ -339,7 +339,7 @@ func TestBranchFilter(t *testing.T) {
Branch: "dev",
Event: model.EventPush,
},
Last: &model.Pipeline{},
Prev: &model.Pipeline{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},
@@ -382,7 +382,7 @@ func TestRootWhenFilter(t *testing.T) {
Forge: getMockForge(t),
Repo: &model.Repo{},
Curr: &model.Pipeline{Event: "tag"},
Last: &model.Pipeline{},
Prev: &model.Pipeline{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},
@@ -432,7 +432,7 @@ func TestZeroSteps(t *testing.T) {
Forge: getMockForge(t),
Repo: &model.Repo{},
Curr: pipeline,
Last: &model.Pipeline{},
Prev: &model.Pipeline{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},
@@ -472,7 +472,7 @@ func TestZeroStepsAsMultiPipelineDeps(t *testing.T) {
Forge: getMockForge(t),
Repo: &model.Repo{},
Curr: pipeline,
Last: &model.Pipeline{},
Prev: &model.Pipeline{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},
@@ -530,7 +530,7 @@ func TestZeroStepsAsMultiPipelineTransitiveDeps(t *testing.T) {
Forge: getMockForge(t),
Repo: &model.Repo{},
Curr: pipeline,
Last: &model.Pipeline{},
Prev: &model.Pipeline{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},

View File

@@ -102,6 +102,7 @@ func apiRoutes(e *gin.RouterGroup) {
repo.DELETE("/pipelines/:number", session.MustRepoAdmin(), api.DeletePipeline)
repo.GET("/pipelines/:number", api.GetPipeline)
repo.GET("/pipelines/:number/config", api.GetPipelineConfig)
repo.GET("/pipelines/:number/metadata", session.MustPush, api.GetPipelineMetadata)
// requires push permissions
repo.POST("/pipelines/:number", session.MustPush, api.PostPipeline)