diff --git a/.golangci.yaml b/.golangci.yaml index a33401c8c3..be360a1f28 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -104,6 +104,7 @@ linters: - pkg: go.woodpecker-ci.org/woodpecker/v3/cli - pkg: go.woodpecker-ci.org/woodpecker/v3/cmd/agent - pkg: go.woodpecker-ci.org/woodpecker/v3/cmd/cli + - pkg: go.woodpecker-ci.org/woodpecker/v3/pipeline/backend - pkg: go.woodpecker-ci.org/woodpecker/v3/woodpecker-go/woodpecker rpc: list-mode: lax diff --git a/cmd/server/openapi/docs.go b/cmd/server/openapi/docs.go index 8caf9e5204..617841b10a 100644 --- a/cmd/server/openapi/docs.go +++ b/cmd/server/openapi/docs.go @@ -6046,6 +6046,9 @@ const docTemplate = `{ "name": { "type": "string" }, + "org_id": { + "type": "integer" + }, "owner": { "type": "string" }, diff --git a/e2e/scenarios/fixtures.go b/e2e/scenarios/fixtures.go index 8837e5351b..be965b97af 100644 --- a/e2e/scenarios/fixtures.go +++ b/e2e/scenarios/fixtures.go @@ -182,7 +182,7 @@ func loadMultiWorkflowScenario(t *testing.T, dirName string) Scenario { require.NotEmpty(t, files, "no YAML files in multi-workflow dir %s", dirName) require.NotEmpty(t, s.Name, "scenario.json missing 'name' in %s", dirName) - s.Files = forge_types.SortByName(files) + s.Files = files if s.Event == "" { s.Event = model.EventPush } diff --git a/e2e/scenarios/infra_test.go b/e2e/scenarios/infra_test.go index 8b0031cefa..03f54cc865 100644 --- a/e2e/scenarios/infra_test.go +++ b/e2e/scenarios/infra_test.go @@ -89,4 +89,11 @@ func TestInfraSmoke(t *testing.T) { finished := setup.WaitForPipeline(t, env.Store, createdPipeline.ID) assert.Equal(t, model.StatusSuccess, finished.Status, "pipeline should succeed") + + workflows, err := env.Store.WorkflowGetTree(finished) + require.NoError(t, err) + require.Len(t, workflows, 1, "smoke test expects exactly one workflow") + assert.Equal(t, "woodpecker", workflows[0].Name) + assert.Equal(t, model.StatusSuccess, workflows[0].State) + assert.Greater(t, workflows[0].AgentID, int64(0), "workflow should record agent that ran it") } diff --git a/server/pipeline/step_builder/step_builder.go b/pipeline/frontend/builder/builder.go similarity index 61% rename from server/pipeline/step_builder/step_builder.go rename to pipeline/frontend/builder/builder.go index beb1784796..598499b830 100644 --- a/server/pipeline/step_builder/step_builder.go +++ b/pipeline/frontend/builder/builder.go @@ -13,12 +13,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package step_builder +package builder import ( "fmt" "maps" - "path/filepath" "slices" "strconv" "strings" @@ -36,36 +35,22 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/linter" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/matrix" yaml_types "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/types" - forge_types "go.woodpecker-ci.org/woodpecker/v3/server/forge/types" - "go.woodpecker-ci.org/woodpecker/v3/server/model" ) -// StepBuilder Takes the hook data and the yaml and returns the internal data model. -type StepBuilder struct { - Repo *model.Repo // TODO: get rid of server dependency - Curr *model.Pipeline // TODO: get rid of server dependency - Prev *model.Pipeline // TODO: get rid of server dependency - Host string - Yamls []*forge_types.FileMeta +// PipelineBuilder Takes the yaml configs and some metadata and returns the internal data model to execute a pipeline. +type PipelineBuilder struct { + Yamls []*YamlFile Envs map[string]string - Forge metadata.ServerForge DefaultLabels map[string]string RepoTrusted *metadata.TrustedConfiguration TrustedClonePlugins []string PrivilegedPlugins []string CompilerOptions []compiler.Option + GetWorkflowMetadata func(workflow *Workflow) metadata.Metadata } -type Item struct { - Workflow *model.Workflow // TODO: get rid of server dependency - Labels map[string]string - DependsOn []string - RunsOn []string - Config *backend_types.Config -} - -func (b *StepBuilder) Build() (items []*Item, errorsAndWarnings error) { - b.Yamls = forge_types.SortByName(b.Yamls) +func (b *PipelineBuilder) Build() (items []*Item, errorsAndWarnings error) { + b.Yamls = SortYamlFilesByName(b.Yamls) pidSequence := 1 @@ -80,9 +65,8 @@ func (b *StepBuilder) Build() (items []*Item, errorsAndWarnings error) { } for i, axis := range axes { - workflow := &model.Workflow{ + workflow := &Workflow{ PID: pidSequence, - State: model.StatusPending, Environ: axis, Name: SanitizePath(y.Name), } @@ -112,8 +96,8 @@ func (b *StepBuilder) Build() (items []*Item, errorsAndWarnings error) { return items, errorsAndWarnings } -func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.Axis, data string) (item *Item, errorsAndWarnings error) { - workflowMetadata := MetadataFromStruct(b.Forge, b.Repo, b.Curr, b.Prev, workflow, b.Host) +func (b *PipelineBuilder) genItemForWorkflow(workflow *Workflow, axis matrix.Axis, data string) (item *Item, errorsAndWarnings error) { + workflowMetadata := b.GetWorkflowMetadata(workflow) environ := b.environmentVariables(workflowMetadata, axis) // add global environment variables for substituting @@ -140,9 +124,9 @@ func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.A // lint pipeline errorsAndWarnings = multierr.Append(errorsAndWarnings, linter.New( linter.WithTrusted(linter.TrustedConfiguration{ - Network: b.Repo.Trusted.Network, - Volumes: b.Repo.Trusted.Volumes, - Security: b.Repo.Trusted.Security, + Network: b.RepoTrusted.Network, + Volumes: b.RepoTrusted.Volumes, + Security: b.RepoTrusted.Security, }), linter.PrivilegedPlugins(b.PrivilegedPlugins), linter.WithTrustedClonePlugins(b.TrustedClonePlugins), @@ -182,7 +166,8 @@ func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.A Config: ir, Labels: parsed.Labels, DependsOn: parsed.DependsOn, - RunsOn: parsed.RunsOn, //nolint:staticcheck // TODO: remove in next major. + // TODO: remove in next major. + RunsOn: parsed.RunsOn, //nolint:staticcheck } if len(item.Labels) == 0 { item.Labels = make(map[string]string, len(b.DefaultLabels)) @@ -197,74 +182,35 @@ func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.A item.RunsOn = append(item.RunsOn, "success") } - // "woodpecker-ci.org" namespace is reserved for internal use + // "woodpecker-ci.org" namespace is reserved for internal use — drop any + // user-defined labels that try to use it. for key := range item.Labels { if strings.HasPrefix(key, pipeline.InternalLabelPrefix) { - log.Debug().Str("forge", b.Forge.Name()).Str("repo", b.Repo.FullName).Str("label", key).Msg("dropped pipeline label with reserved prefix woodpecker-ci.org") + log.Debug().Str("label", key).Msg("dropped pipeline label with reserved prefix woodpecker-ci.org") delete(item.Labels, key) } } - // Add Woodpecker managed labels to the pipeline - item.Labels[pipeline.LabelForgeRemoteID] = b.Forge.Name() - item.Labels[pipeline.LabelRepoForgeID] = string(b.Repo.ForgeRemoteID) - item.Labels[pipeline.LabelRepoID] = strconv.FormatInt(b.Repo.ID, 10) - item.Labels[pipeline.LabelRepoName] = b.Repo.Name - item.Labels[pipeline.LabelRepoFullName] = b.Repo.FullName - item.Labels[pipeline.LabelBranch] = b.Repo.Branch - item.Labels[pipeline.LabelOrgID] = strconv.FormatInt(b.Repo.OrgID, 10) - - for stageI := range item.Config.Stages { - for stepI := range item.Config.Stages[stageI].Steps { - item.Config.Stages[stageI].Steps[stepI].WorkflowLabels = item.Labels - item.Config.Stages[stageI].Steps[stepI].OrgID = b.Repo.OrgID - } - } + // Stamp Woodpecker-managed internal labels onto the item so that the server + // and backends can use them for routing, observability, etc. + item.Labels[pipeline.LabelForgeRemoteID] = workflowMetadata.Forge.Type + item.Labels[pipeline.LabelRepoForgeID] = workflowMetadata.Repo.RemoteID + item.Labels[pipeline.LabelRepoID] = strconv.FormatInt(workflowMetadata.Repo.ID, 10) + item.Labels[pipeline.LabelRepoName] = workflowMetadata.Repo.Name + item.Labels[pipeline.LabelRepoFullName] = workflowMetadata.Repo.Owner + "/" + workflowMetadata.Repo.Name + item.Labels[pipeline.LabelBranch] = workflowMetadata.Repo.Branch + item.Labels[pipeline.LabelOrgID] = strconv.FormatInt(workflowMetadata.Repo.OrgID, 10) return item, errorsAndWarnings } -func filterItemsWithMissingDependencies(items []*Item) []*Item { - itemsToRemove := make([]*Item, 0) - - for _, item := range items { - for _, dep := range item.DependsOn { - if !containsItemWithName(dep, items) { - itemsToRemove = append(itemsToRemove, item) - } - } - } - - if len(itemsToRemove) > 0 { - filtered := make([]*Item, 0) - for _, item := range items { - if !containsItemWithName(item.Workflow.Name, itemsToRemove) { - filtered = append(filtered, item) - } - } - // Recursive to handle transitive deps - return filterItemsWithMissingDependencies(filtered) - } - - return items -} - -func containsItemWithName(name string, items []*Item) bool { - for _, item := range items { - if name == item.Workflow.Name { - return true - } - } - return false -} - -func (b *StepBuilder) environmentVariables(metadata metadata.Metadata, axis matrix.Axis) map[string]string { +func (b *PipelineBuilder) environmentVariables(metadata metadata.Metadata, axis matrix.Axis) map[string]string { environ := metadata.Environ() maps.Copy(environ, axis) return environ } -func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, environ map[string]string, metadata metadata.Metadata, workflowID int64) (*backend_types.Config, error) { +func (b *PipelineBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, environ map[string]string, metadata metadata.Metadata, workflowID int64) (*backend_types.Config, error) { options := []compiler.Option{} options = append(options, compiler.WithEnviron(environ), @@ -288,11 +234,3 @@ func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, envi return compiler.New(options...).Compile(parsed) } - -func SanitizePath(path string) string { - path = filepath.Base(path) - path = strings.TrimSuffix(path, ".yml") - path = strings.TrimSuffix(path, ".yaml") - path = strings.TrimPrefix(path, ".") - return path -} diff --git a/server/pipeline/step_builder/step_builder_test.go b/pipeline/frontend/builder/builder_test.go similarity index 67% rename from server/pipeline/step_builder/step_builder_test.go rename to pipeline/frontend/builder/builder_test.go index e8f9814c0f..3fb49b830e 100644 --- a/server/pipeline/step_builder/step_builder_test.go +++ b/pipeline/frontend/builder/builder_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package step_builder +package builder import ( "testing" @@ -23,30 +23,21 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/metadata" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/compiler" - "go.woodpecker-ci.org/woodpecker/v3/server/forge" - "go.woodpecker-ci.org/woodpecker/v3/server/forge/mocks" - forge_types "go.woodpecker-ci.org/woodpecker/v3/server/forge/types" - "go.woodpecker-ci.org/woodpecker/v3/server/model" ) func TestGlobalEnvsubst(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), + m := &testMetadata{} + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, Envs: map[string]string{ "KEY_K": "VALUE_V", "IMAGE": "scratch", }, RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{}, - Curr: &model.Pipeline{ - Message: "aaa", - Event: model.EventPush, - }, - Prev: &model.Pipeline{}, - Host: "", - Yamls: []*forge_types.FileMeta{ + Yamls: []*YamlFile{ {Data: []byte(` when: event: push @@ -66,21 +57,16 @@ steps: func TestMissingGlobalEnvsubst(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), + m := &testMetadata{} + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, Envs: map[string]string{ "KEY_K": "VALUE_V", "NO_IMAGE": "scratch", }, RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{}, - Curr: &model.Pipeline{ - Message: "aaa", - Event: model.EventPush, - }, - Prev: &model.Pipeline{}, - Host: "", - Yamls: []*forge_types.FileMeta{ + Yamls: []*YamlFile{ {Data: []byte(` when: event: push @@ -100,17 +86,12 @@ steps: func TestMultilineEnvsubst(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), - RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{}, - Curr: &model.Pipeline{ - Message: `aaa -bbb`, - }, - Prev: &model.Pipeline{}, - Host: "", - Yamls: []*forge_types.FileMeta{ + m := &testMetadata{} + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, + Yamls: []*YamlFile{ {Data: []byte(` when: event: push @@ -139,16 +120,14 @@ steps: func TestMultiPipeline(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), - Repo: &model.Repo{}, - RepoTrusted: &metadata.TrustedConfiguration{}, - Curr: &model.Pipeline{ - Event: model.EventPush, - }, - Prev: &model.Pipeline{}, - Host: "", - Yamls: []*forge_types.FileMeta{ + m := &testMetadata{ + pipelineEvent: "push", + } + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, + Yamls: []*YamlFile{ {Data: []byte(` when: event: push @@ -174,16 +153,14 @@ steps: func TestDependsOn(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), - Repo: &model.Repo{}, - RepoTrusted: &metadata.TrustedConfiguration{}, - Curr: &model.Pipeline{ - Event: model.EventPush, - }, - Prev: &model.Pipeline{}, - Host: "", - Yamls: []*forge_types.FileMeta{ + m := &testMetadata{ + pipelineEvent: "push", + } + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, + Yamls: []*YamlFile{ {Name: "lint", Data: []byte(` when: event: push @@ -232,16 +209,14 @@ depends_on: func TestRunsOn(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), - RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{}, - Curr: &model.Pipeline{ - Event: model.EventPush, - }, - Prev: &model.Pipeline{}, - Host: "", - Yamls: []*forge_types.FileMeta{ + m := &testMetadata{ + pipelineEvent: "push", + } + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, + Yamls: []*YamlFile{ {Data: []byte(` when: event: push @@ -256,6 +231,7 @@ steps: items, err := b.Build() assert.NoError(t, err) + assert.Len(t, items, 1, "Should have generated 1 pipeline") assert.Len(t, items[0].RunsOn, 2, "Should run on success and failure") assert.ElementsMatchf(t, []string{"success", "failure"}, items[0].RunsOn, "Should run on failure") } @@ -263,16 +239,14 @@ steps: func TestPipelineName(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), - RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{Config: ".woodpecker"}, - Curr: &model.Pipeline{ - Event: model.EventPush, - }, - Prev: &model.Pipeline{}, - Host: "", - Yamls: []*forge_types.FileMeta{ + m := &testMetadata{ + pipelineEvent: "push", + } + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, + Yamls: []*YamlFile{ {Name: ".woodpecker/lint.yml", Data: []byte(` when: event: push @@ -292,25 +266,24 @@ steps: items, err := b.Build() assert.NoError(t, err) + assert.Len(t, items, 2, "Should have generated 2 pipelines") pipelineNames := []string{items[0].Workflow.Name, items[1].Workflow.Name} - assert.True(t, containsItemWithName("lint", items) && containsItemWithName("test", items), + assert.True(t, ContainsItemWithName("lint", items) && ContainsItemWithName("test", items), "Pipeline name should be 'lint' and 'test' but are '%v'", pipelineNames) } func TestBranchFilter(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), - RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{}, - Curr: &model.Pipeline{ - Branch: "dev", - Event: model.EventPush, - }, - Prev: &model.Pipeline{}, - Host: "", - Yamls: []*forge_types.FileMeta{ + m := &testMetadata{ + pipelineEvent: "push", + branch: "dev", + } + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, + Yamls: []*YamlFile{ {Data: []byte(` when: event: push @@ -332,20 +305,19 @@ steps: items, err := b.Build() assert.NoError(t, err) assert.Len(t, items, 1, "Should have generated 1 pipeline") - assert.Equal(t, model.StatusPending, items[0].Workflow.State, "Should run on dev branch") } func TestRootWhenFilter(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), - RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{}, - Curr: &model.Pipeline{Event: "tag"}, - Prev: &model.Pipeline{}, - Host: "", - Yamls: []*forge_types.FileMeta{ + m := &testMetadata{ + pipelineEvent: "tag", + } + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, + Yamls: []*YamlFile{ {Data: []byte(` when: event: @@ -378,19 +350,12 @@ steps: func TestZeroSteps(t *testing.T) { t.Parallel() - pipeline := &model.Pipeline{ - Branch: "dev", - Event: model.EventPush, - } + m := &testMetadata{} - b := StepBuilder{ - Forge: getMockForge(t), - RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{}, - Curr: pipeline, - Prev: &model.Pipeline{}, - Host: "", - Yamls: []*forge_types.FileMeta{ + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, + Yamls: []*YamlFile{ {Data: []byte(` when: event: push @@ -412,19 +377,14 @@ steps: func TestZeroStepsAsMultiPipelineTransitiveDeps(t *testing.T) { t.Parallel() - pipeline := &model.Pipeline{ - Branch: "dev", - Event: model.EventPush, + m := &testMetadata{ + pipelineEvent: "push", } - b := StepBuilder{ - Forge: getMockForge(t), - RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{}, - Curr: pipeline, - Prev: &model.Pipeline{}, - Host: "", - Yamls: []*forge_types.FileMeta{ + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, + Yamls: []*YamlFile{ {Name: "zerostep", Data: []byte(` when: event: push @@ -508,14 +468,14 @@ func TestSanitizePath(t *testing.T) { func TestMatrix(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), - RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{}, - Curr: &model.Pipeline{Event: model.EventPush}, - Prev: &model.Pipeline{}, - Host: "", - Yamls: []*forge_types.FileMeta{ + m := &testMetadata{ + pipelineEvent: "push", + } + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, + Yamls: []*YamlFile{ {Data: []byte(` when: event: push @@ -549,14 +509,12 @@ steps: func TestMissingWorkflowDeps(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), - RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{}, - Curr: &model.Pipeline{Event: model.EventPush}, - Prev: &model.Pipeline{}, - Host: "", - Yamls: []*forge_types.FileMeta{ + m := &testMetadata{} + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, + Yamls: []*YamlFile{ { Name: "workflow-with-missing-deps", Data: []byte(` @@ -580,13 +538,12 @@ depends_on: func TestInvalidYAML(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: nil, - RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{}, - Curr: &model.Pipeline{Event: model.EventPush}, - Prev: &model.Pipeline{}, - Yamls: []*forge_types.FileMeta{ + m := &testMetadata{} + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, + Yamls: []*YamlFile{ {Name: "broken-yaml", Data: []byte(` when: event: push @@ -605,21 +562,20 @@ steps: func TestEnvVarPrecedence(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), + m := &testMetadata{ + pipelineEvent: "push", + repo: "actual-repo", + } + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, Envs: map[string]string{ "CUSTOM_VAR": "global-value", "CI_REPO_NAME": "should-not-override", "ANOTHER_CUSTOM": "global-value-2", }, RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{Name: "actual-repo"}, - Curr: &model.Pipeline{ - Event: model.EventPush, - Message: "test", - }, - Prev: &model.Pipeline{}, - Yamls: []*forge_types.FileMeta{ + Yamls: []*YamlFile{ {Data: []byte(` when: event: push @@ -645,17 +601,18 @@ steps: func TestLabelMerging(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), - RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{Name: "test-repo"}, - Curr: &model.Pipeline{Event: model.EventPush}, - Prev: &model.Pipeline{}, + m := &testMetadata{ + pipelineEvent: "push", + } + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, DefaultLabels: map[string]string{ "default-label": "default-value", "override-me": "default", }, - Yamls: []*forge_types.FileMeta{ + Yamls: []*YamlFile{ {Data: []byte(` when: event: push @@ -691,18 +648,19 @@ steps: func TestCompilerOptions(t *testing.T) { t.Parallel() - b := StepBuilder{ - Forge: getMockForge(t), - RepoTrusted: &metadata.TrustedConfiguration{}, - Repo: &model.Repo{}, - Curr: &model.Pipeline{Event: model.EventPush}, - Prev: &model.Pipeline{}, + m := &testMetadata{ + pipelineEvent: "push", + } + + b := PipelineBuilder{ + GetWorkflowMetadata: m.GetWorkflowMetadata, + RepoTrusted: &metadata.TrustedConfiguration{}, CompilerOptions: []compiler.Option{ compiler.WithEnviron(map[string]string{ "KEY": "VALUE", }), }, - Yamls: []*forge_types.FileMeta{ + Yamls: []*YamlFile{ {Data: []byte(` skip_clone: true when: @@ -722,9 +680,22 @@ steps: assert.Equal(t, "VALUE", items[0].Config.Stages[0].Steps[0].Environment["KEY"], "Environment variable should be set") } -func getMockForge(t *testing.T) forge.Forge { - forge := mocks.NewMockForge(t) - forge.On("Name").Return("mock") - forge.On("URL").Return("https://codeberg.org") - return forge +type testMetadata struct { + pipelineEvent metadata.Event + branch string + repo string +} + +func (t *testMetadata) GetWorkflowMetadata(w *Workflow) metadata.Metadata { + return metadata.Metadata{ + Repo: metadata.Repo{ + Name: t.repo, + }, + Curr: metadata.Pipeline{ + Event: t.pipelineEvent, + Commit: metadata.Commit{ + Branch: t.branch, + }, + }, + } } diff --git a/pipeline/frontend/builder/types.go b/pipeline/frontend/builder/types.go new file mode 100644 index 0000000000..8179e8b7db --- /dev/null +++ b/pipeline/frontend/builder/types.go @@ -0,0 +1,54 @@ +// Copyright 2026 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package builder + +import ( + "sort" + + backend_types "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" +) + +type Item struct { + Workflow *Workflow + Labels map[string]string + DependsOn []string + RunsOn []string + Config *backend_types.Config +} + +type Workflow struct { + ID int64 `json:"id"` + PID int `json:"pid"` + Name string `json:"name"` + Environ map[string]string `json:"environ,omitempty"` + AxisID int `json:"-"` +} + +type YamlFile struct { + Name string + Data []byte +} + +type yamlFileList []*YamlFile + +func (a yamlFileList) Len() int { return len(a) } +func (a yamlFileList) Less(i, j int) bool { return a[i].Name < a[j].Name } +func (a yamlFileList) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +func SortYamlFilesByName(fm []*YamlFile) []*YamlFile { + l := yamlFileList(fm) + sort.Sort(l) + return l +} diff --git a/server/forge/types/meta_test.go b/pipeline/frontend/builder/types_test.go similarity index 81% rename from server/forge/types/meta_test.go rename to pipeline/frontend/builder/types_test.go index 071607ea30..8270b53d0c 100644 --- a/server/forge/types/meta_test.go +++ b/pipeline/frontend/builder/types_test.go @@ -12,16 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package types +package builder_test import ( "testing" "github.com/stretchr/testify/assert" + + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/builder" ) func TestSortByName(t *testing.T) { - fm := []*FileMeta{ + fm := []*builder.YamlFile{ { Name: "a", }, @@ -33,7 +35,7 @@ func TestSortByName(t *testing.T) { }, } - assert.Equal(t, []*FileMeta{ + assert.Equal(t, []*builder.YamlFile{ { Name: "a", }, @@ -43,5 +45,5 @@ func TestSortByName(t *testing.T) { { Name: "c", }, - }, SortByName(fm)) + }, builder.SortYamlFilesByName(fm)) } diff --git a/pipeline/frontend/builder/utils.go b/pipeline/frontend/builder/utils.go new file mode 100644 index 0000000000..94403d6683 --- /dev/null +++ b/pipeline/frontend/builder/utils.go @@ -0,0 +1,62 @@ +// Copyright 2025 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package builder + +import ( + "path/filepath" + "strings" +) + +func SanitizePath(path string) string { + path = filepath.Base(path) + path = strings.TrimSuffix(path, ".yml") + path = strings.TrimSuffix(path, ".yaml") + path = strings.TrimPrefix(path, ".") + return path +} + +func filterItemsWithMissingDependencies(items []*Item) []*Item { + itemsToRemove := make([]*Item, 0) + + for _, item := range items { + for _, dep := range item.DependsOn { + if !ContainsItemWithName(dep, items) { + itemsToRemove = append(itemsToRemove, item) + } + } + } + + if len(itemsToRemove) > 0 { + filtered := make([]*Item, 0) + for _, item := range items { + if !ContainsItemWithName(item.Workflow.Name, itemsToRemove) { + filtered = append(filtered, item) + } + } + // Recursive to handle transitive deps + return filterItemsWithMissingDependencies(filtered) + } + + return items +} + +func ContainsItemWithName(name string, items []*Item) bool { + for _, item := range items { + if name == item.Workflow.Name { + return true + } + } + return false +} diff --git a/pipeline/frontend/metadata/types.go b/pipeline/frontend/metadata/types.go index f39602ddee..cc775ca544 100644 --- a/pipeline/frontend/metadata/types.go +++ b/pipeline/frontend/metadata/types.go @@ -32,6 +32,7 @@ type ( ID int64 `json:"id,omitempty"` Name string `json:"name,omitempty"` Owner string `json:"owner,omitempty"` + OrgID int64 `json:"org_id,omitempty"` RemoteID string `json:"remote_id,omitempty"` ForgeURL string `json:"forge_url,omitempty"` CloneURL string `json:"clone_url,omitempty"` diff --git a/server/api/pipeline.go b/server/api/pipeline.go index 03be0b6251..f454168a71 100644 --- a/server/api/pipeline.go +++ b/server/api/pipeline.go @@ -31,7 +31,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v3/server" "go.woodpecker-ci.org/woodpecker/v3/server/model" "go.woodpecker-ci.org/woodpecker/v3/server/pipeline" - "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/step_builder" + "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/metadata" "go.woodpecker-ci.org/woodpecker/v3/server/router/middleware/session" "go.woodpecker-ci.org/woodpecker/v3/server/store" "go.woodpecker-ci.org/woodpecker/v3/server/store/types" @@ -458,8 +458,8 @@ func GetPipelineMetadata(c *gin.Context) { return } - metadata := step_builder.MetadataFromStruct(forge, repo, currentPipeline, prevPipeline, nil, server.Config.Server.Host) - c.JSON(http.StatusOK, metadata) + m := metadata.NewServerMetadata(forge, repo, currentPipeline, prevPipeline, server.Config.Server.Host).GetWorkflowMetadata(nil) + c.JSON(http.StatusOK, m) } // CancelPipeline diff --git a/server/api/pipeline_test.go b/server/api/pipeline_test.go index 807bf63918..852fa86548 100644 --- a/server/api/pipeline_test.go +++ b/server/api/pipeline_test.go @@ -329,11 +329,11 @@ func TestCreatePipeline(t *testing.T) { mockStore.On("GetUser", int64(1)).Return(fakeUser, nil) mockStore.On("CreatePipeline", mock.Anything).Return(nil) - mockStore.On("GetPipelineLastBefore", fakeRepo, "main", mock.Anything).Return(nil, nil).Maybe() + mockStore.On("GetPipelineLastBefore", fakeRepo, "main", mock.Anything).Return(nil, types.ErrRecordNotExist).Maybe() mockStore.On("ConfigPersist", mock.Anything).Return(&model.Config{ID: 1}, nil).Maybe() mockStore.On("ConfigFindIdentical", mock.Anything, mock.Anything).Return(nil, nil).Maybe() mockStore.On("PipelineConfigCreate", mock.Anything).Return(nil).Maybe() - mockStore.On("WorkflowsCreate", mock.Anything).Return(nil).Maybe() + mockStore.On("WorkflowsCreate", mock.Anything).Return(nil) mockStore.On("UpdatePipeline", mock.Anything).Return(nil).Maybe() w := httptest.NewRecorder() @@ -401,11 +401,11 @@ func TestCreatePipeline(t *testing.T) { mockStore.On("GetUser", int64(1)).Return(fakeUser, nil) mockStore.On("CreatePipeline", mock.Anything).Return(nil) - mockStore.On("GetPipelineLastBefore", fakeRepo, "main", mock.Anything).Return(nil, nil).Maybe() + mockStore.On("GetPipelineLastBefore", fakeRepo, "main", mock.Anything).Return(&model.Pipeline{}, nil).Maybe() mockStore.On("ConfigPersist", mock.Anything).Return(&model.Config{ID: 1}, nil).Maybe() mockStore.On("ConfigFindIdentical", mock.Anything, mock.Anything).Return(nil, nil).Maybe() mockStore.On("PipelineConfigCreate", mock.Anything).Return(nil).Maybe() - mockStore.On("WorkflowsCreate", mock.Anything).Return(nil).Maybe() + mockStore.On("WorkflowsCreate", mock.Anything).Return(nil) mockStore.On("UpdatePipeline", mock.Anything).Return(nil).Maybe() w := httptest.NewRecorder() diff --git a/server/forge/types/meta.go b/server/forge/types/meta.go index 5e1f9652d0..ad0efbb8bd 100644 --- a/server/forge/types/meta.go +++ b/server/forge/types/meta.go @@ -14,22 +14,8 @@ package types -import "sort" - // FileMeta represents a file in version control. type FileMeta struct { Name string Data []byte } - -type fileMetaList []*FileMeta - -func (a fileMetaList) Len() int { return len(a) } -func (a fileMetaList) Less(i, j int) bool { return a[i].Name < a[j].Name } -func (a fileMetaList) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - -func SortByName(fm []*FileMeta) []*FileMeta { - l := fileMetaList(fm) - sort.Sort(l) - return l -} diff --git a/server/pipeline/config.go b/server/pipeline/config.go index 3ad427f9ff..9afc6d98e7 100644 --- a/server/pipeline/config.go +++ b/server/pipeline/config.go @@ -15,16 +15,16 @@ package pipeline import ( + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/builder" forge_types "go.woodpecker-ci.org/woodpecker/v3/server/forge/types" "go.woodpecker-ci.org/woodpecker/v3/server/model" - "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/step_builder" "go.woodpecker-ci.org/woodpecker/v3/server/store" ) func findOrPersistPipelineConfig(store store.Store, currentPipeline *model.Pipeline, forgeYamlConfig *forge_types.FileMeta) (*model.Config, error) { return store.ConfigPersist(&model.Config{ RepoID: currentPipeline.RepoID, - Name: step_builder.SanitizePath(forgeYamlConfig.Name), + Name: builder.SanitizePath(forgeYamlConfig.Name), Data: forgeYamlConfig.Data, }) } diff --git a/server/pipeline/create.go b/server/pipeline/create.go index 09093551eb..2e8585c456 100644 --- a/server/pipeline/create.go +++ b/server/pipeline/create.go @@ -57,7 +57,7 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline return nil, errors.New(msg) } - // If the forge has a refresh token, the current access token + // If the repoUser has a refresh token, the current access token // may be stale. Therefore, we should refresh prior to dispatching // the pipeline. forge.Refresh(ctx, _forge, _store, repoUser) @@ -111,7 +111,11 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline return nil, ErrFiltered } - pipeline = setPipelineStepsOnPipeline(pipeline, pipelineItems) + enrichPipelineItemSteps(pipelineItems, repo) + pipeline, err = saveWorkflowsFromPipelineBuilder(_store, pipeline, pipelineItems) + if err != nil { + return nil, fmt.Errorf("saveWorkflowsFromPipelineBuilder failed: %w", err) + } // persist the pipeline config for historical correctness, restarts, etc var configs []*model.Config @@ -131,10 +135,7 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline return nil, errors.New(msg) } - if err := prepareStart(ctx, _forge, _store, pipeline, repoUser, repo); err != nil { - log.Error().Err(err).Str("repo", repo.FullName).Msgf("error preparing pipeline for %s#%d", repo.FullName, pipeline.Number) - return nil, err - } + publishPipeline(ctx, _forge, pipeline, repo, repoUser) if pipeline.Status == model.StatusBlocked { return pipeline, nil diff --git a/server/pipeline/items.go b/server/pipeline/items.go index 00e2c024cb..420c998488 100644 --- a/server/pipeline/items.go +++ b/server/pipeline/items.go @@ -18,25 +18,28 @@ import ( "context" "database/sql" "errors" + "fmt" "maps" "github.com/rs/zerolog/log" pipeline_errors "go.woodpecker-ci.org/woodpecker/v3/pipeline/errors" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/builder" pipeline_metadata "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/metadata" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/yaml/compiler" "go.woodpecker-ci.org/woodpecker/v3/server" "go.woodpecker-ci.org/woodpecker/v3/server/forge" forge_types "go.woodpecker-ci.org/woodpecker/v3/server/forge/types" "go.woodpecker-ci.org/woodpecker/v3/server/model" - "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/step_builder" + "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/metadata" "go.woodpecker-ci.org/woodpecker/v3/server/store" ) -func parsePipeline(ctx context.Context, forge forge.Forge, store store.Store, currentPipeline *model.Pipeline, user *model.User, repo *model.Repo, yamls []*forge_types.FileMeta, envs map[string]string) ([]*step_builder.Item, error) { +func parsePipeline(ctx context.Context, forge forge.Forge, store store.Store, currentPipeline *model.Pipeline, user *model.User, repo *model.Repo, forgeYamls []*forge_types.FileMeta, envs map[string]string) ([]*builder.Item, error) { netrc, err := forge.Netrc(user, repo) if err != nil { log.Error().Err(err).Msg("failed to generate netrc file") + netrc = &model.Netrc{} } // get the previous pipeline so that we can send status change notifications @@ -48,7 +51,7 @@ func parsePipeline(ctx context.Context, forge forge.Forge, store store.Store, cu secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) secs, err := secretService.SecretListPipeline(ctx, repo, currentPipeline, netrc) if err != nil { - log.Error().Err(err).Msgf("error getting secrets for %s#%d", repo.FullName, currentPipeline.Number) + return nil, fmt.Errorf("error getting secrets for %s#%d: %w", repo.FullName, currentPipeline.Number, err) } var secrets []compiler.Secret @@ -69,7 +72,7 @@ func parsePipeline(ctx context.Context, forge forge.Forge, store store.Store, cu registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) regs, err := registryService.RegistryListPipeline(ctx, repo, currentPipeline, netrc) if err != nil { - log.Error().Err(err).Msgf("error getting registry credentials for %s#%d", repo.FullName, currentPipeline.Number) + return nil, fmt.Errorf("error getting registry credentials for %s#%d: %w", repo.FullName, currentPipeline.Number, err) } var registries []compiler.Registry @@ -87,7 +90,10 @@ func parsePipeline(ctx context.Context, forge forge.Forge, store store.Store, cu environmentService := server.Config.Services.Manager.EnvironmentService() if environmentService != nil { - globals, _ := environmentService.EnvironList(repo) + globals, err := environmentService.EnvironList(repo) + if err != nil { + return nil, fmt.Errorf("failed to list global environment for repo %s: %w", repo.FullName, err) + } for _, global := range globals { envs[global.Name] = global.Value } @@ -95,14 +101,20 @@ func parsePipeline(ctx context.Context, forge forge.Forge, store store.Store, cu maps.Copy(envs, currentPipeline.AdditionalVariables) - b := step_builder.StepBuilder{ - Repo: repo, - Curr: currentPipeline, - Prev: prev, + serverMetadata := metadata.NewServerMetadata(forge, repo, currentPipeline, prev, server.Config.Server.Host) + + yamls := make([]*builder.YamlFile, 0, len(forgeYamls)) + for _, forgeYaml := range forgeYamls { + yamls = append(yamls, &builder.YamlFile{ + Name: forgeYaml.Name, + Data: forgeYaml.Data, + }) + } + + b := builder.PipelineBuilder{ + GetWorkflowMetadata: serverMetadata.GetWorkflowMetadata, Envs: envs, - Host: server.Config.Server.Host, Yamls: yamls, - Forge: forge, TrustedClonePlugins: append(repo.NetrcTrustedPlugins, server.Config.Pipeline.TrustedClonePlugins...), PrivilegedPlugins: server.Config.Pipeline.PrivilegedPlugins, RepoTrusted: &pipeline_metadata.TrustedConfiguration{ @@ -146,7 +158,7 @@ func parsePipeline(ctx context.Context, forge forge.Forge, store store.Store, cu func createPipelineItems(c context.Context, forge forge.Forge, store store.Store, currentPipeline *model.Pipeline, user *model.User, repo *model.Repo, yamls []*forge_types.FileMeta, envs map[string]string, -) (*model.Pipeline, []*step_builder.Item, error) { +) (*model.Pipeline, []*builder.Item, error) { pipelineItems, err := parsePipeline(c, forge, store, currentPipeline, user, repo, yamls, envs) if pipeline_errors.HasBlockingErrors(err) { currentPipeline, uErr := UpdateToStatusError(store, *currentPipeline, err) @@ -159,18 +171,38 @@ func createPipelineItems(c context.Context, forge forge.Forge, store store.Store return currentPipeline, nil, err } else if err != nil { currentPipeline.Errors = pipeline_errors.GetPipelineErrors(err) - err = updatePipelinePending(c, forge, store, currentPipeline, repo, user) + if err := updatePipelinePending(c, forge, store, currentPipeline, repo, user); err != nil { + return nil, nil, err + } } - currentPipeline = setPipelineStepsOnPipeline(currentPipeline, pipelineItems) + enrichPipelineItemSteps(pipelineItems, repo) + currentPipeline, err = saveWorkflowsFromPipelineBuilder(store, currentPipeline, pipelineItems) return currentPipeline, pipelineItems, err } -// setPipelineStepsOnPipeline is the link between pipeline representation in "pipeline package" and server -// to be specific this func currently is used to convert the pipeline.Item list (crafted by StepBuilder.Build()) into -// a pipeline that can be stored in the database by the server. -func setPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*step_builder.Item) *model.Pipeline { +// enrichPipelineItemSteps stamps server-side fields onto the backend step +// definitions inside each item's compiled config. +// +// TODO(6444): OrgID and WorkflowLabels on backend/types.Step are Kubernetes-specific +// and should be moved to step.BackendOptions so that generic step types carry +// no backend-specific fields. +func enrichPipelineItemSteps(items []*builder.Item, repo *model.Repo) { + for _, item := range items { + for stageI := range item.Config.Stages { + for stepI := range item.Config.Stages[stageI].Steps { + item.Config.Stages[stageI].Steps[stepI].WorkflowLabels = item.Labels + item.Config.Stages[stageI].Steps[stepI].OrgID = repo.OrgID + } + } + } +} + +// saveWorkflowsFromPipelineBuilder is the link between pipeline representation in "pipeline package" and server +// to be specific this func currently is used to convert the pipeline.Item list (crafted by PipelineBuilder.Build()) into +// a pipeline that can be stored in the database by the server and save converted workflows. +func saveWorkflowsFromPipelineBuilder(store store.Store, pipeline *model.Pipeline, pipelineItems []*builder.Item) (*model.Pipeline, error) { var pidSequence int for _, item := range pipelineItems { if pidSequence < item.Workflow.PID { @@ -181,7 +213,23 @@ func setPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*step_ // the workflows in the pipeline should be empty as only we do populate them, // but if a pipeline was already loaded form database it might contain things, so we just clean it pipeline.Workflows = nil + for _, item := range pipelineItems { + workflow := &model.Workflow{ + ID: item.Workflow.ID, + Name: item.Workflow.Name, + PID: item.Workflow.PID, + PipelineID: pipeline.ID, + State: model.StatusPending, + Environ: item.Workflow.Environ, + AxisID: item.Workflow.AxisID, + } + + if pipeline.Status == model.StatusBlocked { + workflow.State = model.StatusBlocked + } + + // gather all workflow steps through stages as flat list for _, stage := range item.Config.Stages { for _, step := range stage.Steps { pidSequence++ @@ -195,18 +243,25 @@ func setPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*step_ Failure: step.Failure, Type: model.StepType(step.Type), } + if pipeline.Status == model.StatusBlocked { step.State = model.StatusBlocked } - item.Workflow.Children = append(item.Workflow.Children, step) + workflow.Children = append(workflow.Children, step) } } - if pipeline.Status == model.StatusBlocked { - item.Workflow.State = model.StatusBlocked - } - item.Workflow.PipelineID = pipeline.ID - pipeline.Workflows = append(pipeline.Workflows, item.Workflow) + + pipeline.Workflows = append(pipeline.Workflows, workflow) } - return pipeline + if err := store.WorkflowsCreate(pipeline.Workflows); err != nil { + return nil, err + } + + // now thread IDs back to the builder items + for i, wf := range pipeline.Workflows { + pipelineItems[i].Workflow.ID = wf.ID + } + + return pipeline, nil } diff --git a/server/pipeline/items_test.go b/server/pipeline/items_test.go index 645ced5d1c..f20c6e220b 100644 --- a/server/pipeline/items_test.go +++ b/server/pipeline/items_test.go @@ -19,13 +19,14 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" - backend_types "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" + backend_types "go.woodpecker-ci.org/woodpecker/v3/pipeline/backend/types" //nolint:depguard // needed to construct builder.Item.Config in tests; will be resolved when backend-specific fields move to BackendOptions (see enrichPipelineItemSteps TODO) + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/builder" "go.woodpecker-ci.org/woodpecker/v3/server" forge_mocks "go.woodpecker-ci.org/woodpecker/v3/server/forge/mocks" forge_types "go.woodpecker-ci.org/woodpecker/v3/server/forge/types" "go.woodpecker-ci.org/woodpecker/v3/server/model" - "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/step_builder" manager_mocks "go.woodpecker-ci.org/woodpecker/v3/server/services/mocks" registry_service_mocks "go.woodpecker-ci.org/woodpecker/v3/server/services/registry/mocks" secret_service_mocks "go.woodpecker-ci.org/woodpecker/v3/server/services/secret/mocks" @@ -40,8 +41,9 @@ func TestSetPipelineStepsOnPipeline(t *testing.T) { Event: model.EventPush, } - pipelineItems := []*step_builder.Item{{ - Workflow: &model.Workflow{ + pipelineItems := []*builder.Item{{ + Workflow: &builder.Workflow{ + ID: 1, PID: 1, }, Config: &backend_types.Config{ @@ -63,7 +65,12 @@ func TestSetPipelineStepsOnPipeline(t *testing.T) { }, }, }} - pipeline = setPipelineStepsOnPipeline(pipeline, pipelineItems) + + s := store_mocks.NewMockStore(t) + s.On("WorkflowsCreate", mock.Anything).Return(nil) + + pipeline, err := saveWorkflowsFromPipelineBuilder(s, pipeline, pipelineItems) + require.NoError(t, err) if len(pipeline.Workflows) != 1 { t.Fatal("Should generate three in total") } diff --git a/server/pipeline/step_builder/metadata.go b/server/pipeline/metadata/metadata.go similarity index 59% rename from server/pipeline/step_builder/metadata.go rename to server/pipeline/metadata/metadata.go index 42067e4738..d4d0e893bb 100644 --- a/server/pipeline/step_builder/metadata.go +++ b/server/pipeline/metadata/metadata.go @@ -12,59 +12,80 @@ // See the License for the specific language governing permissions and // limitations under the License. -package step_builder +package metadata import ( "fmt" "net/url" "strings" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/builder" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/metadata" "go.woodpecker-ci.org/woodpecker/v3/server/model" "go.woodpecker-ci.org/woodpecker/v3/version" ) -// MetadataFromStruct return the metadata from a pipeline will run with. -func MetadataFromStruct(forge metadata.ServerForge, repo *model.Repo, pipeline, prev *model.Pipeline, workflow *model.Workflow, sysURL string) metadata.Metadata { - host := sysURL - uri, err := url.Parse(sysURL) +type ServerMetadata struct { + forge metadata.ServerForge + repo *model.Repo + pipeline *model.Pipeline + previousPipeline *model.Pipeline + sysURL string +} + +func NewServerMetadata(forge metadata.ServerForge, repo *model.Repo, pipeline, previousPipeline *model.Pipeline, sysURL string) *ServerMetadata { + return &ServerMetadata{ + forge: forge, + repo: repo, + pipeline: pipeline, + previousPipeline: previousPipeline, + sysURL: sysURL, + } +} + +// GetWorkflowMetadata return the metadata from a pipeline will run with. +// TODO: builder should depend on metadata not the other way around +func (s *ServerMetadata) GetWorkflowMetadata(workflow *builder.Workflow) metadata.Metadata { + host := s.sysURL + uri, err := url.Parse(s.sysURL) if err == nil { host = uri.Host } fForge := metadata.Forge{} - if forge != nil { + if s.forge != nil { fForge = metadata.Forge{ - Type: forge.Name(), - URL: forge.URL(), + Type: s.forge.Name(), + URL: s.forge.URL(), } } fRepo := metadata.Repo{} - if repo != nil { + if s.repo != nil { fRepo = metadata.Repo{ - ID: repo.ID, - Name: repo.Name, - Owner: repo.Owner, - RemoteID: fmt.Sprint(repo.ForgeRemoteID), - ForgeURL: repo.ForgeURL, - CloneURL: repo.Clone, - CloneSSHURL: repo.CloneSSH, - Private: repo.IsSCMPrivate, - Branch: repo.Branch, + ID: s.repo.ID, + Name: s.repo.Name, + Owner: s.repo.Owner, + OrgID: s.repo.OrgID, + RemoteID: fmt.Sprint(s.repo.ForgeRemoteID), + ForgeURL: s.repo.ForgeURL, + CloneURL: s.repo.Clone, + CloneSSHURL: s.repo.CloneSSH, + Private: s.repo.IsSCMPrivate, + Branch: s.repo.Branch, Trusted: metadata.TrustedConfiguration{ - Network: repo.Trusted.Network, - Volumes: repo.Trusted.Volumes, - Security: repo.Trusted.Security, + Network: s.repo.Trusted.Network, + Volumes: s.repo.Trusted.Volumes, + Security: s.repo.Trusted.Security, }, } - if idx := strings.LastIndex(repo.FullName, "/"); idx != -1 { - if fRepo.Name == "" && repo.FullName != "" { - fRepo.Name = repo.FullName[idx+1:] + if idx := strings.LastIndex(s.repo.FullName, "/"); idx != -1 { + if fRepo.Name == "" && s.repo.FullName != "" { + fRepo.Name = s.repo.FullName[idx+1:] } - if fRepo.Owner == "" && repo.FullName != "" { - fRepo.Owner = repo.FullName[:idx] + if fRepo.Owner == "" && s.repo.FullName != "" { + fRepo.Owner = s.repo.FullName[:idx] } } } @@ -80,13 +101,13 @@ func MetadataFromStruct(forge metadata.ServerForge, repo *model.Repo, pipeline, return metadata.Metadata{ Repo: fRepo, - Curr: metadataPipelineFromModelPipeline(pipeline, true), - Prev: metadataPipelineFromModelPipeline(prev, false), + Curr: metadataPipelineFromModelPipeline(s.pipeline, true), + Prev: metadataPipelineFromModelPipeline(s.previousPipeline, false), Workflow: fWorkflow, Step: metadata.Step{}, Sys: metadata.System{ Name: "woodpecker", - URL: sysURL, + URL: s.sysURL, Host: host, Platform: "", // will be set by pipeline platform option or by agent Version: version.Version, diff --git a/server/pipeline/step_builder/metadata_test.go b/server/pipeline/metadata/metadata_test.go similarity index 93% rename from server/pipeline/step_builder/metadata_test.go rename to server/pipeline/metadata/metadata_test.go index 60ef96a3b4..c399fef572 100644 --- a/server/pipeline/step_builder/metadata_test.go +++ b/server/pipeline/metadata/metadata_test.go @@ -12,19 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -package step_builder +package metadata import ( "testing" "github.com/stretchr/testify/assert" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/builder" "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/metadata" "go.woodpecker-ci.org/woodpecker/v3/server/forge/mocks" "go.woodpecker-ci.org/woodpecker/v3/server/model" ) -func TestMetadataFromStruct(t *testing.T) { +func TestGetWorkflowMetadata(t *testing.T) { forge := mocks.NewMockForge(t) forge.On("Name").Return("gitea") forge.On("URL").Return("https://gitea.com") @@ -34,7 +35,7 @@ func TestMetadataFromStruct(t *testing.T) { forge metadata.ServerForge repo *model.Repo pipeline, prev *model.Pipeline - workflow *model.Workflow + workflow *builder.Workflow sysURL string expectedMetadata metadata.Metadata expectedEnviron map[string]string @@ -60,7 +61,7 @@ func TestMetadataFromStruct(t *testing.T) { repo: &model.Repo{FullName: "testUser/testRepo", ForgeURL: "https://gitea.com/testUser/testRepo", Clone: "https://gitea.com/testUser/testRepo.git", CloneSSH: "git@gitea.com:testUser/testRepo.git", Branch: "main", IsSCMPrivate: true}, pipeline: &model.Pipeline{Number: 3, ChangedFiles: []string{"test.go", "markdown file.md"}}, prev: &model.Pipeline{Number: 2}, - workflow: &model.Workflow{Name: "hello"}, + workflow: &builder.Workflow{Name: "hello"}, sysURL: "https://example.com", expectedMetadata: metadata.Metadata{ Forge: metadata.Forge{Type: "gitea", URL: "https://gitea.com"}, @@ -91,7 +92,7 @@ func TestMetadataFromStruct(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - result := MetadataFromStruct(testCase.forge, testCase.repo, testCase.pipeline, testCase.prev, testCase.workflow, testCase.sysURL) + result := NewServerMetadata(testCase.forge, testCase.repo, testCase.pipeline, testCase.prev, testCase.sysURL).GetWorkflowMetadata(testCase.workflow) assert.EqualValues(t, testCase.expectedMetadata, result) assert.EqualValues(t, testCase.expectedEnviron, result.Environ()) }) diff --git a/server/pipeline/queue.go b/server/pipeline/queue.go index b8003e9239..ceff84c30d 100644 --- a/server/pipeline/queue.go +++ b/server/pipeline/queue.go @@ -20,24 +20,21 @@ import ( "fmt" "maps" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/builder" "go.woodpecker-ci.org/woodpecker/v3/rpc" "go.woodpecker-ci.org/woodpecker/v3/server" "go.woodpecker-ci.org/woodpecker/v3/server/model" - "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/step_builder" ) -func queuePipeline(ctx context.Context, repo *model.Repo, pipelineItems []*step_builder.Item) error { +func queuePipeline(ctx context.Context, repo *model.Repo, activePipeline *model.Pipeline, pipelineItems []*builder.Item) error { var tasks []*model.Task for _, item := range pipelineItems { - if item.Workflow.State == model.StatusSkipped { - continue - } task := &model.Task{ ID: fmt.Sprint(item.Workflow.ID), PID: item.Workflow.PID, Name: item.Workflow.Name, Labels: make(map[string]string), - PipelineID: item.Workflow.PipelineID, + PipelineID: activePipeline.ID, RepoID: repo.ID, } maps.Copy(task.Labels, item.Labels) @@ -63,7 +60,7 @@ func queuePipeline(ctx context.Context, repo *model.Repo, pipelineItems []*step_ return server.Config.Services.Scheduler.PushAtOnce(ctx, tasks) } -func getTaskDependencies(dependsOn []string, items []*step_builder.Item) (taskIDs []string) { +func getTaskDependencies(dependsOn []string, items []*builder.Item) (taskIDs []string) { for _, dep := range dependsOn { for _, pipelineItem := range items { if pipelineItem.Workflow.Name == dep { diff --git a/server/pipeline/restart.go b/server/pipeline/restart.go index 4dc2ae3f80..a7db5e145e 100644 --- a/server/pipeline/restart.go +++ b/server/pipeline/restart.go @@ -95,11 +95,7 @@ func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipelin return nil, errors.New(msg) } - if err := prepareStart(ctx, forge, store, newPipeline, user, repo); err != nil { - msg := fmt.Sprintf("failure to prepare pipeline for %s", repo.FullName) - log.Error().Err(err).Msg(msg) - return nil, errors.New(msg) - } + publishPipeline(ctx, forge, newPipeline, repo, user) newPipeline, err = start(ctx, forge, store, newPipeline, user, repo, pipelineItems) if err != nil { diff --git a/server/pipeline/start.go b/server/pipeline/start.go index cb7b6480f5..2339147c36 100644 --- a/server/pipeline/start.go +++ b/server/pipeline/start.go @@ -19,14 +19,14 @@ import ( "github.com/rs/zerolog/log" + "go.woodpecker-ci.org/woodpecker/v3/pipeline/frontend/builder" "go.woodpecker-ci.org/woodpecker/v3/server/forge" "go.woodpecker-ci.org/woodpecker/v3/server/model" - "go.woodpecker-ci.org/woodpecker/v3/server/pipeline/step_builder" "go.woodpecker-ci.org/woodpecker/v3/server/store" ) // start a pipeline, make sure it was stored persistent in the store before. -func start(ctx context.Context, forge forge.Forge, store store.Store, activePipeline *model.Pipeline, user *model.User, repo *model.Repo, pipelineItems []*step_builder.Item) (*model.Pipeline, error) { +func start(ctx context.Context, forge forge.Forge, store store.Store, activePipeline *model.Pipeline, user *model.User, repo *model.Repo, pipelineItems []*builder.Item) (*model.Pipeline, error) { // call to cancel previous pipelines if needed if err := cancelPreviousPipelines(ctx, forge, store, activePipeline, repo, user); err != nil { // should be not breaking @@ -35,7 +35,7 @@ func start(ctx context.Context, forge forge.Forge, store store.Store, activePipe publishPipeline(ctx, forge, activePipeline, repo, user) - if err := queuePipeline(ctx, repo, pipelineItems); err != nil { + if err := queuePipeline(ctx, repo, activePipeline, pipelineItems); err != nil { log.Error().Err(err).Msg("queuePipeline") return nil, err } @@ -43,16 +43,6 @@ func start(ctx context.Context, forge forge.Forge, store store.Store, activePipe return activePipeline, nil } -func prepareStart(ctx context.Context, forge forge.Forge, store store.Store, activePipeline *model.Pipeline, user *model.User, repo *model.Repo) error { - if err := store.WorkflowsCreate(activePipeline.Workflows); err != nil { - log.Error().Err(err).Str("repo", repo.FullName).Msgf("error persisting steps for %s#%d", repo.FullName, activePipeline.Number) - return err - } - - publishPipeline(ctx, forge, activePipeline, repo, user) - return nil -} - func publishPipeline(ctx context.Context, forge forge.Forge, pipeline *model.Pipeline, repo *model.Repo, repoUser *model.User) { if err := publishToTopic(ctx, pipeline, repo); err != nil { log.Error().Err(err).Msg("could not push pipeline status change to pubsub provider")