diff --git a/.vscode/settings.json b/.vscode/settings.json index daee9de8e..ce460ae9c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,6 +9,9 @@ "go.lintTool": "golangci-lint", "go.lintFlags": ["--fast"], "go.buildTags": "test", + "gopls": { // cspell:words gopls + "buildFlags": ["-tags=test"] + }, "eslint.workingDirectories": ["./web"], "prettier.ignorePath": "./web/.prettierignore", // Enable the ESlint flat config support diff --git a/cli/exec/exec.go b/cli/exec/exec.go index 68db9aabe..7b5659c15 100644 --- a/cli/exec/exec.go +++ b/cli/exec/exec.go @@ -32,6 +32,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/pipeline" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/docker" + "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/dummy" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/kubernetes" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/local" backendTypes "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" @@ -228,6 +229,7 @@ func execWithAxis(c *cli.Context, file, repoPath string, axis matrix.Axis) error kubernetes.New(), docker.New(), local.New(), + dummy.New(), } backendEngine, err := backend.FindBackend(backendCtx, backends, c.String("backend-engine")) if err != nil { diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 062fc0729..8c4aa3f14 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -17,6 +17,7 @@ package main import ( "go.woodpecker-ci.org/woodpecker/v2/cmd/agent/core" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/docker" + "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/dummy" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/kubernetes" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/local" backendTypes "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" @@ -27,5 +28,6 @@ func main() { kubernetes.New(), docker.New(), local.New(), + dummy.New(), }) } diff --git a/docs/docs/92-development/09-testing.md b/docs/docs/92-development/09-testing.md new file mode 100644 index 000000000..ba613c108 --- /dev/null +++ b/docs/docs/92-development/09-testing.md @@ -0,0 +1,81 @@ +# Testing + +## Backend + +### Unit Tests + +[We use default golang unit tests](https://go.dev/doc/tutorial/add-a-test) +with [`"github.com/stretchr/testify/assert"`](https://pkg.go.dev/github.com/stretchr/testify@v1.9.0/assert) to simplify testing. + +### Integration Tests + +### Dummy backend + +There is a special backend called **`dummy`** which does not execute any commands, but emulates how a typical backend should behave. +To enable it you need to build the agent or cli with the `test` build tag. + +An example pipeline config would be: + +```yaml +when: + event: manual + +steps: + - name: echo + image: dummy + commands: echo "hello woodpecker" + environment: + SLEEP: '1s' + +services: + echo: + image: dummy + commands: echo "i am a sevice" +``` + +This could be executed via `woodpecker-cli --log-level trace exec --backend-engine dummy example.yaml`: + +```none +9:18PM DBG pipeline/pipeline.go:94 > executing 2 stages, in order of: CLI=exec +9:18PM DBG pipeline/pipeline.go:104 > stage CLI=exec StagePos=0 Steps=echo +9:18PM DBG pipeline/pipeline.go:104 > stage CLI=exec StagePos=1 Steps=echo +9:18PM TRC pipeline/backend/dummy/dummy.go:75 > create workflow environment taskUUID=01J10P578JQE6E25VV1EQF0745 +9:18PM DBG pipeline/pipeline.go:176 > prepare CLI=exec step=echo +9:18PM DBG pipeline/pipeline.go:203 > executing CLI=exec step=echo +9:18PM TRC pipeline/backend/dummy/dummy.go:81 > start step echo taskUUID=01J10P578JQE6E25VV1EQF0745 +9:18PM TRC pipeline/backend/dummy/dummy.go:167 > tail logs of step echo taskUUID=01J10P578JQE6E25VV1EQF0745 +9:18PM DBG pipeline/pipeline.go:209 > complete CLI=exec step=echo +[echo:L0:0s] StepName: echo +[echo:L1:0s] StepType: service +[echo:L2:0s] StepUUID: 01J10P578JQE6E25VV1A2DNQN9 +[echo:L3:0s] StepCommands: +[echo:L4:0s] ------------------ +[echo:L5:0s] echo ja +[echo:L6:0s] ------------------ +[echo:L7:0s] 9:18PM DBG pipeline/pipeline.go:176 > prepare CLI=exec step=echo +9:18PM DBG pipeline/pipeline.go:203 > executing CLI=exec step=echo +9:18PM TRC pipeline/backend/dummy/dummy.go:81 > start step echo taskUUID=01J10P578JQE6E25VV1EQF0745 +9:18PM TRC pipeline/backend/dummy/dummy.go:167 > tail logs of step echo taskUUID=01J10P578JQE6E25VV1EQF0745 +[echo:L0:0s] StepName: echo +[echo:L1:0s] StepType: commands +[echo:L2:0s] StepUUID: 01J10P578JQE6E25VV1DFSXX1Y +[echo:L3:0s] StepCommands: +[echo:L4:0s] ------------------ +[echo:L5:0s] echo ja +[echo:L6:0s] ------------------ +[echo:L7:0s] 9:18PM TRC pipeline/backend/dummy/dummy.go:108 > wait for step echo taskUUID=01J10P578JQE6E25VV1EQF0745 +9:18PM TRC pipeline/backend/dummy/dummy.go:187 > stop step echo taskUUID=01J10P578JQE6E25VV1EQF0745 +9:18PM DBG pipeline/pipeline.go:209 > complete CLI=exec step=echo +9:18PM TRC pipeline/backend/dummy/dummy.go:208 > delete workflow environment taskUUID=01J10P578JQE6E25VV1EQF0745 +``` + +There are also environment variables to alter step behaviour: + +- `SLEEP: 10` will let the step wait 10 seconds +- `EXPECT_TYPE` allows to check if a step is a `clone`, `service`, `plugin` or `commands` +- `STEP_START_FAIL: true` if set will simulate a step to fail before actually being started (e.g. happens when the container image can not be pulled) +- `STEP_TAIL_FAIL: true` if set will error when we simulate to read from stdout for logs +- `STEP_EXIT_CODE: 2` if set will be used as exit code, default is 0 +- `STEP_OOM_KILLED: true` simulates a step being killed by memory constrains + +You can let the setup of a whole workflow fail by setting it's UUID to `WorkflowSetupShouldFail`. diff --git a/pipeline/backend/dummy/dummy.go b/pipeline/backend/dummy/dummy.go new file mode 100644 index 000000000..4eadd1c65 --- /dev/null +++ b/pipeline/backend/dummy/dummy.go @@ -0,0 +1,240 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build test +// +build test + +package dummy + +import ( + "context" + "fmt" + "io" + "strconv" + "strings" + "sync" + "time" + + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" + + backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" +) + +type dummy struct { + kv sync.Map +} + +const ( + // Step names to control behavior of dummy backend. + WorkflowSetupFailUUID = "WorkflowSetupShouldFail" + EnvKeyStepSleep = "SLEEP" + EnvKeyStepType = "EXPECT_TYPE" + EnvKeyStepStartFail = "STEP_START_FAIL" + EnvKeyStepExitCode = "STEP_EXIT_CODE" + EnvKeyStepTailFail = "STEP_TAIL_FAIL" + EnvKeyStepOOMKilled = "STEP_OOM_KILLED" + + // Internal const. + stepStateStarted = "started" + stepStateDone = "done" + testServiceTimeout = 1 * time.Second +) + +// New returns a dummy backend. +func New() backend.Backend { + return &dummy{ + kv: sync.Map{}, + } +} + +func (e *dummy) Name() string { + return "dummy" +} + +func (e *dummy) IsAvailable(_ context.Context) bool { + return true +} + +func (e *dummy) Flags() []cli.Flag { + return nil +} + +// Load new client for Docker Backend using environment variables. +func (e *dummy) Load(_ context.Context) (*backend.BackendInfo, error) { + return &backend.BackendInfo{ + Platform: "dummy", + }, nil +} + +func (e *dummy) SetupWorkflow(_ context.Context, _ *backend.Config, taskUUID string) error { + if taskUUID == WorkflowSetupFailUUID { + return fmt.Errorf("expected fail to setup workflow") + } + log.Trace().Str("taskUUID", taskUUID).Msg("create workflow environment") + e.kv.Store("task_"+taskUUID, "setup") + return nil +} + +func (e *dummy) StartStep(_ context.Context, step *backend.Step, taskUUID string) error { + log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name) + + // internal state checks + _, exist := e.kv.Load("task_" + taskUUID) + if !exist { + return fmt.Errorf("expect env of workflow %s to exist but found none to destroy", taskUUID) + } + stepState, stepExist := e.kv.Load(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID)) + if stepExist { + // Detect issues like https://github.com/woodpecker-ci/woodpecker/issues/3494 + return fmt.Errorf("StartStep detected already started step '%s' (%s) in state: %s", step.Name, step.UUID, stepState) + } + + if stepStartFail, _ := strconv.ParseBool(step.Environment[EnvKeyStepStartFail]); stepStartFail { + return fmt.Errorf("expected fail to start step") + } + + expectStepType, testStepType := step.Environment[EnvKeyStepType] + if testStepType && string(step.Type) != expectStepType { + return fmt.Errorf("expected step type '%s' but got '%s'", expectStepType, step.Type) + } + + e.kv.Store(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID), stepStateStarted) + return nil +} + +func (e *dummy) WaitStep(ctx context.Context, step *backend.Step, taskUUID string) (*backend.State, error) { + log.Trace().Str("taskUUID", taskUUID).Msgf("wait for step %s", step.Name) + + _, exist := e.kv.Load("task_" + taskUUID) + if !exist { + err := fmt.Errorf("expect env of workflow %s to exist but found none to destroy", taskUUID) + return &backend.State{Error: err}, err + } + + // check state + stepState, stepExist := e.kv.Load(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID)) + if !stepExist { + err := fmt.Errorf("WaitStep expect step '%s' (%s) to be created but found none", step.Name, step.UUID) + return &backend.State{Error: err}, err + } + if stepState != stepStateStarted { + err := fmt.Errorf("WaitStep expect step '%s' (%s) to be '%s' but it is: %s", step.Name, step.UUID, stepStateStarted, stepState) + return &backend.State{Error: err}, err + } + + // extend wait time logic + if sleep, sleepExist := step.Environment[EnvKeyStepSleep]; sleepExist { + toSleep, err := time.ParseDuration(sleep) + if err != nil { + err = fmt.Errorf("WaitStep fail to parse sleep duration: %w", err) + return &backend.State{Error: err}, err + } + time.Sleep(toSleep) + } else { + if step.Type == backend.StepTypeService { + select { + case <-time.NewTimer(testServiceTimeout).C: + err := fmt.Errorf("WaitStep fail due to timeout of service after 1 second") + return &backend.State{Error: err}, err + case <-ctx.Done(): + // context for service closed ... we can move forward + } + } else { + time.Sleep(time.Nanosecond) + } + } + + e.kv.Store(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID), stepStateDone) + + oomKilled, _ := strconv.ParseBool(step.Environment[EnvKeyStepOOMKilled]) + exitCode := 0 + + if code, exist := step.Environment[EnvKeyStepExitCode]; exist { + exitCode, _ = strconv.Atoi(strings.TrimSpace(code)) + } + + return &backend.State{ + ExitCode: exitCode, + Exited: true, + OOMKilled: oomKilled, + }, nil +} + +func (e *dummy) TailStep(_ context.Context, step *backend.Step, taskUUID string) (io.ReadCloser, error) { + log.Trace().Str("taskUUID", taskUUID).Msgf("tail logs of step %s", step.Name) + + _, exist := e.kv.Load("task_" + taskUUID) + if !exist { + return nil, fmt.Errorf("expect env of workflow %s to exist but found none to destroy", taskUUID) + } + + // check state + stepState, stepExist := e.kv.Load(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID)) + if !stepExist { + return nil, fmt.Errorf("WaitStep expect step '%s' (%s) to be created but found none", step.Name, step.UUID) + } + if stepState != stepStateStarted { + return nil, fmt.Errorf("WaitStep expect step '%s' (%s) to be '%s' but it is: %s", step.Name, step.UUID, stepStateStarted, stepState) + } + + if tailShouldFail, _ := strconv.ParseBool(step.Environment[EnvKeyStepTailFail]); tailShouldFail { + return nil, fmt.Errorf("expected fail to read stdout of step") + } + + return io.NopCloser(strings.NewReader(dummyExecStepOutput(step))), nil +} + +func (e *dummy) DestroyStep(_ context.Context, step *backend.Step, taskUUID string) error { + log.Trace().Str("taskUUID", taskUUID).Msgf("stop step %s", step.Name) + + _, exist := e.kv.Load("task_" + taskUUID) + if !exist { + return fmt.Errorf("expect env of workflow %s to exist but found none to destroy", taskUUID) + } + + // check state + stepState, stepExist := e.kv.Load(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID)) + if !stepExist { + return fmt.Errorf("WaitStep expect step '%s' (%s) to be created but found none", step.Name, step.UUID) + } + if stepState != stepStateDone { + return fmt.Errorf("WaitStep expect step '%s' (%s) to be '%s' but it is: %s", step.Name, step.UUID, stepStateDone, stepState) + } + + e.kv.Delete(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID)) + return nil +} + +func (e *dummy) DestroyWorkflow(_ context.Context, _ *backend.Config, taskUUID string) error { + log.Trace().Str("taskUUID", taskUUID).Msgf("delete workflow environment") + + _, exist := e.kv.Load("task_" + taskUUID) + if !exist { + return fmt.Errorf("expect env of workflow %s to exist but found none to destroy", taskUUID) + } + e.kv.Delete("task_" + taskUUID) + return nil +} + +func dummyExecStepOutput(step *backend.Step) string { + return fmt.Sprintf(`StepName: %s +StepType: %s +StepUUID: %s +StepCommands: +------------------ +%s +------------------ +`, step.Name, step.Type, step.UUID, strings.Join(step.Commands, "\n")) +} diff --git a/pipeline/backend/dummy/dummy_noop.go b/pipeline/backend/dummy/dummy_noop.go new file mode 100644 index 000000000..5a5a23cbb --- /dev/null +++ b/pipeline/backend/dummy/dummy_noop.go @@ -0,0 +1,78 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !test +// +build !test + +package dummy + +import ( + "context" + "errors" + "io" + + "github.com/urfave/cli/v2" + + "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" +) + +type noop struct{} + +var ErrOnCompileExcluded = errors.New("the dummy backend engine was excluded on compile time") + +// New returns a dummy backend. +func New() types.Backend { + return &noop{} +} + +func (e *noop) Name() string { + return "dummy" +} + +func (e *noop) IsAvailable(context.Context) bool { + return false +} + +func (e *noop) Flags() []cli.Flag { + return nil +} + +// Load new client for Docker Backend using environment variables. +func (e *noop) Load(context.Context) (*types.BackendInfo, error) { + return nil, ErrOnCompileExcluded +} + +func (e *noop) SetupWorkflow(context.Context, *types.Config, string) error { + return ErrOnCompileExcluded +} + +func (e *noop) StartStep(context.Context, *types.Step, string) error { + return ErrOnCompileExcluded +} + +func (e *noop) WaitStep(context.Context, *types.Step, string) (*types.State, error) { + return nil, ErrOnCompileExcluded +} + +func (e *noop) TailStep(context.Context, *types.Step, string) (io.ReadCloser, error) { + return nil, ErrOnCompileExcluded +} + +func (e *noop) DestroyStep(context.Context, *types.Step, string) error { + return ErrOnCompileExcluded +} + +func (e *noop) DestroyWorkflow(context.Context, *types.Config, string) error { + return ErrOnCompileExcluded +} diff --git a/pipeline/backend/dummy/dummy_test.go b/pipeline/backend/dummy/dummy_test.go new file mode 100644 index 000000000..faa1abbc8 --- /dev/null +++ b/pipeline/backend/dummy/dummy_test.go @@ -0,0 +1,171 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build test +// +build test + +package dummy_test + +import ( + "context" + "io" + "testing" + + "github.com/stretchr/testify/assert" + + "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/dummy" + "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" +) + +func TestSmalPipelineDummyRun(t *testing.T) { + dummyEngine := dummy.New() + ctx := context.Background() + + assert.True(t, dummyEngine.IsAvailable(ctx)) + assert.EqualValues(t, "dummy", dummyEngine.Name()) + _, err := dummyEngine.Load(ctx) + assert.NoError(t, err) + + assert.Error(t, dummyEngine.SetupWorkflow(ctx, nil, dummy.WorkflowSetupFailUUID)) + + t.Run("expect fail of step func with non setup workflow", func(t *testing.T) { + step := &types.Step{Name: "step1", UUID: "SID_1"} + nonExistWorkflowID := "WID_NONE" + + err := dummyEngine.StartStep(ctx, step, nonExistWorkflowID) + assert.Error(t, err) + + _, err = dummyEngine.TailStep(ctx, step, nonExistWorkflowID) + assert.Error(t, err) + + _, err = dummyEngine.WaitStep(ctx, step, nonExistWorkflowID) + assert.Error(t, err) + + err = dummyEngine.DestroyStep(ctx, step, nonExistWorkflowID) + assert.Error(t, err) + }) + + t.Run("step exec successfully", func(t *testing.T) { + step := &types.Step{ + Name: "step1", + UUID: "SID_1", + Type: types.StepTypeCommands, + Environment: map[string]string{}, + Commands: []string{"echo ja", "echo nein"}, + } + workflowUUID := "WID_1" + + assert.NoError(t, dummyEngine.SetupWorkflow(ctx, nil, workflowUUID)) + + assert.NoError(t, dummyEngine.StartStep(ctx, step, workflowUUID)) + + reader, err := dummyEngine.TailStep(ctx, step, workflowUUID) + assert.NoError(t, err) + log, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.EqualValues(t, `StepName: step1 +StepType: commands +StepUUID: SID_1 +StepCommands: +------------------ +echo ja +echo nein +------------------ +`, string(log)) + + state, err := dummyEngine.WaitStep(ctx, step, workflowUUID) + assert.NoError(t, err) + assert.NoError(t, state.Error) + assert.EqualValues(t, 0, state.ExitCode) + + assert.NoError(t, dummyEngine.DestroyStep(ctx, step, workflowUUID)) + + assert.NoError(t, dummyEngine.DestroyWorkflow(ctx, nil, workflowUUID)) + }) + + t.Run("step exec error", func(t *testing.T) { + step := &types.Step{ + Name: "dummy", + UUID: "SID_2", + Type: types.StepTypePlugin, + Environment: map[string]string{dummy.EnvKeyStepType: "plugin", dummy.EnvKeyStepExitCode: "1"}, + } + workflowUUID := "WID_1" + + assert.NoError(t, dummyEngine.SetupWorkflow(ctx, nil, workflowUUID)) + + assert.NoError(t, dummyEngine.StartStep(ctx, step, workflowUUID)) + + _, err := dummyEngine.TailStep(ctx, step, workflowUUID) + assert.NoError(t, err) + + state, err := dummyEngine.WaitStep(ctx, step, workflowUUID) + assert.NoError(t, err) + assert.NoError(t, state.Error) + assert.EqualValues(t, 1, state.ExitCode) + + assert.NoError(t, dummyEngine.DestroyStep(ctx, step, workflowUUID)) + + assert.NoError(t, dummyEngine.DestroyWorkflow(ctx, nil, workflowUUID)) + }) + + t.Run("step tail error", func(t *testing.T) { + step := &types.Step{ + Name: "dummy", + UUID: "SID_2", + Environment: map[string]string{dummy.EnvKeyStepTailFail: "true"}, + } + workflowUUID := "WID_1" + + assert.NoError(t, dummyEngine.SetupWorkflow(ctx, nil, workflowUUID)) + + assert.NoError(t, dummyEngine.StartStep(ctx, step, workflowUUID)) + + _, err := dummyEngine.TailStep(ctx, step, workflowUUID) + assert.Error(t, err) + + _, err = dummyEngine.WaitStep(ctx, step, workflowUUID) + assert.NoError(t, err) + + assert.NoError(t, dummyEngine.DestroyStep(ctx, step, workflowUUID)) + + assert.NoError(t, dummyEngine.DestroyWorkflow(ctx, nil, workflowUUID)) + }) + + t.Run("step start fail", func(t *testing.T) { + step := &types.Step{ + Name: "dummy", + UUID: "SID_2", + Type: types.StepTypeService, + Environment: map[string]string{dummy.EnvKeyStepType: "service", dummy.EnvKeyStepStartFail: "true"}, + } + workflowUUID := "WID_1" + + assert.NoError(t, dummyEngine.SetupWorkflow(ctx, nil, workflowUUID)) + + assert.Error(t, dummyEngine.StartStep(ctx, step, workflowUUID)) + + _, err := dummyEngine.TailStep(ctx, step, workflowUUID) + assert.Error(t, err) + + state, err := dummyEngine.WaitStep(ctx, step, workflowUUID) + assert.Error(t, err) + assert.Error(t, state.Error) + assert.EqualValues(t, 0, state.ExitCode) + + assert.Error(t, dummyEngine.DestroyStep(ctx, step, workflowUUID)) + + assert.NoError(t, dummyEngine.DestroyWorkflow(ctx, nil, workflowUUID)) + }) +}