diff --git a/cli/exec/exec.go b/cli/exec/exec.go index de662f762..7703d8f17 100644 --- a/cli/exec/exec.go +++ b/cli/exec/exec.go @@ -216,7 +216,7 @@ func execWithAxis(c *cli.Context, file, repoPath string, axis matrix.Axis) error return err } - if err = engine.Load(backendCtx); err != nil { + if _, err = engine.Load(backendCtx); err != nil { return err } diff --git a/cmd/agent/agent.go b/cmd/agent/agent.go index aac2a3453..989e55c84 100644 --- a/cmd/agent/agent.go +++ b/cmd/agent/agent.go @@ -22,7 +22,6 @@ import ( "fmt" "net/http" "os" - "runtime" "strings" "sync" "time" @@ -57,8 +56,6 @@ func run(c *cli.Context) error { hostname, _ = os.Hostname() } - platform := runtime.GOOS + "/" + runtime.GOARCH - counter.Polling = c.Int("max-workflows") counter.Running = 0 @@ -155,7 +152,15 @@ func run(c *cli.Context) error { return err } - agentConfig.AgentID, err = client.RegisterAgent(ctx, platform, engine.Name(), version.String(), parallel) + // load engine (e.g. init api client) + engInfo, err := engine.Load(backendCtx) + if err != nil { + log.Error().Err(err).Msg("cannot load backend engine") + return err + } + log.Debug().Msgf("loaded %s backend engine", engine.Name()) + + agentConfig.AgentID, err = client.RegisterAgent(ctx, engInfo.Platform, engine.Name(), version.String(), parallel) if err != nil { return err } @@ -164,7 +169,7 @@ func run(c *cli.Context) error { labels := map[string]string{ "hostname": hostname, - "platform": platform, + "platform": engInfo.Platform, "backend": engine.Name(), "repo": "*", // allow all repos by default } @@ -195,13 +200,6 @@ func run(c *cli.Context) error { } }() - // load engine (e.g. init api client) - if err := engine.Load(backendCtx); err != nil { - log.Error().Err(err).Msg("cannot load backend engine") - return err - } - log.Debug().Msgf("loaded %s backend engine", engine.Name()) - for i := 0; i < parallel; i++ { i := i go func() { @@ -226,7 +224,7 @@ func run(c *cli.Context) error { log.Info().Msgf( "Starting Woodpecker agent with version '%s' and backend '%s' using platform '%s' running up to %d pipelines in parallel", - version.String(), engine.Name(), platform, parallel) + version.String(), engine.Name(), engInfo.Platform, parallel) wg.Wait() return nil diff --git a/pipeline/backend/common/script.go b/pipeline/backend/common/script.go index 8e955b19f..1df8742dd 100644 --- a/pipeline/backend/common/script.go +++ b/pipeline/backend/common/script.go @@ -16,12 +16,11 @@ package common import ( "encoding/base64" - "runtime" ) -func GenerateContainerConf(commands []string) (env map[string]string, entry, cmd []string) { +func GenerateContainerConf(commands []string, goos string) (env map[string]string, entry, cmd []string) { env = make(map[string]string) - if runtime.GOOS == "windows" { + if goos == "windows" { env["CI_SCRIPT"] = base64.StdEncoding.EncodeToString([]byte(generateScriptWindows(commands))) env["HOME"] = "c:\\root" env["SHELL"] = "powershell.exe" diff --git a/pipeline/backend/common/script_win_test.go b/pipeline/backend/common/script_win_test.go new file mode 100644 index 000000000..bbd1bfb41 --- /dev/null +++ b/pipeline/backend/common/script_win_test.go @@ -0,0 +1,58 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGenerateScriptWin(t *testing.T) { + testdata := []struct { + from []string + want string + }{ + { + from: []string{"echo %PATH%", "go build", "go test"}, + want: ` +$ErrorActionPreference = 'Stop'; +&cmd /c "mkdir c:\root"; +if ($Env:CI_NETRC_MACHINE) { +$netrc=[string]::Format("{0}\_netrc",$Env:HOME); +"machine $Env:CI_NETRC_MACHINE" >> $netrc; +"login $Env:CI_NETRC_USERNAME" >> $netrc; +"password $Env:CI_NETRC_PASSWORD" >> $netrc; +}; +[Environment]::SetEnvironmentVariable("CI_NETRC_PASSWORD",$null); +[Environment]::SetEnvironmentVariable("CI_SCRIPT",$null); + +Write-Output ('+ "echo %PATH%"'); +& echo %PATH%; if ($LASTEXITCODE -ne 0) {exit $LASTEXITCODE} + +Write-Output ('+ "go build"'); +& go build; if ($LASTEXITCODE -ne 0) {exit $LASTEXITCODE} + +Write-Output ('+ "go test"'); +& go test; if ($LASTEXITCODE -ne 0) {exit $LASTEXITCODE} + +`, + }, + } + for _, test := range testdata { + script := generateScriptWindows(test.from) + assert.EqualValues(t, test.want, script, "Want encoded script for %s", test.from) + } +} diff --git a/pipeline/backend/docker/convert.go b/pipeline/backend/docker/convert.go index 815b61a7a..2b097c2e7 100644 --- a/pipeline/backend/docker/convert.go +++ b/pipeline/backend/docker/convert.go @@ -27,7 +27,7 @@ import ( ) // returns a container configuration. -func toConfig(step *types.Step) *container.Config { +func (e *docker) toConfig(step *types.Step) *container.Config { config := &container.Config{ Image: step.Image, Labels: map[string]string{"wp_uuid": step.UUID}, @@ -37,7 +37,7 @@ func toConfig(step *types.Step) *container.Config { } if len(step.Commands) != 0 { - env, entry, cmd := common.GenerateContainerConf(step.Commands) + env, entry, cmd := common.GenerateContainerConf(step.Commands, e.info.OSType) for k, v := range env { step.Environment[k] = v } @@ -76,9 +76,6 @@ func toHostConfig(step *types.Step) *container.HostConfig { Sysctls: step.Sysctls, } - // if len(step.VolumesFrom) != 0 { - // config.VolumesFrom = step.VolumesFrom - // } if len(step.NetworkMode) != 0 { config.NetworkMode = container.NetworkMode(step.NetworkMode) } diff --git a/pipeline/backend/docker/docker.go b/pipeline/backend/docker/docker.go index 7f19f5bc8..4e180c7c6 100644 --- a/pipeline/backend/docker/docker.go +++ b/pipeline/backend/docker/docker.go @@ -20,7 +20,6 @@ import ( "net/http" "os" "path/filepath" - "runtime" "strings" "github.com/docker/docker/api/types" @@ -43,6 +42,7 @@ type docker struct { enableIPv6 bool network string volumes []string + info types.Info } const ( @@ -94,10 +94,10 @@ func httpClientOfOpts(dockerCertPath string, verifyTLS bool) *http.Client { } // Load new client for Docker Engine using environment variables. -func (e *docker) Load(ctx context.Context) error { +func (e *docker) Load(ctx context.Context) (*backend.EngineInfo, error) { c, ok := ctx.Value(backend.CliContext).(*cli.Context) if !ok { - return backend.ErrNoCliContextFound + return nil, backend.ErrNoCliContextFound } var dockerClientOpts []client.Opt @@ -115,10 +115,15 @@ func (e *docker) Load(ctx context.Context) error { cl, err := client.NewClientWithOpts(dockerClientOpts...) if err != nil { - return err + return nil, err } e.client = cl + e.info, err = cl.Info(ctx) + if err != nil { + return nil, err + } + e.enableIPv6 = c.Bool("backend-docker-ipv6") e.network = c.String("backend-docker-network") @@ -137,7 +142,9 @@ func (e *docker) Load(ctx context.Context) error { e.volumes = append(e.volumes, strings.Join(parts, ":")) } - return nil + return &backend.EngineInfo{ + Platform: e.info.OSType + "/" + normalizeArchType(e.info.Architecture), + }, nil } func (e *docker) SetupWorkflow(_ context.Context, conf *backend.Config, taskUUID string) error { @@ -154,7 +161,7 @@ func (e *docker) SetupWorkflow(_ context.Context, conf *backend.Config, taskUUID } networkDriver := networkDriverBridge - if runtime.GOOS == "windows" { + if e.info.OSType == "windows" { networkDriver = networkDriverNAT } for _, n := range conf.Networks { @@ -172,7 +179,7 @@ func (e *docker) SetupWorkflow(_ context.Context, conf *backend.Config, taskUUID func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID string) error { log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name) - config := toConfig(step) + config := e.toConfig(step) hostConfig := toHostConfig(step) containerName := toContainerName(step) @@ -262,9 +269,6 @@ func (e *docker) WaitStep(ctx context.Context, step *backend.Step, taskUUID stri if err != nil { return nil, err } - // if info.State.Running { - // TODO - // } return &backend.State{ Exited: true, @@ -360,3 +364,15 @@ func isErrContainerNotFoundOrNotRunning(err error) bool { // Error: No such container: ... return err != nil && (strings.Contains(err.Error(), "No such container") || strings.Contains(err.Error(), "is not running")) } + +// normalizeArchType converts the arch type reported by docker info into +// the runtime.GOARCH format +// TODO: find out if we we need to convert other arch types too +func normalizeArchType(s string) string { + switch s { + case "x86_64": + return "amd64" + default: + return s + } +} diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index 5b9709392..263b55a27 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "os" + "runtime" "strings" "time" @@ -46,6 +47,7 @@ type kube struct { ctx context.Context client kubernetes.Interface config *Config + goos string } type Config struct { @@ -104,10 +106,10 @@ func (e *kube) IsAvailable(context.Context) bool { return len(host) > 0 } -func (e *kube) Load(context.Context) error { +func (e *kube) Load(context.Context) (*types.EngineInfo, error) { config, err := configFromCliContext(e.ctx) if err != nil { - return err + return nil, err } e.config = config @@ -120,12 +122,16 @@ func (e *kube) Load(context.Context) error { } if err != nil { - return err + return nil, err } e.client = kubeClient - return nil + // TODO(2693): use info resp of kubeClient to define platform var + e.goos = runtime.GOOS + return &types.EngineInfo{ + Platform: runtime.GOOS + "/" + runtime.GOARCH, + }, nil } // Setup the pipeline environment. @@ -183,7 +189,7 @@ func (e *kube) SetupWorkflow(ctx context.Context, conf *types.Config, taskUUID s // Start the pipeline step. func (e *kube) StartStep(ctx context.Context, step *types.Step, taskUUID string) error { - pod, err := Pod(e.config.Namespace, step, e.config.PodLabels, e.config.PodAnnotations) + pod, err := Pod(e.config.Namespace, step, e.config.PodLabels, e.config.PodAnnotations, e.goos) if err != nil { return err } diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index 09e0b6d79..6671d1dcd 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -28,7 +28,7 @@ import ( "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" ) -func Pod(namespace string, step *types.Step, labels, annotations map[string]string) (*v1.Pod, error) { +func Pod(namespace string, step *types.Step, labels, annotations map[string]string, goos string) (*v1.Pod, error) { var ( vols []v1.Volume volMounts []v1.VolumeMount @@ -66,7 +66,7 @@ func Pod(namespace string, step *types.Step, labels, annotations map[string]stri } if len(step.Commands) != 0 { - scriptEnv, entry, cmds := common.GenerateContainerConf(step.Commands) + scriptEnv, entry, cmds := common.GenerateContainerConf(step.Commands, goos) for k, v := range scriptEnv { step.Environment[k] = v } diff --git a/pipeline/backend/local/local.go b/pipeline/backend/local/local.go index f90a44b0a..5543fed0e 100644 --- a/pipeline/backend/local/local.go +++ b/pipeline/backend/local/local.go @@ -60,10 +60,12 @@ func (e *local) IsAvailable(context.Context) bool { return true } -func (e *local) Load(context.Context) error { +func (e *local) Load(context.Context) (*types.EngineInfo, error) { e.loadClone() - return nil + return &types.EngineInfo{ + Platform: runtime.GOOS + "/" + runtime.GOARCH, + }, nil } // SetupWorkflow the pipeline environment. diff --git a/pipeline/backend/types/engine.go b/pipeline/backend/types/engine.go index b06e9a981..d35abeb47 100644 --- a/pipeline/backend/types/engine.go +++ b/pipeline/backend/types/engine.go @@ -29,7 +29,7 @@ type Engine interface { IsAvailable(ctx context.Context) bool // Load loads the backend engine. - Load(ctx context.Context) error + Load(ctx context.Context) (*EngineInfo, error) // SetupWorkflow sets up the workflow environment. SetupWorkflow(ctx context.Context, conf *Config, taskUUID string) error @@ -50,3 +50,8 @@ type Engine interface { // DestroyWorkflow destroys the workflow environment. DestroyWorkflow(ctx context.Context, conf *Config, taskUUID string) error } + +// EngineInfo represents the reported information of a loaded engine +type EngineInfo struct { + Platform string +}