github/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2025-09-03 22:10:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #28 from laszlocph/dryness

Browse Source 
Dryness
This commit is contained in:
Laszlo Fogas
2019-06-05 06:51:45 +02:00
committed by GitHub
parent 21b32cc3aa 8199fe4a21
commit bb00a8d7e3
13 changed files with 354 additions and 541 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/drone-server/server.go
View File

@@ -671,7 +671,6 @@ func setupEvilGlobals(c *cli.Context, v store.Store, r remote.Remote) {
droneserver.Config.Services.Secrets = setupSecretService(c, v) droneserver.Config.Services.Secrets = setupSecretService(c, v)
droneserver.Config.Services.Senders = sender.New(v, v) droneserver.Config.Services.Senders = sender.New(v, v)
droneserver.Config.Services.Environ = setupEnvironService(c, v) droneserver.Config.Services.Environ = setupEnvironService(c, v)
droneserver.Config.Services.Limiter = setupLimiter(c, v)
if endpoint := c.String("gating-service"); endpoint != "" { if endpoint := c.String("gating-service"); endpoint != "" {
droneserver.Config.Services.Senders = sender.NewRemote(endpoint) droneserver.Config.Services.Senders = sender.NewRemote(endpoint)

4
cmd/drone-server/setup.go
View File

@@ -66,10 +66,6 @@ func setupEnvironService(c *cli.Context, s store.Store) model.EnvironService {
return nil return nil
} }
func setupLimiter(c *cli.Context, s store.Store) model.Limiter {
return new(model.NoLimit)
}
func setupPubsub(c *cli.Context) {} func setupPubsub(c *cli.Context) {}
func setupStream(c *cli.Context) {} func setupStream(c *cli.Context) {}
func setupGatingService(c *cli.Context) {} func setupGatingService(c *cli.Context) {}

41
model/limit.go
View File

@@ -1,41 +0,0 @@
// Copyright 2018 Drone.IO Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
// Limiter defines an interface for limiting repository creation.
// This could be used, for example, to limit repository creation to
// a specific organization or a specific set of users.
type Limiter interface {
LimitUser(*User) error
LimitRepo(*User, *Repo) error
LimitRepos(*User, []*Repo) []*Repo
LimitBuild(*User, *Repo, *Build) error
}
// NoLimit implements the Limiter interface without enforcing any
// actual limits. All limiting functions are no-ops.
type NoLimit struct{}
// LimitUser is a no-op for limiting user creation.
func (NoLimit) LimitUser(*User) error { return nil }
// LimitRepo is a no-op for limiting repo creation.
func (NoLimit) LimitRepo(*User, *Repo) error { return nil }
// LimitRepos is a no-op for limiting repository listings.
func (NoLimit) LimitRepos(user *User, repos []*Repo) []*Repo { return repos }
// LimitBuild is a no-op for limiting build creation.
func (NoLimit) LimitBuild(*User, *Repo, *Build) error { return nil }

173
server/build.go
View File

@@ -17,7 +17,6 @@ package server
import ( import (
"bytes" "bytes"
"context" "context"
"encoding/json"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@@ -25,13 +24,11 @@ import (
"time" "time"
"github.com/Sirupsen/logrus" "github.com/Sirupsen/logrus"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/rpc" "github.com/gin-gonic/gin"
"github.com/laszlocph/drone-oss-08/cncd/pubsub"
"github.com/laszlocph/drone-oss-08/cncd/queue" "github.com/laszlocph/drone-oss-08/cncd/queue"
"github.com/laszlocph/drone-oss-08/remote" "github.com/laszlocph/drone-oss-08/remote"
"github.com/laszlocph/drone-oss-08/shared/httputil" "github.com/laszlocph/drone-oss-08/shared/httputil"
"github.com/laszlocph/drone-oss-08/store" "github.com/laszlocph/drone-oss-08/store"
"github.com/gin-gonic/gin"
"github.com/laszlocph/drone-oss-08/model" "github.com/laszlocph/drone-oss-08/model"
"github.com/laszlocph/drone-oss-08/router/middleware/session" "github.com/laszlocph/drone-oss-08/router/middleware/session"
@@ -270,14 +267,6 @@ func PostApproval(c *gin.Context) {
build.Reviewed = time.Now().Unix() build.Reviewed = time.Now().Unix()
build.Reviewer = user.Login build.Reviewer = user.Login
//
//
// This code is copied pasted until I have a chance
// to refactor into a proper function. Lots of changes
// and technical debt. No judgement please!
//
//
// fetch the build file from the database // fetch the build file from the database
conf, err := Config.Storage.Config.ConfigLoad(build.ConfigID) conf, err := Config.Storage.Config.ConfigLoad(build.ConfigID)
if err != nil { if err != nil {
@@ -326,7 +315,7 @@ func PostApproval(c *gin.Context) {
} }
}() }()
b := builder{ b := procBuilder{
Repo: repo, Repo: repo,
Curr: build, Curr: build,
Last: last, Last: last,
@@ -337,7 +326,7 @@ func PostApproval(c *gin.Context) {
Yaml: conf.Data, Yaml: conf.Data,
Envs: envs, Envs: envs,
} }
items, err := b.Build() buildItems, err := b.Build()
if err != nil { if err != nil {
build.Status = model.StatusError build.Status = model.StatusError
build.Started = time.Now().Unix() build.Started = time.Now().Unix()
@@ -347,73 +336,14 @@ func PostApproval(c *gin.Context) {
return return
} }
var pcounter = len(items) setBuildProcs(build, buildItems)
for _, item := range items { err = store.FromContext(c).ProcCreate(build.Procs)
build.Procs = append(build.Procs, item.Proc) if err != nil {
item.Proc.BuildID = build.ID logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err)
for _, stage := range item.Config.Stages {
var gid int
for _, step := range stage.Steps {
pcounter++
if gid == 0 {
gid = pcounter
}
proc := &model.Proc{
BuildID: build.ID,
Name: step.Alias,
PID: pcounter,
PPID: item.Proc.PID,
PGID: gid,
State: model.StatusPending,
}
build.Procs = append(build.Procs, proc)
}
}
} }
store.FromContext(c).ProcCreate(build.Procs)
// publishToTopic(c, build, repo)
// publish topic queueBuild(build, repo, buildItems)
//
buildCopy := *build
buildCopy.Procs = model.Tree(buildCopy.Procs)
message := pubsub.Message{
Labels: map[string]string{
"repo": repo.FullName,
"private": strconv.FormatBool(repo.IsPrivate),
},
}
message.Data, _ = json.Marshal(model.Event{
Type: model.Enqueued,
Repo: *repo,
Build: buildCopy,
})
// TODO remove global reference
Config.Services.Pubsub.Publish(c, "topic/events", message)
//
// end publish topic
//
for _, item := range items {
task := new(queue.Task)
task.ID = fmt.Sprint(item.Proc.ID)
task.Labels = map[string]string{}
task.Labels["platform"] = item.Platform
for k, v := range item.Labels {
task.Labels[k] = v
}
task.Data, _ = json.Marshal(rpc.Pipeline{
ID: fmt.Sprint(item.Proc.ID),
Config: item.Config,
Timeout: b.Repo.Timeout,
})
Config.Services.Logs.Open(context.Background(), task.ID)
Config.Services.Queue.Push(context.Background(), task)
}
} }
func PostDecline(c *gin.Context) { func PostDecline(c *gin.Context) {
@@ -463,15 +393,8 @@ func GetBuildQueue(c *gin.Context) {
c.JSON(200, out) c.JSON(200, out)
} }
// // PostBuild restarts a build
//
//
//
//
//
func PostBuild(c *gin.Context) { func PostBuild(c *gin.Context) {
remote_ := remote.FromContext(c) remote_ := remote.FromContext(c)
repo := session.Repo(c) repo := session.Repo(c)
@@ -581,7 +504,7 @@ func PostBuild(c *gin.Context) {
} }
} }
b := builder{ b := procBuilder{
Repo: repo, Repo: repo,
Curr: build, Curr: build,
Last: last, Last: last,
@@ -592,7 +515,7 @@ func PostBuild(c *gin.Context) {
Yaml: conf.Data, Yaml: conf.Data,
Envs: buildParams, Envs: buildParams,
} }
items, err := b.Build() buildItems, err := b.Build()
if err != nil { if err != nil {
build.Status = model.StatusError build.Status = model.StatusError
build.Started = time.Now().Unix() build.Started = time.Now().Unix()
@@ -602,30 +525,7 @@ func PostBuild(c *gin.Context) {
return return
} }
var pcounter = len(items) setBuildProcs(build, buildItems)
for _, item := range items {
build.Procs = append(build.Procs, item.Proc)
item.Proc.BuildID = build.ID
for _, stage := range item.Config.Stages {
var gid int
for _, step := range stage.Steps {
pcounter++
if gid == 0 {
gid = pcounter
}
proc := &model.Proc{
BuildID: build.ID,
Name: step.Alias,
PID: pcounter,
PPID: item.Proc.PID,
PGID: gid,
State: model.StatusPending,
}
build.Procs = append(build.Procs, proc)
}
}
}
err = store.FromContext(c).ProcCreate(build.Procs) err = store.FromContext(c).ProcCreate(build.Procs)
if err != nil { if err != nil {
@@ -637,55 +537,12 @@ func PostBuild(c *gin.Context) {
c.JSON(500, build) c.JSON(500, build)
return return
} }
c.JSON(202, build) c.JSON(202, build)
// publishToTopic(c, build, repo)
// publish topic queueBuild(build, repo, buildItems)
//
buildCopy := *build
buildCopy.Procs = model.Tree(buildCopy.Procs)
message := pubsub.Message{
Labels: map[string]string{
"repo": repo.FullName,
"private": strconv.FormatBool(repo.IsPrivate),
},
}
message.Data, _ = json.Marshal(model.Event{
Type: model.Enqueued,
Repo: *repo,
Build: buildCopy,
})
// TODO remove global reference
Config.Services.Pubsub.Publish(c, "topic/events", message)
//
// end publish topic
//
for _, item := range items {
task := new(queue.Task)
task.ID = fmt.Sprint(item.Proc.ID)
task.Labels = map[string]string{}
task.Labels["platform"] = item.Platform
for k, v := range item.Labels {
task.Labels[k] = v
}
task.Data, _ = json.Marshal(rpc.Pipeline{
ID: fmt.Sprint(item.Proc.ID),
Config: item.Config,
Timeout: b.Repo.Timeout,
})
Config.Services.Logs.Open(context.Background(), task.ID)
Config.Services.Queue.Push(context.Background(), task)
}
} }
//
///
//
func DeleteBuildLogs(c *gin.Context) { func DeleteBuildLogs(c *gin.Context) {
repo := session.Repo(c) repo := session.Repo(c)
user := session.User(c) user := session.User(c)

372
server/hook.go
View File

@@ -20,10 +20,8 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"math/rand" "math/rand"
"net/url"
"regexp" "regexp"
"strconv" "strconv"
"strings"
"time" "time"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
@@ -34,26 +32,13 @@ import (
"github.com/laszlocph/drone-oss-08/shared/httputil" "github.com/laszlocph/drone-oss-08/shared/httputil"
"github.com/laszlocph/drone-oss-08/shared/token" "github.com/laszlocph/drone-oss-08/shared/token"
"github.com/laszlocph/drone-oss-08/store" "github.com/laszlocph/drone-oss-08/store"
"github.com/drone/envsubst"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/backend"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml" "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/compiler"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/linter"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/matrix"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/rpc" "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/rpc"
"github.com/laszlocph/drone-oss-08/cncd/pubsub" "github.com/laszlocph/drone-oss-08/cncd/pubsub"
"github.com/laszlocph/drone-oss-08/cncd/queue" "github.com/laszlocph/drone-oss-08/cncd/queue"
) )
//
// CANARY IMPLEMENTATION
//
// This file is a complete disaster because I'm trying to wedge in some
// experimental code. Please pardon our appearance during renovations.
//
var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`) var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`)
func init() { func init() {
@@ -157,54 +142,30 @@ func PostHook(c *gin.Context) {
} }
} }
// fetch the build file from the database // fetch the build file from the remote
confb, err := remote.FileBackoff(remote_, user, repo, build, repo.Config) remoteYamlConfig, err := remote.FileBackoff(remote_, user, repo, build, repo.Config)
if err != nil { if err != nil {
logrus.Errorf("error: %s: cannot find %s in %s: %s", repo.FullName, repo.Config, build.Ref, err) logrus.Errorf("error: %s: cannot find %s in %s: %s", repo.FullName, repo.Config, build.Ref, err)
c.AbortWithError(404, err) c.AbortWithError(404, err)
return return
} }
sha := shasum(confb) conf, err := findOrPersistPipelineConfig(repo, remoteYamlConfig)
conf, err := Config.Storage.Config.ConfigFind(repo, sha)
if err != nil { if err != nil {
conf = &model.Config{ logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err)
RepoID: repo.ID, c.AbortWithError(500, err)
Data: string(confb), return
Hash: sha,
}
err = Config.Storage.Config.ConfigCreate(conf)
if err != nil {
// retry in case we receive two hooks at the same time
conf, err = Config.Storage.Config.ConfigFind(repo, sha)
if err != nil {
logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err)
c.AbortWithError(500, err)
return
}
}
} }
build.ConfigID = conf.ID build.ConfigID = conf.ID
netrc, err := remote_.Netrc(user, repo) // verify that pipeline can be built at all
if err != nil { parsedPipelineConfig, err := yaml.ParseString(conf.Data)
c.String(500, "Failed to generate netrc file. %s", err)
return
}
// verify the branches can be built vs skipped
branches, err := yaml.ParseString(conf.Data)
if err == nil { if err == nil {
if !branches.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy { if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy {
c.String(200, "Branch does not match restrictions defined in yaml") c.String(200, "Branch does not match restrictions defined in yaml")
return return
} }
} }
// update some build fields
build.RepoID = repo.ID
build.Verified = true
build.Status = model.StatusPending
if repo.IsGated { if repo.IsGated {
allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf) allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf)
if !allowed { if !allowed {
@@ -212,12 +173,11 @@ func PostHook(c *gin.Context) {
} }
} }
if err = Config.Services.Limiter.LimitBuild(user, repo, build); err != nil { // update some build fields
c.String(403, "Build blocked by limiter") build.RepoID = repo.ID
return build.Verified = true
} build.Status = model.StatusPending
build.Trim()
err = store.CreateBuild(c, build, build.Procs...) err = store.CreateBuild(c, build, build.Procs...)
if err != nil { if err != nil {
logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err) logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err)
@@ -231,6 +191,12 @@ func PostHook(c *gin.Context) {
return return
} }
netrc, err := remote_.Netrc(user, repo)
if err != nil {
c.String(500, "Failed to generate netrc file. %s", err)
return
}
envs := map[string]string{} envs := map[string]string{}
if Config.Services.Environ != nil { if Config.Services.Environ != nil {
globals, _ := Config.Services.Environ.EnvironList(repo) globals, _ := Config.Services.Environ.EnvironList(repo)
@@ -249,14 +215,9 @@ func PostHook(c *gin.Context) {
logrus.Debugf("Error getting registry credentials for %s#%d. %s", repo.FullName, build.Number, err) logrus.Debugf("Error getting registry credentials for %s#%d. %s", repo.FullName, build.Number, err)
} }
// get the previous build so that we can send // get the previous build so that we can send status change notifications
// on status change notifications
last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
//
// BELOW: NEW
//
defer func() { defer func() {
uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number)
err = remote_.Status(user, repo, build, uri) err = remote_.Status(user, repo, build, uri)
@@ -265,7 +226,7 @@ func PostHook(c *gin.Context) {
} }
}() }()
b := builder{ b := procBuilder{
Repo: repo, Repo: repo,
Curr: build, Curr: build,
Last: last, Last: last,
@@ -276,7 +237,7 @@ func PostHook(c *gin.Context) {
Link: httputil.GetURL(c.Request), Link: httputil.GetURL(c.Request),
Yaml: conf.Data, Yaml: conf.Data,
} }
items, err := b.Build() buildItems, err := b.Build()
if err != nil { if err != nil {
build.Status = model.StatusError build.Status = model.StatusError
build.Started = time.Now().Unix() build.Started = time.Now().Unix()
@@ -286,9 +247,42 @@ func PostHook(c *gin.Context) {
return return
} }
var pcounter = len(items) setBuildProcs(build, buildItems)
for _, item := range items { err = store.FromContext(c).ProcCreate(build.Procs)
if err != nil {
logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err)
}
publishToTopic(c, build, repo)
queueBuild(build, repo, buildItems)
}
func findOrPersistPipelineConfig(repo *model.Repo, remoteYamlConfig []byte) (*model.Config, error) {
sha := shasum(remoteYamlConfig)
conf, err := Config.Storage.Config.ConfigFind(repo, sha)
if err != nil {
conf = &model.Config{
RepoID: repo.ID,
Data: string(remoteYamlConfig),
Hash: sha,
}
err = Config.Storage.Config.ConfigCreate(conf)
if err != nil {
// retry in case we receive two hooks at the same time
conf, err = Config.Storage.Config.ConfigFind(repo, sha)
if err != nil {
return nil, err
}
}
}
return conf, nil
}
func setBuildProcs(build *model.Build, buildItems []*buildItem) {
pcounter := len(buildItems)
for _, item := range buildItems {
build.Procs = append(build.Procs, item.Proc) build.Procs = append(build.Procs, item.Proc)
item.Proc.BuildID = build.ID item.Proc.BuildID = build.ID
@@ -311,14 +305,9 @@ func PostHook(c *gin.Context) {
} }
} }
} }
err = store.FromContext(c).ProcCreate(build.Procs) }
if err != nil {
logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err)
}
// func publishToTopic(c *gin.Context, build *model.Build, repo *model.Repo) {
// publish topic
//
message := pubsub.Message{ message := pubsub.Message{
Labels: map[string]string{ Labels: map[string]string{
"repo": repo.FullName, "repo": repo.FullName,
@@ -332,13 +321,11 @@ func PostHook(c *gin.Context) {
Repo: *repo, Repo: *repo,
Build: buildCopy, Build: buildCopy,
}) })
// TODO remove global reference
Config.Services.Pubsub.Publish(c, "topic/events", message) Config.Services.Pubsub.Publish(c, "topic/events", message)
// }
// end publish topic
//
for _, item := range items { func queueBuild(build *model.Build, repo *model.Repo, buildItems []*buildItem) {
for _, item := range buildItems {
task := new(queue.Task) task := new(queue.Task)
task.ID = fmt.Sprint(item.Proc.ID) task.ID = fmt.Sprint(item.Proc.ID)
task.Labels = map[string]string{} task.Labels = map[string]string{}
@@ -346,12 +333,12 @@ func PostHook(c *gin.Context) {
task.Labels[k] = v task.Labels[k] = v
} }
task.Labels["platform"] = item.Platform task.Labels["platform"] = item.Platform
task.Labels["repo"] = b.Repo.FullName task.Labels["repo"] = repo.FullName
task.Data, _ = json.Marshal(rpc.Pipeline{ task.Data, _ = json.Marshal(rpc.Pipeline{
ID: fmt.Sprint(item.Proc.ID), ID: fmt.Sprint(item.Proc.ID),
Config: item.Config, Config: item.Config,
Timeout: b.Repo.Timeout, Timeout: repo.Timeout,
}) })
Config.Services.Logs.Open(context.Background(), task.ID) Config.Services.Logs.Open(context.Background(), task.ID)
@@ -359,237 +346,6 @@ func PostHook(c *gin.Context) {
} }
} }
// return the metadata from the cli context.
func metadataFromStruct(repo *model.Repo, build, last *model.Build, proc *model.Proc, link string) frontend.Metadata {
host := link
uri, err := url.Parse(link)
if err == nil {
host = uri.Host
}
return frontend.Metadata{
Repo: frontend.Repo{
Name: repo.FullName,
Link: repo.Link,
Remote: repo.Clone,
Private: repo.IsPrivate,
Branch: repo.Branch,
},
Curr: frontend.Build{
Number: build.Number,
Parent: build.Parent,
Created: build.Created,
Started: build.Started,
Finished: build.Finished,
Status: build.Status,
Event: build.Event,
Link: build.Link,
Target: build.Deploy,
Commit: frontend.Commit{
Sha: build.Commit,
Ref: build.Ref,
Refspec: build.Refspec,
Branch: build.Branch,
Message: build.Message,
Author: frontend.Author{
Name: build.Author,
Email: build.Email,
Avatar: build.Avatar,
},
},
},
Prev: frontend.Build{
Number: last.Number,
Created: last.Created,
Started: last.Started,
Finished: last.Finished,
Status: last.Status,
Event: last.Event,
Link: last.Link,
Target: last.Deploy,
Commit: frontend.Commit{
Sha: last.Commit,
Ref: last.Ref,
Refspec: last.Refspec,
Branch: last.Branch,
Message: last.Message,
Author: frontend.Author{
Name: last.Author,
Email: last.Email,
Avatar: last.Avatar,
},
},
},
Job: frontend.Job{
Number: proc.PID,
Matrix: proc.Environ,
},
Sys: frontend.System{
Name: "drone",
Link: link,
Host: host,
Arch: "linux/amd64",
},
}
}
type builder struct {
Repo *model.Repo
Curr *model.Build
Last *model.Build
Netrc *model.Netrc
Secs []*model.Secret
Regs []*model.Registry
Link string
Yaml string
Envs map[string]string
}
type buildItem struct {
Proc *model.Proc
Platform string
Labels map[string]string
Config *backend.Config
}
func (b *builder) Build() ([]*buildItem, error) {
axes, err := matrix.ParseString(b.Yaml)
if err != nil {
return nil, err
}
if len(axes) == 0 {
axes = append(axes, matrix.Axis{})
}
var items []*buildItem
for i, axis := range axes {
proc := &model.Proc{
BuildID: b.Curr.ID,
PID: i + 1,
PGID: i + 1,
State: model.StatusPending,
Environ: axis,
}
metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link)
environ := metadata.Environ()
for k, v := range metadata.EnvironDrone() {
environ[k] = v
}
for k, v := range axis {
environ[k] = v
}
var secrets []compiler.Secret
for _, sec := range b.Secs {
if !sec.Match(b.Curr.Event) {
continue
}
secrets = append(secrets, compiler.Secret{
Name: sec.Name,
Value: sec.Value,
Match: sec.Images,
})
}
y := b.Yaml
s, err := envsubst.Eval(y, func(name string) string {
env := environ[name]
if strings.Contains(env, "\n") {
env = fmt.Sprintf("%q", env)
}
return env
})
if err != nil {
return nil, err
}
y = s
parsed, err := yaml.ParseString(y)
if err != nil {
return nil, err
}
metadata.Sys.Arch = parsed.Platform
if metadata.Sys.Arch == "" {
metadata.Sys.Arch = "linux/amd64"
}
lerr := linter.New(
linter.WithTrusted(b.Repo.IsTrusted),
).Lint(parsed)
if lerr != nil {
return nil, lerr
}
var registries []compiler.Registry
for _, reg := range b.Regs {
registries = append(registries, compiler.Registry{
Hostname: reg.Address,
Username: reg.Username,
Password: reg.Password,
Email: reg.Email,
})
}
ir := compiler.New(
compiler.WithEnviron(environ),
compiler.WithEnviron(b.Envs),
compiler.WithEscalated(Config.Pipeline.Privileged...),
compiler.WithResourceLimit(Config.Pipeline.Limits.MemSwapLimit, Config.Pipeline.Limits.MemLimit, Config.Pipeline.Limits.ShmSize, Config.Pipeline.Limits.CPUQuota, Config.Pipeline.Limits.CPUShares, Config.Pipeline.Limits.CPUSet),
compiler.WithVolumes(Config.Pipeline.Volumes...),
compiler.WithNetworks(Config.Pipeline.Networks...),
compiler.WithLocal(false),
compiler.WithOption(
compiler.WithNetrc(
b.Netrc.Login,
b.Netrc.Password,
b.Netrc.Machine,
),
b.Repo.IsPrivate,
),
compiler.WithRegistry(registries...),
compiler.WithSecret(secrets...),
compiler.WithPrefix(
fmt.Sprintf(
"%d_%d",
proc.ID,
rand.Int(),
),
),
compiler.WithEnviron(proc.Environ),
compiler.WithProxy(),
compiler.WithWorkspaceFromURL("/drone", b.Repo.Link),
compiler.WithMetadata(metadata),
).Compile(parsed)
// for _, sec := range b.Secs {
// if !sec.MatchEvent(b.Curr.Event) {
// continue
// }
// if b.Curr.Verified || sec.SkipVerify {
// ir.Secrets = append(ir.Secrets, &backend.Secret{
// Mask: sec.Conceal,
// Name: sec.Name,
// Value: sec.Value,
// })
// }
// }
item := &buildItem{
Proc: proc,
Config: ir,
Labels: parsed.Labels,
Platform: metadata.Sys.Arch,
}
if item.Labels == nil {
item.Labels = map[string]string{}
}
items = append(items, item)
}
return items, nil
}
func shasum(raw []byte) string { func shasum(raw []byte) string {
sum := sha256.Sum256(raw) sum := sha256.Sum256(raw)
return fmt.Sprintf("%x", sum) return fmt.Sprintf("%x", sum)

2
server/hook_test.go
View File

@@ -21,7 +21,7 @@ import (
) )
func TestMultilineEnvsubst(t *testing.T) { func TestMultilineEnvsubst(t *testing.T) {
b := builder{ b := procBuilder{
Repo: &model.Repo{}, Repo: &model.Repo{},
Curr: &model.Build{ Curr: &model.Build{
Message: `aaa Message: `aaa

7
server/login.go
View File

@@ -19,12 +19,12 @@ import (
"net/http" "net/http"
"time" "time"
"github.com/gorilla/securecookie"
"github.com/laszlocph/drone-oss-08/model" "github.com/laszlocph/drone-oss-08/model"
"github.com/laszlocph/drone-oss-08/remote" "github.com/laszlocph/drone-oss-08/remote"
"github.com/laszlocph/drone-oss-08/shared/httputil" "github.com/laszlocph/drone-oss-08/shared/httputil"
"github.com/laszlocph/drone-oss-08/shared/token" "github.com/laszlocph/drone-oss-08/shared/token"
"github.com/laszlocph/drone-oss-08/store" "github.com/laszlocph/drone-oss-08/store"
"github.com/gorilla/securecookie"
"github.com/Sirupsen/logrus" "github.com/Sirupsen/logrus"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
@@ -95,11 +95,6 @@ func HandleAuth(c *gin.Context) {
), ),
} }
if err = Config.Services.Limiter.LimitUser(u); err != nil {
c.String(403, "User activation blocked by limiter")
return
}
// insert the user into the database // insert the user into the database
if err := store.CreateUser(c, u); err != nil { if err := store.CreateUser(c, u); err != nil {
logrus.Errorf("cannot insert %s. %s", u.Login, err) logrus.Errorf("cannot insert %s. %s", u.Login, err)

263
server/procBuilder.go Normal file
View File

@@ -0,0 +1,263 @@
// Copyright 2018 Drone.IO Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package server
import (
"fmt"
"math/rand"
"net/url"
"strings"
"github.com/drone/envsubst"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/backend"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/compiler"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/linter"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/matrix"
"github.com/laszlocph/drone-oss-08/model"
)
// Takes the hook data and the yaml and returns in internal data model
type procBuilder struct {
Repo *model.Repo
Curr *model.Build
Last *model.Build
Netrc *model.Netrc
Secs []*model.Secret
Regs []*model.Registry
Link string
Yaml string
Envs map[string]string
}
type buildItem struct {
Proc *model.Proc
Platform string
Labels map[string]string
Config *backend.Config
}
func (b *procBuilder) Build() ([]*buildItem, error) {
axes, err := matrix.ParseString(b.Yaml)
if err != nil {
return nil, err
}
if len(axes) == 0 {
axes = append(axes, matrix.Axis{})
}
var items []*buildItem
for i, axis := range axes {
proc := &model.Proc{
BuildID: b.Curr.ID,
PID: i + 1,
PGID: i + 1,
State: model.StatusPending,
Environ: axis,
}
metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link)
environ := metadata.Environ()
for k, v := range metadata.EnvironDrone() {
environ[k] = v
}
for k, v := range axis {
environ[k] = v
}
var secrets []compiler.Secret
for _, sec := range b.Secs {
if !sec.Match(b.Curr.Event) {
continue
}
secrets = append(secrets, compiler.Secret{
Name: sec.Name,
Value: sec.Value,
Match: sec.Images,
})
}
y := b.Yaml
s, err := envsubst.Eval(y, func(name string) string {
env := environ[name]
if strings.Contains(env, "\n") {
env = fmt.Sprintf("%q", env)
}
return env
})
if err != nil {
return nil, err
}
y = s
parsed, err := yaml.ParseString(y)
if err != nil {
return nil, err
}
metadata.Sys.Arch = parsed.Platform
if metadata.Sys.Arch == "" {
metadata.Sys.Arch = "linux/amd64"
}
lerr := linter.New(
linter.WithTrusted(b.Repo.IsTrusted),
).Lint(parsed)
if lerr != nil {
return nil, lerr
}
var registries []compiler.Registry
for _, reg := range b.Regs {
registries = append(registries, compiler.Registry{
Hostname: reg.Address,
Username: reg.Username,
Password: reg.Password,
Email: reg.Email,
})
}
ir := compiler.New(
compiler.WithEnviron(environ),
compiler.WithEnviron(b.Envs),
compiler.WithEscalated(Config.Pipeline.Privileged...),
compiler.WithResourceLimit(Config.Pipeline.Limits.MemSwapLimit, Config.Pipeline.Limits.MemLimit, Config.Pipeline.Limits.ShmSize, Config.Pipeline.Limits.CPUQuota, Config.Pipeline.Limits.CPUShares, Config.Pipeline.Limits.CPUSet),
compiler.WithVolumes(Config.Pipeline.Volumes...),
compiler.WithNetworks(Config.Pipeline.Networks...),
compiler.WithLocal(false),
compiler.WithOption(
compiler.WithNetrc(
b.Netrc.Login,
b.Netrc.Password,
b.Netrc.Machine,
),
b.Repo.IsPrivate,
),
compiler.WithRegistry(registries...),
compiler.WithSecret(secrets...),
compiler.WithPrefix(
fmt.Sprintf(
"%d_%d",
proc.ID,
rand.Int(),
),
),
compiler.WithEnviron(proc.Environ),
compiler.WithProxy(),
compiler.WithWorkspaceFromURL("/drone", b.Repo.Link),
compiler.WithMetadata(metadata),
).Compile(parsed)
// for _, sec := range b.Secs {
// if !sec.MatchEvent(b.Curr.Event) {
// continue
// }
// if b.Curr.Verified || sec.SkipVerify {
// ir.Secrets = append(ir.Secrets, &backend.Secret{
// Mask: sec.Conceal,
// Name: sec.Name,
// Value: sec.Value,
// })
// }
// }
item := &buildItem{
Proc: proc,
Config: ir,
Labels: parsed.Labels,
Platform: metadata.Sys.Arch,
}
if item.Labels == nil {
item.Labels = map[string]string{}
}
items = append(items, item)
}
return items, nil
}
// return the metadata from the cli context.
func metadataFromStruct(repo *model.Repo, build, last *model.Build, proc *model.Proc, link string) frontend.Metadata {
host := link
uri, err := url.Parse(link)
if err == nil {
host = uri.Host
}
return frontend.Metadata{
Repo: frontend.Repo{
Name: repo.FullName,
Link: repo.Link,
Remote: repo.Clone,
Private: repo.IsPrivate,
Branch: repo.Branch,
},
Curr: frontend.Build{
Number: build.Number,
Parent: build.Parent,
Created: build.Created,
Started: build.Started,
Finished: build.Finished,
Status: build.Status,
Event: build.Event,
Link: build.Link,
Target: build.Deploy,
Commit: frontend.Commit{
Sha: build.Commit,
Ref: build.Ref,
Refspec: build.Refspec,
Branch: build.Branch,
Message: build.Message,
Author: frontend.Author{
Name: build.Author,
Email: build.Email,
Avatar: build.Avatar,
},
},
},
Prev: frontend.Build{
Number: last.Number,
Created: last.Created,
Started: last.Started,
Finished: last.Finished,
Status: last.Status,
Event: last.Event,
Link: last.Link,
Target: last.Deploy,
Commit: frontend.Commit{
Sha: last.Commit,
Ref: last.Ref,
Refspec: last.Refspec,
Branch: last.Branch,
Message: last.Message,
Author: frontend.Author{
Name: last.Author,
Email: last.Email,
Avatar: last.Avatar,
},
},
},
Job: frontend.Job{
Number: proc.PID,
Matrix: proc.Environ,
},
Sys: frontend.System{
Name: "drone",
Link: link,
Host: host,
Arch: "linux/amd64",
},
}
}

5
server/repo.go
View File

@@ -41,11 +41,6 @@ func PostRepo(c *gin.Context) {
return return
} }
if err := Config.Services.Limiter.LimitRepo(user, repo); err != nil {
c.String(403, "Repository activation blocked by limiter")
return
}
repo.IsActive = true repo.IsActive = true
repo.UserID = user.ID repo.UserID = user.ID
if !repo.AllowPush && !repo.AllowPull && !repo.AllowDeploy && !repo.AllowTag { if !repo.AllowPush && !repo.AllowPull && !repo.AllowDeploy && !repo.AllowTag {

1
server/rpc.go
View File

@@ -55,7 +55,6 @@ var Config = struct {
Secrets model.SecretService Secrets model.SecretService
Registries model.RegistryService Registries model.RegistryService
Environ model.EnvironService Environ model.EnvironService
Limiter model.Limiter
} }
Storage struct { Storage struct {
// Users model.UserStore // Users model.UserStore

11
server/sync.go
View File

@@ -28,10 +28,9 @@ type Syncer interface {
} }
type syncer struct { type syncer struct {
remote remote.Remote remote remote.Remote
store store.Store store store.Store
perms model.PermStore perms model.PermStore
limiter model.Limiter
} }
func (s *syncer) Sync(user *model.User) error { func (s *syncer) Sync(user *model.User) error {
@@ -41,10 +40,6 @@ func (s *syncer) Sync(user *model.User) error {
return err return err
} }
if s.limiter != nil {
repos = s.limiter.LimitRepos(user, repos)
}
var perms []*model.Perm var perms []*model.Perm
for _, repo := range repos { for _, repo := range repos {
perm := model.Perm{ perm := model.Perm{

14
server/user.go
View File

@@ -46,10 +46,9 @@ func GetFeed(c *gin.Context) {
store.FromContext(c).UpdateUser(user) store.FromContext(c).UpdateUser(user)
sync := syncer{ sync := syncer{
remote: remote.FromContext(c), remote: remote.FromContext(c),
store: store.FromContext(c), store: store.FromContext(c),
perms: store.FromContext(c), perms: store.FromContext(c),
limiter: Config.Services.Limiter,
} }
if err := sync.Sync(user); err != nil { if err := sync.Sync(user); err != nil {
logrus.Debugf("sync error: %s: %s", user.Login, err) logrus.Debugf("sync error: %s: %s", user.Login, err)
@@ -89,10 +88,9 @@ func GetRepos(c *gin.Context) {
store.FromContext(c).UpdateUser(user) store.FromContext(c).UpdateUser(user)
sync := syncer{ sync := syncer{
remote: remote.FromContext(c), remote: remote.FromContext(c),
store: store.FromContext(c), store: store.FromContext(c),
perms: store.FromContext(c), perms: store.FromContext(c),
limiter: Config.Services.Limiter,
} }
if err := sync.Sync(user); err != nil { if err := sync.Sync(user); err != nil {
logrus.Debugf("sync error: %s: %s", user.Login, err) logrus.Debugf("sync error: %s: %s", user.Login, err)

1
store/datastore/builds.go
View File

@@ -72,6 +72,7 @@ func (db *datastore) GetBuildQueue() ([]*model.Feed, error) {
} }
func (db *datastore) CreateBuild(build *model.Build, procs ...*model.Proc) error { func (db *datastore) CreateBuild(build *model.Build, procs ...*model.Proc) error {
build.Trim()
id, err := db.incrementRepoRetry(build.RepoID) id, err := db.incrementRepoRetry(build.RepoID)
if err != nil { if err != nil {
return err return err