Rework log streaming and related functions (#1802)

closes #1801
closes #1815 
closes #1144
closes  #983
closes  #557
closes #1827
regression of #1791

# TODO
- [x] adjust log model
- [x] add migration for logs
- [x] send log line via grpc using step-id
- [x] save log-line to db
- [x] stream log-lines to UI
- [x] use less structs for log-data
- [x] make web UI work
  - [x] display logs loaded from db
  - [x] display streaming logs
- [ ] ~~make migration work~~ -> dedicated pull (#1828)

# TESTED
- [x] new logs are stored in database
- [x] log retrieval via cli (of new logs) works
- [x] log streaming works (tested via curl & webui)
- [x] log retrieval via web (of new logs) works

---------

Co-authored-by: 6543 <6543@obermui.de>

agent/logger.go

@@ -50,9 +50,8 @@ func (r *Runner) createLogger(_ context.Context, logger zerolog.Logger, uploads
 
 		loglogger.Debug().Msg("log stream opened")
 
-		limitedPart := io.LimitReader(part, maxLogsUpload)
-		logStream := rpc.NewLineWriter(r.client, work.ID, step.Alias, secrets...)
-		if _, err := io.Copy(logStream, limitedPart); err != nil {
+		logStream := rpc.NewLineWriter(r.client, step.UUID, secrets...)
+		if _, err := io.Copy(logStream, part); err != nil {
 			log.Error().Err(err).Msg("copy limited logStream part")
 		}
 

agent/rpc/client_grpc.go

@@ -278,14 +278,14 @@ func (c *client) Update(ctx context.Context, id string, state rpc.State) (err er
 }
 
 // Log writes the pipeline log entry.
-func (c *client) Log(ctx context.Context, id string, line *rpc.Line) (err error) {
+func (c *client) Log(ctx context.Context, logEntry *rpc.LogEntry) (err error) {
 	req := new(proto.LogRequest)
-	req.Id = id
-	req.Line = new(proto.Line)
-	req.Line.Out = line.Out
-	req.Line.Pos = int32(line.Pos)
-	req.Line.Step = line.Step
-	req.Line.Time = line.Time
+	req.LogEntry = new(proto.LogEntry)
+	req.LogEntry.StepUuid = logEntry.StepUUID
+	req.LogEntry.Data = logEntry.Data
+	req.LogEntry.Line = int32(logEntry.Line)
+	req.LogEntry.Time = logEntry.Time
+	req.LogEntry.Type = int32(logEntry.Type)
 	for {
 		_, err = c.client.Log(ctx, req)
 		if err == nil {

agent/runner.go

@@ -31,14 +31,6 @@ import (
 	"github.com/woodpecker-ci/woodpecker/shared/utils"
 )
 
-// TODO: Implement log streaming.
-// Until now we need to limit the size of the logs and files that we upload.
-// The maximum grpc payload size is 4194304. So we need to set these limits below the maximum.
-const (
-	maxLogsUpload = 2000000 // this is per step
-	maxFileUpload = 1000000
-)
-
 type Runner struct {
 	client   rpc.Peer
 	filter   rpc.Filter