Move cncd/{logging,pubsub,queue}/ to server/{logging,pubsub,queue}/ (#346)

* Move cncd/{logging,pubsub,queue}/ to server/{logging,pubsub,queue}/ * Update REAMDEs and include history Co-authored-by: Anbraten <anton@ju60.de> Co-authored-by: Anbraten <anton@ju60.de>
2025-10-22 00:10:49 +00:00 · 2021-09-23 15:29:09 -05:00
parent 780c902a6b
commit a0d008e071
30 changed files with 51 additions and 33 deletions
--- a/server/logging/LICENSE
+++ b/server/logging/LICENSE
@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2017, Brad Rydzewski
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--- a/server/logging/README.md
+++ b/server/logging/README.md
@@ -0,0 +1,12 @@
+# logging package
+
+Go package provides a common interface for storing and streaming logs.
+
+## History
+
+This was originally published in: https://github.com/cncd/logging
+Then it was included in: https://github.com/drone-ci/drone/cncd/logging
+
+## Documentation:
+
+https://godoc.org/github.com/woodpecker-ci/woodpecker/server/logging
--- a/server/logging/log.go
+++ b/server/logging/log.go
@@ -0,0 +1,143 @@
+package logging
+
+import (
+	"context"
+	"io"
+	"sync"
+)
+
+// TODO (bradrydzewski) writing to subscribers is currently a blocking
+// operation and does not protect against slow clients from locking
+// the stream. This should be resolved.
+
+// TODO (bradrydzewski) implement a mux.Info to fetch information and
+// statistics for the multiplexier. Streams, subscribers, etc
+// mux.Info()
+
+// TODO (bradrydzewski) refactor code to place publisher and subscriber
+// operations in separate files with more encapsulated logic.
+// sub.push()
+// sub.join()
+// sub.start()... event loop
+
+type subscriber struct {
+	handler Handler
+}
+
+type stream struct {
+	sync.Mutex
+
+	path string
+	hist []*Entry
+	subs map[*subscriber]struct{}
+	done chan struct{}
+	wait sync.WaitGroup
+}
+
+type log struct {
+	sync.Mutex
+
+	streams map[string]*stream
+}
+
+// New returns a new logger.
+func New() Log {
+	return &log{
+		streams: map[string]*stream{},
+	}
+}
+
+func (l *log) Open(c context.Context, path string) error {
+	l.Lock()
+	_, ok := l.streams[path]
+	if !ok {
+		l.streams[path] = &stream{
+			path: path,
+			subs: make(map[*subscriber]struct{}),
+			done: make(chan struct{}),
+		}
+	}
+	l.Unlock()
+	return nil
+}
+
+func (l *log) Write(c context.Context, path string, entry *Entry) error {
+	l.Lock()
+	s, ok := l.streams[path]
+	l.Unlock()
+	if !ok {
+		return ErrNotFound
+	}
+	s.Lock()
+	s.hist = append(s.hist, entry)
+	for sub := range s.subs {
+		go sub.handler(entry)
+	}
+	s.Unlock()
+	return nil
+}
+
+func (l *log) Tail(c context.Context, path string, handler Handler) error {
+	l.Lock()
+	s, ok := l.streams[path]
+	l.Unlock()
+	if !ok {
+		return ErrNotFound
+	}
+
+	sub := &subscriber{
+		handler: handler,
+	}
+	s.Lock()
+	if len(s.hist) != 0 {
+		sub.handler(s.hist...)
+	}
+	s.subs[sub] = struct{}{}
+	s.Unlock()
+
+	select {
+	case <-c.Done():
+	case <-s.done:
+	}
+
+	s.Lock()
+	delete(s.subs, sub)
+	s.Unlock()
+	return nil
+}
+
+func (l *log) Close(c context.Context, path string) error {
+	l.Lock()
+	s, ok := l.streams[path]
+	l.Unlock()
+	if !ok {
+		return ErrNotFound
+	}
+
+	s.Lock()
+	close(s.done)
+	s.Unlock()
+
+	l.Lock()
+	delete(l.streams, path)
+	l.Unlock()
+	return nil
+}
+
+func (l *log) Snapshot(c context.Context, path string, w io.Writer) error {
+	l.Lock()
+	s, ok := l.streams[path]
+	l.Unlock()
+	if !ok {
+		return ErrNotFound
+	}
+	s.Lock()
+	for _, entry := range s.hist {
+		w.Write(entry.Data)
+		w.Write(cr)
+	}
+	s.Unlock()
+	return nil
+}
+
+var cr = []byte{'\n'}
--- a/server/logging/log_test.go
+++ b/server/logging/log_test.go
@@ -0,0 +1,52 @@
+package logging
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestLogging(t *testing.T) {
+	var (
+		wg sync.WaitGroup
+
+		testPath  = "test"
+		testEntry = &Entry{
+			Data: []byte("test"),
+		}
+	)
+
+	ctx, cancel := context.WithCancel(
+		context.Background(),
+	)
+
+	logger := New()
+	logger.Open(ctx, testPath)
+	go func() {
+		logger.Tail(ctx, testPath, func(entry ...*Entry) { wg.Done() })
+	}()
+	go func() {
+		logger.Tail(ctx, testPath, func(entry ...*Entry) { wg.Done() })
+	}()
+
+	<-time.After(500 * time.Millisecond)
+
+	wg.Add(4)
+	go func() {
+		logger.Write(ctx, testPath, testEntry)
+		logger.Write(ctx, testPath, testEntry)
+	}()
+
+	wg.Wait()
+
+	wg.Add(1)
+	go func() {
+		logger.Tail(ctx, testPath, func(entry ...*Entry) { wg.Done() })
+	}()
+
+	<-time.After(500 * time.Millisecond)
+
+	wg.Wait()
+	cancel()
+}
--- a/server/logging/logging.go
+++ b/server/logging/logging.go
@@ -0,0 +1,80 @@
+package logging
+
+import (
+	"context"
+	"errors"
+	"io"
+)
+
+// ErrNotFound is returned when the log does not exist.
+var ErrNotFound = errors.New("stream: not found")
+
+// Entry defines a log entry.
+type Entry struct {
+	// ID identifies this message.
+	ID string `json:"id,omitempty"`
+
+	// Data is the actual data in the entry.
+	Data []byte `json:"data"`
+
+	// Tags represents the key-value pairs the
+	// entry is tagged with.
+	Tags map[string]string `json:"tags,omitempty"`
+}
+
+// Handler defines a callback function for handling log entries.
+type Handler func(...*Entry)
+
+// Log defines a log multiplexer.
+type Log interface {
+	// Open opens the log.
+	Open(c context.Context, path string) error
+
+	// Write writes the entry to the log.
+	Write(c context.Context, path string, entry *Entry) error
+
+	// Tail tails the log.
+	Tail(c context.Context, path string, handler Handler) error
+
+	// Close closes the log.
+	Close(c context.Context, path string) error
+
+	// Snapshot snapshots the stream to Writer w.
+	Snapshot(c context.Context, path string, w io.Writer) error
+
+	// Info returns runtime information about the multiplexer.
+	// Info(c context.Context) (interface{}, error)
+}
+
+// // global streamer
+// var global = New()
+//
+// // Set sets a default global logger.
+// func Set(log Log) {
+// 	global = log
+// }
+//
+// // Open opens the log stream.
+// func Open(c context.Context, path string) error {
+// 	return global.Open(c, path)
+// }
+//
+// // Write writes the log entry to the stream.
+// func Write(c context.Context, path string, entry *Entry) error {
+// 	return global.Write(c, path, entry)
+// }
+//
+// // Tail tails the log stream.
+// func Tail(c context.Context, path string, handler Handler) error {
+// 	return global.Tail(c, path, handler)
+// }
+//
+// // Close closes the log stream.
+// func Close(c context.Context, path string) error {
+// 	return global.Close(c, path)
+// }
+//
+// // Snapshot snapshots the stream to Writer w.
+// func Snapshot(c context.Context, path string, w io.Writer) error {
+// 	return global.Snapshot(c, path, w)
+// }