diff --git a/ci/run.py b/ci/run.py index b385041..dea1d99 100755 --- a/ci/run.py +++ b/ci/run.py @@ -205,20 +205,44 @@ class Libjwt(Project): for cmd in cmds: shell(cmd) +class Libhiredis(Project): + def __init__(self): + super(Libhiredis, self).__init__('hiredis') + + def branch(self): + return 'v1.1.0' + + @property + def url(self): + return 'https://github.com/redis/hiredis.git' + + @chdir + def compile_and_install(self): + cmds = [ + 'sudo make', + 'sudo make install', + ] + + for cmd in cmds: + shell(cmd) + def fetch_and_build(): libsearpc = Libsearpc() libjwt = Libjwt() + libhiredis = Libhiredis() libevhtp = Libevhtp() ccnet = CcnetServer() seafile = SeafileServer() libsearpc.clone() libjwt.clone() + libhiredis.clone() libevhtp.clone() ccnet.clone() libsearpc.compile_and_install() libjwt.compile_and_install() + libhiredis.compile_and_install() libevhtp.compile_and_install() seafile.compile_and_install() diff --git a/common/obj-cache.c b/common/obj-cache.c new file mode 100644 index 0000000..716df41 --- /dev/null +++ b/common/obj-cache.c @@ -0,0 +1,189 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include "common.h" + +#define DEBUG_FLAG SEAFILE_DEBUG_OTHER +#include "log.h" +#include "redis-cache.h" +#include "obj-cache.h" + +#define DEFAULT_MEMCACHED_EXPIRY 24 * 3600 +#define DEFAULT_MAX_CONNECTIONS 100 + +typedef struct CacheOption { + char *cache_provider; + char *redis_host; + char *redis_passwd; + int redis_port; + int redis_max_connections; + int redis_expiry; +} CacheOption; + +static void +cache_option_free (CacheOption *option) +{ + if (!option) + return; + g_free (option->cache_provider); + g_free (option->redis_host); + g_free (option->redis_passwd); + g_free (option); +} + +static void +load_cache_option_from_env (CacheOption *option) +{ + const char *cache_provider, *redis_host, *redis_port, *redis_passwd, *redis_max_conn, *redis_expiry; + + cache_provider = g_getenv("CACHE_PROVIDER"); + redis_host = g_getenv("REDIS_HOST"); + redis_port = g_getenv("REDIS_PORT"); + redis_passwd = g_getenv("REDIS_PASSWORD"); + redis_max_conn = g_getenv("REDIS_MAX_CONNECTIONS"); + redis_expiry = g_getenv("REDIS_EXPIRY"); + + if (!cache_provider) { + return; + } + + if (cache_provider) { + g_free (option->cache_provider); + option->cache_provider = g_strdup (cache_provider); + } + if (redis_host) { + g_free (option->redis_host); + option->redis_host = g_strdup (redis_host); + } + if (redis_port) { + option->redis_port = atoi (redis_port); + } + if (redis_passwd) { + g_free (option->redis_passwd); + option->redis_passwd = g_strdup (redis_passwd); + } + if (redis_max_conn) { + option->redis_max_connections = atoi (redis_max_conn); + } + if (redis_expiry) { + option->redis_expiry = atoi (redis_expiry); + } +} + +ObjCache * +objcache_new (GKeyFile *config) +{ + ObjCache *cache = NULL; + GError *error = NULL; + CacheOption *option = g_new0 (CacheOption, 1); + int redis_port; + int redis_expiry; + int redis_max_connections; + + redis_expiry = DEFAULT_MEMCACHED_EXPIRY; + redis_port = 6379; + redis_max_connections = DEFAULT_MAX_CONNECTIONS; + + option->redis_port = redis_port; + option->redis_max_connections = redis_max_connections; + option->redis_expiry = redis_expiry; + + load_cache_option_from_env (option); + + if (g_strcmp0 (option->cache_provider, "redis") == 0) { + cache = redis_cache_new (option->redis_host, option->redis_passwd, option->redis_port, option->redis_expiry, option->redis_max_connections); + } else if (option->cache_provider){ + seaf_warning ("Unsupported cache provider: %s\n", option->cache_provider); + } + + cache_option_free (option); + + return cache; +} + +void * +objcache_get_object (ObjCache *cache, const char *obj_id, size_t *len) +{ + return cache->get_object (cache, obj_id, len); +} + +int +objcache_set_object (ObjCache *cache, + const char *obj_id, + const void *object, + int len, + int expiry) +{ + return cache->set_object (cache, obj_id, object, len, expiry); +} + +gboolean +objcache_test_object (ObjCache *cache, const char *obj_id) +{ + return cache->test_object (cache, obj_id); +} + +int +objcache_delete_object (ObjCache *cache, const char *obj_id) +{ + return cache->delete_object (cache, obj_id); +} + +int +objcache_set_object_existence (ObjCache *cache, const char *obj_id, int val, int expiry, const char *existence_prefix) +{ + char *key; + char buf[8]; + int n; + int ret; + + key = g_strdup_printf ("%s%s", existence_prefix, obj_id); + n = snprintf (buf, sizeof(buf), "%d", val); + + ret = cache->set_object (cache, key, buf, n+1, expiry); + + g_free (key); + return ret; +} + +int +objcache_get_object_existence (ObjCache *cache, const char *obj_id, int *val_out, const char *existence_prefix) +{ + char *key; + size_t len; + char *val; + int ret = 0; + + key = g_strdup_printf ("%s%s", existence_prefix, obj_id); + + val = cache->get_object (cache, key, &len); + if (!val) + ret = -1; + else + *val_out = atoi(val); + + g_free (key); + g_free (val); + return ret; +} + +int +objcache_delete_object_existence (ObjCache *cache, const char *obj_id, const char *existence_prefix) +{ + char *key; + int ret; + + key = g_strdup_printf ("%s%s", existence_prefix, obj_id); + + ret = cache->delete_object (cache, key); + + g_free (key); + return ret; +} + +int +objcache_publish (ObjCache *cache, const char *channel, const char *msg) +{ + int ret; + ret = cache->publish (cache, channel, msg); + return ret; +} diff --git a/common/obj-cache.h b/common/obj-cache.h new file mode 100644 index 0000000..4bf35b8 --- /dev/null +++ b/common/obj-cache.h @@ -0,0 +1,72 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#ifndef OBJ_CACHE_H +#define OBJ_CACHE_H + +#define DEFAULT_MEMCACHED_EXPIRY 24 * 3600 + +#define TYPE_REDIS 0x02 + +typedef struct ObjCache ObjCache; + +struct ObjCache { + void* (*get_object) (ObjCache *cache, + const char *obj_id, + size_t *len); + + int (*set_object) (ObjCache *cache, + const char *obj_id, + const void *object, + int len, + int expiry); + + gboolean (*test_object) (ObjCache *cache, + const char *obj_id); + + int (*delete_object) (ObjCache *cache, + const char *obj_id); + + int (*publish) (ObjCache *cache, + const char *channel, + const char *msg); + + int mc_expiry; + char *host; + int port; + char cache_type; + + void *priv; +}; + +ObjCache * +objcache_new (); + +void * +objcache_get_object (struct ObjCache *cache, const char *obj_id, size_t *len); + +int +objcache_set_object (struct ObjCache *cache, + const char *obj_id, + const void *object, + int len, + int expiry); + +gboolean +objcache_test_object (struct ObjCache *cache, const char *obj_id); + +int +objcache_delete_object (struct ObjCache *cache, const char *obj_id); + +int +objcache_set_object_existence (struct ObjCache *cache, const char *obj_id, int val, int expiry, const char *existence_prefix); + +int +objcache_get_object_existence (struct ObjCache *cache, const char *obj_id, int *val_out, const char *existence_prefix); + +int +objcache_delete_object_existence (struct ObjCache *cache, const char *obj_id, const char *existence_prefix); + +int +objcache_publish (ObjCache *cache, const char *channel, const char *msg); + +#endif diff --git a/common/redis-cache.c b/common/redis-cache.c new file mode 100644 index 0000000..80eb000 --- /dev/null +++ b/common/redis-cache.c @@ -0,0 +1,404 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include "common.h" + +#include +#include "redis-cache.h" + +#define DEBUG_FLAG SEAFILE_DEBUG_OTHER +#include "log.h" + +struct _RedisConnectionPool { + char *host; + int port; + GPtrArray *connections; + pthread_mutex_t lock; + int max_connections; +}; +typedef struct _RedisConnectionPool RedisConnectionPool; + +struct _RedisConnection { + gboolean is_available; + redisContext *ac; + gint64 ctime; /* Used to clean up unused connection. */ + gboolean release; /* If TRUE, the connection will be released. */ +}; +typedef struct _RedisConnection RedisConnection; + +typedef struct RedisPriv { + RedisConnectionPool *redis_pool; + char *passwd; +} RedisPriv; + +static int +redis_auth (RedisConnection *conn, const char *passwd) +{ + redisReply *reply; + int ret = 0; + + if (!passwd) { + return 0; + } + + reply = redisCommand(conn->ac, "AUTH %s", passwd); + if (!reply) { + seaf_warning ("Failed to auth redis server.\n"); + ret = -1; + goto out; + } + + if (reply->type != REDIS_REPLY_STATUS || + g_strcmp0 (reply->str, "OK") != 0) { + if (reply->type == REDIS_REPLY_ERROR) { + seaf_warning ("Failed to auth redis: %s.\n", reply->str); + } + ret = -1; + goto out; + } + +out: + freeReplyObject (reply); + return ret; +} + + +static RedisConnection * +redis_connection_new (const char *host, const char *passwd, int port) +{ + RedisConnection *conn = g_new0 (RedisConnection, 1); + + conn->ac = redisConnect(host, port); + if (!conn->ac || conn->ac->err) { + if (conn->ac) { + seaf_warning ("Failed to connect to redis : %s\n", conn->ac->errstr); + redisFree (conn->ac); + } else { + seaf_warning ("Can't allocate redis context\n"); + } + g_free (conn); + return NULL; + } + + if (redis_auth (conn, passwd) < 0) { + redisFree (conn->ac); + g_free (conn); + return NULL; + } + conn->ctime = (gint64)time(NULL); + + return conn; +} + +static void +redis_connection_free (RedisConnection *conn) +{ + if (!conn) + return; + + if (conn->ac) + redisFree(conn->ac); + + g_free (conn); +} + +static RedisConnectionPool * +redis_connection_pool_new (const char *host, int port, int max_connections) +{ + RedisConnectionPool *pool = g_new0 (RedisConnectionPool, 1); + pool->host = g_strdup(host); + pool->port = port; + pool->connections = g_ptr_array_sized_new (max_connections); + pool->max_connections = max_connections; + pthread_mutex_init (&pool->lock, NULL); + return pool; +} + +static RedisConnection * +redis_connection_pool_get_connection (RedisConnectionPool *pool, const char *passwd) +{ + RedisConnection *conn = NULL; + + if (pool->max_connections == 0) { + conn = redis_connection_new (pool->host, passwd, pool->port); + return conn; + } + + pthread_mutex_lock (&pool->lock); + + guint i, size = pool->connections->len; + for (i = 0; i < size; ++i) { + conn = g_ptr_array_index (pool->connections, i); + if (!conn->is_available) { + continue; + } + conn->is_available = FALSE; + goto out; + } + conn = NULL; + if (size < pool->max_connections) { + conn = redis_connection_new (pool->host, passwd, pool->port); + if (conn) { + conn->is_available = FALSE; + g_ptr_array_add (pool->connections, conn); + } + } else { + seaf_warning ("The number of redis connections exceeds the limit. The maximum connections is %d.\n", pool->max_connections); + } + +out: + pthread_mutex_unlock (&pool->lock); + return conn; +} + +static void +redis_connection_pool_return_connection (RedisConnectionPool *pool, RedisConnection *conn) +{ + if (!conn) + return; + + if (pool->max_connections == 0) { + redis_connection_free (conn); + return; + } + + if (conn->release) { + pthread_mutex_lock (&pool->lock); + g_ptr_array_remove (pool->connections, conn); + pthread_mutex_unlock (&pool->lock); + redis_connection_free (conn); + return; + } + + pthread_mutex_lock (&pool->lock); + conn->is_available = TRUE; + pthread_mutex_unlock (&pool->lock); +} + +void * +redis_cache_get_object (ObjCache *cache, const char *obj_id, size_t *len) +{ + RedisConnection *conn; + char *object = NULL; + redisReply *reply; + RedisPriv *priv = cache->priv; + RedisConnectionPool *pool = priv->redis_pool; + + conn = redis_connection_pool_get_connection (pool, priv->passwd); + if (!conn) { + seaf_warning ("Failed to get redis connection to host %s.\n", cache->host); + return NULL; + } + + reply = redisCommand(conn->ac, "GET %s", obj_id); + if (!reply) { + seaf_warning ("Failed to get object %s from redis cache.\n", obj_id); + conn->release = TRUE; + goto out; + } + if (reply->type != REDIS_REPLY_STRING) { + if (reply->type == REDIS_REPLY_ERROR) { + conn->release = TRUE; + seaf_warning ("Failed to get %s from redis cache: %s.\n", + obj_id, reply->str); + } + goto out; + } + + *len = reply->len; + object = g_memdup (reply->str, reply->len); + +out: + freeReplyObject(reply); + redis_connection_pool_return_connection (pool, conn); + + return object; +} + +int +redis_cache_set_object (ObjCache *cache, + const char *obj_id, + const void *object, + int len, + int expiry) +{ + RedisConnection *conn; + redisReply *reply; + int ret = 0; + RedisPriv *priv = cache->priv; + RedisConnectionPool *pool = priv->redis_pool; + + conn = redis_connection_pool_get_connection (pool, priv->passwd); + if (!conn) { + seaf_warning ("Failed to get redis connection to host %s.\n", cache->host); + return -1; + } + + if (expiry <= 0) + expiry = cache->mc_expiry; + reply = redisCommand(conn->ac, "SET %s %b EX %d", obj_id, object, len, expiry); + if (!reply) { + seaf_warning ("Failed to set object %s to redis cache.\n", obj_id); + ret = -1; + conn->release = TRUE; + goto out; + } + if (reply->type != REDIS_REPLY_STATUS || + g_strcmp0 (reply->str, "OK") != 0) { + if (reply->type == REDIS_REPLY_ERROR) { + conn->release = TRUE; + seaf_warning ("Failed to set %s to redis: %s.\n", + obj_id, reply->str); + } + ret = -1; + } + +out: + freeReplyObject(reply); + redis_connection_pool_return_connection (pool, conn); + + return ret; +} + +gboolean +redis_cache_test_object (ObjCache *cache, const char *obj_id) +{ + RedisConnection *conn; + redisReply *reply; + gboolean ret = FALSE; + RedisPriv *priv = cache->priv; + RedisConnectionPool *pool = priv->redis_pool; + + conn = redis_connection_pool_get_connection (pool, priv->passwd); + if (!conn) { + seaf_warning ("Failed to get redis connection to host %s.\n", cache->host); + return ret; + } + + reply = redisCommand(conn->ac, "EXISTS %s", obj_id); + if (!reply) { + seaf_warning ("Failed to test object %s from redis cache.\n", obj_id); + conn->release = TRUE; + goto out; + } + if (reply->type != REDIS_REPLY_INTEGER || + reply->integer != 1) { + if (reply->type == REDIS_REPLY_ERROR) { + conn->release = TRUE; + seaf_warning ("Failed to test %s from redis: %s.\n", + obj_id, reply->str); + } + goto out; + } + + ret = TRUE; + +out: + freeReplyObject(reply); + redis_connection_pool_return_connection (pool, conn); + + return ret; +} + +int +redis_cache_delete_object (ObjCache *cache, const char *obj_id) +{ + RedisConnection *conn; + redisReply *reply; + int ret = 0; + RedisPriv *priv = cache->priv; + RedisConnectionPool *pool = priv->redis_pool; + + conn = redis_connection_pool_get_connection (pool, priv->passwd); + if (!conn) { + seaf_warning ("Failed to get redis connection to host %s.\n", cache->host); + return -1; + } + + reply = redisCommand(conn->ac, "DEL %s", obj_id); + if (!reply) { + seaf_warning ("Failed to delete object %s from redis cache.\n", obj_id); + ret = -1; + conn->release = TRUE; + goto out; + } + if (reply->type != REDIS_REPLY_INTEGER || + reply->integer != 1) { + if (reply->type == REDIS_REPLY_ERROR) { + conn->release = TRUE; + seaf_warning ("Failed to del %s from redis: %s.\n", + obj_id, reply->str); + } + ret = -1; + } + +out: + freeReplyObject(reply); + redis_connection_pool_return_connection (pool, conn); + + return ret; +} + +int +redis_cache_publish (ObjCache *cache, const char *channel, const char *msg) +{ + RedisConnection *conn; + redisReply *reply; + int ret = 0; + RedisPriv *priv = cache->priv; + RedisConnectionPool *pool = priv->redis_pool; + + conn = redis_connection_pool_get_connection (pool, priv->passwd); + if (!conn) { + seaf_warning ("Failed to get redis connection to host %s.\n", cache->host); + return -1; + } + + reply = redisCommand(conn->ac, "PUBLISH %s %s", channel, msg); + if (!reply) { + seaf_warning ("Failed to publish metrics to redis channel.\n"); + ret = -1; + conn->release = TRUE; + goto out; + } + if (reply->type != REDIS_REPLY_INTEGER || + reply->integer < 0) { + if (reply->type == REDIS_REPLY_ERROR) { + conn->release = TRUE; + seaf_warning ("Failed to publish metrics to redis channel.\n"); + } + ret = -1; + } + +out: + freeReplyObject(reply); + redis_connection_pool_return_connection (pool, conn); + + return ret; +} + +ObjCache * +redis_cache_new (const char *host, const char *passwd, + int port, int redis_expiry, + int max_connections) +{ + ObjCache *cache = g_new0 (ObjCache, 1); + RedisPriv *priv = g_new0 (RedisPriv, 1); + + priv->redis_pool = redis_connection_pool_new (host, port, max_connections); + + cache->priv = priv; + + cache->host = g_strdup (host); + priv->passwd = g_strdup (passwd); + cache->port = port; + cache->mc_expiry = redis_expiry; + cache->cache_type = TYPE_REDIS; + + cache->get_object = redis_cache_get_object; + cache->set_object = redis_cache_set_object; + cache->test_object = redis_cache_test_object; + cache->delete_object = redis_cache_delete_object; + cache->publish = redis_cache_publish; + + return cache; +} diff --git a/common/redis-cache.h b/common/redis-cache.h new file mode 100644 index 0000000..f665fd1 --- /dev/null +++ b/common/redis-cache.h @@ -0,0 +1,14 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#ifndef REDIS_CACHE_H +#define REDIS_CACHE_H + +#include "obj-cache.h" + +ObjCache * +redis_cache_new (const char *host, const char *passwd, + int port, int mc_expiry, + int max_connections); + + +#endif diff --git a/configure.ac b/configure.ac index 7ba9cee..aaebf74 100644 --- a/configure.ac +++ b/configure.ac @@ -198,6 +198,7 @@ ZDB_REQUIRED=2.10 CURL_REQUIRED=7.17 FUSE_REQUIRED=2.7.3 ZLIB_REQUIRED=1.2.0 +LIHIBREDIS_REQUIRED=0.15.0 PKG_CHECK_MODULES(SSL, [openssl]) AC_SUBST(SSL_CFLAGS) @@ -266,6 +267,10 @@ if test "${compile_httpserver}" = "yes"; then AC_DEFINE([HAVE_EVHTP], [1], [Define to 1 if httpserver is enabled.]) fi +PKG_CHECK_MODULES(LIBHIREDIS, [hiredis >= $LIHIBREDIS_REQUIRED]) +AC_SUBST(LIBHIREDIS_CFLAGS) +AC_SUBST(LIBHIREDIS_LIBS) + PKG_CHECK_MODULES(CURL, [libcurl >= $CURL_REQUIRED]) AC_SUBST(CURL_CFLAGS) AC_SUBST(CURL_LIBS) diff --git a/fileserver/fileop.go b/fileserver/fileop.go index 9e4e163..cd66225 100644 --- a/fileserver/fileop.go +++ b/fileserver/fileop.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "mime" "mime/multipart" "net" @@ -1380,7 +1379,7 @@ func writeBlockDataToTmpFile(r *http.Request, fsm *recvData, formFiles map[strin tmpFile, err := repomgr.GetUploadTmpFile(repoID, filePath) if err != nil || tmpFile == "" { tmpDir := filepath.Join(httpTempDir, "cluster-shared") - f, err = ioutil.TempFile(tmpDir, filename) + f, err = os.CreateTemp(tmpDir, filename) if err != nil { return err } diff --git a/fileserver/fileserver.go b/fileserver/fileserver.go index efd5d93..70308e8 100644 --- a/fileserver/fileserver.go +++ b/fileserver/fileserver.go @@ -8,7 +8,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "net/http" "os" "os/signal" @@ -23,12 +22,12 @@ import ( "github.com/haiwen/seafile-server/fileserver/blockmgr" "github.com/haiwen/seafile-server/fileserver/commitmgr" "github.com/haiwen/seafile-server/fileserver/fsmgr" + "github.com/haiwen/seafile-server/fileserver/metrics" "github.com/haiwen/seafile-server/fileserver/option" "github.com/haiwen/seafile-server/fileserver/repomgr" "github.com/haiwen/seafile-server/fileserver/searpc" "github.com/haiwen/seafile-server/fileserver/share" "github.com/haiwen/seafile-server/fileserver/utils" - _ "github.com/mattn/go-sqlite3" log "github.com/sirupsen/logrus" "gopkg.in/ini.v1" @@ -255,7 +254,7 @@ func loadDBOptionFromFile() (*DBOption, error) { // registerCA registers CA to verify server cert. func registerCA(capath string) { rootCertPool := x509.NewCertPool() - pem, err := ioutil.ReadFile(capath) + pem, err := os.ReadFile(capath) if err != nil { log.Fatal(err) } @@ -415,6 +414,8 @@ func main() { initUpload() + metrics.Init() + router := newHTTPRouter() go handleSignals() @@ -436,6 +437,7 @@ func handleSignals() { signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM, os.Interrupt) <-signalChan + metrics.Stop() removePidfile(pidFilePath) os.Exit(0) } @@ -541,6 +543,10 @@ func newHTTPRouter() *mux.Router { r.Handle("/debug/pprof/goroutine", &profileHandler{pprof.Handler("goroutine")}) r.Handle("/debug/pprof/threadcreate", &profileHandler{pprof.Handler("threadcreate")}) r.Handle("/debug/pprof/trace", &traceHandler{}) + + if option.HasRedisOptions { + r.Use(metrics.MetricMiddleware) + } return r } diff --git a/fileserver/go.mod b/fileserver/go.mod index 98048a0..f4f185e 100644 --- a/fileserver/go.mod +++ b/fileserver/go.mod @@ -1,22 +1,23 @@ module github.com/haiwen/seafile-server/fileserver -go 1.17 +go 1.22 require ( github.com/dgraph-io/ristretto v0.2.0 + github.com/go-redis/redis/v8 v8.11.5 github.com/go-sql-driver/mysql v1.5.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 github.com/json-iterator/go v1.1.12 - github.com/mattn/go-sqlite3 v1.14.0 github.com/sirupsen/logrus v1.8.1 golang.org/x/text v0.3.8 gopkg.in/ini.v1 v1.55.0 ) require ( - github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect diff --git a/fileserver/go.sum b/fileserver/go.sum index beb2382..c22e0f8 100644 --- a/fileserver/go.sum +++ b/fileserver/go.sum @@ -1,7 +1,5 @@ -github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= -github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -9,8 +7,14 @@ github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dr github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= +github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= @@ -26,12 +30,16 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= -github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -43,51 +51,27 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ= gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/fileserver/metrics/metrics.go b/fileserver/metrics/metrics.go new file mode 100644 index 0000000..19eb74d --- /dev/null +++ b/fileserver/metrics/metrics.go @@ -0,0 +1,167 @@ +package metrics + +import ( + "container/list" + "context" + "encoding/json" + "fmt" + "net/http" + "runtime/debug" + "sync" + "time" + + "github.com/dgraph-io/ristretto/z" + "github.com/go-redis/redis/v8" + "github.com/haiwen/seafile-server/fileserver/option" + + log "github.com/sirupsen/logrus" +) + +const ( + RedisChannel = "metric_channel" + ComponentName = "go_fileserver" + MetricInterval = 30 * time.Second +) + +type MetricMgr struct { + sync.Mutex + inFlightRequestList *list.List +} + +type RequestInfo struct { + urlPath string + method string + start time.Time +} + +func (m *MetricMgr) AddReq(urlPath, method string) *list.Element { + req := new(RequestInfo) + req.urlPath = urlPath + req.method = method + req.start = time.Now() + + m.Lock() + defer m.Unlock() + e := m.inFlightRequestList.PushBack(req) + + return e +} + +func (m *MetricMgr) DecReq(e *list.Element) { + m.Lock() + defer m.Unlock() + + m.inFlightRequestList.Remove(e) +} + +var ( + client *redis.Client + closer *z.Closer + + metricMgr *MetricMgr +) + +func Init() { + if !option.HasRedisOptions { + return + } + metricMgr = new(MetricMgr) + metricMgr.inFlightRequestList = list.New() + + closer = z.NewCloser(1) + go metricsHandler() +} + +func Stop() { + if !option.HasRedisOptions { + return + } + closer.SignalAndWait() +} + +func metricsHandler() { + defer closer.Done() + defer func() { + if err := recover(); err != nil { + log.Errorf("panic: %v\n%s", err, debug.Stack()) + } + }() + + server := fmt.Sprintf("%s:%d", option.RedisHost, option.RedisPort) + opt := &redis.Options{ + Addr: server, + Password: option.RedisPasswd, + } + opt.PoolSize = 1 + + client = redis.NewClient(opt) + + ticker := time.NewTicker(MetricInterval) + defer ticker.Stop() + + for { + select { + case <-closer.HasBeenClosed(): + return + case <-ticker.C: + err := publishMetrics() + if err != nil { + log.Warnf("Failed to publish metrics to redis channel: %v", err) + continue + } + } + } +} + +func MetricMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + req := metricMgr.AddReq(r.URL.Path, r.Method) + next.ServeHTTP(w, r) + metricMgr.DecReq(req) + }) +} + +type MetricMessage struct { + MetricName string `json:"metric_name"` + MetricValue any `json:"metric_value"` + MetricType string `json:"metric_type"` + ComponentName string `json:"component_name"` + MetricHelp string `json:"metric_help"` + NodeName string `json:"node_name"` +} + +func publishMetrics() error { + metricMgr.Lock() + inFlightRequestCount := metricMgr.inFlightRequestList.Len() + metricMgr.Unlock() + + msg := &MetricMessage{MetricName: "in_flight_request_total", + MetricValue: inFlightRequestCount, + MetricType: "gauge", + ComponentName: ComponentName, + MetricHelp: "The number of currently running http requests.", + } + + data, err := json.Marshal(msg) + if err != nil { + return err + } + + err = publishRedisMsg(RedisChannel, data) + if err != nil { + return err + } + + return nil +} + +func publishRedisMsg(channel string, msg []byte) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := client.Publish(ctx, channel, msg).Err() + if err != nil { + return fmt.Errorf("failed to publish redis message: %w", err) + } + return nil +} diff --git a/fileserver/option/option.go b/fileserver/option/option.go index 356a041..abb726a 100644 --- a/fileserver/option/option.go +++ b/fileserver/option/option.go @@ -54,6 +54,15 @@ var ( // quota options DefaultQuota int64 + // redis options + HasRedisOptions bool + RedisHost string + RedisPasswd string + RedisPort uint32 + RedisExpiry uint32 + RedisMaxConn uint32 + RedisTimeout time.Duration + // Profile password ProfilePassword string EnableProfiling bool @@ -81,6 +90,11 @@ func initDefaultOptions() { VerifyClientBlocks = true FsIdListRequestTimeout = -1 DBOpTimeout = 60 * time.Second + RedisHost = "127.0.0.1" + RedisPort = 6379 + RedisExpiry = 24 * 3600 + RedisMaxConn = 100 + RedisTimeout = 1 * time.Second } func LoadFileServerOptions(centralDir string) { @@ -140,6 +154,8 @@ func LoadFileServerOptions(centralDir string) { } } + loadCacheOptionFromEnv() + GroupTableName = os.Getenv("SEAFILE_MYSQL_DB_GROUP_TABLE_NAME") if GroupTableName == "" { GroupTableName = "Group" @@ -262,6 +278,45 @@ func parseQuota(quotaStr string) int64 { return quota } +func loadCacheOptionFromEnv() { + cacheProvider := os.Getenv("CACHE_PROVIDER") + if cacheProvider != "redis" { + return + } + + HasRedisOptions = true + + redisHost := os.Getenv("REDIS_HOST") + if redisHost != "" { + RedisHost = redisHost + } + redisPort := os.Getenv("REDIS_PORT") + if redisPort != "" { + port, err := strconv.ParseUint(redisPort, 10, 32) + if err != nil { + RedisPort = uint32(port) + } + } + redisPasswd := os.Getenv("REDIS_PASSWORD") + if redisPasswd != "" { + RedisPasswd = redisPasswd + } + redisMaxConn := os.Getenv("REDIS_MAX_CONNECTIONS") + if redisMaxConn != "" { + maxConn, err := strconv.ParseUint(redisMaxConn, 10, 32) + if err != nil { + RedisMaxConn = uint32(maxConn) + } + } + redisExpiry := os.Getenv("REDIS_EXPIRY") + if redisExpiry != "" { + expiry, err := strconv.ParseUint(redisExpiry, 10, 32) + if err != nil { + RedisExpiry = uint32(expiry) + } + } +} + func LoadSeahubConfig() error { JWTPrivateKey = os.Getenv("JWT_PRIVATE_KEY") if JWTPrivateKey == "" { diff --git a/fileserver/sync_api.go b/fileserver/sync_api.go index ea1bbc8..5cbc9d7 100644 --- a/fileserver/sync_api.go +++ b/fileserver/sync_api.go @@ -9,7 +9,7 @@ import ( "errors" "fmt" "html" - "io/ioutil" + "io" "net" "net/http" "strconv" @@ -469,7 +469,7 @@ func recvFSCB(rsp http.ResponseWriter, r *http.Request) *appError { err := fmt.Errorf("Failed to get repo store id by repo id %s: %v", repoID, err) return &appError{err, "", http.StatusInternalServerError} } - fsBuf, err := ioutil.ReadAll(r.Body) + fsBuf, err := io.ReadAll(r.Body) if err != nil { return &appError{nil, err.Error(), http.StatusBadRequest} } @@ -950,7 +950,7 @@ func putCommitCB(rsp http.ResponseWriter, r *http.Request) *appError { return appErr } - data, err := ioutil.ReadAll(r.Body) + data, err := io.ReadAll(r.Body) if err != nil { return &appError{nil, err.Error(), http.StatusBadRequest} } diff --git a/server/Makefile.am b/server/Makefile.am index 51cd66d..f97d7af 100644 --- a/server/Makefile.am +++ b/server/Makefile.am @@ -13,6 +13,7 @@ AM_CFLAGS = -DPKGDATADIR=\"$(pkgdatadir)\" \ @MSVC_CFLAGS@ \ @LIBARCHIVE_CFLAGS@ \ @MYSQL_CFLAGS@ \ + @LIBHIREDIS_CFLAGS@ \ -Wall bin_PROGRAMS = seaf-server @@ -37,7 +38,8 @@ noinst_HEADERS = web-accesstoken-mgr.h seafile-session.h \ index-blocks-mgr.h \ http-tx-mgr.h \ notif-mgr.h \ - change-set.h + change-set.h \ + metric-mgr.h seaf_server_SOURCES = \ seaf-server.c \ @@ -60,6 +62,7 @@ seaf_server_SOURCES = \ http-tx-mgr.c \ notif-mgr.c \ change-set.c \ + metric-mgr.c \ ../common/seaf-db.c \ ../common/branch-mgr.c ../common/fs-mgr.c \ ../common/config-mgr.c \ @@ -81,6 +84,8 @@ seaf_server_SOURCES = \ ../common/block-backend.c \ ../common/block-backend-fs.c \ ../common/merge-new.c \ + ../common/obj-cache.c \ + ../common/redis-cache.c \ ../common/block-tx-utils.c seaf_server_LDADD = $(top_builddir)/lib/libseafile_common.la \ @@ -89,4 +94,4 @@ seaf_server_LDADD = $(top_builddir)/lib/libseafile_common.la \ @SEARPC_LIBS@ @JANSSON_LIBS@ ${LIB_WS32} @ZLIB_LIBS@ \ @LIBARCHIVE_LIBS@ @LIB_ICONV@ \ @LDAP_LIBS@ @MYSQL_LIBS@ -lsqlite3 \ - @CURL_LIBS@ @JWT_LIBS@ @ARGON2_LIBS@ + @CURL_LIBS@ @JWT_LIBS@ @LIBHIREDIS_LIBS@ @ARGON2_LIBS@ diff --git a/server/access-file.c b/server/access-file.c index 924b54a..a5f63f3 100644 --- a/server/access-file.c +++ b/server/access-file.c @@ -2315,15 +2315,61 @@ on_error: } */ +static evhtp_res +request_finish_cb (evhtp_request_t *req, void *arg) +{ + RequestInfo *info = arg; + struct timeval end, intv; + + seaf_metric_manager_in_flight_request_dec (seaf->metric_mgr); + + if (!info) + return EVHTP_RES_OK; + + g_free (info->url_path); + g_free (info->method); + g_free (info); + return EVHTP_RES_OK; +} + +static evhtp_res +access_headers_cb (evhtp_request_t *req, evhtp_headers_t *hdr, void *arg) +{ + htp_method method = evhtp_request_get_method (req); + const char *method_str = htparser_get_methodstr_m (method); + RequestInfo *info = NULL; + info = g_new0 (RequestInfo, 1); + info->url_path = g_strdup (req->uri->path->full); + info->method = g_strdup (method_str); + + gettimeofday (&info->start, NULL); + + seaf_metric_manager_in_flight_request_inc (seaf->metric_mgr); + evhtp_set_hook (&req->hooks, evhtp_hook_on_request_fini, request_finish_cb, info); + req->cbarg = info; + + return EVHTP_RES_OK; +} + int access_file_init (evhtp_t *htp) { - evhtp_set_regex_cb (htp, "^/files/.*", access_cb, NULL); - evhtp_set_regex_cb (htp, "^/blks/.*", access_blks_cb, NULL); - evhtp_set_regex_cb (htp, "^/zip/.*", access_zip_cb, NULL); - evhtp_set_regex_cb (htp, "^/f/.*", access_link_cb, NULL); + evhtp_callback_t *cb; + + cb = evhtp_set_regex_cb (htp, "^/files/.*", access_cb, NULL); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, access_headers_cb, NULL); + + cb = evhtp_set_regex_cb (htp, "^/blks/.*", access_blks_cb, NULL); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, access_headers_cb, NULL); + + cb = evhtp_set_regex_cb (htp, "^/zip/.*", access_zip_cb, NULL); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, access_headers_cb, NULL); + + cb = evhtp_set_regex_cb (htp, "^/f/.*", access_link_cb, NULL); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, access_headers_cb, NULL); //evhtp_set_regex_cb (htp, "^/d/.*", access_dir_link_cb, NULL); - evhtp_set_regex_cb (htp, "^/repos/[\\da-z]{8}-[\\da-z]{4}-[\\da-z]{4}-[\\da-z]{4}-[\\da-z]{12}/files/.*", access_v2_cb, NULL); + cb = evhtp_set_regex_cb (htp, "^/repos/[\\da-z]{8}-[\\da-z]{4}-[\\da-z]{4}-[\\da-z]{4}-[\\da-z]{12}/files/.*", access_v2_cb, NULL); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, access_headers_cb, NULL); return 0; } diff --git a/server/http-server.c b/server/http-server.c index eb3d6a6..25a73b1 100644 --- a/server/http-server.c +++ b/server/http-server.c @@ -3005,38 +3005,82 @@ out: json_decref (repo_array); } +static evhtp_res +http_request_finish_cb (evhtp_request_t *req, void *arg) +{ + RequestInfo *info = arg; + struct timeval end, intv; + + seaf_metric_manager_in_flight_request_dec (seaf->metric_mgr); + + if (!info) + return EVHTP_RES_OK; + + g_free (info->url_path); + g_free (info->method); + g_free (info); + return EVHTP_RES_OK; +} + +static evhtp_res +http_request_start_cb (evhtp_request_t *req, evhtp_headers_t *hdr, void *arg) +{ + htp_method method = evhtp_request_get_method (req); + const char *method_str = htparser_get_methodstr_m (method); + RequestInfo *info = NULL; + info = g_new0 (RequestInfo, 1); + info->url_path = g_strdup (req->uri->path->full); + info->method = g_strdup (method_str); + + gettimeofday (&info->start, NULL); + + seaf_metric_manager_in_flight_request_inc (seaf->metric_mgr); + evhtp_set_hook (&req->hooks, evhtp_hook_on_request_fini, http_request_finish_cb, info); + req->cbarg = info; + + return EVHTP_RES_OK; +} + static void http_request_init (HttpServerStruct *server) { HttpServer *priv = server->priv; + evhtp_callback_t *cb; - evhtp_set_cb (priv->evhtp, + cb = evhtp_set_cb (priv->evhtp, GET_PROTO_PATH, get_protocol_cb, NULL); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, GET_CHECK_QUOTA_REGEX, get_check_quota_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, OP_PERM_CHECK_REGEX, get_check_permission_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, HEAD_COMMIT_OPER_REGEX, head_commit_oper_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, GET_HEAD_COMMITS_MULTI_REGEX, head_commits_multi_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, COMMIT_OPER_REGEX, commit_oper_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, GET_FS_OBJ_ID_REGEX, get_fs_obj_id_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); // evhtp_set_regex_cb (priv->evhtp, // START_FS_OBJ_ID_REGEX, start_fs_obj_id_cb, @@ -3050,37 +3094,45 @@ http_request_init (HttpServerStruct *server) // RETRIEVE_FS_OBJ_ID_REGEX, retrieve_fs_obj_id_cb, // priv); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, BLOCK_OPER_REGEX, block_oper_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, POST_CHECK_FS_REGEX, post_check_fs_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, POST_CHECK_BLOCK_REGEX, post_check_block_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, POST_RECV_FS_REGEX, post_recv_fs_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, POST_PACK_FS_REGEX, post_pack_fs_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, GET_BLOCK_MAP_REGEX, get_block_map_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, GET_JWT_TOKEN_REGEX, get_jwt_token_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); - evhtp_set_regex_cb (priv->evhtp, + cb = evhtp_set_regex_cb (priv->evhtp, GET_ACCESSIBLE_REPO_LIST_REGEX, get_accessible_repo_list_cb, priv); + evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, http_request_start_cb, NULL); /* Web access file */ access_file_init (priv->evhtp); diff --git a/server/http-server.h b/server/http-server.h index d9eb3a9..da7e209 100644 --- a/server/http-server.h +++ b/server/http-server.h @@ -4,6 +4,8 @@ #ifdef HAVE_EVHTP #include +#include "metric-mgr.h" + struct _SeafileSession; struct _HttpServer; @@ -23,6 +25,12 @@ struct _HttpServerStruct { gboolean verify_client_blocks; }; +typedef struct RequestInfo { + struct timeval start; + char *url_path; + char *method; +} RequestInfo; + typedef struct _HttpServerStruct HttpServerStruct; HttpServerStruct * @@ -40,6 +48,7 @@ send_statistic_msg (const char *repo_id, char *user, char *operation, guint64 by char * get_client_ip_addr (void *data); + #endif #endif diff --git a/server/metric-mgr.c b/server/metric-mgr.c new file mode 100644 index 0000000..2bd4020 --- /dev/null +++ b/server/metric-mgr.c @@ -0,0 +1,140 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include "common.h" + +#include "utils.h" +#include "log.h" + +#include +#include + +#include "seafile-session.h" +#include "metric-mgr.h" +#include "obj-cache.h" + +#define PUBLISH_INTERVAL 30 /* 30 seconds*/ +#define REDIS_CHANNEL "metric_channel" +#define COMPONENT_NAME "fileserver" + +struct _SeafMetricManagerPriv { + int in_flight_request_count; + + struct ObjCache *cache; +}; + +SeafMetricManager* +seaf_metric_manager_new (struct _SeafileSession *seaf) +{ + SeafMetricManager *mgr = g_new0 (SeafMetricManager, 1); + + mgr->priv = g_new0 (SeafMetricManagerPriv, 1); + mgr->seaf = seaf; + + // redis cache + mgr->priv->cache = objcache_new (); + + return mgr; +} + +static void * +publish_metrics (void *data); + +int +seaf_metric_manager_start (SeafMetricManager *mgr) +{ + pthread_t tid; + int rc; + + rc = pthread_create (&tid, NULL, publish_metrics, mgr); + if (rc != 0) { + seaf_warning ("Failed to create publish metrics worker thread: %s.\n", + strerror(rc)); + return -1; + } + + return 0; +} + +void +seaf_metric_manager_in_flight_request_inc (SeafMetricManager *mgr) +{ + SeafMetricManagerPriv *priv = mgr->priv; + + g_atomic_int_inc (&priv->in_flight_request_count); +} + +void +seaf_metric_manager_in_flight_request_dec (SeafMetricManager *mgr) +{ + SeafMetricManagerPriv *priv = mgr->priv; + g_atomic_int_dec_and_test (&priv->in_flight_request_count); +} + +static int +publish_redis_msg (SeafMetricManager *mgr, const char *msg) +{ + SeafMetricManagerPriv *priv = mgr->priv; + + if (!priv->cache) { + return 0; + } + + int ret = objcache_publish (priv->cache, REDIS_CHANNEL, msg); + + return ret; +} + +static int +publish_in_flight_request (SeafMetricManager *mgr) +{ + int ret = 0; + json_t *obj = NULL; + char *msg = NULL; + SeafMetricManagerPriv *priv = mgr->priv; + + obj = json_object (); + + json_object_set_new (obj, "metric_name", json_string("in_flight_request_total")); + json_object_set_new (obj, "metric_value", json_integer (priv->in_flight_request_count)); + json_object_set_new (obj, "metric_type", json_string("gauge")); + json_object_set_new (obj, "component_name", json_string(COMPONENT_NAME)); + json_object_set_new (obj, "metric_help", json_string("The number of currently running http requests.")); + + msg = json_dumps (obj, JSON_COMPACT); + + ret = publish_redis_msg (mgr, msg); + + json_decref (obj); + g_free (msg); + return ret; +} + +static void +do_publish_metrics (SeafMetricManager *mgr) +{ + int rc; + + // Don't publish metrics when use go fileserver. + if (seaf->go_fileserver) { + return; + } + + rc = publish_in_flight_request (mgr); + if (rc < 0) { + seaf_warning ("Failed to publish in flight request\n"); + return; + } +} + +static void * +publish_metrics (void *data) +{ + SeafMetricManager *mgr = data; + + while (1) { + do_publish_metrics (mgr); + sleep(PUBLISH_INTERVAL); + } + + return NULL; +} diff --git a/server/metric-mgr.h b/server/metric-mgr.h new file mode 100644 index 0000000..5ce756e --- /dev/null +++ b/server/metric-mgr.h @@ -0,0 +1,30 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#ifndef SEAF_METRIC_MGR_H +#define SEAF_METRIC_MGR_H + +struct _SeafMetricManager; + +typedef struct _SeafMetricManager SeafMetricManager; + +typedef struct _SeafMetricManagerPriv SeafMetricManagerPriv; + +struct _SeafMetricManager { + struct _SeafileSession *seaf; + + SeafMetricManagerPriv *priv; +}; + +SeafMetricManager* +seaf_metric_manager_new (struct _SeafileSession *seaf); + +int +seaf_metric_manager_start (SeafMetricManager *mgr); + +void +seaf_metric_manager_in_flight_request_inc (SeafMetricManager *mgr); + +void +seaf_metric_manager_in_flight_request_dec (SeafMetricManager *mgr); + +#endif diff --git a/server/seafile-session.c b/server/seafile-session.c index 7cc59fb..6092347 100644 --- a/server/seafile-session.c +++ b/server/seafile-session.c @@ -322,6 +322,10 @@ seafile_session_new(const char *central_config_dir, } } + session->metric_mgr = seaf_metric_manager_new (session); + if (!session->metric_mgr) + goto onerror; + return session; onerror: @@ -510,6 +514,11 @@ seafile_session_start (SeafileSession *session) #endif } + if (seaf_metric_manager_start (session->metric_mgr) < 0) { + seaf_warning ("Failed to start metric manager.\n"); + return -1; + } + return 0; } diff --git a/server/seafile-session.h b/server/seafile-session.h index a5574b9..9117ff2 100644 --- a/server/seafile-session.h +++ b/server/seafile-session.h @@ -30,6 +30,7 @@ #include "index-blocks-mgr.h" #include "notif-mgr.h" #include "http-tx-mgr.h" +#include "metric-mgr.h" #include @@ -95,6 +96,9 @@ struct _SeafileSession { char *notif_server_private_key; char *notif_url; + // For metric + SeafMetricManager *metric_mgr; + gboolean log_to_stdout; gboolean is_repair; diff --git a/server/upload-file.c b/server/upload-file.c index d7a946e..5012a57 100755 --- a/server/upload-file.c +++ b/server/upload-file.c @@ -1839,6 +1839,8 @@ upload_finish_cb (evhtp_request_t *req, void *arg) RecvFSM *fsm = arg; GList *ptr; + seaf_metric_manager_in_flight_request_dec (seaf->metric_mgr); + if (!fsm) return EVHTP_RES_OK; @@ -2635,6 +2637,8 @@ upload_headers_cb (evhtp_request_t *req, evhtp_headers_t *hdr, void *arg) pthread_mutex_unlock (&pg_lock); } + seaf_metric_manager_in_flight_request_inc (seaf->metric_mgr); + /* Set up per-request hooks, so that we can read file data piece by piece. */ evhtp_set_hook (&req->hooks, evhtp_hook_on_read, upload_read_cb, fsm); evhtp_set_hook (&req->hooks, evhtp_hook_on_request_fini, upload_finish_cb, fsm);