haiwen/seafile-server
1
0
Fork 0
mirror of https://github.com/haiwen/seafile-server.git synced 2025-09-02 16:04:26 +00:00
Code Issues Releases Wiki Activity

Support traffic statistics.

Browse Source
This commit is contained in:
cuihaikuo
2018-08-10 17:47:19 +08:00
parent 1991a68bcf
commit ac5f26fef8
8 changed files with 198 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
common/mq-mgr.c
View File

@@ -138,3 +138,13 @@ seaf_mq_manager_publish_event (SeafMqManager *mgr, const char *content)
ccnet_message_free (msg);
}
void
seaf_mq_manager_publish_stats_event (SeafMqManager *mgr, const char *content)
{
static const char *app = "seaf_server.stats";
CcnetMessage *msg = create_message (mgr, app, content, 0);
_send_message (mgr, msg);
ccnet_message_free (msg);
}

3
common/mq-mgr.h
View File

@@ -56,4 +56,7 @@ seaf_mq_manager_publish_notification (SeafMqManager *mgr,
void
seaf_mq_manager_publish_event (SeafMqManager *mgr, const char *content);
void
seaf_mq_manager_publish_stats_event (SeafMqManager *mgr, const char *content);
#endif

89
server/access-file.c
View File

@@ -27,6 +27,7 @@
#include "seafile-session.h"
#include "access-file.h"
#include "zip-download-mgr.h"
#include "http-server.h"
#define FILE_TYPE_MAP_DEFAULT_LEN 1
#define BUFFER_SIZE 1024 * 64
@@ -47,6 +48,8 @@ typedef struct SendBlockData {
char store_id[37];
int repo_version;
char *user;
bufferevent_data_cb saved_read_cb;
bufferevent_data_cb saved_write_cb;
bufferevent_event_cb saved_event_cb;
@@ -66,6 +69,9 @@ typedef struct SendfileData {
char store_id[37];
int repo_version;
char *user;
char *token_type;
bufferevent_data_cb saved_read_cb;
bufferevent_data_cb saved_write_cb;
bufferevent_event_cb saved_event_cb;
@@ -83,6 +89,9 @@ typedef struct SendFileRangeData {
char store_id[37];
int repo_version;
char *user;
char *token_type;
bufferevent_data_cb saved_read_cb;
bufferevent_data_cb saved_write_cb;
bufferevent_event_cb saved_event_cb;
@@ -92,10 +101,14 @@ typedef struct SendFileRangeData {
typedef struct SendDirData {
evhtp_request_t *req;
size_t remain;
guint64 total_size;
int zipfd;
char *zipfile;
char *token;
char *user;
char *token_type;
char repo_id[37];
bufferevent_data_cb saved_read_cb;
bufferevent_data_cb saved_write_cb;
@@ -142,6 +155,7 @@ free_sendblock_data (SendBlockData *data)
}
g_free (data->block_id);
g_free (data->user);
g_free (data);
}
@@ -157,6 +171,8 @@ free_sendfile_data (SendfileData *data)
EVP_CIPHER_CTX_free (data->ctx);
seafile_unref (data->file);
g_free (data->user);
g_free (data->token_type);
g_free (data->crypt);
g_free (data);
}
@@ -170,6 +186,8 @@ free_send_file_range_data (SendFileRangeData *data)
}
seafile_unref (data->file);
g_free (data->user);
g_free (data->token_type);
g_free (data);
}
@@ -180,6 +198,8 @@ free_senddir_data (SendDirData *data)
zip_download_mgr_del_zip_progress (seaf->zip_download_mgr, data->token);
g_free (data->user);
g_free (data->token_type);
g_free (data->token);
g_free (data);
}
@@ -232,6 +252,8 @@ write_block_data_cb (struct bufferevent *bev, void *ctx)
evhtp_send_reply_end (data->req);
send_statistic_msg (data->store_id, data->user, "web-file-download", (guint64)data->bsize);
free_sendblock_data (data);
return;
}
@@ -317,6 +339,15 @@ next:
evhtp_send_reply_end (data->req);
if (g_strcmp0(data->token_type, "view") != 0) {
char *oper = "web-file-download";
if (g_strcmp0(data->token_type, "download-link") == 0)
oper = "link-file-download";
send_statistic_msg(data->store_id, data->user, oper,
(guint64)data->file->file_size);
}
free_sendfile_data (data);
return;
}
@@ -414,6 +445,13 @@ write_dir_data_cb (struct bufferevent *bev, void *ctx)
evhtp_send_reply_end (data->req);
char *oper = "web-file-download";
if (g_strcmp0(data->token_type, "download-dir-link") == 0 ||
g_strcmp0(data->token_type, "download-multi-link") == 0)
oper = "link-file-download";
send_statistic_msg(data->repo_id, data->user, oper, data->total_size);
free_senddir_data (data);
return;
}
@@ -503,7 +541,7 @@ test_firefox (evhtp_request_t *req)
static int
do_file(evhtp_request_t *req, SeafRepo *repo, const char *file_id,
const char *filename, const char *operation,
SeafileCryptKey *crypt_key)
SeafileCryptKey *crypt_key, const char *user)
{
Seafile *file;
char *type = NULL;
@@ -561,7 +599,8 @@ do_file(evhtp_request_t *req, SeafRepo *repo, const char *file_id,
evhtp_headers_add_header (req->headers_out,
evhtp_header_new("Content-Length", file_size, 1, 1));
if (strcmp(operation, "download") == 0) {
if (strcmp(operation, "download") == 0 ||
strcmp(operation, "download-link") == 0) {
/* Safari doesn't support 'utf8', 'utf-8' is compatible with most of browsers. */
snprintf(cont_filename, SEAF_PATH_MAX,
"attachment;filename*=\"utf-8\' \'%s\"", filename);
@@ -600,6 +639,8 @@ do_file(evhtp_request_t *req, SeafRepo *repo, const char *file_id,
data->req = req;
data->file = file;
data->crypt = crypt;
data->user = g_strdup(user);
data->token_type = g_strdup (operation);
memcpy (data->store_id, repo->store_id, 36);
data->repo_version = repo->version;
@@ -758,6 +799,14 @@ next:
bufferevent_write (bev, buf, n);
if (data->range_remain == 0) {
if (data->start_off + n >= data->file->file_size) {
char *oper = "web-file-download";
if (g_strcmp0(data->token_type, "download-link") == 0)
oper = "link-file-download";
send_statistic_msg (data->store_id, data->user, oper,
(guint64)data->file->file_size);
}
finish_file_range_request (bev, data);
}
@@ -868,7 +917,8 @@ set_resp_disposition (evhtp_request_t *req, const char *operation,
static int
do_file_range (evhtp_request_t *req, SeafRepo *repo, const char *file_id,
const char *filename, const char *operation, const char *byte_ranges)
const char *filename, const char *operation, const char *byte_ranges,
const char *user)
{
Seafile *file;
SendFileRangeData *data = NULL;
@@ -947,6 +997,8 @@ do_file_range (evhtp_request_t *req, SeafRepo *repo, const char *file_id,
data->blk_idx = -1;
data->start_off = start;
data->range_remain = end-start+1;
data->user = g_strdup(user);
data->token_type = g_strdup (operation);
memcpy (data->store_id, repo->store_id, 36);
data->repo_version = repo->version;
@@ -979,7 +1031,8 @@ do_file_range (evhtp_request_t *req, SeafRepo *repo, const char *file_id,
static int
start_download_zip_file (evhtp_request_t *req, const char *token,
const char *zipname, char *zipfile)
const char *zipname, char *zipfile,
const char *repo_id, const char *user, const char *token_type)
{
SeafStat st;
char file_size[255];
@@ -1017,6 +1070,10 @@ start_download_zip_file (evhtp_request_t *req, const char *token,
data->zipfile = zipfile;
data->token = g_strdup (token);
data->remain = st.st_size;
data->total_size = (guint64)st.st_size;
data->user = g_strdup (user);
data->token_type = g_strdup (token_type);
snprintf(data->repo_id, sizeof(data->repo_id), "%s", repo_id);
/* We need to overwrite evhtp's callback functions to
* write file data piece by piece.
@@ -1085,7 +1142,9 @@ access_zip_cb (evhtp_request_t *req, void *arg)
json_error_t jerror;
char *filename = NULL;
char *repo_id = NULL;
char *user = NULL;
char *zip_file_path;
char *token_type = NULL;
const char *error = NULL;
int error_code;
@@ -1155,7 +1214,10 @@ access_zip_cb (evhtp_request_t *req, void *arg)
goto out;
}
int ret = start_download_zip_file (req, token, filename, zip_file_path);
g_object_get (info, "username", &user, NULL);
g_object_get (info, "repo_id", &repo_id, NULL);
g_object_get (info, "op", &token_type, NULL);
int ret = start_download_zip_file (req, token, filename, zip_file_path, repo_id, user, token_type);
if (ret < 0) {
error = "Internal server error\n";
error_code = EVHTP_RES_SERVERR;
@@ -1173,6 +1235,10 @@ out:
g_free (filename);
if (repo_id)
g_free (repo_id);
if (user)
g_free (user);
if (token_type)
g_free (token_type);
if (error) {
evbuffer_add_printf(req->buffer_out, "%s\n", error);
@@ -1219,7 +1285,8 @@ access_cb(evhtp_request_t *req, void *arg)
user = seafile_web_access_get_username (webaccess);
if (strcmp(operation, "view") != 0 &&
strcmp(operation, "download") != 0) {
strcmp(operation, "download") != 0 &&
strcmp(operation, "download-link") != 0) {
error = "Bad access token";
goto bad_req;
}
@@ -1252,11 +1319,11 @@ access_cb(evhtp_request_t *req, void *arg)
}
if (!repo->encrypted && byte_ranges) {
if (do_file_range (req, repo, data, filename, operation, byte_ranges) < 0) {
if (do_file_range (req, repo, data, filename, operation, byte_ranges, user) < 0) {
error = "Internal server error\n";
goto bad_req;
}
} else if (do_file(req, repo, data, filename, operation, key) < 0) {
} else if (do_file(req, repo, data, filename, operation, key, user) < 0) {
error = "Internal server error\n";
goto bad_req;
}
@@ -1286,7 +1353,7 @@ bad_req:
}
static int
do_block(evhtp_request_t *req, SeafRepo *repo, const char *file_id,
do_block(evhtp_request_t *req, SeafRepo *repo, const char *user, const char *file_id,
const char *blk_id)
{
Seafile *file;
@@ -1346,6 +1413,7 @@ do_block(evhtp_request_t *req, SeafRepo *repo, const char *file_id,
data = g_new0 (SendBlockData, 1);
data->req = req;
data->block_id = g_strdup(blk_id);
data->user = g_strdup(user);
memcpy (data->store_id, repo->store_id, 36);
data->repo_version = repo->version;
@@ -1385,6 +1453,7 @@ access_blks_cb(evhtp_request_t *req, void *arg)
const char *repo_id = NULL;
const char *id = NULL;
const char *operation = NULL;
const char *user = NULL;
char *repo_role = NULL;
SeafileWebAccess *webaccess = NULL;
@@ -1427,7 +1496,7 @@ access_blks_cb(evhtp_request_t *req, void *arg)
}
if (strcmp(operation, "downloadblks") == 0) {
if (do_block(req, repo, id, blkid) < 0) {
if (do_block(req, repo, user, id, blkid) < 0) {
error = "Internal server error\n";
goto bad_req;
}

62
server/http-server.c
View File

@@ -62,11 +62,21 @@ struct _HttpServer {
pthread_mutex_t vir_repo_info_cache_lock;
uint32_t cevent_id; /* Used for sending activity events. */
uint32_t stats_event_id; /* Used for sending events for statistics. */
event_t *reap_timer;
};
typedef struct _HttpServer HttpServer;
struct _StatsEventData {
char *etype;
char *user;
char *operation;
char repo_id[37];
guint64 bytes;
};
typedef struct _StatsEventData StatsEventData;
typedef struct TokenInfo {
char *repo_id;
char *email;
@@ -471,6 +481,18 @@ free_repo_event_data (RepoEventData *data)
g_free (data);
}
static void
free_stats_event_data (StatsEventData *data)
{
if (!data)
return;
g_free (data->etype);
g_free (data->user);
g_free (data->operation);
g_free (data);
}
static void
publish_repo_event (CEvent *event, void *data)
{
@@ -488,6 +510,22 @@ publish_repo_event (CEvent *event, void *data)
free_repo_event_data (rdata);
}
static void
publish_stats_event (CEvent *event, void *data)
{
StatsEventData *rdata = event->data;
GString *buf = g_string_new (NULL);
g_string_printf (buf, "%s\t%s\t%s\t%"G_GUINT64_FORMAT,
rdata->etype, rdata->user,
rdata->repo_id, rdata->bytes);
seaf_mq_manager_publish_stats_event (seaf->mq_mgr, buf->str);
g_string_free (buf, TRUE);
free_stats_event_data (rdata);
}
static void
on_repo_oper (HttpServer *htp_server, const char *etype,
const char *repo_id, char *user, char *ip, char *client_name)
@@ -514,6 +552,19 @@ on_repo_oper (HttpServer *htp_server, const char *etype,
}
}
void
send_statistic_msg (const char *repo_id, char *user, char *operation, guint64 bytes)
{
StatsEventData *rdata = g_new0 (StatsEventData, 1);
memcpy (rdata->repo_id, repo_id, 36);
rdata->etype = g_strdup (operation);
rdata->user = g_strdup (user);
rdata->bytes = bytes;
cevent_manager_add_event (seaf->ev_mgr, seaf->http_server->priv->stats_event_id, rdata);
}
char *
get_client_ip_addr (evhtp_request_t *req)
{
@@ -1454,12 +1505,13 @@ get_block_cb (evhtp_request_t *req, void *arg)
char *store_id = NULL;
HttpServer *htp_server = arg;
BlockMetadata *blk_meta = NULL;
char *username = NULL;
char **parts = g_strsplit (req->uri->path->full + 1, "/", 0);
repo_id = parts[1];
block_id = parts[3];
int token_status = validate_token (htp_server, req, repo_id, NULL, FALSE);
int token_status = validate_token (htp_server, req, repo_id, &username, FALSE);
if (token_status != EVHTP_RES_OK) {
evhtp_send_reply (req, token_status);
goto out;
@@ -1505,12 +1557,14 @@ get_block_cb (evhtp_request_t *req, void *arg)
evhtp_send_reply (req, EVHTP_RES_OK);
}
g_free (block_con);
send_statistic_msg (store_id, username, "sync-file-download", (guint64)rsize);
free_handle:
seaf_block_manager_close_block (seaf->block_mgr, blk_handle);
seaf_block_manager_block_handle_free (seaf->block_mgr, blk_handle);
out:
g_free (username);
g_free (blk_meta);
g_free (store_id);
g_strfreev (parts);
@@ -1602,6 +1656,8 @@ put_send_block_cb (evhtp_request_t *req, void *arg)
evhtp_send_reply (req, EVHTP_RES_OK);
send_statistic_msg (store_id, username, "sync-file-upload", (guint64)blk_len);
out:
g_free (username);
g_free (store_id);
@@ -2199,6 +2255,10 @@ seaf_http_server_start (HttpServerStruct *server)
(cevent_handler)publish_repo_event,
NULL);
server->priv->stats_event_id = cevent_manager_register (seaf->ev_mgr,
(cevent_handler)publish_stats_event,
NULL);
int ret = pthread_create (&server->priv->thread_id, NULL, http_server_run, server);
if (ret != 0)
return -1;

3
server/http-server.h
View File

@@ -35,4 +35,7 @@ int
seaf_http_server_invalidate_tokens (HttpServerStruct *htp_server,
const GList *tokens);
void
send_statistic_msg (const char *repo_id, char *user, char *operation, guint64 bytes);
#endif

35
server/upload-file.c
View File

@@ -25,6 +25,7 @@
#include "seafile-session.h"
#include "upload-file.h"
#include "http-status-codes.h"
#include "http-server.h"
#include "seafile-error.h"
@@ -74,6 +75,8 @@ typedef struct RecvFSM {
char *progress_id;
Progress *progress;
char *token_type; /* For sending statistic type */
gboolean need_idx_progress;
} RecvFSM;
@@ -639,6 +642,12 @@ upload_api_cb(evhtp_request_t *req, void *arg)
g_free (ret_json);
send_success_reply (req);
char *oper = "web-file-upload";
if (g_strcmp0(fsm->token_type, "upload-link") == 0)
oper = "link-file-upload";
send_statistic_msg(fsm->repo_id, fsm->user, oper, (guint64)content_len);
return;
error:
@@ -701,6 +710,8 @@ upload_raw_blks_api_cb(evhtp_request_t *req, void *arg)
}
goto error;
}
guint64 content_len = (guint64)get_content_length(req);
send_statistic_msg(fsm->repo_id, fsm->user, "web-file-upload", content_len);
evbuffer_add (req->buffer_out, "\"OK\"", 4);
send_success_reply (req);
@@ -1118,6 +1129,11 @@ upload_ajax_cb(evhtp_request_t *req, void *arg)
}
evhtp_send_reply (req, EVHTP_RES_OK);
char *oper = "web-file-upload";
if (g_strcmp0(fsm->token_type, "upload-link") == 0)
oper = "link-file-upload";
send_statistic_msg(fsm->repo_id, fsm->user, oper, (guint64)content_len);
return;
error:
@@ -1308,6 +1324,8 @@ update_api_cb(evhtp_request_t *req, void *arg)
evbuffer_add(req->buffer_out, new_file_id, strlen(new_file_id));
send_success_reply (req);
send_statistic_msg(fsm->repo_id, fsm->user, "web-file-upload", (guint64)content_len);
g_free (new_file_id);
return;
@@ -1701,12 +1719,14 @@ update_ajax_cb(evhtp_request_t *req, void *arg)
}
goto error;
}
send_statistic_msg(fsm->repo_id, fsm->user, "web-file-upload", (guint64)content_len);
char *json_ret = format_update_json_ret (filename, new_file_id, size);
evbuffer_add (req->buffer_out, json_ret, strlen(json_ret));
send_success_reply (req);
g_free (new_file_id);
g_free (filename);
g_free (json_ret);
@@ -1756,6 +1776,7 @@ upload_finish_cb (evhtp_request_t *req, void *arg)
g_free (fsm->user);
g_free (fsm->boundary);
g_free (fsm->input_name);
g_free (fsm->token_type);
g_hash_table_destroy (fsm->form_kvs);
@@ -2197,7 +2218,8 @@ static int
check_access_token (const char *token,
const char *url_op,
char **repo_id,
char **user)
char **user,
char **token_type)
{
SeafileWebAccess *webaccess;
const char *op;
@@ -2211,6 +2233,12 @@ check_access_token (const char *token,
* token with op = "update" can only be used for "update-*" operations.
*/
op = seafile_web_access_get_op (webaccess);
if (token_type)
*token_type = g_strdup (op);
if (g_strcmp0(op, "upload-link") == 0)
op = "upload";
if (strncmp (url_op, op, strlen(op)) != 0) {
g_object_unref (webaccess);
return -1;
@@ -2258,6 +2286,7 @@ upload_headers_cb (evhtp_request_t *req, evhtp_headers_t *hdr, void *arg)
gint64 content_len;
char *progress_id = NULL;
char *err_msg = NULL;
char *token_type = NULL;
RecvFSM *fsm = NULL;
Progress *progress = NULL;
@@ -2280,7 +2309,7 @@ upload_headers_cb (evhtp_request_t *req, evhtp_headers_t *hdr, void *arg)
}
char *url_op = parts[0];
if (check_access_token (token, url_op, &repo_id, &user) < 0) {
if (check_access_token (token, url_op, &repo_id, &user, &token_type) < 0) {
err_msg = "Access denied";
goto err;
}
@@ -2307,6 +2336,7 @@ upload_headers_cb (evhtp_request_t *req, evhtp_headers_t *hdr, void *arg)
fsm->boundary = boundary;
fsm->repo_id = repo_id;
fsm->user = user;
fsm->token_type = token_type;
fsm->line = evbuffer_new ();
fsm->form_kvs = g_hash_table_new_full (g_str_hash, g_str_equal,
g_free, g_free);
@@ -2348,6 +2378,7 @@ err:
g_free (repo_id);
g_free (user);
g_free (boundary);
g_free (token_type);
g_free (progress_id);
g_strfreev (parts);
return EVHTP_RES_OK;

8
server/web-accesstoken-mgr.c
View File

@@ -142,8 +142,12 @@ seaf_web_at_manager_get_access_token (SeafWebAccessTokenManager *mgr,
strcmp(op, "downloadblks") != 0 &&
strcmp(op, "download-dir") != 0 &&
strcmp(op, "download-multi") != 0 &&
strcmp(op, "download-link") != 0 &&
strcmp(op, "download-dir-link") != 0 &&
strcmp(op, "download-multi-link") != 0 &&
strcmp(op, "upload") != 0 &&
strcmp(op, "update") != 0 &&
strcmp(op, "upload-link") != 0 &&
strcmp(op, "upload-blks-api") != 0 &&
strcmp(op, "upload-blks-aj") != 0 &&
strcmp(op, "update-blks-api") != 0 &&
@@ -173,7 +177,9 @@ seaf_web_at_manager_get_access_token (SeafWebAccessTokenManager *mgr,
pthread_mutex_unlock (&mgr->priv->lock);
if (strcmp(op, "download-dir") == 0 ||
strcmp(op, "download-multi") == 0) {
strcmp(op, "download-multi") == 0 ||
strcmp(op, "download-dir-link") == 0 ||
strcmp(op, "download-multi-link") == 0) {
webaccess = g_object_new (SEAFILE_TYPE_WEB_ACCESS,
"repo_id", info->repo_id,

3
server/zip-download-mgr.c
View File

@@ -497,7 +497,8 @@ zip_download_mgr_start_zip_task (ZipDownloadMgr *mgr,
obj->repo = repo;
obj->user = g_strdup (seafile_web_access_get_username (info));
if (strcmp (operation, "download-dir") == 0) {
if (strcmp (operation, "download-dir") == 0 ||
strcmp (operation, "download-dir-link") == 0) {
obj->type = DOWNLOAD_DIR;
ret = parse_download_dir_data (obj, data);
if (ret < 0) {