diff --git a/common/mq-mgr.c b/common/mq-mgr.c index 5c15079..dc17f67 100644 --- a/common/mq-mgr.c +++ b/common/mq-mgr.c @@ -138,3 +138,13 @@ seaf_mq_manager_publish_event (SeafMqManager *mgr, const char *content) ccnet_message_free (msg); } +void +seaf_mq_manager_publish_stats_event (SeafMqManager *mgr, const char *content) +{ + static const char *app = "seaf_server.stats"; + + CcnetMessage *msg = create_message (mgr, app, content, 0); + _send_message (mgr, msg); + + ccnet_message_free (msg); +} diff --git a/common/mq-mgr.h b/common/mq-mgr.h index 3c1d184..ac28861 100644 --- a/common/mq-mgr.h +++ b/common/mq-mgr.h @@ -56,4 +56,7 @@ seaf_mq_manager_publish_notification (SeafMqManager *mgr, void seaf_mq_manager_publish_event (SeafMqManager *mgr, const char *content); +void +seaf_mq_manager_publish_stats_event (SeafMqManager *mgr, const char *content); + #endif diff --git a/server/access-file.c b/server/access-file.c index 67f4648..fca4670 100644 --- a/server/access-file.c +++ b/server/access-file.c @@ -27,6 +27,7 @@ #include "seafile-session.h" #include "access-file.h" #include "zip-download-mgr.h" +#include "http-server.h" #define FILE_TYPE_MAP_DEFAULT_LEN 1 #define BUFFER_SIZE 1024 * 64 @@ -47,6 +48,8 @@ typedef struct SendBlockData { char store_id[37]; int repo_version; + char *user; + bufferevent_data_cb saved_read_cb; bufferevent_data_cb saved_write_cb; bufferevent_event_cb saved_event_cb; @@ -66,6 +69,9 @@ typedef struct SendfileData { char store_id[37]; int repo_version; + char *user; + char *token_type; + bufferevent_data_cb saved_read_cb; bufferevent_data_cb saved_write_cb; bufferevent_event_cb saved_event_cb; @@ -83,6 +89,9 @@ typedef struct SendFileRangeData { char store_id[37]; int repo_version; + char *user; + char *token_type; + bufferevent_data_cb saved_read_cb; bufferevent_data_cb saved_write_cb; bufferevent_event_cb saved_event_cb; @@ -92,10 +101,14 @@ typedef struct SendFileRangeData { typedef struct SendDirData { evhtp_request_t *req; size_t remain; + guint64 total_size; int zipfd; char *zipfile; char *token; + char *user; + char *token_type; + char repo_id[37]; bufferevent_data_cb saved_read_cb; bufferevent_data_cb saved_write_cb; @@ -142,6 +155,7 @@ free_sendblock_data (SendBlockData *data) } g_free (data->block_id); + g_free (data->user); g_free (data); } @@ -157,6 +171,8 @@ free_sendfile_data (SendfileData *data) EVP_CIPHER_CTX_free (data->ctx); seafile_unref (data->file); + g_free (data->user); + g_free (data->token_type); g_free (data->crypt); g_free (data); } @@ -170,6 +186,8 @@ free_send_file_range_data (SendFileRangeData *data) } seafile_unref (data->file); + g_free (data->user); + g_free (data->token_type); g_free (data); } @@ -180,6 +198,8 @@ free_senddir_data (SendDirData *data) zip_download_mgr_del_zip_progress (seaf->zip_download_mgr, data->token); + g_free (data->user); + g_free (data->token_type); g_free (data->token); g_free (data); } @@ -232,6 +252,8 @@ write_block_data_cb (struct bufferevent *bev, void *ctx) evhtp_send_reply_end (data->req); + send_statistic_msg (data->store_id, data->user, "web-file-download", (guint64)data->bsize); + free_sendblock_data (data); return; } @@ -317,6 +339,15 @@ next: evhtp_send_reply_end (data->req); + if (g_strcmp0(data->token_type, "view") != 0) { + char *oper = "web-file-download"; + if (g_strcmp0(data->token_type, "download-link") == 0) + oper = "link-file-download"; + + send_statistic_msg(data->store_id, data->user, oper, + (guint64)data->file->file_size); + } + free_sendfile_data (data); return; } @@ -414,6 +445,13 @@ write_dir_data_cb (struct bufferevent *bev, void *ctx) evhtp_send_reply_end (data->req); + char *oper = "web-file-download"; + if (g_strcmp0(data->token_type, "download-dir-link") == 0 || + g_strcmp0(data->token_type, "download-multi-link") == 0) + oper = "link-file-download"; + + send_statistic_msg(data->repo_id, data->user, oper, data->total_size); + free_senddir_data (data); return; } @@ -503,7 +541,7 @@ test_firefox (evhtp_request_t *req) static int do_file(evhtp_request_t *req, SeafRepo *repo, const char *file_id, const char *filename, const char *operation, - SeafileCryptKey *crypt_key) + SeafileCryptKey *crypt_key, const char *user) { Seafile *file; char *type = NULL; @@ -561,7 +599,8 @@ do_file(evhtp_request_t *req, SeafRepo *repo, const char *file_id, evhtp_headers_add_header (req->headers_out, evhtp_header_new("Content-Length", file_size, 1, 1)); - if (strcmp(operation, "download") == 0) { + if (strcmp(operation, "download") == 0 || + strcmp(operation, "download-link") == 0) { /* Safari doesn't support 'utf8', 'utf-8' is compatible with most of browsers. */ snprintf(cont_filename, SEAF_PATH_MAX, "attachment;filename*=\"utf-8\' \'%s\"", filename); @@ -600,6 +639,8 @@ do_file(evhtp_request_t *req, SeafRepo *repo, const char *file_id, data->req = req; data->file = file; data->crypt = crypt; + data->user = g_strdup(user); + data->token_type = g_strdup (operation); memcpy (data->store_id, repo->store_id, 36); data->repo_version = repo->version; @@ -758,6 +799,14 @@ next: bufferevent_write (bev, buf, n); if (data->range_remain == 0) { + if (data->start_off + n >= data->file->file_size) { + char *oper = "web-file-download"; + if (g_strcmp0(data->token_type, "download-link") == 0) + oper = "link-file-download"; + + send_statistic_msg (data->store_id, data->user, oper, + (guint64)data->file->file_size); + } finish_file_range_request (bev, data); } @@ -868,7 +917,8 @@ set_resp_disposition (evhtp_request_t *req, const char *operation, static int do_file_range (evhtp_request_t *req, SeafRepo *repo, const char *file_id, - const char *filename, const char *operation, const char *byte_ranges) + const char *filename, const char *operation, const char *byte_ranges, + const char *user) { Seafile *file; SendFileRangeData *data = NULL; @@ -947,6 +997,8 @@ do_file_range (evhtp_request_t *req, SeafRepo *repo, const char *file_id, data->blk_idx = -1; data->start_off = start; data->range_remain = end-start+1; + data->user = g_strdup(user); + data->token_type = g_strdup (operation); memcpy (data->store_id, repo->store_id, 36); data->repo_version = repo->version; @@ -979,7 +1031,8 @@ do_file_range (evhtp_request_t *req, SeafRepo *repo, const char *file_id, static int start_download_zip_file (evhtp_request_t *req, const char *token, - const char *zipname, char *zipfile) + const char *zipname, char *zipfile, + const char *repo_id, const char *user, const char *token_type) { SeafStat st; char file_size[255]; @@ -1017,6 +1070,10 @@ start_download_zip_file (evhtp_request_t *req, const char *token, data->zipfile = zipfile; data->token = g_strdup (token); data->remain = st.st_size; + data->total_size = (guint64)st.st_size; + data->user = g_strdup (user); + data->token_type = g_strdup (token_type); + snprintf(data->repo_id, sizeof(data->repo_id), "%s", repo_id); /* We need to overwrite evhtp's callback functions to * write file data piece by piece. @@ -1085,7 +1142,9 @@ access_zip_cb (evhtp_request_t *req, void *arg) json_error_t jerror; char *filename = NULL; char *repo_id = NULL; + char *user = NULL; char *zip_file_path; + char *token_type = NULL; const char *error = NULL; int error_code; @@ -1155,7 +1214,10 @@ access_zip_cb (evhtp_request_t *req, void *arg) goto out; } - int ret = start_download_zip_file (req, token, filename, zip_file_path); + g_object_get (info, "username", &user, NULL); + g_object_get (info, "repo_id", &repo_id, NULL); + g_object_get (info, "op", &token_type, NULL); + int ret = start_download_zip_file (req, token, filename, zip_file_path, repo_id, user, token_type); if (ret < 0) { error = "Internal server error\n"; error_code = EVHTP_RES_SERVERR; @@ -1173,6 +1235,10 @@ out: g_free (filename); if (repo_id) g_free (repo_id); + if (user) + g_free (user); + if (token_type) + g_free (token_type); if (error) { evbuffer_add_printf(req->buffer_out, "%s\n", error); @@ -1219,7 +1285,8 @@ access_cb(evhtp_request_t *req, void *arg) user = seafile_web_access_get_username (webaccess); if (strcmp(operation, "view") != 0 && - strcmp(operation, "download") != 0) { + strcmp(operation, "download") != 0 && + strcmp(operation, "download-link") != 0) { error = "Bad access token"; goto bad_req; } @@ -1252,11 +1319,11 @@ access_cb(evhtp_request_t *req, void *arg) } if (!repo->encrypted && byte_ranges) { - if (do_file_range (req, repo, data, filename, operation, byte_ranges) < 0) { + if (do_file_range (req, repo, data, filename, operation, byte_ranges, user) < 0) { error = "Internal server error\n"; goto bad_req; } - } else if (do_file(req, repo, data, filename, operation, key) < 0) { + } else if (do_file(req, repo, data, filename, operation, key, user) < 0) { error = "Internal server error\n"; goto bad_req; } @@ -1286,7 +1353,7 @@ bad_req: } static int -do_block(evhtp_request_t *req, SeafRepo *repo, const char *file_id, +do_block(evhtp_request_t *req, SeafRepo *repo, const char *user, const char *file_id, const char *blk_id) { Seafile *file; @@ -1346,6 +1413,7 @@ do_block(evhtp_request_t *req, SeafRepo *repo, const char *file_id, data = g_new0 (SendBlockData, 1); data->req = req; data->block_id = g_strdup(blk_id); + data->user = g_strdup(user); memcpy (data->store_id, repo->store_id, 36); data->repo_version = repo->version; @@ -1385,6 +1453,7 @@ access_blks_cb(evhtp_request_t *req, void *arg) const char *repo_id = NULL; const char *id = NULL; const char *operation = NULL; + const char *user = NULL; char *repo_role = NULL; SeafileWebAccess *webaccess = NULL; @@ -1427,7 +1496,7 @@ access_blks_cb(evhtp_request_t *req, void *arg) } if (strcmp(operation, "downloadblks") == 0) { - if (do_block(req, repo, id, blkid) < 0) { + if (do_block(req, repo, user, id, blkid) < 0) { error = "Internal server error\n"; goto bad_req; } diff --git a/server/http-server.c b/server/http-server.c index b0e2a3b..e9533be 100644 --- a/server/http-server.c +++ b/server/http-server.c @@ -62,11 +62,21 @@ struct _HttpServer { pthread_mutex_t vir_repo_info_cache_lock; uint32_t cevent_id; /* Used for sending activity events. */ + uint32_t stats_event_id; /* Used for sending events for statistics. */ event_t *reap_timer; }; typedef struct _HttpServer HttpServer; +struct _StatsEventData { + char *etype; + char *user; + char *operation; + char repo_id[37]; + guint64 bytes; +}; +typedef struct _StatsEventData StatsEventData; + typedef struct TokenInfo { char *repo_id; char *email; @@ -471,6 +481,18 @@ free_repo_event_data (RepoEventData *data) g_free (data); } +static void +free_stats_event_data (StatsEventData *data) +{ + if (!data) + return; + + g_free (data->etype); + g_free (data->user); + g_free (data->operation); + g_free (data); +} + static void publish_repo_event (CEvent *event, void *data) { @@ -488,6 +510,22 @@ publish_repo_event (CEvent *event, void *data) free_repo_event_data (rdata); } +static void +publish_stats_event (CEvent *event, void *data) +{ + StatsEventData *rdata = event->data; + + GString *buf = g_string_new (NULL); + g_string_printf (buf, "%s\t%s\t%s\t%"G_GUINT64_FORMAT, + rdata->etype, rdata->user, + rdata->repo_id, rdata->bytes); + + seaf_mq_manager_publish_stats_event (seaf->mq_mgr, buf->str); + + g_string_free (buf, TRUE); + free_stats_event_data (rdata); +} + static void on_repo_oper (HttpServer *htp_server, const char *etype, const char *repo_id, char *user, char *ip, char *client_name) @@ -514,6 +552,19 @@ on_repo_oper (HttpServer *htp_server, const char *etype, } } +void +send_statistic_msg (const char *repo_id, char *user, char *operation, guint64 bytes) +{ + StatsEventData *rdata = g_new0 (StatsEventData, 1); + + memcpy (rdata->repo_id, repo_id, 36); + rdata->etype = g_strdup (operation); + rdata->user = g_strdup (user); + rdata->bytes = bytes; + + cevent_manager_add_event (seaf->ev_mgr, seaf->http_server->priv->stats_event_id, rdata); +} + char * get_client_ip_addr (evhtp_request_t *req) { @@ -1454,12 +1505,13 @@ get_block_cb (evhtp_request_t *req, void *arg) char *store_id = NULL; HttpServer *htp_server = arg; BlockMetadata *blk_meta = NULL; + char *username = NULL; char **parts = g_strsplit (req->uri->path->full + 1, "/", 0); repo_id = parts[1]; block_id = parts[3]; - int token_status = validate_token (htp_server, req, repo_id, NULL, FALSE); + int token_status = validate_token (htp_server, req, repo_id, &username, FALSE); if (token_status != EVHTP_RES_OK) { evhtp_send_reply (req, token_status); goto out; @@ -1505,12 +1557,14 @@ get_block_cb (evhtp_request_t *req, void *arg) evhtp_send_reply (req, EVHTP_RES_OK); } g_free (block_con); + send_statistic_msg (store_id, username, "sync-file-download", (guint64)rsize); free_handle: seaf_block_manager_close_block (seaf->block_mgr, blk_handle); seaf_block_manager_block_handle_free (seaf->block_mgr, blk_handle); out: + g_free (username); g_free (blk_meta); g_free (store_id); g_strfreev (parts); @@ -1602,6 +1656,8 @@ put_send_block_cb (evhtp_request_t *req, void *arg) evhtp_send_reply (req, EVHTP_RES_OK); + send_statistic_msg (store_id, username, "sync-file-upload", (guint64)blk_len); + out: g_free (username); g_free (store_id); @@ -2199,6 +2255,10 @@ seaf_http_server_start (HttpServerStruct *server) (cevent_handler)publish_repo_event, NULL); + server->priv->stats_event_id = cevent_manager_register (seaf->ev_mgr, + (cevent_handler)publish_stats_event, + NULL); + int ret = pthread_create (&server->priv->thread_id, NULL, http_server_run, server); if (ret != 0) return -1; diff --git a/server/http-server.h b/server/http-server.h index eb04aec..9e766e6 100644 --- a/server/http-server.h +++ b/server/http-server.h @@ -35,4 +35,7 @@ int seaf_http_server_invalidate_tokens (HttpServerStruct *htp_server, const GList *tokens); +void +send_statistic_msg (const char *repo_id, char *user, char *operation, guint64 bytes); + #endif diff --git a/server/upload-file.c b/server/upload-file.c index ad60ead..f673777 100644 --- a/server/upload-file.c +++ b/server/upload-file.c @@ -25,6 +25,7 @@ #include "seafile-session.h" #include "upload-file.h" #include "http-status-codes.h" +#include "http-server.h" #include "seafile-error.h" @@ -74,6 +75,8 @@ typedef struct RecvFSM { char *progress_id; Progress *progress; + char *token_type; /* For sending statistic type */ + gboolean need_idx_progress; } RecvFSM; @@ -639,6 +642,12 @@ upload_api_cb(evhtp_request_t *req, void *arg) g_free (ret_json); send_success_reply (req); + + char *oper = "web-file-upload"; + if (g_strcmp0(fsm->token_type, "upload-link") == 0) + oper = "link-file-upload"; + send_statistic_msg(fsm->repo_id, fsm->user, oper, (guint64)content_len); + return; error: @@ -701,6 +710,8 @@ upload_raw_blks_api_cb(evhtp_request_t *req, void *arg) } goto error; } + guint64 content_len = (guint64)get_content_length(req); + send_statistic_msg(fsm->repo_id, fsm->user, "web-file-upload", content_len); evbuffer_add (req->buffer_out, "\"OK\"", 4); send_success_reply (req); @@ -1118,6 +1129,11 @@ upload_ajax_cb(evhtp_request_t *req, void *arg) } evhtp_send_reply (req, EVHTP_RES_OK); + char *oper = "web-file-upload"; + if (g_strcmp0(fsm->token_type, "upload-link") == 0) + oper = "link-file-upload"; + send_statistic_msg(fsm->repo_id, fsm->user, oper, (guint64)content_len); + return; error: @@ -1308,6 +1324,8 @@ update_api_cb(evhtp_request_t *req, void *arg) evbuffer_add(req->buffer_out, new_file_id, strlen(new_file_id)); send_success_reply (req); + send_statistic_msg(fsm->repo_id, fsm->user, "web-file-upload", (guint64)content_len); + g_free (new_file_id); return; @@ -1701,12 +1719,14 @@ update_ajax_cb(evhtp_request_t *req, void *arg) } goto error; } + send_statistic_msg(fsm->repo_id, fsm->user, "web-file-upload", (guint64)content_len); char *json_ret = format_update_json_ret (filename, new_file_id, size); evbuffer_add (req->buffer_out, json_ret, strlen(json_ret)); send_success_reply (req); + g_free (new_file_id); g_free (filename); g_free (json_ret); @@ -1756,6 +1776,7 @@ upload_finish_cb (evhtp_request_t *req, void *arg) g_free (fsm->user); g_free (fsm->boundary); g_free (fsm->input_name); + g_free (fsm->token_type); g_hash_table_destroy (fsm->form_kvs); @@ -2197,7 +2218,8 @@ static int check_access_token (const char *token, const char *url_op, char **repo_id, - char **user) + char **user, + char **token_type) { SeafileWebAccess *webaccess; const char *op; @@ -2211,6 +2233,12 @@ check_access_token (const char *token, * token with op = "update" can only be used for "update-*" operations. */ op = seafile_web_access_get_op (webaccess); + if (token_type) + *token_type = g_strdup (op); + + if (g_strcmp0(op, "upload-link") == 0) + op = "upload"; + if (strncmp (url_op, op, strlen(op)) != 0) { g_object_unref (webaccess); return -1; @@ -2258,6 +2286,7 @@ upload_headers_cb (evhtp_request_t *req, evhtp_headers_t *hdr, void *arg) gint64 content_len; char *progress_id = NULL; char *err_msg = NULL; + char *token_type = NULL; RecvFSM *fsm = NULL; Progress *progress = NULL; @@ -2280,7 +2309,7 @@ upload_headers_cb (evhtp_request_t *req, evhtp_headers_t *hdr, void *arg) } char *url_op = parts[0]; - if (check_access_token (token, url_op, &repo_id, &user) < 0) { + if (check_access_token (token, url_op, &repo_id, &user, &token_type) < 0) { err_msg = "Access denied"; goto err; } @@ -2307,6 +2336,7 @@ upload_headers_cb (evhtp_request_t *req, evhtp_headers_t *hdr, void *arg) fsm->boundary = boundary; fsm->repo_id = repo_id; fsm->user = user; + fsm->token_type = token_type; fsm->line = evbuffer_new (); fsm->form_kvs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); @@ -2348,6 +2378,7 @@ err: g_free (repo_id); g_free (user); g_free (boundary); + g_free (token_type); g_free (progress_id); g_strfreev (parts); return EVHTP_RES_OK; diff --git a/server/web-accesstoken-mgr.c b/server/web-accesstoken-mgr.c index 89152e6..08b27c0 100644 --- a/server/web-accesstoken-mgr.c +++ b/server/web-accesstoken-mgr.c @@ -142,8 +142,12 @@ seaf_web_at_manager_get_access_token (SeafWebAccessTokenManager *mgr, strcmp(op, "downloadblks") != 0 && strcmp(op, "download-dir") != 0 && strcmp(op, "download-multi") != 0 && + strcmp(op, "download-link") != 0 && + strcmp(op, "download-dir-link") != 0 && + strcmp(op, "download-multi-link") != 0 && strcmp(op, "upload") != 0 && strcmp(op, "update") != 0 && + strcmp(op, "upload-link") != 0 && strcmp(op, "upload-blks-api") != 0 && strcmp(op, "upload-blks-aj") != 0 && strcmp(op, "update-blks-api") != 0 && @@ -173,7 +177,9 @@ seaf_web_at_manager_get_access_token (SeafWebAccessTokenManager *mgr, pthread_mutex_unlock (&mgr->priv->lock); if (strcmp(op, "download-dir") == 0 || - strcmp(op, "download-multi") == 0) { + strcmp(op, "download-multi") == 0 || + strcmp(op, "download-dir-link") == 0 || + strcmp(op, "download-multi-link") == 0) { webaccess = g_object_new (SEAFILE_TYPE_WEB_ACCESS, "repo_id", info->repo_id, diff --git a/server/zip-download-mgr.c b/server/zip-download-mgr.c index 120d454..d1431fc 100644 --- a/server/zip-download-mgr.c +++ b/server/zip-download-mgr.c @@ -497,7 +497,8 @@ zip_download_mgr_start_zip_task (ZipDownloadMgr *mgr, obj->repo = repo; obj->user = g_strdup (seafile_web_access_get_username (info)); - if (strcmp (operation, "download-dir") == 0) { + if (strcmp (operation, "download-dir") == 0 || + strcmp (operation, "download-dir-link") == 0) { obj->type = DOWNLOAD_DIR; ret = parse_download_dir_data (obj, data); if (ret < 0) {