Support traffic statistics.

common/mq-mgr.c

@@ -138,3 +138,13 @@ seaf_mq_manager_publish_event (SeafMqManager *mgr, const char *content)
     ccnet_message_free (msg);
 }
 
+void
+seaf_mq_manager_publish_stats_event (SeafMqManager *mgr, const char *content)
+{
+    static const char *app = "seaf_server.stats";
+
+    CcnetMessage *msg = create_message (mgr, app, content, 0);
+    _send_message (mgr, msg);
+
+    ccnet_message_free (msg);
+}

common/mq-mgr.h

@@ -56,4 +56,7 @@ seaf_mq_manager_publish_notification (SeafMqManager *mgr,
 void
 seaf_mq_manager_publish_event (SeafMqManager *mgr, const char *content);
 
+void
+seaf_mq_manager_publish_stats_event (SeafMqManager *mgr, const char *content);
+
 #endif

server/access-file.c

@@ -27,6 +27,7 @@
 #include "seafile-session.h"
 #include "access-file.h"
 #include "zip-download-mgr.h"
+#include "http-server.h"
 
 #define FILE_TYPE_MAP_DEFAULT_LEN 1
 #define BUFFER_SIZE 1024 * 64
@@ -47,6 +48,8 @@ typedef struct SendBlockData {
     char store_id[37];
     int repo_version;
 
+    char *user;
+
     bufferevent_data_cb saved_read_cb;
     bufferevent_data_cb saved_write_cb;
     bufferevent_event_cb saved_event_cb;
@@ -66,6 +69,9 @@ typedef struct SendfileData {
     char store_id[37];
     int repo_version;
 
+    char *user;
+    char *token_type;
+
     bufferevent_data_cb saved_read_cb;
     bufferevent_data_cb saved_write_cb;
     bufferevent_event_cb saved_event_cb;
@@ -83,6 +89,9 @@ typedef struct SendFileRangeData {
     char store_id[37];
     int repo_version;
 
+    char *user;
+    char *token_type;
+
     bufferevent_data_cb saved_read_cb;
     bufferevent_data_cb saved_write_cb;
     bufferevent_event_cb saved_event_cb;
@@ -92,10 +101,14 @@ typedef struct SendFileRangeData {
 typedef struct SendDirData {
     evhtp_request_t *req;
     size_t remain;
+    guint64 total_size;
 
     int zipfd;
     char *zipfile;
     char *token;
+    char *user;
+    char *token_type;
+    char repo_id[37];
 
     bufferevent_data_cb saved_read_cb;
     bufferevent_data_cb saved_write_cb;
@@ -142,6 +155,7 @@ free_sendblock_data (SendBlockData *data)
     }
 
     g_free (data->block_id);
+    g_free (data->user);
     g_free (data);
 }
 
@@ -157,6 +171,8 @@ free_sendfile_data (SendfileData *data)
         EVP_CIPHER_CTX_free (data->ctx);
 
     seafile_unref (data->file);
+    g_free (data->user);
+    g_free (data->token_type);
     g_free (data->crypt);
     g_free (data);
 }
@@ -170,6 +186,8 @@ free_send_file_range_data (SendFileRangeData *data)
     }
 
     seafile_unref (data->file);
+    g_free (data->user);
+    g_free (data->token_type);
     g_free (data);
 }
 
@@ -180,6 +198,8 @@ free_senddir_data (SendDirData *data)
 
     zip_download_mgr_del_zip_progress (seaf->zip_download_mgr, data->token);
 
+    g_free (data->user);
+    g_free (data->token_type);
     g_free (data->token);
     g_free (data);
 }
@@ -232,6 +252,8 @@ write_block_data_cb (struct bufferevent *bev, void *ctx)
 
         evhtp_send_reply_end (data->req);
 
+        send_statistic_msg (data->store_id, data->user, "web-file-download", (guint64)data->bsize);
+
         free_sendblock_data (data);
         return;
     }
@@ -317,6 +339,15 @@ next:
 
             evhtp_send_reply_end (data->req);
 
+            if (g_strcmp0(data->token_type, "view") != 0) {
+                char *oper = "web-file-download";
+                if (g_strcmp0(data->token_type, "download-link") == 0)
+                    oper = "link-file-download";
+
+                send_statistic_msg(data->store_id, data->user, oper,
+                                   (guint64)data->file->file_size);
+            }
+
             free_sendfile_data (data);
             return;
         }
@@ -414,6 +445,13 @@ write_dir_data_cb (struct bufferevent *bev, void *ctx)
 
             evhtp_send_reply_end (data->req);
 
+            char *oper = "web-file-download";
+            if (g_strcmp0(data->token_type, "download-dir-link") == 0 ||
+                g_strcmp0(data->token_type, "download-multi-link") == 0)
+                oper = "link-file-download";
+
+            send_statistic_msg(data->repo_id, data->user, oper, data->total_size);
+
             free_senddir_data (data);
             return;
         }
@@ -503,7 +541,7 @@ test_firefox (evhtp_request_t *req)
 static int
 do_file(evhtp_request_t *req, SeafRepo *repo, const char *file_id,
         const char *filename, const char *operation,
-        SeafileCryptKey *crypt_key)
+        SeafileCryptKey *crypt_key, const char *user)
 {
     Seafile *file;
     char *type = NULL;
@@ -561,7 +599,8 @@ do_file(evhtp_request_t *req, SeafRepo *repo, const char *file_id,
     evhtp_headers_add_header (req->headers_out,
                               evhtp_header_new("Content-Length", file_size, 1, 1));
 
-    if (strcmp(operation, "download") == 0) {
+    if (strcmp(operation, "download") == 0 ||
+        strcmp(operation, "download-link") == 0) {
         /* Safari doesn't support 'utf8', 'utf-8' is compatible with most of browsers. */
         snprintf(cont_filename, SEAF_PATH_MAX,
                  "attachment;filename*=\"utf-8\' \'%s\"", filename);
@@ -600,6 +639,8 @@ do_file(evhtp_request_t *req, SeafRepo *repo, const char *file_id,
     data->req = req;
     data->file = file;
     data->crypt = crypt;
+    data->user = g_strdup(user);
+    data->token_type = g_strdup (operation);
 
     memcpy (data->store_id, repo->store_id, 36);
     data->repo_version = repo->version;
@@ -758,6 +799,14 @@ next:
 
     bufferevent_write (bev, buf, n);
     if (data->range_remain == 0) {
+        if (data->start_off + n >= data->file->file_size) {
+            char *oper = "web-file-download";
+            if (g_strcmp0(data->token_type, "download-link") == 0)
+                oper = "link-file-download";
+
+            send_statistic_msg (data->store_id, data->user, oper,
+                                (guint64)data->file->file_size);
+        }
         finish_file_range_request (bev, data);
     }
 
@@ -868,7 +917,8 @@ set_resp_disposition (evhtp_request_t *req, const char *operation,
 
 static int
 do_file_range (evhtp_request_t *req, SeafRepo *repo, const char *file_id,
-               const char *filename, const char *operation, const char *byte_ranges)
+               const char *filename, const char *operation, const char *byte_ranges,
+               const char *user)
 {
     Seafile *file;
     SendFileRangeData *data = NULL;
@@ -947,6 +997,8 @@ do_file_range (evhtp_request_t *req, SeafRepo *repo, const char *file_id,
     data->blk_idx = -1;
     data->start_off = start;
     data->range_remain = end-start+1;
+    data->user = g_strdup(user);
+    data->token_type = g_strdup (operation);
 
     memcpy (data->store_id, repo->store_id, 36);
     data->repo_version = repo->version;
@@ -979,7 +1031,8 @@ do_file_range (evhtp_request_t *req, SeafRepo *repo, const char *file_id,
 
 static int
 start_download_zip_file (evhtp_request_t *req, const char *token,
-                         const char *zipname, char *zipfile)
+                         const char *zipname, char *zipfile,
+                         const char *repo_id, const char *user, const char *token_type)
 {
     SeafStat st;
     char file_size[255];
@@ -1017,6 +1070,10 @@ start_download_zip_file (evhtp_request_t *req, const char *token,
     data->zipfile = zipfile;
     data->token = g_strdup (token);
     data->remain = st.st_size;
+    data->total_size = (guint64)st.st_size;
+    data->user = g_strdup (user);
+    data->token_type = g_strdup (token_type);
+    snprintf(data->repo_id, sizeof(data->repo_id), "%s", repo_id);
 
     /* We need to overwrite evhtp's callback functions to
      * write file data piece by piece.
@@ -1085,7 +1142,9 @@ access_zip_cb (evhtp_request_t *req, void *arg)
     json_error_t jerror;
     char *filename = NULL;
     char *repo_id = NULL;
+    char *user = NULL;
     char *zip_file_path;
+    char *token_type = NULL;
     const char *error = NULL;
     int error_code;
 
@@ -1155,7 +1214,10 @@ access_zip_cb (evhtp_request_t *req, void *arg)
         goto out;
     }
 
-    int ret = start_download_zip_file (req, token, filename, zip_file_path);
+    g_object_get (info, "username", &user, NULL);
+    g_object_get (info, "repo_id", &repo_id, NULL);
+    g_object_get (info, "op", &token_type, NULL);
+    int ret = start_download_zip_file (req, token, filename, zip_file_path, repo_id, user, token_type);
     if (ret < 0) {
         error = "Internal server error\n";
         error_code = EVHTP_RES_SERVERR;
@@ -1173,6 +1235,10 @@ out:
         g_free (filename);
     if (repo_id)
         g_free (repo_id);
+    if (user)
+        g_free (user);
+    if (token_type)
+        g_free (token_type);
 
     if (error) {
         evbuffer_add_printf(req->buffer_out, "%s\n", error);
@@ -1219,7 +1285,8 @@ access_cb(evhtp_request_t *req, void *arg)
     user = seafile_web_access_get_username (webaccess);
 
     if (strcmp(operation, "view") != 0 &&
-        strcmp(operation, "download") != 0) {
+        strcmp(operation, "download") != 0 &&
+        strcmp(operation, "download-link") != 0) {
         error = "Bad access token";
         goto bad_req;
     }
@@ -1252,11 +1319,11 @@ access_cb(evhtp_request_t *req, void *arg)
     }
 
     if (!repo->encrypted && byte_ranges) {
-        if (do_file_range (req, repo, data, filename, operation, byte_ranges) < 0) {
+        if (do_file_range (req, repo, data, filename, operation, byte_ranges, user) < 0) {
             error = "Internal server error\n";
             goto bad_req;
         }
-    } else if (do_file(req, repo, data, filename, operation, key) < 0) {
+    } else if (do_file(req, repo, data, filename, operation, key, user) < 0) {
         error = "Internal server error\n";
         goto bad_req;
     }
@@ -1286,7 +1353,7 @@ bad_req:
 }
 
 static int
-do_block(evhtp_request_t *req, SeafRepo *repo, const char *file_id,
+do_block(evhtp_request_t *req, SeafRepo *repo, const char *user, const char *file_id,
          const char *blk_id)
 {
     Seafile *file;
@@ -1346,6 +1413,7 @@ do_block(evhtp_request_t *req, SeafRepo *repo, const char *file_id,
     data = g_new0 (SendBlockData, 1);
     data->req = req;
     data->block_id = g_strdup(blk_id);
+    data->user = g_strdup(user);
 
     memcpy (data->store_id, repo->store_id, 36);
     data->repo_version = repo->version;
@@ -1385,6 +1453,7 @@ access_blks_cb(evhtp_request_t *req, void *arg)
     const char *repo_id = NULL;
     const char *id = NULL;
     const char *operation = NULL;
+    const char *user = NULL;
 
     char *repo_role = NULL;
     SeafileWebAccess *webaccess = NULL;
@@ -1427,7 +1496,7 @@ access_blks_cb(evhtp_request_t *req, void *arg)
     }
 
     if (strcmp(operation, "downloadblks") == 0) {
-        if (do_block(req, repo, id, blkid) < 0) {
+        if (do_block(req, repo, user, id, blkid) < 0) {
             error = "Internal server error\n";
             goto bad_req;
         }

server/http-server.c

@@ -62,11 +62,21 @@ struct _HttpServer {
     pthread_mutex_t vir_repo_info_cache_lock;
 
     uint32_t cevent_id;         /* Used for sending activity events. */
+    uint32_t stats_event_id;         /* Used for sending events for statistics. */
 
     event_t *reap_timer;
 };
 typedef struct _HttpServer HttpServer;
 
+struct _StatsEventData {
+    char *etype;
+    char *user;
+    char *operation;
+    char repo_id[37];
+    guint64 bytes;
+};
+typedef struct _StatsEventData StatsEventData;
+
 typedef struct TokenInfo {
     char *repo_id;
     char *email;
@@ -471,6 +481,18 @@ free_repo_event_data (RepoEventData *data)
     g_free (data);
 }
 
+static void
+free_stats_event_data (StatsEventData *data)
+{
+    if (!data)
+        return;
+
+    g_free (data->etype);
+    g_free (data->user);
+    g_free (data->operation);
+    g_free (data);
+}
+
 static void
 publish_repo_event (CEvent *event, void *data)
 {
@@ -488,6 +510,22 @@ publish_repo_event (CEvent *event, void *data)
     free_repo_event_data (rdata);
 }
 
+static void
+publish_stats_event (CEvent *event, void *data)
+{
+    StatsEventData *rdata = event->data;
+
+    GString *buf = g_string_new (NULL);
+    g_string_printf (buf, "%s\t%s\t%s\t%"G_GUINT64_FORMAT,
+                     rdata->etype, rdata->user,
+                     rdata->repo_id, rdata->bytes);
+
+    seaf_mq_manager_publish_stats_event (seaf->mq_mgr, buf->str);
+
+    g_string_free (buf, TRUE);
+    free_stats_event_data (rdata);
+}
+
 static void
 on_repo_oper (HttpServer *htp_server, const char *etype,
               const char *repo_id, char *user, char *ip, char *client_name)
@@ -514,6 +552,19 @@ on_repo_oper (HttpServer *htp_server, const char *etype,
     }
 }
 
+void
+send_statistic_msg (const char *repo_id, char *user, char *operation, guint64 bytes)
+{
+    StatsEventData *rdata = g_new0 (StatsEventData, 1);
+
+    memcpy (rdata->repo_id, repo_id, 36);
+    rdata->etype = g_strdup (operation);
+    rdata->user = g_strdup (user);
+    rdata->bytes = bytes;
+
+    cevent_manager_add_event (seaf->ev_mgr, seaf->http_server->priv->stats_event_id, rdata);
+}
+
 char *
 get_client_ip_addr (evhtp_request_t *req)
 {
@@ -1454,12 +1505,13 @@ get_block_cb (evhtp_request_t *req, void *arg)
     char *store_id = NULL;
     HttpServer *htp_server = arg;
     BlockMetadata *blk_meta = NULL;
+    char *username = NULL;
 
     char **parts = g_strsplit (req->uri->path->full + 1, "/", 0);
     repo_id = parts[1];
     block_id = parts[3];
 
-    int token_status = validate_token (htp_server, req, repo_id, NULL, FALSE);
+    int token_status = validate_token (htp_server, req, repo_id, &username, FALSE);
     if (token_status != EVHTP_RES_OK) {
         evhtp_send_reply (req, token_status);
         goto out;
@@ -1505,12 +1557,14 @@ get_block_cb (evhtp_request_t *req, void *arg)
         evhtp_send_reply (req, EVHTP_RES_OK);
     }
     g_free (block_con);
+    send_statistic_msg (store_id, username, "sync-file-download", (guint64)rsize);
 
 free_handle:
     seaf_block_manager_close_block (seaf->block_mgr, blk_handle);
     seaf_block_manager_block_handle_free (seaf->block_mgr, blk_handle);
 
 out:
+    g_free (username);
     g_free (blk_meta);
     g_free (store_id);
     g_strfreev (parts);
@@ -1602,6 +1656,8 @@ put_send_block_cb (evhtp_request_t *req, void *arg)
 
     evhtp_send_reply (req, EVHTP_RES_OK);
 
+    send_statistic_msg (store_id, username, "sync-file-upload", (guint64)blk_len);
+
 out:
     g_free (username);
     g_free (store_id);
@@ -2199,6 +2255,10 @@ seaf_http_server_start (HttpServerStruct *server)
                                     (cevent_handler)publish_repo_event,
                                                        NULL);
 
+    server->priv->stats_event_id = cevent_manager_register (seaf->ev_mgr,
+                                    (cevent_handler)publish_stats_event,
+                                                       NULL);
+
    int ret = pthread_create (&server->priv->thread_id, NULL, http_server_run, server);
    if (ret != 0)
        return -1;

server/http-server.h

@@ -35,4 +35,7 @@ int
 seaf_http_server_invalidate_tokens (HttpServerStruct *htp_server,
                                     const GList *tokens);
 
+void
+send_statistic_msg (const char *repo_id, char *user, char *operation, guint64 bytes);
+
 #endif

server/upload-file.c

@@ -25,6 +25,7 @@
 #include "seafile-session.h"
 #include "upload-file.h"
 #include "http-status-codes.h"
+#include "http-server.h"
 
 #include "seafile-error.h"
 
@@ -74,6 +75,8 @@ typedef struct RecvFSM {
     char *progress_id;
     Progress *progress;
 
+    char *token_type; /* For sending statistic type */
+
     gboolean need_idx_progress;
 } RecvFSM;
 
@@ -639,6 +642,12 @@ upload_api_cb(evhtp_request_t *req, void *arg)
     g_free (ret_json);
 
     send_success_reply (req);
+
+    char *oper = "web-file-upload";
+    if (g_strcmp0(fsm->token_type, "upload-link") == 0)
+        oper = "link-file-upload";
+    send_statistic_msg(fsm->repo_id, fsm->user, oper, (guint64)content_len);
+
     return;
 
 error:
@@ -701,6 +710,8 @@ upload_raw_blks_api_cb(evhtp_request_t *req, void *arg)
         }
         goto error;
     }
+    guint64 content_len = (guint64)get_content_length(req);
+    send_statistic_msg(fsm->repo_id, fsm->user, "web-file-upload", content_len);
 
     evbuffer_add (req->buffer_out, "\"OK\"", 4);
     send_success_reply (req);
@@ -1118,6 +1129,11 @@ upload_ajax_cb(evhtp_request_t *req, void *arg)
     }
     evhtp_send_reply (req, EVHTP_RES_OK);
 
+    char *oper = "web-file-upload";
+    if (g_strcmp0(fsm->token_type, "upload-link") == 0)
+        oper = "link-file-upload";
+    send_statistic_msg(fsm->repo_id, fsm->user, oper, (guint64)content_len);
+
     return;
 
 error:
@@ -1308,6 +1324,8 @@ update_api_cb(evhtp_request_t *req, void *arg)
     evbuffer_add(req->buffer_out, new_file_id, strlen(new_file_id));
     send_success_reply (req);
 
+    send_statistic_msg(fsm->repo_id, fsm->user, "web-file-upload", (guint64)content_len);
+
     g_free (new_file_id);
     return;
 
@@ -1701,12 +1719,14 @@ update_ajax_cb(evhtp_request_t *req, void *arg)
         }
         goto error;
     }
+    send_statistic_msg(fsm->repo_id, fsm->user, "web-file-upload", (guint64)content_len);
 
     char *json_ret = format_update_json_ret (filename, new_file_id, size);
 
     evbuffer_add (req->buffer_out, json_ret, strlen(json_ret));
     send_success_reply (req);
 
+
     g_free (new_file_id);
     g_free (filename);
     g_free (json_ret);
@@ -1756,6 +1776,7 @@ upload_finish_cb (evhtp_request_t *req, void *arg)
     g_free (fsm->user);
     g_free (fsm->boundary);
     g_free (fsm->input_name);
+    g_free (fsm->token_type);
 
     g_hash_table_destroy (fsm->form_kvs);
 
@@ -2197,7 +2218,8 @@ static int
 check_access_token (const char *token,
                     const char *url_op,
                     char **repo_id,
-                    char **user)
+                    char **user,
+                    char **token_type)
 {
     SeafileWebAccess *webaccess;
     const char *op;
@@ -2211,6 +2233,12 @@ check_access_token (const char *token,
      * token with op = "update" can only be used for "update-*" operations.
      */
     op = seafile_web_access_get_op (webaccess);
+    if (token_type)
+        *token_type = g_strdup (op);
+
+    if (g_strcmp0(op, "upload-link") == 0)
+        op = "upload";
+
     if (strncmp (url_op, op, strlen(op)) != 0) {
         g_object_unref (webaccess);
         return -1;
@@ -2258,6 +2286,7 @@ upload_headers_cb (evhtp_request_t *req, evhtp_headers_t *hdr, void *arg)
     gint64 content_len;
     char *progress_id = NULL;
     char *err_msg = NULL;
+    char *token_type = NULL;
     RecvFSM *fsm = NULL;
     Progress *progress = NULL;
 
@@ -2280,7 +2309,7 @@ upload_headers_cb (evhtp_request_t *req, evhtp_headers_t *hdr, void *arg)
     }
     char *url_op = parts[0];
 
-    if (check_access_token (token, url_op, &repo_id, &user) < 0) {
+    if (check_access_token (token, url_op, &repo_id, &user, &token_type) < 0) {
         err_msg = "Access denied";
         goto err;
     }
@@ -2307,6 +2336,7 @@ upload_headers_cb (evhtp_request_t *req, evhtp_headers_t *hdr, void *arg)
     fsm->boundary = boundary;
     fsm->repo_id = repo_id;
     fsm->user = user;
+    fsm->token_type = token_type;
     fsm->line = evbuffer_new ();
     fsm->form_kvs = g_hash_table_new_full (g_str_hash, g_str_equal,
                                            g_free, g_free);
@@ -2348,6 +2378,7 @@ err:
     g_free (repo_id);
     g_free (user);
     g_free (boundary);
+    g_free (token_type);
     g_free (progress_id);
     g_strfreev (parts);
     return EVHTP_RES_OK;

server/web-accesstoken-mgr.c

@@ -142,8 +142,12 @@ seaf_web_at_manager_get_access_token (SeafWebAccessTokenManager *mgr,
         strcmp(op, "downloadblks") != 0 &&
         strcmp(op, "download-dir") != 0 &&
         strcmp(op, "download-multi") != 0 &&
+        strcmp(op, "download-link") != 0 &&
+        strcmp(op, "download-dir-link") != 0 &&
+        strcmp(op, "download-multi-link") != 0 &&
         strcmp(op, "upload") != 0 &&
         strcmp(op, "update") != 0 &&
+        strcmp(op, "upload-link") != 0 &&
         strcmp(op, "upload-blks-api") != 0 &&
         strcmp(op, "upload-blks-aj") != 0 &&
         strcmp(op, "update-blks-api") != 0 &&
@@ -173,7 +177,9 @@ seaf_web_at_manager_get_access_token (SeafWebAccessTokenManager *mgr,
     pthread_mutex_unlock (&mgr->priv->lock);
 
     if (strcmp(op, "download-dir") == 0 ||
-        strcmp(op, "download-multi") == 0) {
+        strcmp(op, "download-multi") == 0 ||
+        strcmp(op, "download-dir-link") == 0 ||
+        strcmp(op, "download-multi-link") == 0) {
 
         webaccess = g_object_new (SEAFILE_TYPE_WEB_ACCESS,
                                   "repo_id", info->repo_id,

server/zip-download-mgr.c

@@ -497,7 +497,8 @@ zip_download_mgr_start_zip_task (ZipDownloadMgr *mgr,
     obj->repo = repo;
     obj->user = g_strdup (seafile_web_access_get_username (info));
 
-    if (strcmp (operation, "download-dir") == 0) {
+    if (strcmp (operation, "download-dir") == 0 ||
+        strcmp (operation, "download-dir-link") == 0) {
         obj->type = DOWNLOAD_DIR;
         ret = parse_download_dir_data (obj, data);
         if (ret < 0) {