Merge branch '7.1'

common/seaf-db.c

@@ -12,12 +12,21 @@
 #include <sqlite3.h>
 #include <pthread.h>
 
+struct DBConnPool {
+    GPtrArray *connections;
+    pthread_mutex_t lock;
+    int max_connections;
+};
+typedef struct DBConnPool DBConnPool;
+
 struct SeafDB {
     int type;
+    DBConnPool *pool;
 };
 
 typedef struct DBConnection {
-    /* Empty */
+    gboolean is_available;
+    DBConnPool *pool;
 } DBConnection;
 
 struct SeafDBRow {
@@ -77,6 +86,97 @@ static int
 mysql_db_row_get_column_int (SeafDBRow *row, int idx);
 static gint64
 mysql_db_row_get_column_int64 (SeafDBRow *row, int idx);
+static gboolean
+mysql_db_connection_ping (DBConnection *vconn);
+
+static DBConnPool *
+init_conn_pool_common (int max_connections)
+{
+    DBConnPool *pool = g_new0(DBConnPool, 1);
+    pool->connections = g_ptr_array_sized_new (max_connections);
+    pthread_mutex_init (&pool->lock, NULL);
+    pool->max_connections = max_connections;
+
+    return pool;
+}
+
+static DBConnection *
+mysql_conn_pool_get_connection (SeafDB *db)
+{
+    DBConnPool *pool = db->pool;
+    DBConnection *conn = NULL;
+
+    if (pool->max_connections == 0) {
+        conn = mysql_db_get_connection (db);
+        conn->pool = pool;
+        return conn;
+    }
+
+    pthread_mutex_lock (&pool->lock);
+
+    guint i, size = pool->connections->len;
+    for (i = 0; i < size; ++i) {
+        conn = g_ptr_array_index (pool->connections, i);
+        if (conn->is_available && mysql_db_connection_ping (conn)) {
+            conn->is_available = FALSE;
+            goto out;
+        }
+    }
+    conn = NULL;
+    if (size < pool->max_connections) {
+        conn = mysql_db_get_connection (db);
+        if (conn) {
+            conn->pool = pool;
+            conn->is_available = FALSE;
+            g_ptr_array_add (pool->connections, conn);
+        }
+    }
+
+out:
+    pthread_mutex_unlock (&pool->lock);
+    return conn;
+}
+
+static void
+mysql_conn_pool_release_connection (DBConnection *conn)
+{
+    if (!conn)
+        return;
+
+    if (conn->pool->max_connections == 0) {
+        mysql_db_release_connection (conn);
+        return;
+    }
+
+    pthread_mutex_lock (&conn->pool->lock);
+    conn->is_available = TRUE;
+    pthread_mutex_unlock (&conn->pool->lock);
+}
+
+#define KEEPALIVE_INTERVAL 30
+static void *
+mysql_conn_keepalive (void *arg)
+{
+    DBConnPool *pool = arg;
+    DBConnection *conn = NULL;
+
+    while (1) {
+        pthread_mutex_lock (&pool->lock);
+
+        guint i, size = pool->connections->len;
+        for (i = 0; i < size; ++i) {
+            conn = g_ptr_array_index (pool->connections, i);
+            if (conn->is_available) {
+                mysql_db_connection_ping (conn);
+            }
+        }
+        pthread_mutex_unlock (&pool->lock);
+
+        sleep (KEEPALIVE_INTERVAL);
+    }
+
+    return NULL;
+}
 
 SeafDB *
 seaf_db_new_mysql (const char *host,
@@ -96,8 +196,8 @@ seaf_db_new_mysql (const char *host,
         return NULL;
     db->type = SEAF_DB_TYPE_MYSQL;
 
-    db_ops.get_connection = mysql_db_get_connection;
-    db_ops.release_connection = mysql_db_release_connection;
+    db_ops.get_connection = mysql_conn_pool_get_connection;
+    db_ops.release_connection = mysql_conn_pool_release_connection;
     db_ops.execute_sql_no_stmt = mysql_db_execute_sql_no_stmt;
     db_ops.execute_sql = mysql_db_execute_sql;
     db_ops.query_foreach_row = mysql_db_query_foreach_row;
@@ -106,6 +206,16 @@ seaf_db_new_mysql (const char *host,
     db_ops.row_get_column_int = mysql_db_row_get_column_int;
     db_ops.row_get_column_int64 = mysql_db_row_get_column_int64;
 
+    db->pool = init_conn_pool_common (max_connections);
+
+    pthread_t tid;
+    int ret = pthread_create (&tid, NULL, mysql_conn_keepalive, db->pool);
+    if (ret != 0) {
+        seaf_warning ("Failed to create mysql connection keepalive thread.\n");
+        return NULL;
+    }
+    pthread_detach (tid);
+
     return db;
 }
 
@@ -535,6 +645,14 @@ typedef struct MySQLDBConnection {
     MYSQL *db_conn;
 } MySQLDBConnection;
 
+static gboolean
+mysql_db_connection_ping (DBConnection *vconn)
+{
+    MySQLDBConnection *conn = (MySQLDBConnection *)vconn;
+
+    return (mysql_ping (conn->db_conn) == 0);
+}
+
 static SeafDB *
 mysql_db_new (const char *host,
               int port,

common/seaf-utils.c

@@ -103,11 +103,15 @@ mysql_db_start (SeafileSession *session)
     charset = seaf_key_file_get_string (session->config,
                                      "database", "connection_charset", NULL);
 
+    if (error)
+        g_clear_error (&error);
     max_connections = g_key_file_get_integer (session->config,
                                               "database", "max_connections",
-                                              NULL);
-    if (max_connections <= 0)
+                                              &error);
+    if (error || max_connections < 0) {
+        g_clear_error (&error);
         max_connections = DEFAULT_MAX_CONNECTIONS;
+    }
 
     session->db = seaf_db_new_mysql (host, port, user, passwd, db, unix_socket, use_ssl, charset, max_connections);
     if (!session->db) {
@@ -121,8 +125,6 @@ mysql_db_start (SeafileSession *session)
     g_free (db);
     g_free (unix_socket);
     g_free (charset);
-    if (error)
-        g_clear_error (&error);
 
     return 0;
 }

controller/seafile-controller.c

@@ -9,6 +9,8 @@
 #include <errno.h>
 #include <string.h>
 #include <getopt.h>
+#include <stdbool.h>
+#include <fcntl.h>
 
 #include <glib.h>
 
@@ -68,7 +70,7 @@ controller_exit (int code)
 
 /* returns the pid of the newly created process */
 static int
-spawn_process (char *argv[])
+spawn_process (char *argv[], bool is_python_process)
 {
     char **ptr = argv;
     GString *buf = g_string_new(argv[0]);
@@ -78,12 +80,31 @@ spawn_process (char *argv[])
     seaf_message ("spawn_process: %s\n", buf->str);
     g_string_free (buf, TRUE);
 
+    int pipefd[2] = {0, 0};
+    if (is_python_process) {
+        if (pipe(pipefd) < 0) {
+            seaf_warning("Failed to create pipe.\n");
+        }
+        fcntl(pipefd[0], F_SETFL, O_NONBLOCK);
+    }
+
     pid_t pid = fork();
 
     if (pid == 0) {
+        if (is_python_process) {
+            if (pipefd[0] > 0 && pipefd[1] > 0) {
+                close(pipefd[0]);
+                dup2(pipefd[1], 2);
+            }
+        }
         /* child process */
         execvp (argv[0], argv);
         seaf_warning ("failed to execvp %s\n", argv[0]);
+        
+        if (pipefd[1] > 0) {
+            close(pipefd[1]);
+        }
+
         exit(-1);
     } else {
         /* controller */
@@ -92,6 +113,16 @@ spawn_process (char *argv[])
         else
             seaf_message ("spawned %s, pid %d\n", argv[0], pid);
 
+        if (is_python_process) {
+            char child_stderr[1024] = {0};
+            if (pipefd[0] > 0 && pipefd[1] > 0){
+                close(pipefd[1]);
+                sleep(1);
+                while (read(pipefd[0], child_stderr, sizeof(child_stderr)) > 0)
+                    seaf_warning("%s", child_stderr);
+                close(pipefd[0]);
+            }
+        }
         return (int)pid;
     }
 }
@@ -171,7 +202,7 @@ start_seaf_server ()
         "-p", ctl->rpc_pipe_path,
         NULL};
 
-    int pid = spawn_process (argv);
+    int pid = spawn_process (argv, false);
     if (pid <= 0) {
         seaf_warning ("Failed to spawn seaf-server\n");
         return -1;
@@ -321,7 +352,7 @@ start_seafevents() {
         NULL
     };
 
-    int pid = spawn_process (argv);
+    int pid = spawn_process (argv, true);
 
     if (pid <= 0) {
         seaf_warning ("Failed to spawn seafevents.\n");
@@ -355,7 +386,7 @@ start_seafdav() {
         NULL
     };
 
-    int pid = spawn_process (argv);
+    int pid = spawn_process (argv, true);
 
     if (pid <= 0) {
         seaf_warning ("Failed to spawn seafdav\n");

python/seaserv/api.py

@@ -675,6 +675,15 @@ class SeafileAPI(object):
         """
         return '{"is_syncable":true}'
 
+    def is_dir_downloadable(self, repo_id, dir_path, user, repo_perm):
+        """
+        Check if the permission of the dir is downloadable.
+        {"is_downloadable": false, "undownloadable_path":"path"}
+        - is_downloadable: true if the dir is downloadable, false if not.
+        - undownloadable_path: the undownloadable path of the repo if the path is not downloadable.
+        """
+        return '{"is_downloadable":true}'
+
     # token
     def generate_repo_token(self, repo_id, username):
         """Generate a token for sync a repo

server/access-file.c

@@ -550,6 +550,7 @@ do_file(evhtp_request_t *req, SeafRepo *repo, const char *file_id,
     unsigned char enc_key[32], enc_iv[16];
     SeafileCrypt *crypt = NULL;
     SendfileData *data;
+    char *policy = "sandbox";
 
     file = seaf_fs_manager_get_seafile(seaf->fs_mgr,
                                        repo->store_id, repo->version, file_id);
@@ -575,6 +576,9 @@ do_file(evhtp_request_t *req, SeafRepo *repo, const char *file_id,
                              evhtp_header_new("Access-Control-Allow-Origin",
                                               "*", 1, 1));
 
+    evhtp_headers_add_header(req->headers_out,
+                             evhtp_header_new("Content-Security-Policy",
+                                              policy, 1, 1));
 
     type = parse_content_type(filename);
     if (type != NULL) {
@@ -922,6 +926,7 @@ do_file_range (evhtp_request_t *req, SeafRepo *repo, const char *file_id,
     SendFileRangeData *data = NULL;
     guint64 start;
     guint64 end;
+    char *policy = "sandbox";
 
     file = seaf_fs_manager_get_seafile(seaf->fs_mgr,
                                        repo->store_id, repo->version, file_id);
@@ -949,6 +954,10 @@ do_file_range (evhtp_request_t *req, SeafRepo *repo, const char *file_id,
     evhtp_headers_add_header (req->headers_out,
                               evhtp_header_new ("Accept-Ranges", "bytes", 0, 0));
 
+    evhtp_headers_add_header(req->headers_out,
+                             evhtp_header_new("Content-Security-Policy",
+                                              policy, 1, 1));
+
     char *content_type = NULL;
     char *type = parse_content_type (filename);
     if (type != NULL) {

server/fileserver-config.c

@@ -20,6 +20,13 @@ fileserver_config_get_integer(GKeyFile *config, char *key, GError **error)
     return g_key_file_get_integer (config, group, key, error);
 }
 
+int
+fileserver_config_get_int64(GKeyFile *config, char *key, GError **error)
+{
+    const char *group = get_group_name(config);
+    return g_key_file_get_int64 (config, group, key, error);
+}
+
 char *
 fileserver_config_get_string(GKeyFile *config, char *key, GError **error)
 {

server/fileserver-config.h

@@ -9,6 +9,9 @@ fileserver_config_get_integer(GKeyFile *config, char *key, GError **error);
 char *
 fileserver_config_get_string(GKeyFile *config, char *key, GError **error);
 
+int
+fileserver_config_get_int64(GKeyFile *config, char *key, GError **error);
+
 gboolean
 fileserver_config_get_boolean(GKeyFile *config, char *key, GError **error);
 

server/gc/fsck.c

@@ -176,6 +176,8 @@ fsck_check_dir_recursive (const char *id, const char *parent_dir, FsckData *fsck
             } else {
                 if (check_blocks (seaf_dent->id, fsck_data, &io_error) < 0) {
                     if (io_error) {
+                        seaf_message ("Failed to check blocks for repo[%.8s] file %s(%.8s).\n",
+                                      fsck_data->repo->id, path, seaf_dent->id);
                         g_free (path);
                         goto out;
                     }

server/http-server.c

@@ -41,6 +41,11 @@
 #define HOST "host"
 #define PORT "port"
 
+#define HTTP_TEMP_FILE_SCAN_INTERVAL  3600 /*1h*/
+#define HTTP_TEMP_FILE_DEFAULT_TTL 3600 * 24 * 3 /*3days*/
+#define HTTP_TEMP_FILE_TTL "http_temp_file_ttl"
+#define HTTP_SCAN_INTERVAL "http_temp_scan_interval"
+
 #define INIT_INFO "If you see this page, Seafile HTTP syncing component works."
 #define PROTO_VERSION "{\"version\": 2}"
 
@@ -2129,7 +2134,7 @@ filter_group_repos (GList *repos)
                              NULL);
         srepo_tmp = g_hash_table_lookup (table, repo_id);
         if (srepo_tmp) {
-            g_object_get (srepo, "permission", &permission_prev,
+            g_object_get (srepo_tmp, "permission", &permission_prev,
                           NULL);
             if (g_strcmp0 (permission, "rw") == 0 && g_strcmp0 (permission_prev, "r") == 0) {
                 g_object_unref (srepo_tmp);
@@ -2508,6 +2513,95 @@ seaf_http_server_new (struct _SeafileSession *session)
     return server;
 }
 
+gint64
+get_last_modify_time (const char *path)
+{
+    struct stat st;
+    if (stat (path, &st) < 0) {
+        return -1;
+    }
+
+    return st.st_mtime;
+}
+
+static gint64
+check_httptemp_dir_recursive (const char *parent_dir, gint64 expired_time)
+{
+    char *full_path;
+    const char *dname;
+    gint64 cur_time;
+    gint64 last_modify = -1;
+    GDir *dir = NULL;
+    gint64 file_num = 0;
+
+    dir = g_dir_open (parent_dir, 0, NULL);
+
+    while ((dname = g_dir_read_name(dir)) != NULL) {
+        full_path = g_build_path ("/", parent_dir, dname, NULL);
+
+        if (g_file_test (full_path, G_FILE_TEST_IS_DIR)) {
+            file_num += check_httptemp_dir_recursive (full_path, expired_time);
+        } else {
+            cur_time = time (NULL);
+            last_modify = get_last_modify_time (full_path);
+            if (last_modify == -1) {
+                g_free (full_path);
+                continue;
+            }
+            /*remove blokc cache from local*/
+            if (last_modify + expired_time <= cur_time) {
+                g_unlink (full_path);
+                file_num ++;
+            }
+        }
+        g_free (full_path);
+    }
+
+    g_dir_close (dir);
+
+    return file_num;
+}
+
+static int
+scan_httptemp_dir (const char *httptemp_dir, gint64 expired_time)
+{
+    return check_httptemp_dir_recursive (httptemp_dir, expired_time);
+}
+
+static void *
+cleanup_expired_httptemp_file (void *arg)
+{
+    GError *error = NULL;
+    HttpServerStruct *server = arg;
+    SeafileSession *session = server->seaf_session;
+    gint64 ttl = 0;
+    gint64 scan_interval = 0;
+    gint64 file_num = 0;
+
+    ttl = fileserver_config_get_int64 (session->config, HTTP_TEMP_FILE_TTL, &error);
+    if (error) {
+        ttl = HTTP_TEMP_FILE_DEFAULT_TTL;
+        g_clear_error (&error);
+    }
+
+    scan_interval = fileserver_config_get_int64 (session->config, HTTP_SCAN_INTERVAL, &error);
+    if (error) {
+        scan_interval = HTTP_TEMP_FILE_SCAN_INTERVAL;
+        g_clear_error (&error);
+    }
+
+    while (TRUE) {
+        sleep (scan_interval);
+        file_num = scan_httptemp_dir (server->http_temp_dir, ttl);
+        if (file_num) {
+            seaf_message ("Clean up %ld http temp files\n", file_num);
+            file_num = 0;
+        }
+    }
+
+    return NULL;
+}
+
 int
 seaf_http_server_start (HttpServerStruct *server)
 {
@@ -2516,6 +2610,13 @@ seaf_http_server_start (HttpServerStruct *server)
        return -1;
 
    pthread_detach (server->priv->thread_id);
+
+   pthread_t tid;
+   ret = pthread_create (&tid, NULL, cleanup_expired_httptemp_file, server);
+   if (ret != 0)
+       return -1;
+
+   pthread_detach (tid);
    return 0;
 }