From 3204e110730a4e4ba6cb3ccea3b2cd52132d7dca Mon Sep 17 00:00:00 2001 From: cuihaikuo Date: Thu, 3 Aug 2017 13:52:28 +0800 Subject: [PATCH 1/4] Add rpc for getting trash repo owner --- common/rpc-service.c | 11 +++++++++++ include/seafile-rpc.h | 3 +++ python/seafile/rpcclient.py | 4 ++++ python/seaserv/api.py | 3 +++ server/repo-mgr.c | 7 +++++++ server/repo-mgr.h | 3 +++ server/seaf-server.c | 5 +++++ 7 files changed, 36 insertions(+) diff --git a/common/rpc-service.c b/common/rpc-service.c index 373f5f1..b14a9d5 100644 --- a/common/rpc-service.c +++ b/common/rpc-service.c @@ -5099,4 +5099,15 @@ seafile_get_file_count_info_by_path (const char *repo_id, return ret; } +char * +seafile_get_trash_repo_owner (const char *repo_id, GError **error) +{ + if (!repo_id) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS, "Argument should not be null"); + return NULL; + } + + return seaf_get_trash_repo_owner (repo_id); +} + #endif /* SEAFILE_SERVER */ diff --git a/include/seafile-rpc.h b/include/seafile-rpc.h index a9c7ccc..916d1c7 100644 --- a/include/seafile-rpc.h +++ b/include/seafile-rpc.h @@ -1038,4 +1038,7 @@ GObject * seafile_get_file_count_info_by_path (const char *repo_id, const char *path, GError **error); + +char * +seafile_get_trash_repo_owner (const char *repo_id, GError **error); #endif diff --git a/python/seafile/rpcclient.py b/python/seafile/rpcclient.py index 0fee19e..320c078 100644 --- a/python/seafile/rpcclient.py +++ b/python/seafile/rpcclient.py @@ -964,3 +964,7 @@ class SeafServerThreadedRpcClient(ccnet.RpcClientBase): @searpc_func("object", ["string", "string"]) def get_file_count_info_by_path(repo_id, path): pass + + @searpc_func("string", ["string"]) + def get_trash_repo_owner(repo_id): + pass diff --git a/python/seaserv/api.py b/python/seaserv/api.py index ac52741..760c471 100644 --- a/python/seaserv/api.py +++ b/python/seaserv/api.py @@ -675,6 +675,9 @@ class SeafileAPI(object): def get_file_count_info_by_path(self, repo_id, path): return seafserv_threaded_rpc.get_file_count_info_by_path(repo_id, path) + def get_trash_repo_owner (self, repo_id): + return seafserv_threaded_rpc.get_trash_repo_owner(repo_id) + seafile_api = SeafileAPI() class CcnetAPI(object): diff --git a/server/repo-mgr.c b/server/repo-mgr.c index 9c8f2e6..4c61f75 100644 --- a/server/repo-mgr.c +++ b/server/repo-mgr.c @@ -3729,3 +3729,10 @@ seaf_get_total_storage (GError **error) return size; } + +char * +seaf_get_trash_repo_owner (const char *repo_id) +{ + char *sql = "SELECT owner_id from RepoTrash WHERE repo_id = ?"; + return seaf_db_statement_get_string(seaf->db, sql, 1, "string", repo_id); +} diff --git a/server/repo-mgr.h b/server/repo-mgr.h index 63c9953..f0f2f45 100644 --- a/server/repo-mgr.h +++ b/server/repo-mgr.h @@ -815,4 +815,7 @@ seaf_get_total_file_number (GError **error); gint64 seaf_get_total_storage (GError **error); + +char * +seaf_get_trash_repo_owner (const char *repo_id); #endif diff --git a/server/seaf-server.c b/server/seaf-server.c index 449ee8f..ae17f42 100644 --- a/server/seaf-server.c +++ b/server/seaf-server.c @@ -368,6 +368,11 @@ static void start_rpc_service (CcnetClient *client, int cloud_mode) "get_file_count_info_by_path", searpc_signature_object__string_string()); + searpc_server_register_function ("seafserv-threaded-rpcserver", + seafile_get_trash_repo_owner, + "get_trash_repo_owner", + searpc_signature_string__string()); + /* share repo to user */ searpc_server_register_function ("seafserv-threaded-rpcserver", seafile_add_share, From 25b599f2b5cf1ed74f148028904ce83ab369e6cc Mon Sep 17 00:00:00 2001 From: lian Date: Thu, 10 Aug 2017 15:01:04 +0800 Subject: [PATCH 2/4] add export-users.sh export users to '../users.xlsx' via `./export-users.sh` --- scripts/export-users.sh | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100755 scripts/export-users.sh diff --git a/scripts/export-users.sh b/scripts/export-users.sh new file mode 100755 index 0000000..6ee6829 --- /dev/null +++ b/scripts/export-users.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +SCRIPT=$(readlink -f "$0") +INSTALLPATH=$(dirname "${SCRIPT}") +TOPDIR=$(dirname "${INSTALLPATH}") +default_ccnet_conf_dir=${TOPDIR}/ccnet +central_config_dir=${TOPDIR}/conf + +function check_python_executable() { + if [[ "$PYTHON" != "" && -x $PYTHON ]]; then + return 0 + fi + + if which python2.7 2>/dev/null 1>&2; then + PYTHON=python2.7 + elif which python27 2>/dev/null 1>&2; then + PYTHON=python27 + else + echo + echo "Can't find a python executable of version 2.7 or above in PATH" + echo "Install python 2.7+ before continue." + echo "Or if you installed it in a non-standard PATH, set the PYTHON enviroment varirable to it" + echo + exit 1 + fi +} + +function read_seafile_data_dir () { + seafile_ini=${default_ccnet_conf_dir}/seafile.ini + if [[ ! -f ${seafile_ini} ]]; then + echo "${seafile_ini} not found. Now quit" + exit 1 + fi + seafile_data_dir=$(cat "${seafile_ini}") + if [[ ! -d ${seafile_data_dir} ]]; then + echo "Your seafile server data directory \"${seafile_data_dir}\" is invalid or doesn't exits." + echo "Please check it first, or create this directory yourself." + echo "" + exit 1; + fi +} + +check_python_executable; +read_seafile_data_dir; + +export CCNET_CONF_DIR=${default_ccnet_conf_dir} +export SEAFILE_CONF_DIR=${seafile_data_dir} +export SEAFILE_CENTRAL_CONF_DIR=${central_config_dir} +export PYTHONPATH=${INSTALLPATH}/seafile/lib/python2.6/site-packages:${INSTALLPATH}/seafile/lib64/python2.6/site-packages:${INSTALLPATH}/seahub/thirdpart:$PYTHONPATH +export PYTHONPATH=${INSTALLPATH}/seafile/lib/python2.7/site-packages:${INSTALLPATH}/seafile/lib64/python2.7/site-packages:$PYTHONPATH + +manage_py=${INSTALLPATH}/seahub/manage.py +exec "$PYTHON" "$manage_py" export_users From cf19311a0d8103ed38ff8d0a306ad8073fb93271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Ccuihaikuo=E2=80=9D?= <“haikuo93@126.com”> Date: Mon, 14 Aug 2017 10:59:08 +0800 Subject: [PATCH 3/4] Add rpc mkdir_with_parents() --- common/rpc-service.c | 24 ++++++++++++++++++++++++ include/seafile-rpc.h | 4 ++++ python/seafile/rpcclient.py | 5 +++++ python/seaserv/api.py | 3 +++ server/seaf-server.c | 5 +++++ 5 files changed, 41 insertions(+) diff --git a/common/rpc-service.c b/common/rpc-service.c index b14a9d5..f5691e5 100644 --- a/common/rpc-service.c +++ b/common/rpc-service.c @@ -5110,4 +5110,28 @@ seafile_get_trash_repo_owner (const char *repo_id, GError **error) return seaf_get_trash_repo_owner (repo_id); } +int +seafile_mkdir_with_parents (const char *repo_id, const char *parent_dir, + const char *new_dir_path, const char *user, + GError **error) +{ + if (!repo_id || !parent_dir || !new_dir_path || !user) { + g_set_error (error, 0, SEAF_ERR_BAD_ARGS, "Argument should not be null"); + return -1; + } + + if (!is_uuid_valid (repo_id)) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS, "Invalid repo id"); + return -1; + } + + if (seaf_repo_manager_mkdir_with_parents (seaf->repo_mgr, repo_id, + parent_dir, new_dir_path, + user, error) < 0) { + return -1; + } + + return 0; +} + #endif /* SEAFILE_SERVER */ diff --git a/include/seafile-rpc.h b/include/seafile-rpc.h index 916d1c7..13e1abd 100644 --- a/include/seafile-rpc.h +++ b/include/seafile-rpc.h @@ -730,6 +730,10 @@ int seafile_post_dir (const char *repo_id, const char *parent_dir, const char *new_dir_name, const char *user, GError **error); +int +seafile_mkdir_with_parents (const char *repo_id, const char *parent_dir, + const char *new_dir_path, const char *user, + GError **error); /** * delete a file/directory from the repo on server. diff --git a/python/seafile/rpcclient.py b/python/seafile/rpcclient.py index 320c078..3e519b2 100644 --- a/python/seafile/rpcclient.py +++ b/python/seafile/rpcclient.py @@ -968,3 +968,8 @@ class SeafServerThreadedRpcClient(ccnet.RpcClientBase): @searpc_func("string", ["string"]) def get_trash_repo_owner(repo_id): pass + + @searpc_func("int", ["string", "string", "string", "string"]) + def seafile_mkdir_with_parents (repo_id, parent_dir, relative_path, username): + pass + mkdir_with_parents = seafile_mkdir_with_parents diff --git a/python/seaserv/api.py b/python/seaserv/api.py index 760c471..84a82e8 100644 --- a/python/seaserv/api.py +++ b/python/seaserv/api.py @@ -678,6 +678,9 @@ class SeafileAPI(object): def get_trash_repo_owner (self, repo_id): return seafserv_threaded_rpc.get_trash_repo_owner(repo_id) + def mkdir_with_parents (self, repo_id, parent_dir, relative_path, username): + return seafserv_threaded_rpc.mkdir_with_parents(repo_id, parent_dir, relative_path, username) + seafile_api = SeafileAPI() class CcnetAPI(object): diff --git a/server/seaf-server.c b/server/seaf-server.c index ae17f42..c70cf6f 100644 --- a/server/seaf-server.c +++ b/server/seaf-server.c @@ -233,6 +233,11 @@ static void start_rpc_service (CcnetClient *client, int cloud_mode) "seafile_post_dir", searpc_signature_int__string_string_string_string()); + searpc_server_register_function ("seafserv-threaded-rpcserver", + seafile_mkdir_with_parents, + "seafile_mkdir_with_parents", + searpc_signature_int__string_string_string_string()); + searpc_server_register_function ("seafserv-threaded-rpcserver", seafile_del_file, "seafile_del_file", From a87db5b58e4cafac926b6d3b657b33211a6c03cb Mon Sep 17 00:00:00 2001 From: Jonathan Xu Date: Wed, 16 Aug 2017 17:44:32 +0800 Subject: [PATCH 4/4] Improved concurrent update handling. --- server/http-server.c | 17 +++++++++-------- server/repo-op.c | 22 +++++++++++++--------- server/size-sched.c | 5 +++-- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/server/http-server.c b/server/http-server.c index 7298c8a..2469cb0 100644 --- a/server/http-server.c +++ b/server/http-server.c @@ -831,7 +831,7 @@ fast_forward_or_merge (const char *repo_id, SeafCommit *base, SeafCommit *new_commit) { -#define MAX_RETRY_COUNT 3 +#define MAX_RETRY_COUNT 10 SeafRepo *repo = NULL; SeafCommit *current_head = NULL, *merged_commit = NULL; @@ -924,16 +924,17 @@ retry: seaf_commit_unref (merged_commit); merged_commit = NULL; - repo = seaf_repo_manager_get_repo (seaf->repo_mgr, repo_id); - if (!repo) { - seaf_warning ("Repo %s doesn't exist.\n", repo_id); - ret = -1; - goto out; - } - if (++retry_cnt <= MAX_RETRY_COUNT) { /* Sleep random time between 100 and 1000 millisecs. */ usleep (g_random_int_range(1, 11) * 100 * 1000); + + repo = seaf_repo_manager_get_repo (seaf->repo_mgr, repo_id); + if (!repo) { + seaf_warning ("Repo %s doesn't exist.\n", repo_id); + ret = -1; + goto out; + } + goto retry; } else { ret = -1; diff --git a/server/repo-op.c b/server/repo-op.c index 1f8918d..fbd5e65 100644 --- a/server/repo-op.c +++ b/server/repo-op.c @@ -442,7 +442,7 @@ gen_new_commit (const char *repo_id, char *new_commit_id, GError **error) { -#define MAX_RETRY_COUNT 3 +#define MAX_RETRY_COUNT 10 SeafRepo *repo = NULL; SeafCommit *new_commit = NULL, *current_head = NULL, *merged_commit = NULL; @@ -560,19 +560,23 @@ retry: seaf_commit_unref (merged_commit); merged_commit = NULL; - repo = seaf_repo_manager_get_repo (seaf->repo_mgr, repo_id); - if (!repo) { - seaf_warning ("Repo %s doesn't exist.\n", repo_id); - g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, "Invalid repo"); - ret = -1; - goto out; - } - if (++retry_cnt <= MAX_RETRY_COUNT) { /* Sleep random time between 100 and 1000 millisecs. */ usleep (g_random_int_range(1, 11) * 100 * 1000); + + repo = seaf_repo_manager_get_repo (seaf->repo_mgr, repo_id); + if (!repo) { + seaf_warning ("Repo %s doesn't exist.\n", repo_id); + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, "Invalid repo"); + ret = -1; + goto out; + } + goto retry; } else { + seaf_warning ("Too many retries for concurrent update on repo %s. Stop retrying.\n", + repo_id); + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, "Concurrent update"); ret = -1; goto out; } diff --git a/server/size-sched.c b/server/size-sched.c index f6f21e4..69d547a 100644 --- a/server/size-sched.c +++ b/server/size-sched.c @@ -6,6 +6,7 @@ #include "seafile-session.h" #include "size-sched.h" +#define DEBUG_FLAG SEAFILE_DEBUG_OTHER #include "log.h" typedef struct SizeSchedulerPriv { @@ -169,8 +170,8 @@ set_repo_size (SeafDB *db, } } else { if (strcmp (old_head_id, cached_head_id) != 0) { - g_message ("[size sched] Size update conflict for repo %s, rollback.\n", - repo_id); + seaf_debug ("[size sched] Size update conflict for repo %s, rollback.\n", + repo_id); ret = SET_SIZE_CONFLICT; goto rollback; }