From ffd92675bea3af8b0b600fbcc93f28273c6f1686 Mon Sep 17 00:00:00 2001 From: cuihaikuo Date: Thu, 29 Jun 2017 14:15:18 +0800 Subject: [PATCH 1/9] Create only one commit when move or copy multiple files --- common/rpc-service.c | 35 +- server/repo-mgr.h | 27 ++ server/repo-op.c | 800 ++++++++++++++++++++++++++++++++++++++----- 3 files changed, 774 insertions(+), 88 deletions(-) diff --git a/common/rpc-service.c b/common/rpc-service.c index a184eb2..1c37116 100644 --- a/common/rpc-service.c +++ b/common/rpc-service.c @@ -3716,11 +3716,19 @@ seafile_copy_file (const char *src_repo_id, rsrc_dir = format_dir_path (norm_src_dir); rdst_dir = format_dir_path (norm_dst_dir); - ret = (GObject *)seaf_repo_manager_copy_file (seaf->repo_mgr, - src_repo_id, rsrc_dir, norm_src_filename, - dst_repo_id, rdst_dir, norm_dst_filename, - user, need_progress, synchronous, - error); + if (strchr(norm_src_filename, '\t') && strchr(norm_dst_filename, '\t')) + ret = (GObject *)seaf_repo_manager_copy_multiple_files (seaf->repo_mgr, + src_repo_id, rsrc_dir, norm_src_filename, + dst_repo_id, rdst_dir, norm_dst_filename, + user, need_progress, synchronous, + error); + + else + ret = (GObject *)seaf_repo_manager_copy_file (seaf->repo_mgr, + src_repo_id, rsrc_dir, norm_src_filename, + dst_repo_id, rdst_dir, norm_dst_filename, + user, need_progress, synchronous, + error); out: g_free (norm_src_dir); @@ -3793,11 +3801,18 @@ seafile_move_file (const char *src_repo_id, rsrc_dir = format_dir_path (norm_src_dir); rdst_dir = format_dir_path (norm_dst_dir); - ret = (GObject *)seaf_repo_manager_move_file (seaf->repo_mgr, - src_repo_id, rsrc_dir, norm_src_filename, - dst_repo_id, rdst_dir, norm_dst_filename, - replace, user, need_progress, synchronous, - error); + if (strchr(norm_src_filename, '\t') && strchr(norm_dst_filename, '\t')) + ret = (GObject *)seaf_repo_manager_move_multiple_files (seaf->repo_mgr, + src_repo_id, rsrc_dir, norm_src_filename, + dst_repo_id, rdst_dir, norm_dst_filename, + replace, user, need_progress, synchronous, + error); + else + ret = (GObject *)seaf_repo_manager_move_file (seaf->repo_mgr, + src_repo_id, rsrc_dir, norm_src_filename, + dst_repo_id, rdst_dir, norm_dst_filename, + replace, user, need_progress, synchronous, + error); out: g_free (norm_src_dir); diff --git a/server/repo-mgr.h b/server/repo-mgr.h index 90b7aa9..63c9953 100644 --- a/server/repo-mgr.h +++ b/server/repo-mgr.h @@ -425,6 +425,19 @@ seaf_repo_manager_copy_file (SeafRepoManager *mgr, int synchronous, GError **error); +SeafileCopyResult * +seaf_repo_manager_copy_multiple_files (SeafRepoManager *mgr, + const char *src_repo_id, + const char *src_dir, + const char *src_filenames, + const char *dst_repo_id, + const char *dst_dir, + const char *dst_filenames, + const char *user, + int need_progress, + int synchronous, + GError **error); + SeafileCopyResult * seaf_repo_manager_move_file (SeafRepoManager *mgr, const char *src_repo_id, @@ -439,6 +452,20 @@ seaf_repo_manager_move_file (SeafRepoManager *mgr, int synchronous, GError **error); +SeafileCopyResult * +seaf_repo_manager_move_multiple_files (SeafRepoManager *mgr, + const char *src_repo_id, + const char *src_dir, + const char *src_filenames, + const char *dst_repo_id, + const char *dst_dir, + const char *dst_filenames, + int replace, + const char *user, + int need_progress, + int synchronous, + GError **error); + int seaf_repo_manager_rename_file (SeafRepoManager *mgr, const char *repo_id, diff --git a/server/repo-op.c b/server/repo-op.c index 0d67e1a..2785444 100644 --- a/server/repo-op.c +++ b/server/repo-op.c @@ -35,6 +35,9 @@ gboolean should_ignore_file(const char *filename, void *data); +static gboolean +is_virtual_repo_and_origin (SeafRepo *repo1, SeafRepo *repo2); + /* * Repo operations. */ @@ -1607,7 +1610,7 @@ seaf_repo_manager_del_file (SeafRepoManager *mgr, } /* Commit. */ - if (file_num) { + if (file_num > 1) { snprintf(buf, SEAF_PATH_MAX, "Deleted \"%s\" and %d more files", desc_file, file_num - 1); } else if (S_ISDIR(mode)) { @@ -1704,7 +1707,8 @@ out: static int put_dirent_and_commit (SeafRepo *repo, const char *path, - SeafDirent *dent, + SeafDirent *dents[], + int n_dents, int replace, const char *user, GError **error) @@ -1712,27 +1716,52 @@ put_dirent_and_commit (SeafRepo *repo, SeafCommit *head_commit = NULL; char *root_id = NULL; char buf[SEAF_PATH_MAX]; - int ret = 0; + int ret = 0, i = 0; GET_COMMIT_OR_FAIL(head_commit, repo->id, repo->version, repo->head->commit_id); + + root_id = head_commit->root_id; + + GList *filenames = NULL; + GList *id_list = NULL; + GList *size_list = NULL; + GList *name_list = NULL; + for (i = 0; i < n_dents; i++) { + filenames = g_list_append (filenames, dents[i]->name); + id_list = g_list_append (id_list, dents[i]->id); + size_list = g_list_append (size_list, &(dents[i]->size)); + } + if (*path == '/') + path = path + 1; + root_id = post_multi_files_recursive (repo, root_id, path, filenames, + id_list, size_list, user, + replace, &name_list); + g_list_free (filenames); + g_list_free (id_list); + g_list_free (size_list); + g_list_free_full (name_list, (GDestroyNotify)g_free); - root_id = do_post_file_replace (repo, - head_commit->root_id, path, - replace, dent); if (!root_id) { - seaf_warning ("[cp file] Failed to cp file %s to %s in repo %s.\n", - dent->name, path, repo->id); + if (n_dents > 1) + seaf_warning ("[cp file] Failed to cp %s and other %d files to %s in repo %s.\n", + dents[0]->name, n_dents - 1, path, repo->id); + else + seaf_warning ("[cp file] Failed to cp %s to %s in repo %s.\n", + dents[0]->name, path, repo->id); g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, - "Failed to cp file"); + "Failed to cp file"); ret = -1; goto out; } /* Commit. */ - if (S_ISDIR(dent->mode)) { - snprintf(buf, sizeof(buf), "Added directory \"%s\"", dent->name); + if (n_dents > 1) { + snprintf(buf, sizeof(buf), "Added \"%s\" and %d more files", + dents[0]->name, n_dents - 1); + } else if (S_ISDIR(dents[0]->mode)) { + snprintf(buf, sizeof(buf), "Added directory \"%s\"", dents[0]->name); } else { - snprintf(buf, sizeof(buf), "Added \"%s\"", dent->name); + snprintf(buf, sizeof(buf), "Added \"%s\"", dents[0]->name); } if (gen_new_commit (repo->id, head_commit, root_id, @@ -1883,7 +1912,12 @@ cross_repo_copy (const char *src_repo_id, { SeafRepo *src_repo = NULL, *dst_repo = NULL; SeafDirent *src_dent = NULL, *dst_dent = NULL; - int ret = 0; + SeafDirent **src_dents = NULL, **dst_dents = NULL; + char **src_names = NULL, **dst_names = NULL; + char *new_id = NULL; + guint64 new_size = 0; + int ret = 0, i = 0; + int file_num = 1; src_repo = seaf_repo_manager_get_repo (seaf->repo_mgr, src_repo_id); if (!src_repo) { @@ -1897,30 +1931,70 @@ cross_repo_copy (const char *src_repo_id, goto out; } - src_dent = get_dirent_by_path (src_repo, NULL, - src_path, src_filename, NULL); - if (!src_dent) { - ret = -1; - goto out; + /* get src dirents */ + if (strchr(src_filename, '\t') && strchr(dst_filename, '\t')) { + src_names = g_strsplit (src_filename, "\t", -1); + dst_names = g_strsplit (dst_filename, "\t", -1); + file_num = g_strv_length (src_names); + + src_dents = g_new0 (SeafDirent *, file_num); + dst_dents = g_new0 (SeafDirent *, file_num); + + for (i = 0; i < file_num; i++) { + if (strcmp(src_names[i], "") == 0) { + ret = -1; + seaf_warning ("[copy files] Bad args: Empty src_filename.\n"); + goto out; + } + src_dents[i] = get_dirent_by_path (src_repo, NULL, + src_path, src_names[i], NULL); + if (!src_dents[i]) { + ret = -1; + seaf_warning ("[copy files] File %s not Found.\n", src_names[i]); + goto out; + } + } + + for (i = 0; i < file_num; i++) { + new_id = copy_recursive (src_repo, dst_repo, + src_dents[i]->id, src_dents[i]->mode, modifier, task, + &new_size); + if (!new_id) { + ret = -1; + seaf_warning ("[copy files] Failed to copy file %s.\n", src_dents[i]->name); + goto out; + } + dst_dents[i] = seaf_dirent_new (dir_version_from_repo_version(dst_repo->version), + new_id, src_dents[i]->mode, dst_names[i], + src_dents[i]->mtime, modifier, new_size); + g_free (new_id); + } + + } else { + src_dent = get_dirent_by_path (src_repo, NULL, + src_path, src_filename, NULL); + if (!src_dent) { + ret = -1; + goto out; + } + new_id = copy_recursive (src_repo, dst_repo, + src_dent->id, src_dent->mode, modifier, task, + &new_size); + if (!new_id) { + ret = -1; + goto out; + } + + dst_dent = seaf_dirent_new (dir_version_from_repo_version(dst_repo->version), + new_id, src_dent->mode, dst_filename, + src_dent->mtime, modifier, new_size); + g_free (new_id); + } - - guint64 new_size = 0; - char *new_id = copy_recursive (src_repo, dst_repo, - src_dent->id, src_dent->mode, modifier, task, - &new_size); - if (!new_id) { - ret = -1; - goto out; - } - - dst_dent = seaf_dirent_new (dir_version_from_repo_version(dst_repo->version), - new_id, src_dent->mode, dst_filename, - src_dent->mtime, modifier, new_size); - g_free (new_id); - if (put_dirent_and_commit (dst_repo, dst_path, - dst_dent, + file_num > 1 ? dst_dents : &dst_dent, + file_num, replace, modifier, NULL) < 0) { @@ -1942,6 +2016,16 @@ out: seaf_dirent_free(src_dent); if (dst_dent) seaf_dirent_free(dst_dent); + if (file_num > 1) { + for (i = 0; i < file_num; i++) { + seaf_dirent_free(src_dents[i]); + seaf_dirent_free(dst_dents[i]); + } + g_free (src_dents); + g_free (dst_dents); + g_strfreev(src_names); + g_strfreev(dst_names); + } if (ret == 0) { update_repo_size (dst_repo_id); @@ -1967,7 +2051,7 @@ is_virtual_repo_and_origin (SeafRepo *repo1, SeafRepo *repo2) static gboolean check_file_count_and_size (SeafRepo *repo, SeafDirent *dent, gint64 total_files, - GError **error) + gint64 *multi_file_size, GError **error) { if (seaf->copy_mgr->max_files > 0 && total_files > seaf->copy_mgr->max_files) { @@ -1998,6 +2082,14 @@ check_file_count_and_size (SeafRepo *repo, SeafDirent *dent, gint64 total_files, repo->store_id, dent->id); return FALSE; } + if (multi_file_size) { + *multi_file_size += size; + if (*multi_file_size > seaf->copy_mgr->max_size) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "Folder or file size is too large"); + return FALSE; + } + } if (size > seaf->copy_mgr->max_size) { g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, @@ -2059,13 +2151,12 @@ seaf_repo_manager_copy_file (SeafRepoManager *mgr, src_canon_path = get_canonical_path (src_path); dst_canon_path = get_canonical_path (dst_path); - /* first check whether a file with file_name already exists in destination dir */ GET_COMMIT_OR_FAIL(dst_head_commit, dst_repo->id, dst_repo->version, dst_repo->head->commit_id); - FAIL_IF_FILE_EXISTS(dst_repo->store_id, dst_repo->version, - dst_head_commit->root_id, dst_canon_path, dst_filename, NULL); + /* FAIL_IF_FILE_EXISTS(dst_repo->store_id, dst_repo->version, + dst_head_commit->root_id, dst_canon_path, dst_filename, NULL); */ /* get src dirent */ src_dent = get_dirent_by_path (src_repo, NULL, @@ -2087,7 +2178,8 @@ seaf_repo_manager_copy_file (SeafRepoManager *mgr, if (put_dirent_and_commit (dst_repo, dst_canon_path, - dst_dent, + &dst_dent, + 1, 0, user, error) < 0) { @@ -2118,7 +2210,7 @@ seaf_repo_manager_copy_file (SeafRepoManager *mgr, goto out; } - if (!check_file_count_and_size (src_repo, src_dent, total_files, error)) { + if (!check_file_count_and_size (src_repo, src_dent, total_files, NULL, error)) { ret = -1; goto out; } @@ -2143,7 +2235,7 @@ seaf_repo_manager_copy_file (SeafRepoManager *mgr, goto out; } } else { - /* Synchronous for cross-repo move */ + /* Synchronous for cross-repo copy */ if (cross_repo_copy (src_repo_id, src_canon_path, src_filename, @@ -2185,10 +2277,231 @@ out: return res; } +SeafileCopyResult * +seaf_repo_manager_copy_multiple_files (SeafRepoManager *mgr, + const char *src_repo_id, + const char *src_path, + const char *src_filenames, + const char *dst_repo_id, + const char *dst_path, + const char *dst_filenames, + const char *user, + int need_progress, + int synchronous, + GError **error) +{ + SeafRepo *src_repo = NULL, *dst_repo = NULL; + SeafDirent **src_dents = NULL, **dst_dents = NULL; + char *src_canon_path = NULL, *dst_canon_path = NULL; + SeafCommit *dst_head_commit = NULL; + int i = 0, ret = 0; + int file_num = 1; + gint64 *file_sizes = NULL; + gboolean background = FALSE; + char *task_id = NULL; + char **src_names = NULL, **dst_names = NULL; + SeafileCopyResult *res = NULL; + + GET_REPO_OR_FAIL(src_repo, src_repo_id); + + if (strcmp(src_repo_id, dst_repo_id) != 0) { + GET_REPO_OR_FAIL(dst_repo, dst_repo_id); + + if (src_repo->encrypted || dst_repo->encrypted) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS, + "Can't copy files between encrypted repo(s)"); + ret = -1; + goto out; + } + + } else { + seaf_repo_ref (src_repo); + dst_repo = src_repo; + } + + src_canon_path = get_canonical_path (src_path); + dst_canon_path = get_canonical_path (dst_path); + + GET_COMMIT_OR_FAIL(dst_head_commit, + dst_repo->id, dst_repo->version, + dst_repo->head->commit_id); + /*FAIL_IF_FILE_EXISTS(dst_repo->store_id, dst_repo->version, + dst_head_commit->root_id, dst_canon_path, dst_filename, NULL);*/ + + /* get src dirents */ + if (strchr(src_filenames, '\t') && strchr(dst_filenames, '\t')) { + src_names = g_strsplit (src_filenames, "\t", -1); + dst_names = g_strsplit (dst_filenames, "\t", -1); + file_num = g_strv_length (src_names); + int dst_file_num = g_strv_length (dst_names); + if (dst_file_num != file_num) { + ret = -1; + seaf_warning ("[copy files] Bad args.\n"); + goto out; + } + + src_dents = g_new0 (SeafDirent *, file_num); + file_sizes = g_new0 (gint64, file_num); + + for (i = 0; i < file_num; i++) { + if (strcmp(src_names[i], "") == 0) { + ret = -1; + seaf_warning ("[copy files] Bad args: Empty src_filenames.\n"); + goto out; + } + src_dents[i] = get_dirent_by_path (src_repo, NULL, + src_canon_path, src_names[i], error); + if (!src_dents[i]) { + ret = -1; + seaf_warning ("[copy files] File %s not Found.\n", src_names[i]); + goto out; + } + file_sizes[i] = (src_dents[i]->version > 0) ? src_dents[i]->size : -1; + } + } else { + ret = -1; + seaf_warning ("[copy files] Bad args: Split filenames with '\\t'.\n"); + goto out; + } + + /* copy file within the same repo */ + if (src_repo == dst_repo || + is_virtual_repo_and_origin (src_repo, dst_repo)) { + dst_dents = g_new0 (SeafDirent *, file_num); + for (i = 0; i < file_num; i++) { + if (strcmp(dst_names[i], "") == 0) { + ret = -1; + seaf_warning ("[copy files] Bad args: Empty dst_filenames.\n"); + goto out; + } + /* duplicate src dirents with new names */ + dst_dents[i] = seaf_dirent_new (dir_version_from_repo_version (dst_repo->version), + src_dents[i]->id, src_dents[i]->mode, dst_names[i], + src_dents[i]->mtime, user, file_sizes[i]); + } + if (put_dirent_and_commit (dst_repo, + dst_canon_path, + dst_dents, + file_num, + 0, + user, + error) < 0) { + if (!error) + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "failed to put dirents"); + ret = -1; + goto out; + } + + seaf_repo_manager_merge_virtual_repo (mgr, src_repo_id, NULL); + + update_repo_size (dst_repo_id); + } else { + /* copy between different repos */ + if (!synchronous) { + background = TRUE; + + gint64 total_files = -1; + gint64 total_files_all = 0; + gint64 total_size_all = 0; + + for (i = 0; i < file_num; i++) { + if (S_ISDIR(src_dents[i]->mode)) + total_files = seaf_fs_manager_count_fs_files (seaf->fs_mgr, + src_repo->store_id, + src_repo->version, + src_dents[i]->id); + else + total_files = 1; + if (total_files < 0) { + seaf_warning ("Failed to get file count.\n"); + ret = -1; + goto out; + } + total_files_all += total_files; + if (!check_file_count_and_size (src_repo, src_dents[i], total_files_all, + &total_size_all, error)) { + ret = -1; + goto out; + } + } + task_id = seaf_copy_manager_add_task (seaf->copy_mgr, + src_repo_id, + src_canon_path, + src_filenames, + dst_repo_id, + dst_canon_path, + dst_filenames, + 0, + user, + total_files_all, + cross_repo_copy, + need_progress); + if (need_progress && !task_id) { + seaf_warning ("Failed to start copy task.\n"); + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "failed to start copy task"); + ret = -1; + goto out; + } + } else { + /* Synchronous for cross-repo copy */ + if (cross_repo_copy (src_repo_id, + src_canon_path, + src_filenames, + dst_repo_id, + dst_canon_path, + dst_filenames, + 0, + user, + NULL) < 0) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "Failed to move"); + ret = -1; + goto out; + } + } // Synchronous copy + } //else diffrent repo + +out: + if (src_repo) seaf_repo_unref (src_repo); + if (dst_repo) seaf_repo_unref (dst_repo); + + if (dst_head_commit) seaf_commit_unref(dst_head_commit); + + if (src_canon_path) g_free (src_canon_path); + if (dst_canon_path) g_free (dst_canon_path); + + if (src_names) + g_strfreev (src_names); + if (dst_names) + g_strfreev (dst_names); + if (file_sizes) + g_free (file_sizes); + if (src_dents) { + for (i = 0; i < file_num; i++) + seaf_dirent_free (src_dents[i]); + g_free (src_dents); + } + if (dst_dents) { + for (i = 0; i < file_num; i++) + seaf_dirent_free (dst_dents[i]); + g_free (dst_dents); + } + if (ret == 0) { + res = seafile_copy_result_new (); + g_object_set (res, "background", background, "task_id", task_id, NULL); + g_free (task_id); + } + + return res; +} + static int move_file_same_repo (const char *repo_id, - const char *src_path, SeafDirent *src_dent, - const char *dst_path, SeafDirent *dst_dent, + const char *src_path, SeafDirent *src_dents[], + const char *dst_path, SeafDirent *dst_dents[], + int file_num, int replace, const char *user, GError **error) @@ -2197,21 +2510,45 @@ move_file_same_repo (const char *repo_id, SeafCommit *head_commit = NULL; char *root_id_after_put = NULL, *root_id = NULL; char buf[SEAF_PATH_MAX]; - int ret = 0; + int ret = 0, i = 0; + GString *filenames_str = NULL; GET_REPO_OR_FAIL(repo, repo_id); GET_COMMIT_OR_FAIL(head_commit, repo->id, repo->version, repo->head->commit_id); - - root_id_after_put = do_post_file_replace (repo, - head_commit->root_id, dst_path, - replace, dst_dent); + + filenames_str = g_string_new (""); + root_id_after_put = head_commit->root_id; + + GList *filenames = NULL; + GList *id_list = NULL; + GList *size_list = NULL; + GList *name_list = NULL; + for (i = 0; i < file_num; i++) { + filenames = g_list_append (filenames, dst_dents[i]->name); + id_list = g_list_append (id_list, dst_dents[i]->id); + size_list = g_list_append (size_list, &(dst_dents[i]->size)); + g_string_append_printf (filenames_str, "%s", src_dents[i]->name); + if ((i + 1) < file_num) + g_string_append_printf (filenames_str, "\t"); + } + if (*dst_path == '/') + dst_path = dst_path + 1; + + root_id_after_put = post_multi_files_recursive (repo, head_commit->root_id, dst_path, filenames, + id_list, size_list, user, + replace, &name_list); + g_list_free (filenames); + g_list_free (id_list); + g_list_free (size_list); + g_list_free_full (name_list, (GDestroyNotify)g_free); + if (!root_id_after_put) { g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS, "move file failed"); ret = -1; goto out; } + root_id = do_del_file (repo, root_id_after_put, src_path, filenames_str->str); - root_id = do_del_file (repo, root_id_after_put, src_path, src_dent->name); if (!root_id) { g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS, "move file failed"); ret = -1; @@ -2219,10 +2556,13 @@ move_file_same_repo (const char *repo_id, } /* Commit. */ - if (S_ISDIR(src_dent->mode)) { - snprintf(buf, SEAF_PATH_MAX, "Moved directory \"%s\"", src_dent->name); + if (file_num > 1) { + snprintf(buf, SEAF_PATH_MAX, "Moved \"%s\" and %d more files", + src_dents[0]->name,file_num - 1); + } else if (S_ISDIR(src_dents[0]->mode)) { + snprintf(buf, SEAF_PATH_MAX, "Moved directory \"%s\"", src_dents[0]->name); } else { - snprintf(buf, SEAF_PATH_MAX, "Moved \"%s\"", src_dent->name); + snprintf(buf, SEAF_PATH_MAX, "Moved \"%s\"", src_dents[0]->name); } if (gen_new_commit (repo_id, head_commit, root_id, @@ -2234,6 +2574,9 @@ out: seaf_repo_unref (repo); if (head_commit) seaf_commit_unref (head_commit); + if (filenames_str) + g_string_free (filenames_str, TRUE); + g_free (root_id_after_put); g_free (root_id); @@ -2253,7 +2596,12 @@ cross_repo_move (const char *src_repo_id, { SeafRepo *src_repo = NULL, *dst_repo = NULL; SeafDirent *src_dent = NULL, *dst_dent = NULL; - int ret = 0; + SeafDirent **src_dents = NULL, **dst_dents = NULL; + char **src_names = NULL, **dst_names = NULL; + char *new_id = NULL; + guint64 new_size = 0; + int ret = 0, i = 0; + int file_num = 1; src_repo = seaf_repo_manager_get_repo (seaf->repo_mgr, src_repo_id); if (!src_repo) { @@ -2267,30 +2615,70 @@ cross_repo_move (const char *src_repo_id, goto out; } - src_dent = get_dirent_by_path (src_repo, NULL, - src_path, src_filename, NULL); - if (!src_dent) { - ret = -1; - goto out; - } + /* get src dirents */ + if (strchr(src_filename, '\t') && strchr(dst_filename, '\t')) { + src_names = g_strsplit (src_filename, "\t", -1); + dst_names = g_strsplit (dst_filename, "\t", -1); + file_num = g_strv_length (src_names); - guint64 new_size = 0; - char *new_id = copy_recursive (src_repo, dst_repo, - src_dent->id, src_dent->mode, modifier, task, - &new_size); - if (!new_id) { - ret = -1; - goto out; - } + src_dents = g_new0 (SeafDirent *, file_num); + dst_dents = g_new0 (SeafDirent *, file_num); - dst_dent = seaf_dirent_new (dir_version_from_repo_version(dst_repo->version), - new_id, src_dent->mode, dst_filename, - src_dent->mtime, modifier, new_size); - g_free (new_id); + for (i = 0; i < file_num; i++) { + if (strcmp(src_names[i], "") == 0) { + ret = -1; + seaf_warning ("[move files] Bad args: Empty src_filename.\n"); + goto out; + } + src_dents[i] = get_dirent_by_path (src_repo, NULL, + src_path, src_names[i], NULL); + if (!src_dents[i]) { + ret = -1; + seaf_warning ("[move files] File %s not Found.\n", src_names[i]); + goto out; + } + } + + for (i = 0; i < file_num; i++) { + new_id = copy_recursive (src_repo, dst_repo, + src_dents[i]->id, src_dents[i]->mode, modifier, task, + &new_size); + if (!new_id) { + ret = -1; + seaf_warning ("[move files] Failed to copy file %s.\n", src_dents[i]->name); + goto out; + } + dst_dents[i] = seaf_dirent_new (dir_version_from_repo_version(dst_repo->version), + new_id, src_dents[i]->mode, dst_names[i], + src_dents[i]->mtime, modifier, new_size); + g_free (new_id); + } + } else { + src_dent = get_dirent_by_path (src_repo, NULL, + src_path, src_filename, NULL); + if (!src_dent) { + ret = -1; + goto out; + } + new_id = copy_recursive (src_repo, dst_repo, + src_dent->id, src_dent->mode, modifier, task, + &new_size); + if (!new_id) { + ret = -1; + goto out; + } + + dst_dent = seaf_dirent_new (dir_version_from_repo_version(dst_repo->version), + new_id, src_dent->mode, dst_filename, + src_dent->mtime, modifier, new_size); + g_free (new_id); + + } if (put_dirent_and_commit (dst_repo, dst_path, - dst_dent, + file_num > 1 ? dst_dents : &dst_dent, + file_num, replace, modifier, NULL) < 0) { @@ -2320,6 +2708,16 @@ out: seaf_dirent_free(src_dent); if (dst_dent) seaf_dirent_free(dst_dent); + if (file_num > 1) { + for (i = 0; i < file_num; i++) { + seaf_dirent_free(src_dents[i]); + seaf_dirent_free(dst_dents[i]); + } + g_free (src_dents); + g_free (dst_dents); + g_strfreev(src_names); + g_strfreev(dst_names); + } if (ret == 0) { update_repo_size (dst_repo_id); @@ -2387,7 +2785,6 @@ seaf_repo_manager_move_file (SeafRepoManager *mgr, ret = -1; goto out; } - gint64 file_size = (src_dent->version > 0) ? src_dent->size : -1; if (src_repo == dst_repo) { @@ -2398,9 +2795,9 @@ seaf_repo_manager_move_file (SeafRepoManager *mgr, /* move file within the same repo */ if (move_file_same_repo (src_repo_id, - src_canon_path, src_dent, - dst_canon_path, dst_dent, - replace, user, error) < 0) { + src_canon_path, &src_dent, + dst_canon_path, &dst_dent, + 1, replace, user, error) < 0) { ret = -1; goto out; } @@ -2421,7 +2818,8 @@ seaf_repo_manager_move_file (SeafRepoManager *mgr, /* add this dirent to dst repo */ if (put_dirent_and_commit (dst_repo, dst_canon_path, - dst_dent, + &dst_dent, + 1, replace, user, error) < 0) { @@ -2460,7 +2858,7 @@ seaf_repo_manager_move_file (SeafRepoManager *mgr, goto out; } - if (!check_file_count_and_size (src_repo, src_dent, total_files, error)) { + if (!check_file_count_and_size (src_repo, src_dent, total_files, NULL, error)) { ret = -1; goto out; } @@ -2524,6 +2922,252 @@ out: return res; } +SeafileCopyResult * +seaf_repo_manager_move_multiple_files (SeafRepoManager *mgr, + const char *src_repo_id, + const char *src_path, + const char *src_filenames, + const char *dst_repo_id, + const char *dst_path, + const char *dst_filenames, + int replace, + const char *user, + int need_progress, + int synchronous, + GError **error) +{ + SeafRepo *src_repo = NULL, *dst_repo = NULL; + SeafDirent **src_dents = NULL, **dst_dents = NULL; + char *src_canon_path = NULL, *dst_canon_path = NULL; + SeafCommit *dst_head_commit = NULL; + int i = 0, ret = 0; + int file_num = 1; + gint64 *file_sizes = NULL; + gboolean background = FALSE; + char *task_id = NULL; + char **src_names = NULL, **dst_names = NULL; + SeafileCopyResult *res = NULL; + + GET_REPO_OR_FAIL(src_repo, src_repo_id); + + if (strcmp(src_repo_id, dst_repo_id) != 0) { + GET_REPO_OR_FAIL(dst_repo, dst_repo_id); + + if (src_repo->encrypted || dst_repo->encrypted) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS, + "Can't copy files between encrypted repo(s)"); + ret = -1; + goto out; + } + + } else { + seaf_repo_ref (src_repo); + dst_repo = src_repo; + } + + src_canon_path = get_canonical_path (src_path); + dst_canon_path = get_canonical_path (dst_path); + + GET_COMMIT_OR_FAIL(dst_head_commit, + dst_repo->id, dst_repo->version, + dst_repo->head->commit_id); + /*FAIL_IF_FILE_EXISTS(dst_repo->store_id, dst_repo->version, + dst_head_commit->root_id, dst_canon_path, dst_filename, NULL);*/ + + /* get src dirents */ + if (strchr(src_filenames, '\t') && strchr(dst_filenames, '\t')) { + src_names = g_strsplit (src_filenames, "\t", -1); + dst_names = g_strsplit (dst_filenames, "\t", -1); + file_num = g_strv_length (src_names); + int dst_file_num = g_strv_length (dst_names); + if (dst_file_num != file_num) { + ret = -1; + seaf_warning ("[move files] Bad args.\n"); + goto out; + } + + src_dents = g_new0 (SeafDirent *, file_num); + file_sizes = g_new0 (gint64, file_num); + + for (i = 0; i < file_num; i++) { + if (strcmp(src_names[i], "") == 0) { + ret = -1; + seaf_warning ("[move files] Bad args: Empty src_filenames.\n"); + goto out; + } + src_dents[i] = get_dirent_by_path (src_repo, NULL, + src_canon_path, src_names[i], error); + if (!src_dents[i]) { + ret = -1; + seaf_warning ("[move files] File %s not Found.\n", src_names[i]); + goto out; + } + file_sizes[i] = (src_dents[i]->version > 0) ? src_dents[i]->size : -1; + + } + } else { + ret = -1; + seaf_warning ("[move files] Bad args: Split filenames with '\\t'.\n"); + goto out; + } + + /* move file within the same repo */ + if (src_repo == dst_repo) { + dst_dents = g_new0 (SeafDirent *, file_num); + for (i = 0; i < file_num; i++) { + if (strcmp(dst_names[i], "") == 0) { + ret = -1; + seaf_warning ("[move files] Bad args: Empty dst_filenames.\n"); + goto out; + } + /* duplicate src dirents with new names */ + dst_dents[i] = seaf_dirent_new (dir_version_from_repo_version (dst_repo->version), + src_dents[i]->id, src_dents[i]->mode, dst_names[i], + src_dents[i]->mtime, user, file_sizes[i]); + } + /* move file within the same repo */ + if (move_file_same_repo (src_repo_id, + src_canon_path, src_dents, + dst_canon_path, dst_dents, + file_num, replace, user, error) < 0) { + ret = -1; + goto out; + } + seaf_repo_manager_cleanup_virtual_repos (mgr, src_repo_id); + seaf_repo_manager_merge_virtual_repo (mgr, src_repo_id, NULL); + + update_repo_size (dst_repo_id); + } else { + /* move between virtual and origin repos */ + if (is_virtual_repo_and_origin (src_repo, dst_repo)) { + dst_dents = g_new0 (SeafDirent *, file_num); + for (i = 0; i < file_num; i++) { + dst_dents[i] = seaf_dirent_new (dir_version_from_repo_version(dst_repo->version), + src_dents[i]->id, src_dents[i]->mode, dst_names[i], + src_dents[i]->mtime, user, src_dents[i]->size); + } + if (put_dirent_and_commit (dst_repo, + dst_path, + dst_dents, + file_num, + replace, + user, + NULL) < 0) { + ret = -1; + goto out; + } + seaf_repo_manager_merge_virtual_repo (mgr, dst_repo->id, NULL); + + if (seaf_repo_manager_del_file (mgr, src_repo->id, src_path, + src_filenames, user, error) < 0) { + ret = -1; + goto out; + } + + seaf_repo_manager_merge_virtual_repo (mgr, src_repo->id, NULL); + + update_repo_size (dst_repo->id); + } else if (!synchronous) { + /* move between different repos */ + background = TRUE; + + gint64 total_files = -1; + gint64 total_files_all = 0; + gint64 total_size_all = 0; + + for (i = 0; i < file_num; i++) { + if (S_ISDIR(src_dents[i]->mode)) + total_files = seaf_fs_manager_count_fs_files (seaf->fs_mgr, + src_repo->store_id, + src_repo->version, + src_dents[i]->id); + else + total_files = 1; + if (total_files < 0) { + seaf_warning ("Failed to get file count.\n"); + ret = -1; + goto out; + } + total_files_all += total_files; + if (!check_file_count_and_size (src_repo, src_dents[i], total_files_all, + &total_size_all, error)) { + ret = -1; + goto out; + } + } + task_id = seaf_copy_manager_add_task (seaf->copy_mgr, + src_repo_id, + src_canon_path, + src_filenames, + dst_repo_id, + dst_canon_path, + dst_filenames, + 0, + user, + total_files_all, + cross_repo_move, + need_progress); + if (need_progress && !task_id) { + seaf_warning ("Failed to start copy task.\n"); + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "failed to start copy task"); + ret = -1; + goto out; + } + } else { + /* Synchronous for cross-repo move */ + if (cross_repo_move (src_repo_id, + src_canon_path, + src_filenames, + dst_repo_id, + dst_canon_path, + dst_filenames, + replace, + user, + NULL) < 0) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "Failed to move"); + ret = -1; + goto out; + } + } // Synchronous move + } //else diffrent repo + +out: + if (src_repo) seaf_repo_unref (src_repo); + if (dst_repo) seaf_repo_unref (dst_repo); + + if (dst_head_commit) seaf_commit_unref(dst_head_commit); + + if (src_canon_path) g_free (src_canon_path); + if (dst_canon_path) g_free (dst_canon_path); + + if (src_names) + g_strfreev (src_names); + if (dst_names) + g_strfreev (dst_names); + if (file_sizes) + g_free (file_sizes); + if (src_dents) { + for (i = 0; i < file_num; i++) + seaf_dirent_free (src_dents[i]); + g_free (src_dents); + } + if (dst_dents) { + for (i = 0; i < file_num; i++) + seaf_dirent_free (dst_dents[i]); + g_free (dst_dents); + } + + if (ret == 0) { + res = seafile_copy_result_new (); + g_object_set (res, "background", background, "task_id", task_id, NULL); + g_free (task_id); + } + + return res; +} + int seaf_repo_manager_post_dir (SeafRepoManager *mgr, const char *repo_id, From cc4ab3885bf1fa7fd221714a46b94a5c431c33df Mon Sep 17 00:00:00 2001 From: cuihaikuo Date: Thu, 29 Jun 2017 23:17:55 +0800 Subject: [PATCH 2/9] A few improvements for posting multiple files --- server/repo-op.c | 110 +++++++++++++++++++++++------------------------ 1 file changed, 54 insertions(+), 56 deletions(-) diff --git a/server/repo-op.c b/server/repo-op.c index 2785444..bb2c4c8 100644 --- a/server/repo-op.c +++ b/server/repo-op.c @@ -714,21 +714,14 @@ out: } static int -add_new_entries (SeafRepo *repo, const char *user, - GList **entries, GList *filenames, GList *id_list, - GList *size_list, int replace_existed, GList **name_list) +add_new_entries (SeafRepo *repo, const char *user, GList **entries, + GList *dents, int replace_existed, GList **name_list) { - GList *ptr1, *ptr2, *ptr3; - char *file, *id; - gint64 *size; + GList *ptr; + SeafDirent *dent; - for (ptr1 = filenames, ptr2 = id_list, ptr3 = size_list; - ptr1 && ptr2 && ptr3; - ptr1 = ptr1->next, ptr2 = ptr2->next, ptr3 = ptr3->next) - { - file = ptr1->data; - id = ptr2->data; - size = ptr3->data; + for (ptr = dents; ptr; ptr = ptr->next) { + dent = ptr->data; char *unique_name; SeafDirent *newdent; @@ -736,28 +729,27 @@ add_new_entries (SeafRepo *repo, const char *user, if (replace_existed) { GList *p; - SeafDirent *dent; - + SeafDirent *tmp_dent; for (p = *entries; p; p = p->next) { - dent = p->data; - if (strcmp(dent->name, file) == 0) { + tmp_dent = p->data; + if (strcmp(tmp_dent->name, dent->name) == 0) { replace = TRUE; *entries = g_list_delete_link (*entries, p); - seaf_dirent_free (dent); + seaf_dirent_free (tmp_dent); break; } } } if (replace) - unique_name = g_strdup (file); + unique_name = g_strdup (dent->name); else - unique_name = generate_unique_filename (file, *entries); + unique_name = generate_unique_filename (dent->name, *entries); if (unique_name != NULL) { newdent = seaf_dirent_new (dir_version_from_repo_version(repo->version), - id, STD_FILE_MODE, unique_name, - (gint64)time(NULL), user, *size); + dent->id, dent->mode, unique_name, + (gint64)time(NULL), user, dent->size); *entries = g_list_insert_sorted (*entries, newdent, compare_dirents); *name_list = g_list_append (*name_list, unique_name); /* No need to free unique_name */ @@ -773,9 +765,7 @@ static char * post_multi_files_recursive (SeafRepo *repo, const char *dir_id, const char *to_path, - GList *filenames, - GList *id_list, - GList *size_list, + GList *dents, const char *user, int replace_existed, GList **name_list) @@ -803,8 +793,7 @@ post_multi_files_recursive (SeafRepo *repo, newentries = dup_seafdir_entries (olddir->entries); if (add_new_entries (repo, user, - &newentries, filenames, id_list, size_list, - replace_existed, name_list) < 0) + &newentries, dents, replace_existed, name_list) < 0) goto out; newdir = seaf_dir_new (NULL, newentries, @@ -832,8 +821,7 @@ post_multi_files_recursive (SeafRepo *repo, if (strcmp(dent->name, to_path_dup) != 0) continue; - id = post_multi_files_recursive (repo, dent->id, remain, filenames, - id_list, size_list, user, + id = post_multi_files_recursive (repo, dent->id, remain, dents, user, replace_existed, name_list); if (id != NULL) { memcpy(dent->id, id, 40); @@ -874,13 +862,37 @@ do_post_multi_files (SeafRepo *repo, int replace_existed, GList **name_list) { + SeafDirent *dent; + GList *dents = NULL; + GList *ptr1, *ptr2, *ptr3; + char *ret; + + for (ptr1 = filenames, ptr2 = id_list, ptr3 = size_list; + ptr1 && ptr2 && ptr3; + ptr1 = ptr1->next, ptr2 = ptr2->next, ptr3 = ptr3->next) { + + char *name = ptr1->data; + char *id = ptr2->data; + gint64 *size = ptr3->data; + + dent = g_new0 (SeafDirent, 1); + dent->name = name; + memcpy(dent->id, id, 40); + dent->id[40] = '\0'; + dent->size = *size; + dent->mode = STD_FILE_MODE; + + dents = g_list_append (dents, dent); + } /* if parent_dir is a absolutely path, we will remove the first '/' */ if (*parent_dir == '/') parent_dir = parent_dir + 1; - return post_multi_files_recursive(repo, root_id, parent_dir, - filenames, id_list, size_list, - user, replace_existed, name_list); + ret = post_multi_files_recursive(repo, root_id, parent_dir, + dents, user, replace_existed, name_list); + g_list_free_full (dents, g_free); + + return ret; } static GList * @@ -1722,23 +1734,16 @@ put_dirent_and_commit (SeafRepo *repo, root_id = head_commit->root_id; - GList *filenames = NULL; - GList *id_list = NULL; - GList *size_list = NULL; + GList *dent_list = NULL; GList *name_list = NULL; - for (i = 0; i < n_dents; i++) { - filenames = g_list_append (filenames, dents[i]->name); - id_list = g_list_append (id_list, dents[i]->id); - size_list = g_list_append (size_list, &(dents[i]->size)); - } + for (i = 0; i < n_dents; i++) + dent_list = g_list_append (dent_list, dents[i]); + if (*path == '/') path = path + 1; - root_id = post_multi_files_recursive (repo, root_id, path, filenames, - id_list, size_list, user, + root_id = post_multi_files_recursive (repo, root_id, path, dent_list, user, replace, &name_list); - g_list_free (filenames); - g_list_free (id_list); - g_list_free (size_list); + g_list_free (dent_list); g_list_free_full (name_list, (GDestroyNotify)g_free); if (!root_id) { @@ -2519,14 +2524,10 @@ move_file_same_repo (const char *repo_id, filenames_str = g_string_new (""); root_id_after_put = head_commit->root_id; - GList *filenames = NULL; - GList *id_list = NULL; - GList *size_list = NULL; + GList *dent_list = NULL; GList *name_list = NULL; for (i = 0; i < file_num; i++) { - filenames = g_list_append (filenames, dst_dents[i]->name); - id_list = g_list_append (id_list, dst_dents[i]->id); - size_list = g_list_append (size_list, &(dst_dents[i]->size)); + dent_list = g_list_append (dent_list, dst_dents[i]); g_string_append_printf (filenames_str, "%s", src_dents[i]->name); if ((i + 1) < file_num) g_string_append_printf (filenames_str, "\t"); @@ -2534,12 +2535,9 @@ move_file_same_repo (const char *repo_id, if (*dst_path == '/') dst_path = dst_path + 1; - root_id_after_put = post_multi_files_recursive (repo, head_commit->root_id, dst_path, filenames, - id_list, size_list, user, + root_id_after_put = post_multi_files_recursive (repo, head_commit->root_id, dst_path, dent_list, user, replace, &name_list); - g_list_free (filenames); - g_list_free (id_list); - g_list_free (size_list); + g_list_free (dent_list); g_list_free_full (name_list, (GDestroyNotify)g_free); if (!root_id_after_put) { From 961e5706536dad149ef4e10fb59f3a38ab3a0385 Mon Sep 17 00:00:00 2001 From: cuihaikuo Date: Fri, 30 Jun 2017 14:30:19 +0800 Subject: [PATCH 3/9] Add get_system_default_repo_id() to seafile_api --- python/seaserv/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/seaserv/api.py b/python/seaserv/api.py index 80ea8ef..a641fff 100644 --- a/python/seaserv/api.py +++ b/python/seaserv/api.py @@ -652,6 +652,9 @@ class SeafileAPI(object): def get_total_storage (self): return seafserv_threaded_rpc.get_total_storage() + def get_system_default_repo_id (self): + return seafserv_threaded_rpc.get_system_default_repo_id() + seafile_api = SeafileAPI() class CcnetAPI(object): From 6f0018c77077dd41bb0946a2ac2a81d79601b1e2 Mon Sep 17 00:00:00 2001 From: cuihaikuo Date: Fri, 30 Jun 2017 16:49:40 +0800 Subject: [PATCH 4/9] Return 'encrypted' in get_trash_repos_by_owner api --- lib/repo.vala | 1 + server/repo-mgr.c | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/lib/repo.vala b/lib/repo.vala index 9784cf1..6e613ec 100644 --- a/lib/repo.vala +++ b/lib/repo.vala @@ -104,6 +104,7 @@ public class TrashRepo : Object { public string owner_id { get; set; } public int64 size { get; set; } public int64 del_time { get; set; } + public bool encrypted { get; set; } } public class SyncInfo : Object { diff --git a/server/repo-mgr.c b/server/repo-mgr.c index f9de055..433a186 100644 --- a/server/repo-mgr.c +++ b/server/repo-mgr.c @@ -2321,6 +2321,15 @@ collect_trash_repo (SeafDBRow *row, void *data) if (!trash_repo) return FALSE; + SeafCommit *commit = seaf_commit_manager_get_commit_compatible (seaf->commit_mgr, + repo_id, head_id); + if (!commit) { + seaf_warning ("Commit %s not found in repo %s\n", head_id, repo_id); + return FALSE; + } + g_object_set (trash_repo, "encrypted", commit->encrypted, NULL); + seaf_commit_unref (commit); + *trash_repos = g_list_prepend (*trash_repos, trash_repo); return TRUE; From 2597fa4df53478993f59a1a39aee426c51828fbe Mon Sep 17 00:00:00 2001 From: Jonathan Xu Date: Fri, 30 Jun 2017 16:48:52 +0800 Subject: [PATCH 5/9] Use 8MB as block size for files uploaded from web. --- common/cdc/cdc.c | 9 +++++++++ common/cdc/cdc.h | 10 ---------- common/fs-mgr.c | 16 ++++++++++------ server/http-server.c | 5 +++-- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/common/cdc/cdc.c b/common/cdc/cdc.c index c11c277..b2222f7 100644 --- a/common/cdc/cdc.c +++ b/common/cdc/cdc.c @@ -22,6 +22,15 @@ #define finger rabin_checksum #define rolling_finger rabin_rolling_checksum +#define BLOCK_SZ (1024*1024*1) +#define BLOCK_MIN_SZ (1024*256) +#define BLOCK_MAX_SZ (1024*1024*4) +#define BLOCK_WIN_SZ 48 + +#define NAME_MAX_SZ 4096 + +#define BREAK_VALUE 0x0013 ///0x0513 + #define READ_SIZE 1024 * 4 #define BYTE_TO_HEX(b) (((b)>=10)?('a'+b-10):('0'+b)) diff --git a/common/cdc/cdc.h b/common/cdc/cdc.h index 9821544..75fb900 100644 --- a/common/cdc/cdc.h +++ b/common/cdc/cdc.h @@ -6,16 +6,6 @@ #include #include -#define BLOCK_SZ (1024*1024*1) -#define BLOCK_MIN_SZ (1024*256) -#define BLOCK_MAX_SZ (1024*1024*4) -#define BLOCK_WIN_SZ 48 - -#define NAME_MAX_SZ 4096 - -#define BREAK_VALUE 0x0013 ///0x0513 - - #ifdef HAVE_MD5 #include "md5.h" #define get_checksum md5 diff --git a/common/fs-mgr.c b/common/fs-mgr.c index 316a614..dd1b112 100644 --- a/common/fs-mgr.c +++ b/common/fs-mgr.c @@ -761,6 +761,10 @@ out: #endif /* SEAFILE_SERVER */ +#define CDC_AVERAGE_BLOCK_SIZE (1 << 23) /* 8MB */ +#define CDC_MIN_BLOCK_SIZE (6 * (1 << 20)) /* 6MB */ +#define CDC_MAX_BLOCK_SIZE (10 * (1 << 20)) /* 10MB */ + int seaf_fs_manager_index_blocks (SeafFSManager *mgr, const char *repo_id, @@ -791,9 +795,9 @@ seaf_fs_manager_index_blocks (SeafFSManager *mgr, #if defined SEAFILE_SERVER && defined FULL_FEATURE if (use_cdc || version == 0) { - cdc.block_sz = calculate_chunk_size (sb.st_size); - cdc.block_min_sz = cdc.block_sz >> 2; - cdc.block_max_sz = cdc.block_sz << 2; + cdc.block_sz = CDC_AVERAGE_BLOCK_SIZE; + cdc.block_min_sz = CDC_MIN_BLOCK_SIZE; + cdc.block_max_sz = CDC_MAX_BLOCK_SIZE; cdc.write_block = seafile_write_chunk; memcpy (cdc.repo_id, repo_id, 36); cdc.version = version; @@ -811,9 +815,9 @@ seaf_fs_manager_index_blocks (SeafFSManager *mgr, } } #else - cdc.block_sz = calculate_chunk_size (sb.st_size); - cdc.block_min_sz = cdc.block_sz >> 2; - cdc.block_max_sz = cdc.block_sz << 2; + cdc.block_sz = CDC_AVERAGE_BLOCK_SIZE; + cdc.block_min_sz = CDC_MIN_BLOCK_SIZE; + cdc.block_max_sz = CDC_MAX_BLOCK_SIZE; cdc.write_block = seafile_write_chunk; memcpy (cdc.repo_id, repo_id, 36); cdc.version = version; diff --git a/server/http-server.c b/server/http-server.c index b25cebd..4d903e0 100644 --- a/server/http-server.c +++ b/server/http-server.c @@ -33,6 +33,7 @@ #define DEFAULT_THREADS 50 #define DEFAULT_MAX_DOWNLOAD_DIR_SIZE 100 * ((gint64)1 << 20) /* 100MB */ #define DEFAULT_MAX_INDEXING_THREADS 1 +#define DEFAULT_FIXED_BLOCK_SIZE ((gint64)1 << 23) /* 8MB */ #define HOST "host" #define PORT "port" @@ -150,11 +151,11 @@ load_http_config (HttpServerStruct *htp_server, SeafileSession *session) "fixed_block_size", &error); if (error){ - htp_server->fixed_block_size = BLOCK_SZ; + htp_server->fixed_block_size = DEFAULT_FIXED_BLOCK_SIZE; g_clear_error(&error); } else { if (fixed_block_size_mb <= 0) - htp_server->fixed_block_size = BLOCK_SZ; + htp_server->fixed_block_size = DEFAULT_FIXED_BLOCK_SIZE; else htp_server->fixed_block_size = fixed_block_size_mb * ((gint64)1 << 20); } From 768c576afc99c2f23b3d9200fc571eec26e7921f Mon Sep 17 00:00:00 2001 From: cuihaikuo Date: Sat, 1 Jul 2017 10:55:10 +0800 Subject: [PATCH 6/9] Add documents for deleting,copying and moving files. --- python/seaserv/api.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/seaserv/api.py b/python/seaserv/api.py index a641fff..66cbc8f 100644 --- a/python/seaserv/api.py +++ b/python/seaserv/api.py @@ -233,9 +233,18 @@ class SeafileAPI(object): return seafserv_threaded_rpc.put_file(repo_id, tmp_file_path, parent_dir, filename, username, head_id) + ''' + If you want to delete multiple files in a batch, @filename should be in + the following format: 'filename1\tfilename2\tfilename3' + ''' def del_file(self, repo_id, parent_dir, filename, username): return seafserv_threaded_rpc.del_file(repo_id, parent_dir, filename, username) + ''' + If you want to move or copy multiple files in a batch, @src_filename and @dst_filename + should be in the following format: 'filename1\tfilename2\tfilename3',make sure the number of files + in @src_filename and @dst_filename parameters match + ''' def copy_file(self, src_repo, src_dir, src_filename, dst_repo, dst_dir, dst_filename, username, need_progress, synchronous=0): return seafserv_threaded_rpc.copy_file(src_repo, src_dir, src_filename, From 36ecb21a584cbaada21e7a5ab0712fad811326bb Mon Sep 17 00:00:00 2001 From: cuihaikuo Date: Tue, 4 Jul 2017 14:50:05 +0800 Subject: [PATCH 7/9] Add rpc for getting file count info --- common/fs-mgr.c | 75 +++++++++++++++++++++++++++++++++++++ common/fs-mgr.h | 8 ++++ common/rpc-service.c | 30 +++++++++++++++ include/seafile-rpc.h | 5 +++ lib/dir.vala | 6 +++ python/seafile/rpcclient.py | 4 ++ python/seaserv/api.py | 3 ++ server/seaf-server.c | 5 +++ 8 files changed, 136 insertions(+) diff --git a/common/fs-mgr.c b/common/fs-mgr.c index dd1b112..68d3e40 100644 --- a/common/fs-mgr.c +++ b/common/fs-mgr.c @@ -2492,6 +2492,42 @@ count_dir_files (SeafFSManager *mgr, const char *repo_id, int version, const cha return count; } +static int +get_file_count_info (SeafFSManager *mgr, + const char *repo_id, + int version, + const char *id, + gint64 *dir_count, + gint64 *file_count, + gint64 *size) +{ + SeafDir *dir; + SeafDirent *seaf_dent; + GList *p; + int ret = 0; + + dir = seaf_fs_manager_get_seafdir (mgr, repo_id, version, id); + if (!dir) + return -1; + + for (p = dir->entries; p; p = p->next) { + seaf_dent = (SeafDirent *)p->data; + + if (S_ISREG(seaf_dent->mode)) { + (*file_count)++; + if (version > 0) + (*size) += seaf_dent->size; + } else if (S_ISDIR(seaf_dent->mode)) { + (*dir_count)++; + ret = get_file_count_info (mgr, repo_id, version, seaf_dent->id, + dir_count, file_count, size); + } + } + seaf_dir_free (dir); + + return ret; +} + int seaf_fs_manager_count_fs_files (SeafFSManager *mgr, const char *repo_id, @@ -3035,3 +3071,42 @@ seaf_fs_manager_remove_store (SeafFSManager *mgr, { return seaf_obj_store_remove_store (mgr->obj_store, store_id); } + +GObject * +seaf_fs_manager_get_file_count_info_by_path (SeafFSManager *mgr, + const char *repo_id, + int version, + const char *root_id, + const char *path, + GError **error) +{ + char *dir_id = NULL; + gint64 file_count = 0, dir_count = 0, size = 0; + SeafileFileCountInfo *info = NULL; + + dir_id = seaf_fs_manager_get_seafdir_id_by_path (mgr, + repo_id, + version, + root_id, + path, NULL); + if (!dir_id) { + seaf_warning ("Path %s doesn't exist or is not a dir in repo %.10s.\n", + path, repo_id); + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS, "Bad path"); + goto out; + } + if (get_file_count_info (mgr, repo_id, version, + dir_id, &dir_count, &file_count, &size) < 0) { + seaf_warning ("Failed to get count info from path %s in repo %.10s.\n", + path, repo_id); + goto out; + } + info = g_object_new (SEAFILE_TYPE_FILE_COUNT_INFO, + "file_count", file_count, + "dir_count", dir_count, + "size", size, NULL); +out: + g_free (dir_id); + + return (GObject *)info; +} diff --git a/common/fs-mgr.h b/common/fs-mgr.h index cf64288..4fe8870 100644 --- a/common/fs-mgr.h +++ b/common/fs-mgr.h @@ -420,4 +420,12 @@ int seaf_fs_manager_remove_store (SeafFSManager *mgr, const char *store_id); +GObject * +seaf_fs_manager_get_file_count_info_by_path (SeafFSManager *mgr, + const char *repo_id, + int version, + const char *root_id, + const char *path, + GError **error); + #endif diff --git a/common/rpc-service.c b/common/rpc-service.c index 1c37116..373f5f1 100644 --- a/common/rpc-service.c +++ b/common/rpc-service.c @@ -5069,4 +5069,34 @@ seafile_get_total_storage (GError **error) return seaf_get_total_storage (error); } +GObject * +seafile_get_file_count_info_by_path (const char *repo_id, + const char *path, + GError **error) +{ + if (!repo_id || !path) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS, "Argument should not be null"); + return NULL; + } + + GObject *ret = NULL; + SeafRepo *repo = NULL; + repo = seaf_repo_manager_get_repo (seaf->repo_mgr, repo_id); + if (!repo) { + seaf_warning ("Failed to get repo %.10s\n", repo_id); + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "Library not exists"); + return NULL; + } + + ret = seaf_fs_manager_get_file_count_info_by_path (seaf->fs_mgr, + repo->store_id, + repo->version, + repo->root_id, + path, error); + seaf_repo_unref (repo); + + return ret; +} + #endif /* SEAFILE_SERVER */ diff --git a/include/seafile-rpc.h b/include/seafile-rpc.h index a01b03c..a9c7ccc 100644 --- a/include/seafile-rpc.h +++ b/include/seafile-rpc.h @@ -1033,4 +1033,9 @@ seafile_get_total_file_number (GError **error); gint64 seafile_get_total_storage (GError **error); + +GObject * +seafile_get_file_count_info_by_path (const char *repo_id, + const char *path, + GError **error); #endif diff --git a/lib/dir.vala b/lib/dir.vala index 2fd8429..c782573 100644 --- a/lib/dir.vala +++ b/lib/dir.vala @@ -16,4 +16,10 @@ namespace Seafile { public int version { set; get; } } + public class FileCountInfo : Object { + public int64 file_count { set; get; } + public int64 dir_count { set; get; } + public int64 size { set; get; } + } + } // namespace diff --git a/python/seafile/rpcclient.py b/python/seafile/rpcclient.py index 8634fee..0fee19e 100644 --- a/python/seafile/rpcclient.py +++ b/python/seafile/rpcclient.py @@ -960,3 +960,7 @@ class SeafServerThreadedRpcClient(ccnet.RpcClientBase): @searpc_func("int64", []) def get_total_storage(): pass + + @searpc_func("object", ["string", "string"]) + def get_file_count_info_by_path(repo_id, path): + pass diff --git a/python/seaserv/api.py b/python/seaserv/api.py index 66cbc8f..dba7d5e 100644 --- a/python/seaserv/api.py +++ b/python/seaserv/api.py @@ -664,6 +664,9 @@ class SeafileAPI(object): def get_system_default_repo_id (self): return seafserv_threaded_rpc.get_system_default_repo_id() + def get_file_count_info_by_path(self, repo_id, path): + return seafserv_threaded_rpc.get_file_count_info_by_path(repo_id, path) + seafile_api = SeafileAPI() class CcnetAPI(object): diff --git a/server/seaf-server.c b/server/seaf-server.c index 7368dc1..449ee8f 100644 --- a/server/seaf-server.c +++ b/server/seaf-server.c @@ -363,6 +363,11 @@ static void start_rpc_service (CcnetClient *client, int cloud_mode) "get_total_storage", searpc_signature_int64__void()); + searpc_server_register_function ("seafserv-threaded-rpcserver", + seafile_get_file_count_info_by_path, + "get_file_count_info_by_path", + searpc_signature_object__string_string()); + /* share repo to user */ searpc_server_register_function ("seafserv-threaded-rpcserver", seafile_add_share, From c41891933228496cadd37619c0357e698c423276 Mon Sep 17 00:00:00 2001 From: cuihaikuo Date: Wed, 5 Jul 2017 14:06:38 +0800 Subject: [PATCH 8/9] Fix a bug that an empty folder can't be packed --- server/pack-dir.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/server/pack-dir.c b/server/pack-dir.c index d3b9d03..b5ae479 100644 --- a/server/pack-dir.c +++ b/server/pack-dir.c @@ -289,6 +289,41 @@ archive_dir (PackDirData *data, seaf_warning ("failed to get dir %s:%s\n", data->store_id, root_id); goto out; } + if (!dir->entries) { + char *pathname = g_build_filename (data->top_dir_name, dirpath, NULL); + struct archive_entry *entry = archive_entry_new (); + gboolean is_windows = data->is_windows; + + if (is_windows && seaf->http_server->windows_encoding) { + char *win_file_name = do_iconv ("UTF-8", + seaf->http_server->windows_encoding, + pathname); + if (!win_file_name) { + seaf_warning ("Failed to convert file name to %s\n", + seaf->http_server->windows_encoding); + ret = -1; + goto out; + } + archive_entry_copy_pathname (entry, win_file_name); + g_free (win_file_name); + + } else { + archive_entry_set_pathname (entry, pathname); + } + + archive_entry_set_filetype (entry, AE_IFDIR); + archive_entry_set_mtime (entry, data->mtime, 0); + archive_entry_set_perm (entry, 0755); + int n = archive_write_header (data->a, entry); + if (n != ARCHIVE_OK) { + seaf_warning ("archive_write_header error: %s\n", archive_error_string(data->a)); + ret = -1; + } + + archive_entry_free (entry); + g_free (pathname); + goto out; + } for (ptr = dir->entries; ptr; ptr = ptr->next) { dent = ptr->data; From e096ccdfa80596665864679a7ce11c5f21692b85 Mon Sep 17 00:00:00 2001 From: Jonathan Xu Date: Wed, 5 Jul 2017 18:13:29 +0800 Subject: [PATCH 9/9] Fix GC bug when specify "-r" and "-D" options together. --- server/gc/seafserv-gc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/gc/seafserv-gc.c b/server/gc/seafserv-gc.c index 0fbe5b9..7504778 100644 --- a/server/gc/seafserv-gc.c +++ b/server/gc/seafserv-gc.c @@ -154,7 +154,7 @@ main(int argc, char *argv[]) } if (rm_garbage) { - delete_garbaged_repos (!rm_garbage); + delete_garbaged_repos (dry_run); return 0; }