diff --git a/ci/run.py b/ci/run.py index 926f426..b524a01 100755 --- a/ci/run.py +++ b/ci/run.py @@ -108,7 +108,6 @@ class Project(object): def projectdir(self): return join(TOPDIR, self.name) - @property def branch(self): return get_project_branch(self) @@ -119,7 +118,7 @@ class Project(object): else: shell( 'git clone --depth=1 --branch {} {}'. - format(self.branch, self.url) + format(self.branch(), self.url) ) @chdir @@ -142,11 +141,17 @@ class Libsearpc(Project): def __init__(self): super(Libsearpc, self).__init__('libsearpc') + def branch(self): + return '7.0' + class CcnetServer(Project): def __init__(self): super(CcnetServer, self).__init__('ccnet-server') + def branch(self): + return '7.0' + class SeafileServer(Project): def __init__(self): diff --git a/common/diff-simple.c b/common/diff-simple.c index 0ed11e2..849fe26 100644 --- a/common/diff-simple.c +++ b/common/diff-simple.c @@ -31,6 +31,7 @@ diff_entry_new_from_dirent (char type, char status, de->status = status; memcpy (de->sha1, sha1, 20); de->name = path; + de->size = dent->size; #ifdef SEAFILE_CLIENT if (type == DIFF_TYPE_COMMITS && @@ -41,7 +42,6 @@ diff_entry_new_from_dirent (char type, char status, de->mtime = dent->mtime; de->mode = dent->mode; de->modifier = g_strdup(dent->modifier); - de->size = dent->size; } #endif @@ -288,6 +288,7 @@ twoway_diff_files (int n, const char *basedir, SeafDirent *files[], void *vdata) if (!dirent_same (tree1, tree2)) { de = diff_entry_new_from_dirent (DIFF_TYPE_COMMITS, DIFF_STATUS_MODIFIED, tree2, basedir); + de->origin_size = tree1->size; *results = g_list_prepend (*results, de); } diff --git a/common/diff-simple.h b/common/diff-simple.h index 29f79bd..0afd963 100644 --- a/common/diff-simple.h +++ b/common/diff-simple.h @@ -40,15 +40,8 @@ typedef struct DiffEntry { unsigned char sha1[20]; /* used for resolve rename */ char *name; char *new_name; /* only used in rename. */ - -#ifdef SEAFILE_CLIENT - /* Fields only used for ADDED, DIR_ADDED, MODIFIED types, - * used in check out files/dirs.*/ - gint64 mtime; - unsigned int mode; - char *modifier; gint64 size; -#endif + gint64 origin_size; /* only used in modified */ } DiffEntry; DiffEntry * diff --git a/common/rpc-service.c b/common/rpc-service.c index d0324b9..ba4a2c2 100644 --- a/common/rpc-service.c +++ b/common/rpc-service.c @@ -4492,12 +4492,16 @@ seafile_set_repo_status(const char *repo_id, int status, GError **error) int seafile_get_repo_status(const char *repo_id, GError **error) { + int status; + if (!is_uuid_valid(repo_id)) { g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS, "Arguments error"); return -1; } - return seaf_repo_manager_get_repo_status(seaf->repo_mgr, repo_id); + status = seaf_repo_manager_get_repo_status(seaf->repo_mgr, repo_id); + + return (status == -1) ? 0 : status; } #endif /* SEAFILE_SERVER */ diff --git a/lib/copy-task.vala b/lib/copy-task.vala index b98ccb9..3d3ef35 100644 --- a/lib/copy-task.vala +++ b/lib/copy-task.vala @@ -5,6 +5,7 @@ public class CopyTask : Object { public int64 total { set; get; } public bool canceled { set; get; } public bool failed { set; get; } + public string failed_reason { set; get; } public bool successful { set; get; } } diff --git a/server/copy-mgr.c b/server/copy-mgr.c index 3d1b1b3..5505104 100644 --- a/server/copy-mgr.c +++ b/server/copy-mgr.c @@ -28,6 +28,7 @@ copy_task_free (CopyTask *task) { if (!task) return; + g_free (task->failed_reason); g_free (task); } @@ -72,11 +73,12 @@ seaf_copy_manager_get_task (SeafCopyManager *mgr, pthread_mutex_lock (&priv->lock); task = g_hash_table_lookup (priv->copy_tasks, task_id); + if (task) { t = seafile_copy_task_new (); g_object_set (t, "done", task->done, "total", task->total, "canceled", task->canceled, "failed", task->failed, - "successful", task->successful, + "failed_reason", task->failed_reason, "successful", task->successful, NULL); if (task->canceled || task->failed || task->successful) g_hash_table_remove(priv->copy_tasks, task_id); diff --git a/server/copy-mgr.h b/server/copy-mgr.h index a02ba1c..ec16ba3 100644 --- a/server/copy-mgr.h +++ b/server/copy-mgr.h @@ -3,6 +3,12 @@ #include +#define COPY_ERR_INTERNAL "Internal error when copy or move" +#define COPY_ERR_BAD_ARG "Invalid arguments" +#define COPY_ERR_TOO_MANY_FILES "Too many files" +#define COPY_ERR_SIZE_TOO_LARGE "Folder or file size is too large" +#define COPY_ERR_QUOTA_IS_FULL "Quota is full" + struct _SeafileSession; struct _SeafCopyManagerPriv; struct _SeafileCopyTask; @@ -23,6 +29,7 @@ struct CopyTask { gint64 total; gint canceled; gboolean failed; + char *failed_reason; gboolean successful; }; typedef struct CopyTask CopyTask; diff --git a/server/gc/fsck.c b/server/gc/fsck.c index 45cf737..c528be6 100644 --- a/server/gc/fsck.c +++ b/server/gc/fsck.c @@ -94,7 +94,6 @@ check_blocks (const char *file_id, FsckData *fsck_data, gboolean *io_error) block_id)) { seaf_warning ("Repo[%.8s] block %s:%s is missing.\n", repo->id, store_id, block_id); ret = -1; - break; } // check block integrity, if not remove it @@ -115,7 +114,6 @@ check_blocks (const char *file_id, FsckData *fsck_data, gboolean *io_error) seaf_message ("Repo[%.8s] block %s is damaged.\n", repo->id, block_id); } ret = -1; - break; } } diff --git a/server/http-server.c b/server/http-server.c index 481a6ad..edfd62b 100644 --- a/server/http-server.c +++ b/server/http-server.c @@ -373,7 +373,7 @@ check_permission (HttpServer *htp_server, const char *repo_id, const char *usern if (strcmp(op, "upload") == 0) { int status = seaf_repo_manager_get_repo_status(seaf->repo_mgr, repo_id); - if (status != REPO_STATUS_NORMAL) + if (status != REPO_STATUS_NORMAL && status != -1) return EVHTP_RES_FORBIDDEN; } diff --git a/server/repo-mgr.c b/server/repo-mgr.c index 7e998b1..a6a5c20 100644 --- a/server/repo-mgr.c +++ b/server/repo-mgr.c @@ -4028,17 +4028,26 @@ seaf_repo_manager_add_upload_tmp_file (SeafRepoManager *mgr, const char *tmp_file, GError **error) { + char *file_path_with_slash = NULL; + + if (file_path[0] == '/') { + file_path_with_slash = g_strdup(file_path); + } else { + file_path_with_slash = g_strconcat("/", file_path, NULL); + } + int ret = seaf_db_statement_query (mgr->seaf->db, "INSERT INTO WebUploadTempFiles " "(repo_id, file_path, tmp_file_path) " "VALUES (?, ?, ?)", 3, "string", repo_id, - "string", file_path, "string", tmp_file); + "string", file_path_with_slash, "string", tmp_file); if (ret < 0) { g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, "Failed to add upload tmp file record to db."); } + g_free (file_path_with_slash); return ret; } @@ -4048,15 +4057,33 @@ seaf_repo_manager_del_upload_tmp_file (SeafRepoManager *mgr, const char *file_path, GError **error) { + char *file_path_with_slash = NULL, *file_path_no_slash = NULL; + + /* Due to a bug in early versions of 7.0, some file_path may be stored in the db without + * a leading slash. To be compatible with those records, we need to check the path + * with and without leading slash. + */ + if (file_path[0] == '/') { + file_path_with_slash = g_strdup(file_path); + file_path_no_slash = g_strdup(file_path+1); + } else { + file_path_with_slash = g_strconcat("/", file_path, NULL); + file_path_no_slash = g_strdup(file_path); + } + int ret = seaf_db_statement_query (mgr->seaf->db, "DELETE FROM WebUploadTempFiles WHERE " - "repo_id = ? AND file_path = ?", - 2, "string", repo_id, "string", file_path); + "repo_id = ? AND file_path IN (?, ?)", + 3, "string", repo_id, + "string", file_path_with_slash, + "string", file_path_no_slash); if (ret < 0) { g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, "Failed to delete upload tmp file record from db."); } + g_free (file_path_with_slash); + g_free (file_path_no_slash); return ret; } @@ -4077,18 +4104,51 @@ seaf_repo_manager_get_upload_tmp_file (SeafRepoManager *mgr, GError **error) { char *tmp_file_path = NULL; + char *file_path_with_slash = NULL, *file_path_no_slash = NULL; + + /* Due to a bug in early versions of 7.0, some file_path may be stored in the db without + * a leading slash. To be compatible with those records, we need to check the path + * with and without leading slash. + * The correct file_path in db should be with a leading slash. + */ + if (file_path[0] == '/') { + file_path_with_slash = g_strdup(file_path); + file_path_no_slash = g_strdup(file_path+1); + } else { + file_path_with_slash = g_strconcat("/", file_path, NULL); + file_path_no_slash = g_strdup(file_path); + } int ret = seaf_db_statement_foreach_row (mgr->seaf->db, "SELECT tmp_file_path FROM WebUploadTempFiles " "WHERE repo_id = ? AND file_path = ?", get_tmp_file_path, &tmp_file_path, - 2, "string", repo_id, "string", file_path); + 2, "string", repo_id, + "string", file_path_with_slash); if (ret < 0) { g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, "Failed to get upload temp file path from db."); - return NULL; + goto out; } + if (!tmp_file_path) { + /* Try file_path without slash. */ + int ret = seaf_db_statement_foreach_row (mgr->seaf->db, + "SELECT tmp_file_path FROM WebUploadTempFiles " + "WHERE repo_id = ? AND file_path = ?", + get_tmp_file_path, &tmp_file_path, + 2, "string", repo_id, + "string", file_path_no_slash); + if (ret < 0) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "Failed to get upload temp file path from db."); + goto out; + } + } + +out: + g_free (file_path_with_slash); + g_free (file_path_no_slash); return tmp_file_path; } diff --git a/server/repo-op.c b/server/repo-op.c index ceae511..427155d 100644 --- a/server/repo-op.c +++ b/server/repo-op.c @@ -40,7 +40,7 @@ is_virtual_repo_and_origin (SeafRepo *repo1, SeafRepo *repo2); static gboolean check_file_count_and_size (SeafRepo *repo, SeafDirent *dent, gint64 total_files, - gint64 *multi_file_size, GError **error); + gint64 *total_size_all, char **err_str); int post_files_and_gen_commit (GList *filenames, @@ -2009,6 +2009,12 @@ get_sub_dirents_hash_map(SeafRepo *repo, const char *parent_dir) return dirent_hash; } +static void +set_failed_reason (char **failed_reason, char *err_str) +{ + *failed_reason = g_strdup (err_str); +} + static int cross_repo_copy (const char *src_repo_id, const char *src_path, @@ -2028,18 +2034,24 @@ cross_repo_copy (const char *src_repo_id, guint64 new_size = 0; int ret = 0, i = 0; int file_num = 1; - GError *error = NULL; GHashTable *dirent_hash = NULL; + gint64 total_size_all = 0; + char *err_str = COPY_ERR_INTERNAL; + int check_quota_ret; src_repo = seaf_repo_manager_get_repo (seaf->repo_mgr, src_repo_id); if (!src_repo) { + err_str = COPY_ERR_INTERNAL; ret = -1; + seaf_warning ("Failed to get source repo.\n"); goto out; } dst_repo = seaf_repo_manager_get_repo (seaf->repo_mgr, dst_repo_id); if (!dst_repo) { + err_str = COPY_ERR_INTERNAL; ret = -1; + seaf_warning ("Failed to get destination repo.\n"); goto out; } @@ -2054,22 +2066,24 @@ cross_repo_copy (const char *src_repo_id, dirent_hash = get_sub_dirents_hash_map (src_repo, src_path); if (!dirent_hash) { + err_str = COPY_ERR_INTERNAL; ret = -1; goto out; } gint64 total_files = -1; gint64 total_files_all = 0; - gint64 total_size_all = 0; /* check filename, size and file count */ for (i = 0; i < file_num; i++) { if (strcmp(src_names[i], "") == 0) { + err_str = COPY_ERR_BAD_ARG; ret = -1; seaf_warning ("[copy files] Bad args: Empty src_filename.\n"); goto out; } src_dents[i] = g_hash_table_lookup (dirent_hash, src_names[i]); if (!src_dents[i]) { + err_str = COPY_ERR_INTERNAL; ret = -1; seaf_warning ("[copy files] File %s not Found.\n", src_names[i]); goto out; @@ -2082,17 +2096,31 @@ cross_repo_copy (const char *src_repo_id, else total_files = 1; if (total_files < 0) { + err_str = COPY_ERR_INTERNAL; seaf_warning ("Failed to get file count.\n"); ret = -1; goto out; } total_files_all += total_files; if (!check_file_count_and_size (src_repo, src_dents[i], total_files_all, - &total_size_all, &error)) { + &total_size_all, &err_str)) { ret = -1; goto out; } } + + check_quota_ret = seaf_quota_manager_check_quota_with_delta (seaf->quota_mgr, dst_repo_id, total_size_all); + if (check_quota_ret != 0) { + if (check_quota_ret == -1) { + err_str = COPY_ERR_INTERNAL; + seaf_warning ("Failed to check quota.\n"); + } else { + err_str = COPY_ERR_QUOTA_IS_FULL; + } + ret = -1; + goto out; + } + if (task) task->total = total_files_all; @@ -2102,6 +2130,7 @@ cross_repo_copy (const char *src_repo_id, src_dents[i]->id, src_dents[i]->mode, modifier, task, &new_size); if (!new_id) { + err_str = COPY_ERR_INTERNAL; ret = -1; seaf_warning ("[copy files] Failed to copy file %s.\n", src_dents[i]->name); goto out; @@ -2116,6 +2145,7 @@ cross_repo_copy (const char *src_repo_id, src_dent = get_dirent_by_path (src_repo, NULL, src_path, src_filename, NULL); if (!src_dent) { + err_str = COPY_ERR_INTERNAL; seaf_warning ("[move file] File %s not Found.\n", src_filename); ret = -1; goto out; @@ -2130,15 +2160,29 @@ cross_repo_copy (const char *src_repo_id, else total_files = 1; if (total_files < 0) { + err_str = COPY_ERR_INTERNAL; seaf_warning ("Failed to get file count.\n"); ret = -1; goto out; } - if (!check_file_count_and_size (src_repo, src_dent, total_files, NULL, &error)) { + if (!check_file_count_and_size (src_repo, src_dent, total_files, &total_size_all, &err_str)) { ret = -1; goto out; } + + check_quota_ret = seaf_quota_manager_check_quota_with_delta (seaf->quota_mgr, dst_repo_id, total_size_all); + if (check_quota_ret != 0) { + if (check_quota_ret == -1) { + err_str = COPY_ERR_INTERNAL; + seaf_warning ("Failed to check quota.\n"); + } else { + err_str = COPY_ERR_QUOTA_IS_FULL; + } + ret = -1; + goto out; + } + if (task) task->total = total_files; @@ -2146,6 +2190,7 @@ cross_repo_copy (const char *src_repo_id, src_dent->id, src_dent->mode, modifier, task, &new_size); if (!new_id) { + err_str = COPY_ERR_INTERNAL; ret = -1; goto out; } @@ -2163,6 +2208,7 @@ cross_repo_copy (const char *src_repo_id, replace, modifier, NULL) < 0) { + err_str = COPY_ERR_INTERNAL; ret = -1; goto out; } @@ -2191,14 +2237,14 @@ out: g_strfreev(src_names); g_strfreev(dst_names); } - if (error) - g_clear_error(&error); if (ret == 0) { update_repo_size (dst_repo_id); } else { - if (task && !task->canceled) + if (task && !task->canceled) { task->failed = TRUE; + set_failed_reason (&(task->failed_reason), err_str); + } } return ret; @@ -2218,52 +2264,48 @@ is_virtual_repo_and_origin (SeafRepo *repo1, SeafRepo *repo2) static gboolean check_file_count_and_size (SeafRepo *repo, SeafDirent *dent, gint64 total_files, - gint64 *multi_file_size, GError **error) + gint64 *total_size_all, char **err_str) { + gint64 total_file_size = 0; + gint64 size = -1; + if (seaf->copy_mgr->max_files > 0 && total_files > seaf->copy_mgr->max_files) { - g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, - "Too many files"); + *err_str = COPY_ERR_TOO_MANY_FILES; seaf_warning("Failed to copy/move file from repo %.8s: Too many files\n", repo->id); return FALSE; } + if (S_ISREG(dent->mode)) { + if (repo->version > 0) + size = dent->size; + else + size = seaf_fs_manager_get_file_size (seaf->fs_mgr, + repo->store_id, + repo->version, + dent->id); + } else { + size = seaf_fs_manager_get_fs_size (seaf->fs_mgr, + repo->store_id, + repo->version, + dent->id); + } + + if (size < 0) { + *err_str = COPY_ERR_INTERNAL; + seaf_warning ("Failed to get dir size of %s:%s.\n", + repo->store_id, dent->id); + return FALSE; + } + + if (total_size_all) { + *total_size_all += size; + total_file_size = *total_size_all; + } + if (seaf->copy_mgr->max_size > 0) { - gint64 size = -1; - - if (S_ISREG(dent->mode)) { - if (repo->version > 0) - size = dent->size; - else - size = seaf_fs_manager_get_file_size (seaf->fs_mgr, - repo->store_id, - repo->version, - dent->id); - } else { - size = seaf_fs_manager_get_fs_size (seaf->fs_mgr, - repo->store_id, - repo->version, - dent->id); - } - if (size < 0) { - seaf_warning ("Failed to get dir size of %s:%s.\n", - repo->store_id, dent->id); - return FALSE; - } - if (multi_file_size) { - *multi_file_size += size; - if (*multi_file_size > seaf->copy_mgr->max_size) { - g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, - "Folder or file size is too large"); - seaf_warning("Failed to copy/move file from repo %.8s: " - "Folder or file size is too large.\n", repo->id); - return FALSE; - } - } - - if (size > seaf->copy_mgr->max_size) { - g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, - "Folder or file size is too large"); + if (total_file_size > seaf->copy_mgr->max_size) { + *err_str = COPY_ERR_SIZE_TOO_LARGE; seaf_warning("Failed to copy/move file from repo %.8s: " "Folder or file size is too large.\n", repo->id); return FALSE; @@ -2728,18 +2770,24 @@ cross_repo_move (const char *src_repo_id, guint64 new_size = 0; int ret = 0, i = 0; int file_num = 1; - GError *error = NULL; GHashTable *dirent_hash = NULL; + gint64 total_size_all = 0; + char *err_str = COPY_ERR_INTERNAL; + int check_quota_ret; src_repo = seaf_repo_manager_get_repo (seaf->repo_mgr, src_repo_id); if (!src_repo) { + err_str = COPY_ERR_INTERNAL; ret = -1; + seaf_warning ("Failed to get source repo.\n"); goto out; } dst_repo = seaf_repo_manager_get_repo (seaf->repo_mgr, dst_repo_id); if (!dst_repo) { + err_str = COPY_ERR_INTERNAL; ret = -1; + seaf_warning ("Failed to get destination repo.\n"); goto out; } @@ -2754,22 +2802,24 @@ cross_repo_move (const char *src_repo_id, dirent_hash = get_sub_dirents_hash_map (src_repo, src_path); if (!dirent_hash) { + err_str = COPY_ERR_INTERNAL; ret = -1; goto out; } gint64 total_files = -1; gint64 total_files_all = 0; - gint64 total_size_all = 0; /* check filename, size and file count */ for (i = 0; i < file_num; i++) { if (strcmp(src_names[i], "") == 0) { + err_str = COPY_ERR_BAD_ARG; ret = -1; seaf_warning ("[move files] Bad args: Empty src_filename.\n"); goto out; } src_dents[i] = g_hash_table_lookup (dirent_hash, src_names[i]); if (!src_dents[i]) { + err_str = COPY_ERR_INTERNAL; ret = -1; seaf_warning ("[move files] File %s not Found.\n", src_names[i]); goto out; @@ -2782,17 +2832,31 @@ cross_repo_move (const char *src_repo_id, else total_files = 1; if (total_files < 0) { + err_str = COPY_ERR_INTERNAL; seaf_warning ("Failed to get file count.\n"); ret = -1; goto out; } total_files_all += total_files; if (!check_file_count_and_size (src_repo, src_dents[i], total_files_all, - &total_size_all, &error)) { + &total_size_all, &err_str)) { ret = -1; goto out; } } + + check_quota_ret = seaf_quota_manager_check_quota_with_delta (seaf->quota_mgr, dst_repo_id, total_size_all); + if (check_quota_ret != 0) { + if (check_quota_ret == -1) { + err_str = COPY_ERR_INTERNAL; + seaf_warning ("Failed to check quota.\n"); + } else { + err_str = COPY_ERR_QUOTA_IS_FULL; + } + ret = -1; + goto out; + } + if (task) task->total = total_files_all; @@ -2802,6 +2866,7 @@ cross_repo_move (const char *src_repo_id, src_dents[i]->id, src_dents[i]->mode, modifier, task, &new_size); if (!new_id) { + err_str = COPY_ERR_INTERNAL; ret = -1; seaf_warning ("[move files] Failed to copy file %s.\n", src_dents[i]->name); goto out; @@ -2815,6 +2880,7 @@ cross_repo_move (const char *src_repo_id, src_dent = get_dirent_by_path (src_repo, NULL, src_path, src_filename, NULL); if (!src_dent) { + err_str = COPY_ERR_INTERNAL; seaf_warning ("[move file] File %s not Found.\n", src_filename); ret = -1; goto out; @@ -2829,15 +2895,29 @@ cross_repo_move (const char *src_repo_id, else total_files = 1; if (total_files < 0) { + err_str = COPY_ERR_INTERNAL; seaf_warning ("Failed to get file count.\n"); ret = -1; goto out; } - if (!check_file_count_and_size (src_repo, src_dent, total_files, NULL, &error)) { + if (!check_file_count_and_size (src_repo, src_dent, total_files, &total_size_all, &err_str)) { ret = -1; goto out; } + + check_quota_ret = seaf_quota_manager_check_quota_with_delta (seaf->quota_mgr, dst_repo_id, total_size_all); + if (check_quota_ret != 0) { + if (check_quota_ret == -1) { + err_str = COPY_ERR_INTERNAL; + seaf_warning ("Failed to check quota.\n"); + } else { + err_str = COPY_ERR_QUOTA_IS_FULL; + } + ret = -1; + goto out; + } + if (task) task->total = total_files; @@ -2845,6 +2925,7 @@ cross_repo_move (const char *src_repo_id, src_dent->id, src_dent->mode, modifier, task, &new_size); if (!new_id) { + err_str = COPY_ERR_INTERNAL; ret = -1; goto out; } @@ -2863,6 +2944,7 @@ cross_repo_move (const char *src_repo_id, replace, modifier, NULL) < 0) { + err_str = COPY_ERR_INTERNAL; ret = -1; goto out; } @@ -2871,6 +2953,7 @@ cross_repo_move (const char *src_repo_id, if (seaf_repo_manager_del_file (seaf->repo_mgr, src_repo_id, src_path, src_filename, modifier, NULL) < 0) { + err_str = COPY_ERR_INTERNAL; ret = -1; goto out; } @@ -2899,14 +2982,14 @@ out: g_strfreev(src_names); g_strfreev(dst_names); } - if (error) - g_clear_error(&error); if (ret == 0) { update_repo_size (dst_repo_id); } else { - if (task && !task->canceled) + if (task && !task->canceled) { task->failed = TRUE; + set_failed_reason (&(task->failed_reason), err_str); + } } return ret; diff --git a/server/size-sched.c b/server/size-sched.c index 39ba398..4058b70 100644 --- a/server/size-sched.c +++ b/server/size-sched.c @@ -5,7 +5,7 @@ #include "seafile-session.h" #include "size-sched.h" - +#include "diff-simple.h" #define DEBUG_FLAG SEAFILE_DEBUG_OTHER #include "log.h" @@ -19,6 +19,12 @@ typedef struct RepoSizeJob { char repo_id[37]; } RepoSizeJob; +typedef struct RepoInfo { + gchar *head_id; + gint64 size; + gint64 file_count; +} RepoInfo; + static void* compute_repo_size (void *vjob); static void @@ -220,15 +226,62 @@ rollback: return ret; } -static char * -get_cached_head_id (SeafDB *db, const char *repo_id) +static gboolean +create_old_repo_info (SeafDBRow *row, void *data) { - char *sql; + RepoInfo **info = data; - sql = "SELECT head_id FROM RepoSize WHERE repo_id=?"; - return seaf_db_statement_get_string (db, sql, 1, "string", repo_id); + const char *head_id = seaf_db_row_get_column_text (row, 0); + gint64 size = seaf_db_row_get_column_int64 (row, 1); + gint64 file_count = seaf_db_row_get_column_int64 (row, 2); + + if (!head_id) + return FALSE; + + *info = g_new0(RepoInfo, 1); + if (!*info) + return FALSE; + (*info)->head_id = g_strdup(head_id); + (*info)->size = size; + (*info)->file_count = file_count; + + return TRUE; } +static RepoInfo* +get_old_repo_info_from_db (SeafDB *db, const char *repo_id, gboolean *is_db_err) +{ + RepoInfo *info = NULL; + char *sql; + + switch (seaf_db_type (db)) { + case SEAF_DB_TYPE_MYSQL: + case SEAF_DB_TYPE_PGSQL: + sql = "select s.head_id,s.size,f.file_count FROM " + "RepoSize s LEFT JOIN RepoFileCount f ON " + "s.repo_id=f.repo_id WHERE " + "s.repo_id=? FOR UPDATE"; + break; + case SEAF_DB_TYPE_SQLITE: + sql = "select s.head_id,s.size,f.file_count FROM " + "RepoSize s LEFT JOIN RepoFileCount f ON " + "s.repo_id=f.repo_id WHERE " + "s.repo_id=?"; + break; + default: + seaf_warning("Unexpected database type.\n"); + *is_db_err = TRUE; + return NULL; + } + int ret = seaf_db_statement_foreach_row (db, sql, + create_old_repo_info, &info, + 1, "string", repo_id); + if (ret < 0) + *is_db_err = TRUE; + + return info; + +} static void* compute_repo_size (void *vjob) @@ -237,12 +290,14 @@ compute_repo_size (void *vjob) SizeScheduler *sched = job->sched; SeafRepo *repo = NULL; SeafCommit *head = NULL; - char *cached_head_id = NULL; + SeafCommit *old_head = NULL; GObject *file_count_info = NULL; gint64 size = 0; gint64 file_count = 0; - GError **error = NULL; int ret; + RepoInfo *info = NULL; + GError *error = NULL; + gboolean is_db_err = FALSE; repo = seaf_repo_manager_get_repo (sched->seaf->repo_mgr, job->repo_id); if (!repo) { @@ -250,8 +305,10 @@ compute_repo_size (void *vjob) return vjob; } - cached_head_id = get_cached_head_id (sched->seaf->db, job->repo_id); - if (g_strcmp0 (cached_head_id, repo->head->commit_id) == 0) + info = get_old_repo_info_from_db(sched->seaf->db, job->repo_id, &is_db_err); + if (is_db_err) + goto out; + if (info && g_strcmp0 (info->head_id, repo->head->commit_id) == 0) goto out; head = seaf_commit_manager_get_commit (sched->seaf->commit_mgr, @@ -263,28 +320,59 @@ compute_repo_size (void *vjob) goto out; } - file_count_info = seaf_fs_manager_get_file_count_info_by_path (seaf->fs_mgr, - repo->store_id, - repo->version, - repo->root_id, - "/", error); + if (info){ + old_head = seaf_commit_manager_get_commit (sched->seaf->commit_mgr, + repo->id, repo->version, + info->head_id); - if (!file_count_info) { - seaf_warning ("[scheduler] failed to get file count info.\n"); - g_clear_error (error); - goto out; + gint64 change_size = 0; + gint64 change_file_count = 0; + GList *diff_entries = NULL; + + ret = diff_commits (old_head, head, &diff_entries, FALSE); + if (ret < 0) { + seaf_warning("[scheduler] failed to do diff.\n"); + goto out; + } + GList *des = NULL; + for (des = diff_entries; des ; des = des->next){ + DiffEntry *diff_entry = des->data; + if (diff_entry->status == DIFF_STATUS_DELETED){ + change_size -= diff_entry->size; + --change_file_count; + } + else if (diff_entry->status == DIFF_STATUS_ADDED){ + change_size += diff_entry->size; + ++change_file_count; + } + else if (diff_entry->status == DIFF_STATUS_MODIFIED) + change_size = change_size + diff_entry->size - diff_entry->origin_size; + } + size = info->size + change_size; + file_count = info->file_count + change_file_count; + + g_list_free_full (diff_entries, (GDestroyNotify)diff_entry_free); + } else { + file_count_info = seaf_fs_manager_get_file_count_info_by_path (seaf->fs_mgr, + repo->store_id, + repo->version, + repo->root_id, + "/", &error); + if (!file_count_info) { + seaf_warning ("[scheduler] failed to get file count info.\n"); + g_clear_error (&error); + goto out; + } + g_object_get (file_count_info, "file_count", &file_count, "size", &size, NULL); + g_object_unref (file_count_info); } - g_object_get (file_count_info, "file_count", &file_count, "size", &size, NULL); - ret = set_repo_size_and_file_count (sched->seaf->db, job->repo_id, repo->head->commit_id, size, file_count); - - g_object_unref (file_count_info); - + if (ret < 0) { seaf_warning ("[scheduler] failed to store repo size and file count %s.\n", job->repo_id); goto out; @@ -293,7 +381,10 @@ compute_repo_size (void *vjob) out: seaf_repo_unref (repo); seaf_commit_unref (head); - g_free (cached_head_id); + seaf_commit_unref (old_head); + if (info) + g_free (info->head_id); + g_free (info); return vjob; } diff --git a/server/upload-file.c b/server/upload-file.c index b09e009..21f955a 100755 --- a/server/upload-file.c +++ b/server/upload-file.c @@ -89,8 +89,8 @@ typedef struct RecvFSM { static GHashTable *upload_progress; static pthread_mutex_t pg_lock; static int -append_block_data_to_tmp_file (RecvFSM *fsm, const char *parent_dir, - const char *file_name); +write_block_data_to_tmp_file (RecvFSM *fsm, const char *parent_dir, + const char *file_name); /* IE8 will set filename to the full path of the uploaded file. * So we need to strip out the basename from it. @@ -456,8 +456,8 @@ upload_api_cb(evhtp_request_t *req, void *arg) goto out; } - if (append_block_data_to_tmp_file (fsm, new_parent_dir, - (char *)fsm->filenames->data) < 0) { + if (write_block_data_to_tmp_file (fsm, new_parent_dir, + (char *)fsm->filenames->data) < 0) { error_code = ERROR_INTERNAL; goto error; } @@ -914,7 +914,7 @@ error: /* } */ static int -copy_block_to_tmp_file (int blk_fd, int tmp_fd) +copy_block_to_tmp_file (int blk_fd, int tmp_fd, gint64 offset) { if (lseek(blk_fd, 0, SEEK_SET) < 0) { seaf_warning ("Failed to rewind block temp file position to start: %s\n", @@ -922,6 +922,12 @@ copy_block_to_tmp_file (int blk_fd, int tmp_fd) return -1; } + if (lseek(tmp_fd, offset, SEEK_SET) <0) { + seaf_warning ("Failed to rewind web upload temp file write position: %s\n", + strerror(errno)); + return -1; + } + char buf[8192]; int buf_len = sizeof(buf); ssize_t len; @@ -945,8 +951,8 @@ copy_block_to_tmp_file (int blk_fd, int tmp_fd) } static int -append_block_data_to_tmp_file (RecvFSM *fsm, const char *parent_dir, - const char *file_name) +write_block_data_to_tmp_file (RecvFSM *fsm, const char *parent_dir, + const char *file_name) { char *abs_path; char *temp_file = NULL; @@ -973,31 +979,40 @@ append_block_data_to_tmp_file (RecvFSM *fsm, const char *parent_dir, seaf->http_server->http_temp_dir, file_name); tmp_fd = g_mkstemp_full (temp_file, O_RDWR, cluster_shared_temp_file_mode); - if (tmp_fd >= 0) { - if (seaf_repo_manager_add_upload_tmp_file (seaf->repo_mgr, - fsm->repo_id, - abs_path, temp_file, - &error) < 0) { - seaf_warning ("%s\n", error->message); - g_clear_error (&error); - close (tmp_fd); - g_unlink (temp_file); - tmp_fd = -1; - ret = -1; - goto out; - } + if (tmp_fd < 0) { + seaf_warning ("Failed to create upload temp file: %s.\n", strerror(errno)); + ret = -1; + goto out; + } + + if (seaf_repo_manager_add_upload_tmp_file (seaf->repo_mgr, + fsm->repo_id, + abs_path, temp_file, + &error) < 0) { + seaf_warning ("%s\n", error->message); + g_clear_error (&error); + close (tmp_fd); + g_unlink (temp_file); + tmp_fd = -1; + ret = -1; + goto out; } } else { - tmp_fd = g_open (temp_file, O_WRONLY | O_APPEND); + tmp_fd = g_open (temp_file, O_WRONLY); + if (tmp_fd < 0) { + seaf_warning ("Failed to open upload temp file: %s.\n", strerror(errno)); + if (errno == ENOENT) { + seaf_message ("Upload temp file %s doesn't exist, remove record from db.\n", + temp_file); + seaf_repo_manager_del_upload_tmp_file (seaf->repo_mgr, fsm->repo_id, + abs_path, &error); + } + ret = -1; + goto out; + } } - if (tmp_fd < 0) { - seaf_warning ("Failed to open upload temp file: %s.\n", strerror(errno)); - ret = -1; - goto out; - } - - if (copy_block_to_tmp_file (fsm->fd, tmp_fd) < 0) { + if (copy_block_to_tmp_file (fsm->fd, tmp_fd, fsm->rstart) < 0) { ret = -1; goto out; } @@ -1105,8 +1120,8 @@ upload_ajax_cb(evhtp_request_t *req, void *arg) goto out; } - if (append_block_data_to_tmp_file (fsm, new_parent_dir, - (char *)fsm->filenames->data) < 0) { + if (write_block_data_to_tmp_file (fsm, new_parent_dir, + (char *)fsm->filenames->data) < 0) { error_code = ERROR_INTERNAL; goto error; } @@ -1294,7 +1309,7 @@ update_api_cb(evhtp_request_t *req, void *arg) goto out; } - if (append_block_data_to_tmp_file (fsm, parent_dir, filename) < 0) { + if (write_block_data_to_tmp_file (fsm, parent_dir, filename) < 0) { send_error_reply (req, EVHTP_RES_SERVERR, "Internal error.\n"); goto out; } @@ -2340,7 +2355,7 @@ check_access_token (const char *token, _repo_id = seafile_web_access_get_repo_id (webaccess); int status = seaf_repo_manager_get_repo_status(seaf->repo_mgr, _repo_id); - if (status != REPO_STATUS_NORMAL) { + if (status != REPO_STATUS_NORMAL && status != -1) { g_object_unref (webaccess); return -1; } diff --git a/server/zip-download-mgr.c b/server/zip-download-mgr.c index eb4626d..a7ec75f 100644 --- a/server/zip-download-mgr.c +++ b/server/zip-download-mgr.c @@ -548,6 +548,10 @@ zip_download_mgr_start_zip_task (ZipDownloadMgr *mgr, } progress = g_new0 (Progress, 1); + /* Set to real total in worker thread. Here to just prevent the client from thinking + * the zip has been finished too early. + */ + progress->total = 1; progress->expire_ts = time(NULL) + PROGRESS_TTL; obj->progress = progress; diff --git a/tests/test_file_operation/test_file_operation.py b/tests/test_file_operation/test_file_operation.py index 2d0c153..b553907 100644 --- a/tests/test_file_operation/test_file_operation.py +++ b/tests/test_file_operation/test_file_operation.py @@ -42,6 +42,20 @@ def test_file_operation(): # test copy_file (asynchronous) t_repo_id2 = api.create_repo('test_file_operation2', '', USER, passwd = None) + usage = api.get_user_self_usage (USER) + api.set_user_quota(USER, usage + 1); + t_copy_file_result2 = api.copy_file(t_repo_id1, '/', file_name, t_repo_id2, '/', file_name, USER, 1, 0) + assert t_copy_file_result2 + assert t_copy_file_result2.background + while True: + time.sleep(0.1) + t_copy_task = api.get_copy_task(t_copy_file_result2.task_id) + assert t_copy_task.failed + assert t_copy_task.failed_reason == 'Quota is full' + if t_copy_task.failed: + break; + + api.set_user_quota(USER, -1); t_copy_file_result2 = api.copy_file(t_repo_id1, '/', file_name, t_repo_id2, '/', file_name, USER, 1, 0) assert t_copy_file_result2 assert t_copy_file_result2.task_id @@ -70,6 +84,21 @@ def test_file_operation(): assert t_file_id is None # test move_file (asynchronous) + usage = api.get_user_self_usage (USER) + api.set_user_quota(USER, usage + 1); + t_move_file_result2 = api.move_file(t_repo_id1, '/', file_name, t_repo_id2, '/' , new_file_name, 1, USER, 1, 0) + assert t_move_file_result2 + assert t_move_file_result2.task_id + assert t_move_file_result2.background + while True: + time.sleep(0.1) + t_move_task = api.get_copy_task(t_move_file_result2.task_id) + assert t_move_task.failed + assert t_move_task.failed_reason == 'Quota is full' + if t_move_task.failed: + break + + api.set_user_quota(USER, -1); t_move_file_result2 = api.move_file(t_repo_id1, '/', file_name, t_repo_id2, '/' , new_file_name, 1, USER, 1, 0) assert t_move_file_result2 assert t_move_file_result2.task_id diff --git a/tests/test_file_operation/test_upload_and_update.py b/tests/test_file_operation/test_upload_and_update.py new file mode 100644 index 0000000..237fe48 --- /dev/null +++ b/tests/test_file_operation/test_upload_and_update.py @@ -0,0 +1,239 @@ +import pytest +import requests +import os +from tests.config import USER +from seaserv import seafile_api as api + +file_name = 'file.txt' +file_name_not_replaced = 'file (1).txt' +file_path = os.getcwd() + '/' + file_name +file_content = 'File content.\r\n' +file_size = len(file_content) + +resumable_file_name = 'resumable.txt' +chunked_part1_name = 'part1.txt' +chunked_part2_name = 'part2.txt' +chunked_part1_path = os.getcwd() + '/' + chunked_part1_name +chunked_part2_path = os.getcwd() + '/' + chunked_part2_name +chunked_part1_content = 'First line.\r\n' +chunked_part2_content = 'Second line.\r\n' +total_size = len(chunked_part1_content) + len(chunked_part2_content) + +#File_id is not used when upload files, but +#the argument obj_id of get_fileserver_access_token shouldn't be NULL. +file_id = '0000000000000000000000000000000000000000' + +def create_test_file(): + fp = open(file_path, 'w') + fp.close() + fp = open(chunked_part1_path, 'w') + fp.close() + fp = open(chunked_part2_path, 'w') + fp.close() + +def assert_upload_response(response, replace, file_exist): + assert response.status_code == 200 + response_json = response.json() + assert response_json[0]['size'] == 0 + assert response_json[0]['id'] == file_id + if file_exist and not replace: + assert response_json[0]['name'] == file_name_not_replaced + else: + assert response_json[0]['name'] == file_name + +def assert_resumable_upload_response(response, repo_id, file_name, upload_complete): + assert response.status_code == 200 + if not upload_complete: + assert response.text == '{"success": true}' + offset = api.get_upload_tmp_file_offset(repo_id, '/' + file_name) + assert offset == len(chunked_part1_content) + else: + response_json = response.json() + assert response_json[0]['size'] == total_size + new_file_id = response_json[0]['id'] + assert len(new_file_id) == 40 and new_file_id != file_id + assert response_json[0]['name'] == resumable_file_name + +def assert_update_response(response, is_json): + assert response.status_code == 200 + if is_json: + response_json = response.json() + assert response_json[0]['size'] == file_size + new_file_id = response_json[0]['id'] + assert len(new_file_id) == 40 and new_file_id != file_id + assert response_json[0]['name'] == file_name + else: + new_file_id = response.text + assert len(new_file_id) == 40 and new_file_id != file_id + +def write_file(file_path, file_content): + fp = open(file_path, 'w') + fp.write(file_content) + fp.close() + +def del_repo_files(repo_id): + api.del_file(repo_id, '/', file_name, USER) + api.del_file(repo_id, '/', file_name_not_replaced, USER) + api.del_file(repo_id, '/', 'subdir', USER) + api.del_file(repo_id, '/', resumable_file_name, USER) + +def del_local_files(): + os.remove(file_path) + os.remove(chunked_part1_path) + os.remove(chunked_part2_path) + +def test_ajax(repo): + create_test_file() + #test upload file to root dir. + token = api.get_fileserver_access_token(repo.id, file_id, 'upload', USER, False) + upload_url_base = 'http://127.0.0.1:8082/upload-aj/'+ token + files = {'file': open(file_path, 'rb'), + 'parent_dir':'/'} + response = requests.post(upload_url_base, files = files) + assert_upload_response(response, False, False) + + #test upload file to root dir when file already exists. + response = requests.post(upload_url_base, files = files) + assert_upload_response(response, False, True) + + #test upload file to subdir. + files = {'file': open(file_path, 'rb'), + 'parent_dir':'/', + 'relative_path':'subdir'} + response = requests.post(upload_url_base, files = files) + assert_upload_response(response, False, False) + + #test upload file to subdir when file already exists. + response = requests.post(upload_url_base, files = files) + assert_upload_response(response, False, True) + + #test resumable upload file + write_file(chunked_part1_path, chunked_part1_content) + write_file(chunked_part2_path, chunked_part2_content) + + token = api.get_fileserver_access_token(repo.id, file_id, + 'upload', USER, False) + headers = {'Content-Range':'bytes 0-{}/{}'.format(str(len(chunked_part1_content) - 1), + str(total_size)), + 'Content-Disposition':'attachment; filename=\"{}\"'.format(resumable_file_name)} + files = {'file': open(chunked_part1_path, 'rb'), + 'parent_dir':'/'} + response = requests.post(upload_url_base, headers = headers, + files = files) + assert_resumable_upload_response(response, repo.id, + resumable_file_name, False) + + headers = {'Content-Range':'bytes {}-{}/{}'.format(str(len(chunked_part1_content)), + str(total_size - 1), + str(total_size)), + 'Content-Disposition':'attachment; filename=\"{}\"'.format(resumable_file_name)} + files = {'file': open(chunked_part2_path, 'rb'), + 'parent_dir':'/'} + response = requests.post(upload_url_base, headers = headers, + files = files) + assert_resumable_upload_response(response, repo.id, + resumable_file_name, True) + + #test update file. + write_file(file_path, file_content) + token = api.get_fileserver_access_token(repo.id, file_id, 'update', USER, False) + update_url_base = 'http://127.0.0.1:8082/update-aj/' + token + files = {'file': open(file_path, 'rb'), + 'target_file':'/' + file_name} + response = requests.post(update_url_base, files = files) + assert_update_response(response, True) + + del_repo_files(repo.id) + del_local_files() + +def test_api(repo): + create_test_file() + params = {'ret-json':'1'} + + #test upload file to root dir. + token = api.get_fileserver_access_token(repo.id, file_id, 'upload', USER, False) + upload_url_base = 'http://127.0.0.1:8082/upload-api/' + token + files = {'file':open(file_path, 'rb'), + 'parent_dir':'/'} + response = requests.post(upload_url_base, params = params, + files = files) + assert_upload_response(response, False, False) + + #test upload file to root dir when file already exists and replace is set. + files = {'file':open(file_path, 'rb'), + 'parent_dir':'/', + 'replace':'1'} + response = requests.post(upload_url_base, params = params, + files = files) + assert_upload_response(response, True, True) + + #test upload file to root dir when file already exists and replace is unset. + files = {'file':open(file_path, 'rb'), + 'parent_dir':'/'} + response = requests.post(upload_url_base, params = params, + files = files) + assert_upload_response(response, False, True) + + #test upload the file to subdir. + files = {'file':open(file_path, 'rb'), + 'parent_dir':'/', + 'relative_path':'subdir'} + response = requests.post(upload_url_base, params = params, + files = files) + assert_upload_response(response, False, False) + + #test upload the file to subdir when file already exists and replace is set. + files = {'file':open(file_path, 'rb'), + 'parent_dir':'/', + 'relative_path':'subdir', + 'replace':'1'} + response = requests.post(upload_url_base, params = params, + files = files) + assert_upload_response(response, True, True) + + #unset test upload the file to subdir when file already exists and replace is unset. + files = {'file':open(file_path, 'rb'), + 'parent_dir':'/', + 'relative_path':'subdir'} + response = requests.post(upload_url_base, params = params, + files = files) + assert_upload_response(response, False, True) + + #test resumable upload file + write_file(chunked_part1_path, chunked_part1_content) + write_file(chunked_part2_path, chunked_part2_content) + + token = api.get_fileserver_access_token(repo.id, file_id, + 'upload', USER, False) + headers = {'Content-Range':'bytes 0-{}/{}'.format(str(len(chunked_part1_content) - 1), + str(total_size)), + 'Content-Disposition':'attachment; filename=\"{}\"'.format(resumable_file_name)} + files = {'file': open(chunked_part1_path, 'rb'), + 'parent_dir':'/'} + response = requests.post(upload_url_base, headers = headers, + files = files, params = params) + assert_resumable_upload_response(response, repo.id, + resumable_file_name, False) + + headers = {'Content-Range':'bytes {}-{}/{}'.format(str(len(chunked_part1_content)), + str(total_size - 1), + str(total_size)), + 'Content-Disposition':'attachment; filename=\"{}\"'.format(resumable_file_name)} + files = {'file': open(chunked_part2_path, 'rb'), + 'parent_dir':'/'} + response = requests.post(upload_url_base, headers = headers, + files = files, params = params) + assert_resumable_upload_response(response, repo.id, + resumable_file_name, True) + + #test update file. + write_file(file_path, file_content) + token = api.get_fileserver_access_token(repo.id, file_id, 'update', USER, False) + update_url_base = 'http://127.0.0.1:8082/update-api/' + token + files = {'file':open(file_path, 'rb'), + 'target_file':'/' + file_name} + response = requests.post(update_url_base, files = files) + assert_update_response(response, False) + + del_repo_files(repo.id) + del_local_files()