From d851199f06fb01bb5ad864cb95ee8222e4342fa9 Mon Sep 17 00:00:00 2001 From: ly1217 Date: Tue, 19 Feb 2019 01:18:31 -0800 Subject: [PATCH] Add byte-range upload. --- common/rpc-service.c | 27 ++ include/seafile-rpc.h | 4 + python/seafile/rpcclient.py | 5 + python/seaserv/api.py | 3 + server/http-server.c | 24 ++ server/http-server.h | 1 + server/repo-mgr.c | 129 +++++++++ server/repo-mgr.h | 25 ++ server/seaf-server.c | 5 + server/upload-file.c | 529 ++++++++++++++++++++++++++++++------ 10 files changed, 671 insertions(+), 81 deletions(-) mode change 100644 => 100755 server/upload-file.c diff --git a/common/rpc-service.c b/common/rpc-service.c index bf61640..36398fa 100644 --- a/common/rpc-service.c +++ b/common/rpc-service.c @@ -5293,6 +5293,33 @@ seafile_org_get_shared_users_by_repo (int org_id, org_id, repo_id); } +/* Resumable file upload. */ + +gint64 +seafile_get_upload_tmp_file_offset (const char *repo_id, const char *file_path, + GError **error) +{ + if (!repo_id || !is_uuid_valid(repo_id)) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS, + "Invalid repo id"); + return -1; + } + + int path_len; + if (!file_path || (path_len = strlen(file_path)) == 0) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS, + "Invalid file path"); + return -1; + } + + char *rfile_path = format_dir_path (file_path); + gint64 ret = seaf_repo_manager_get_upload_tmp_file_offset (seaf->repo_mgr, repo_id, + rfile_path, error); + g_free (rfile_path); + + return ret; +} + char * seafile_convert_repo_path (const char *repo_id, const char *path, diff --git a/include/seafile-rpc.h b/include/seafile-rpc.h index d7e0073..923e81d 100644 --- a/include/seafile-rpc.h +++ b/include/seafile-rpc.h @@ -1103,6 +1103,10 @@ seafile_org_get_shared_users_by_repo (int org_id, const char *repo_id, GError **error); +gint64 +seafile_get_upload_tmp_file_offset (const char *repo_id, const char *file_path, + GError **error); + char * seafile_convert_repo_path (const char *repo_id, const char *path, diff --git a/python/seafile/rpcclient.py b/python/seafile/rpcclient.py index d7ad782..aae1694 100644 --- a/python/seafile/rpcclient.py +++ b/python/seafile/rpcclient.py @@ -989,6 +989,11 @@ class SeafServerThreadedRpcClient(ccnet.RpcClientBase): def get_trash_repo_owner(repo_id): pass + @searpc_func("int64", ["string", "string"]) + def seafile_get_upload_tmp_file_offset(repo_id, file_path): + pass + get_upload_tmp_file_offset = seafile_get_upload_tmp_file_offset + @searpc_func("int", ["string", "string", "string", "string"]) def seafile_mkdir_with_parents (repo_id, parent_dir, relative_path, username): pass diff --git a/python/seaserv/api.py b/python/seaserv/api.py index fb8b2fe..423368a 100644 --- a/python/seaserv/api.py +++ b/python/seaserv/api.py @@ -383,6 +383,9 @@ class SeafileAPI(object): def check_repo_blocks_missing(self, repo_id, blklist): return seafserv_threaded_rpc.check_repo_blocks_missing(repo_id, blklist) + def get_upload_tmp_file_offset (self, repo_id, file_path): + return seafserv_threaded_rpc.get_upload_tmp_file_offset (repo_id, file_path) + # file lock def check_file_lock(self, repo_id, path, user): """ diff --git a/server/http-server.c b/server/http-server.c index f9adcc2..ce3286d 100644 --- a/server/http-server.c +++ b/server/http-server.c @@ -35,6 +35,7 @@ #define DEFAULT_MAX_INDEXING_THREADS 1 #define DEFAULT_MAX_INDEX_PROCESSING_THREADS 3 #define DEFAULT_FIXED_BLOCK_SIZE ((gint64)1 << 23) /* 8MB */ +#define DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE 0600 #define HOST "host" #define PORT "port" @@ -130,6 +131,7 @@ load_http_config (HttpServerStruct *htp_server, SeafileSession *session) char *encoding; int max_indexing_threads; int max_index_processing_threads; + char *cluster_shared_temp_file_mode = NULL; host = fileserver_config_get_string (session->config, HOST, &error); if (!error) { @@ -232,6 +234,28 @@ load_http_config (HttpServerStruct *htp_server, SeafileSession *session) seaf_message ("fileserver: max_index_processing_threads= %d\n", htp_server->max_index_processing_threads); + cluster_shared_temp_file_mode = fileserver_config_get_string (session->config, + "cluster_shared_temp_file_mode", + &error); + if (error) { + htp_server->cluster_shared_temp_file_mode = DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE; + g_clear_error (&error); + } else { + if (!cluster_shared_temp_file_mode) { + htp_server->cluster_shared_temp_file_mode = DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE; + } else { + htp_server->cluster_shared_temp_file_mode = strtol(cluster_shared_temp_file_mode, NULL, 8); + + if (htp_server->cluster_shared_temp_file_mode < 0001 || + htp_server->cluster_shared_temp_file_mode > 0777) + htp_server->cluster_shared_temp_file_mode = DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE; + + g_free (cluster_shared_temp_file_mode); + } + } + seaf_message ("fileserver: cluster_shared_temp_file_mode = %o\n", + htp_server->cluster_shared_temp_file_mode); + encoding = g_key_file_get_string (session->config, "zip", "windows_encoding", &error); diff --git a/server/http-server.h b/server/http-server.h index 9e766e6..3d725e4 100644 --- a/server/http-server.h +++ b/server/http-server.h @@ -21,6 +21,7 @@ struct _HttpServerStruct { int max_indexing_threads; int worker_threads; int max_index_processing_threads; + int cluster_shared_temp_file_mode; }; typedef struct _HttpServerStruct HttpServerStruct; diff --git a/server/repo-mgr.c b/server/repo-mgr.c index dfb21ac..9ebdf0b 100644 --- a/server/repo-mgr.c +++ b/server/repo-mgr.c @@ -1058,6 +1058,12 @@ create_tables_mysql (SeafRepoManager *mgr) if (seaf_db_query (db, sql) < 0) return -1; + sql = "CREATE TABLE IF NOT EXISTS WebUploadTempFiles ( " + "id BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, repo_id CHAR(40) NOT NULL, " + "file_path TEXT NOT NULL, tmp_file_path TEXT NOT NULL) ENGINE=INNODB"; + if (seaf_db_query (db, sql) < 0) + return -1; + return 0; } @@ -1206,6 +1212,11 @@ create_tables_sqlite (SeafRepoManager *mgr) if (seaf_db_query (db, sql) < 0) return -1; + sql = "CREATE TABLE IF NOT EXISTS WebUploadTempFiles (repo_id CHAR(40) NOT NULL, " + "file_path TEXT NOT NULL, tmp_file_path TEXT NOT NULL)"; + if (seaf_db_query (db, sql) < 0) + return -1; + return 0; } @@ -1344,6 +1355,11 @@ create_tables_pgsql (SeafRepoManager *mgr) if (seaf_db_query (db, sql) < 0) return -1; + sql = "CREATE TABLE IF NOT EXISTS WebUploadTempFiles (repo_id CHAR(40) NOT NULL, " + "file_path TEXT NOT NULL, tmp_file_path TEXT NOT NULL)"; + if (seaf_db_query (db, sql) < 0) + return -1; + sql = "CREATE TABLE IF NOT EXISTS RepoInfo (repo_id CHAR(36) PRIMARY KEY, " "name VARCHAR(255) NOT NULL, update_time BIGINT, version INTEGER, " "is_encrypted INTEGER, last_modifier VARCHAR(255))"; @@ -3918,6 +3934,119 @@ seaf_get_total_storage (GError **error) return size; } +int +seaf_repo_manager_add_upload_tmp_file (SeafRepoManager *mgr, + const char *repo_id, + const char *file_path, + const char *tmp_file, + GError **error) +{ + int ret = seaf_db_statement_query (mgr->seaf->db, + "INSERT INTO WebUploadTempFiles " + "(repo_id, file_path, tmp_file_path) " + "VALUES (?, ?, ?)", 3, "string", repo_id, + "string", file_path, "string", tmp_file); + + if (ret < 0) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "Failed to add upload tmp file record to db."); + } + + return ret; +} + +int +seaf_repo_manager_del_upload_tmp_file (SeafRepoManager *mgr, + const char *repo_id, + const char *file_path, + GError **error) +{ + int ret = seaf_db_statement_query (mgr->seaf->db, + "DELETE FROM WebUploadTempFiles WHERE " + "repo_id = ? AND file_path = ?", + 2, "string", repo_id, "string", file_path); + if (ret < 0) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "Failed to delete upload tmp file record from db."); + } + + return ret; +} + +static gboolean +get_tmp_file_path (SeafDBRow *row, void *data) +{ + char **path = data; + + *path = g_strdup (seaf_db_row_get_column_text (row, 0)); + + return FALSE; +} + +char * +seaf_repo_manager_get_upload_tmp_file (SeafRepoManager *mgr, + const char *repo_id, + const char *file_path, + GError **error) +{ + char *tmp_file_path = NULL; + + int ret = seaf_db_statement_foreach_row (mgr->seaf->db, + "SELECT tmp_file_path FROM WebUploadTempFiles " + "WHERE repo_id = ? AND file_path = ?", + get_tmp_file_path, &tmp_file_path, + 2, "string", repo_id, "string", file_path); + if (ret < 0) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "Failed to get upload temp file path from db."); + return NULL; + } + + return tmp_file_path; +} + +gint64 +seaf_repo_manager_get_upload_tmp_file_offset (SeafRepoManager *mgr, + const char *repo_id, + const char *file_path, + GError **error) +{ + char *tmp_file_path = NULL; + SeafStat file_stat; + + tmp_file_path = seaf_repo_manager_get_upload_tmp_file (mgr, repo_id, + file_path, error); + if (*error) { + return -1; + } + + if (!tmp_file_path) + return 0; + + if (seaf_stat (tmp_file_path, &file_stat) < 0) { + if (errno == ENOENT) { + seaf_message ("Temp file %s doesn't exist, remove reocrd from db.\n", + tmp_file_path); + if (seaf_repo_manager_del_upload_tmp_file (mgr, repo_id, + file_path, error) < 0) { + g_free (tmp_file_path); + return -1; + } + return 0; + } + seaf_warning ("Failed to stat temp file %s: %s.\n", + tmp_file_path, strerror(errno)); + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "Failed to stat temp file."); + g_free (tmp_file_path); + return -1; + } + + g_free (tmp_file_path); + + return file_stat.st_size; +} + void seaf_repo_manager_update_repo_info (SeafRepoManager *mgr, const char *repo_id, const char *head_commit_id) diff --git a/server/repo-mgr.h b/server/repo-mgr.h index 52f001e..b8c2d37 100644 --- a/server/repo-mgr.h +++ b/server/repo-mgr.h @@ -818,6 +818,31 @@ seaf_get_total_file_number (GError **error); gint64 seaf_get_total_storage (GError **error); +int +seaf_repo_manager_add_upload_tmp_file (SeafRepoManager *mgr, + const char *repo_id, + const char *file_path, + const char *tmp_file, + GError **error); + +int +seaf_repo_manager_del_upload_tmp_file (SeafRepoManager *mgr, + const char *repo_id, + const char *file_path, + GError **error); + +char * +seaf_repo_manager_get_upload_tmp_file (SeafRepoManager *mgr, + const char *repo_id, + const char *file_path, + GError **error); + +gint64 +seaf_repo_manager_get_upload_tmp_file_offset (SeafRepoManager *mgr, + const char *repo_id, + const char *file_path, + GError **error); + void seaf_repo_manager_update_repo_info (SeafRepoManager *mgr, const char *repo_id, diff --git a/server/seaf-server.c b/server/seaf-server.c index d6cb3a3..62a5069 100644 --- a/server/seaf-server.c +++ b/server/seaf-server.c @@ -587,6 +587,11 @@ static void start_rpc_service (CcnetClient *client, int cloud_mode) "get_virtual_repo", searpc_signature_object__string_string_string()); + searpc_server_register_function ("seafserv-threaded-rpcserver", + seafile_get_upload_tmp_file_offset, + "seafile_get_upload_tmp_file_offset", + searpc_signature_int64__string_string()); + /* Clean trash */ searpc_server_register_function ("seafserv-threaded-rpcserver", diff --git a/server/upload-file.c b/server/upload-file.c old mode 100644 new mode 100755 index c133b0c..f851425 --- a/server/upload-file.c +++ b/server/upload-file.c @@ -78,12 +78,19 @@ typedef struct RecvFSM { char *token_type; /* For sending statistic type */ gboolean need_idx_progress; + + gint64 rstart; + gint64 rend; + gint64 fsize; } RecvFSM; #define MAX_CONTENT_LINE 10240 static GHashTable *upload_progress; static pthread_mutex_t pg_lock; +static int +append_block_data_to_tmp_file (RecvFSM *fsm, const char *parent_dir, + const char *file_name); /* IE8 will set filename to the full path of the uploaded file. * So we need to strip out the basename from it. @@ -167,6 +174,24 @@ send_redirect_reply (evhtp_request_t *req) evhtp_send_reply(req, EVHTP_RES_SEEOTHER); } +static void +send_success_reply_ie8_compatible (evhtp_request_t *req, evhtp_res code) +{ + set_content_length_header (req); + + const char *accept = evhtp_kv_find (req->headers_in, "Accept"); + if (accept && strstr (accept, "application/json") != NULL) { + evhtp_headers_add_header ( + req->headers_out, + evhtp_header_new("Content-Type", "application/json; charset=utf-8", 1, 1)); + } else { + evhtp_headers_add_header ( + req->headers_out, + evhtp_header_new("Content-Type", "text/plain", 1, 1)); + } + evhtp_send_reply (req, code); +} + static void redirect_to_upload_error (evhtp_request_t *req, const char *repo_id, @@ -370,37 +395,27 @@ file_list_to_json (GList *files) } static int -create_relative_path (RecvFSM *fsm, char *parent_dir, char **abs_path) +create_relative_path (RecvFSM *fsm, char *parent_dir, char *relative_path) { int rc = 0; GError *error = NULL; - char *relative_path = g_hash_table_lookup (fsm->form_kvs, "relative_path"); - if (relative_path != NULL && strcmp(relative_path, "") != 0) { - rc = seaf_repo_manager_mkdir_with_parents (seaf->repo_mgr, - fsm->repo_id, - parent_dir, - relative_path, - fsm->user, - &error); - if (rc < 0) { - if (error) { - seaf_warning ("[upload folder] %s.", error->message); - g_clear_error (&error); - } - goto out; - } + if (!relative_path) + return 0; - *abs_path = g_new0 (char, SEAF_PATH_MAX); - if (!*abs_path) { - rc = -1; - goto out; + rc = seaf_repo_manager_mkdir_with_parents (seaf->repo_mgr, + fsm->repo_id, + parent_dir, + relative_path, + fsm->user, + &error); + if (rc < 0) { + if (error) { + seaf_warning ("[upload folder] %s.", error->message); + g_clear_error (&error); } - strcat (*abs_path, parent_dir); - strcat (*abs_path, relative_path); } -out: return rc; } @@ -518,6 +533,7 @@ upload_api_cb(evhtp_request_t *req, void *arg) { RecvFSM *fsm = arg; char *parent_dir, *replace_str; + char *relative_path = NULL, *new_parent_dir = NULL; GError *error = NULL; int error_code = ERROR_INTERNAL; char *filenames_json, *tmp_files_json; @@ -552,7 +568,7 @@ upload_api_cb(evhtp_request_t *req, void *arg) if (!fsm || fsm->state == RECV_ERROR) return; - if (!fsm->files) { + if (!fsm->filenames) { seaf_debug ("[upload] No file uploaded.\n"); send_error_reply (req, EVHTP_RES_BADREQ, "No file.\n"); return; @@ -573,14 +589,65 @@ upload_api_cb(evhtp_request_t *req, void *arg) send_error_reply (req, EVHTP_RES_BADREQ, "Invalid URL.\n"); return; } + relative_path = g_hash_table_lookup (fsm->form_kvs, "relative_path"); + if (relative_path != NULL) { + if (relative_path[0] == '/' || relative_path[0] == '\\') { + seaf_warning ("Invalid relative path %s.\n", relative_path); + send_error_reply (req, EVHTP_RES_BADREQ, "Invalid relative path."); + return; + } + char *tmp_p = get_canonical_path(parent_dir); + char *tmp_r = get_canonical_path(relative_path); + new_parent_dir = g_build_path("/", tmp_p, tmp_r, NULL); + g_free(tmp_p); + g_free(tmp_r); + } else { + new_parent_dir = get_canonical_path(parent_dir); + } + + if (fsm->rstart >= 0) { + if (fsm->filenames->next) { + seaf_debug ("[upload] Breakpoint transfer only support one file in one request.\n"); + send_error_reply (req, EVHTP_RES_BADREQ, "More files in one request.\n"); + goto out; + } + + if (parent_dir[0] != '/') { + seaf_debug ("[upload] Invalid parent dir, should start with /.\n"); + send_error_reply (req, EVHTP_RES_BADREQ, "Invalid parent dir.\n"); + goto out; + } + + if (append_block_data_to_tmp_file (fsm, new_parent_dir, + (char *)fsm->filenames->data) < 0) { + error_code = ERROR_INTERNAL; + goto error; + } + if (fsm->rend != fsm->fsize - 1) { + const char *success_str = "{\"success\": true}"; + evbuffer_add (req->buffer_out, success_str, strlen(success_str)); + send_success_reply_ie8_compatible (req, EVHTP_RES_OK); + goto out; + } + } + + if (!fsm->files) { + seaf_debug ("[upload] No file uploaded.\n"); + send_error_reply (req, EVHTP_RES_BADREQ, "No file.\n"); + goto out; + } if (!check_parent_dir (req, fsm->repo_id, parent_dir)) - return; + goto out; if (!check_tmp_file_list (fsm->files, &error_code)) goto error; - gint64 content_len = get_content_length(req); + gint64 content_len; + if (fsm->fsize > 0) + content_len = fsm->fsize; + else + content_len = get_content_length (req); if (seaf_quota_manager_check_quota_with_delta (seaf->quota_mgr, fsm->repo_id, content_len) != 0) { @@ -588,22 +655,18 @@ upload_api_cb(evhtp_request_t *req, void *arg) goto error; } + rc = create_relative_path (fsm, parent_dir, relative_path); + if (rc < 0) + goto error; + filenames_json = file_list_to_json (fsm->filenames); tmp_files_json = file_list_to_json (fsm->files); - char *abs_path = NULL; - rc = create_relative_path (fsm, parent_dir, &abs_path); - if (rc < 0) { - goto error; - } else if (abs_path) { - parent_dir = abs_path; - } - char *ret_json = NULL; char *task_id = NULL; rc = seaf_repo_manager_post_multi_files (seaf->repo_mgr, fsm->repo_id, - parent_dir, + new_parent_dir, filenames_json, tmp_files_json, fsm->user, @@ -611,8 +674,6 @@ upload_api_cb(evhtp_request_t *req, void *arg) &ret_json, fsm->need_idx_progress ? &task_id : NULL, &error); - if (abs_path) - g_free (abs_path); g_free (filenames_json); g_free (tmp_files_json); if (rc < 0) { @@ -648,9 +709,27 @@ upload_api_cb(evhtp_request_t *req, void *arg) oper = "link-file-upload"; send_statistic_msg(fsm->repo_id, fsm->user, oper, (guint64)content_len); + if (fsm->rstart >= 0 && fsm->rend == fsm->fsize - 1) { + // File upload success, try to remove tmp file from WebUploadTmpFile table + char *abs_path; + + if (new_parent_dir[strlen(new_parent_dir) - 1] == '/') { + abs_path = g_strconcat (new_parent_dir, (char *)fsm->filenames->data, NULL); + } else { + abs_path = g_strconcat (new_parent_dir, "/", (char *)fsm->filenames->data, NULL); + } + + seaf_repo_manager_del_upload_tmp_file (seaf->repo_mgr, fsm->repo_id, abs_path, NULL); + g_free (abs_path); + } + +out: + g_free(new_parent_dir); + return; error: + g_free(new_parent_dir); switch (error_code) { case ERROR_FILENAME: send_error_reply (req, SEAF_HTTP_RES_BADFILENAME, "Invalid filename.\n"); @@ -1000,6 +1079,118 @@ error: /* } */ /* } */ +static int +copy_block_to_tmp_file (int blk_fd, int tmp_fd) +{ + if (lseek(blk_fd, 0, SEEK_SET) < 0) { + seaf_warning ("Failed to rewind block temp file position to start: %s\n", + strerror(errno)); + return -1; + } + + char buf[8192]; + int buf_len = sizeof(buf); + ssize_t len; + + while (TRUE) { + len = readn (blk_fd, buf, buf_len); + if (len < 0) { + seaf_warning ("Failed to read content from block temp file: %s.\n", + strerror(errno)); + return -1; + } else if (len == 0) { + return 0; + } + + if (writen (tmp_fd, buf, len) != len) { + seaf_warning ("Failed to write content to temp file: %s.\n", + strerror(errno)); + return -1; + } + } +} + +static int +append_block_data_to_tmp_file (RecvFSM *fsm, const char *parent_dir, + const char *file_name) +{ + char *abs_path; + char *temp_file = NULL; + GError *error = NULL; + int tmp_fd = -1; + int ret = 0; + HttpServerStruct *htp_server = seaf->http_server; + int cluster_shared_temp_file_mode = htp_server->cluster_shared_temp_file_mode; + + if (parent_dir[strlen(parent_dir) - 1] == '/') { + abs_path = g_strconcat (parent_dir, file_name, NULL); + } else { + abs_path = g_strconcat (parent_dir, "/", file_name, NULL); + } + + temp_file = seaf_repo_manager_get_upload_tmp_file (seaf->repo_mgr, + fsm->repo_id, + abs_path, &error); + if (error) { + seaf_warning ("%s\n", error->message); + g_clear_error (&error); + ret = -1; + goto out; + } + + if (!temp_file) { + temp_file = g_strdup_printf ("%s/cluster-shared/%sXXXXXX", + seaf->http_server->http_temp_dir, + file_name); + tmp_fd = g_mkstemp_full (temp_file, O_RDWR, cluster_shared_temp_file_mode); + if (tmp_fd >= 0) { + if (seaf_repo_manager_add_upload_tmp_file (seaf->repo_mgr, + fsm->repo_id, + abs_path, temp_file, + &error) < 0) { + seaf_warning ("%s\n", error->message); + g_clear_error (&error); + close (tmp_fd); + g_unlink (temp_file); + tmp_fd = -1; + ret = -1; + goto out; + } + } + } else { + tmp_fd = g_open (temp_file, O_WRONLY | O_APPEND); + } + + if (tmp_fd < 0) { + seaf_warning ("Failed to open upload temp file: %s.\n", strerror(errno)); + ret = -1; + goto out; + } + + if (copy_block_to_tmp_file (fsm->fd, tmp_fd) < 0) { + ret = -1; + goto out; + } + + if (fsm->rend == fsm->fsize - 1) { + // For the last block, record tmp_files for upload to seafile and remove + fsm->tmp_files = g_list_prepend (fsm->tmp_files, g_strdup(temp_file)); // for cleaning up + fsm->files = g_list_prepend (fsm->files, g_strdup(temp_file)); // for virus checking, indexing... + } + +out: + g_free (abs_path); + if (tmp_fd >= 0) { + close (tmp_fd); + } + g_free (temp_file); + close (fsm->fd); + g_unlink (fsm->tmp_file); + g_free (fsm->tmp_file); + fsm->tmp_file = NULL; + + return ret; +} /* Handle AJAX file upload. @return an array of json data, e.g. [{"name": "foo.txt"}] @@ -1008,7 +1199,7 @@ static void upload_ajax_cb(evhtp_request_t *req, void *arg) { RecvFSM *fsm = arg; - char *parent_dir; + char *parent_dir = NULL, *relative_path = NULL, *new_parent_dir = NULL; GError *error = NULL; int error_code = ERROR_INTERNAL; char *filenames_json, *tmp_files_json; @@ -1016,7 +1207,7 @@ upload_ajax_cb(evhtp_request_t *req, void *arg) evhtp_headers_add_header (req->headers_out, evhtp_header_new("Access-Control-Allow-Headers", - "x-requested-with, content-type, accept, origin, authorization", 1, 1)); + "x-requested-with, content-type, content-range, content-disposition, accept, origin, authorization", 1, 1)); evhtp_headers_add_header (req->headers_out, evhtp_header_new("Access-Control-Allow-Methods", "GET, POST, PUT, PATCH, DELETE, OPTIONS", 1, 1)); @@ -1042,12 +1233,6 @@ upload_ajax_cb(evhtp_request_t *req, void *arg) if (!fsm || fsm->state == RECV_ERROR) return; - if (!fsm->files) { - seaf_debug ("[upload] No file uploaded.\n"); - send_error_reply (req, EVHTP_RES_BADREQ, "No file.\n"); - return; - } - parent_dir = g_hash_table_lookup (fsm->form_kvs, "parent_dir"); if (!parent_dir) { seaf_debug ("[upload] No parent dir given.\n"); @@ -1055,13 +1240,72 @@ upload_ajax_cb(evhtp_request_t *req, void *arg) return; } - if (!check_parent_dir (req, fsm->repo_id, parent_dir)) + if (!fsm->filenames) { + seaf_debug ("[upload] No file uploaded.\n"); + send_error_reply (req, EVHTP_RES_BADREQ, "No file.\n"); return; + } + + relative_path = g_hash_table_lookup (fsm->form_kvs, "relative_path"); + if (relative_path != NULL) { + if (relative_path[0] == '/' || relative_path[0] == '\\') { + seaf_warning ("Invalid relative path %s.\n", relative_path); + send_error_reply (req, EVHTP_RES_BADREQ, "Invalid relative path."); + return; + } + char *tmp_p = get_canonical_path(parent_dir); + char *tmp_r = get_canonical_path(relative_path); + new_parent_dir = g_build_path("/", tmp_p, tmp_r, NULL); + g_free(tmp_p); + g_free(tmp_r); + } else { + new_parent_dir = get_canonical_path(parent_dir); + } + + if (fsm->rstart >= 0) { + if (fsm->filenames->next) { + seaf_debug ("[upload] Breakpoint transfer only support one file in one request.\n"); + send_error_reply (req, EVHTP_RES_BADREQ, "More files in one request.\n"); + goto out; + } + + if (parent_dir[0] != '/') { + seaf_debug ("[upload] Invalid parent dir, should start with /.\n"); + send_error_reply (req, EVHTP_RES_BADREQ, "Invalid parent dir.\n"); + goto out; + } + + if (append_block_data_to_tmp_file (fsm, new_parent_dir, + (char *)fsm->filenames->data) < 0) { + error_code = ERROR_INTERNAL; + goto error; + } + if (fsm->rend != fsm->fsize - 1) { + const char *success_str = "{\"success\": true}"; + evbuffer_add (req->buffer_out, success_str, strlen(success_str)); + send_success_reply_ie8_compatible (req, EVHTP_RES_OK); + goto out; + } + } + + if (!fsm->files) { + seaf_debug ("[upload] No file uploaded.\n"); + send_error_reply (req, EVHTP_RES_BADREQ, "No file.\n"); + goto out; + } + + if (!check_parent_dir (req, fsm->repo_id, parent_dir)) + goto out; if (!check_tmp_file_list (fsm->files, &error_code)) goto error; - gint64 content_len = get_content_length (req); + gint64 content_len; + if (fsm->fsize > 0) + content_len = fsm->fsize; + else + content_len = get_content_length (req); + if (seaf_quota_manager_check_quota_with_delta (seaf->quota_mgr, fsm->repo_id, content_len) != 0) { @@ -1069,13 +1313,9 @@ upload_ajax_cb(evhtp_request_t *req, void *arg) goto error; } - char *abs_path = NULL; - rc = create_relative_path (fsm, parent_dir, &abs_path); - if (rc < 0) { + rc = create_relative_path (fsm, parent_dir, relative_path); + if (rc < 0) goto error; - } else if (abs_path) { - parent_dir = abs_path; - } filenames_json = file_list_to_json (fsm->filenames); tmp_files_json = file_list_to_json (fsm->files); @@ -1084,7 +1324,7 @@ upload_ajax_cb(evhtp_request_t *req, void *arg) char *task_id = NULL; rc = seaf_repo_manager_post_multi_files (seaf->repo_mgr, fsm->repo_id, - parent_dir, + new_parent_dir, filenames_json, tmp_files_json, fsm->user, @@ -1092,8 +1332,6 @@ upload_ajax_cb(evhtp_request_t *req, void *arg) &ret_json, fsm->need_idx_progress ? &task_id : NULL, &error); - if (abs_path) - g_free (abs_path); g_free (filenames_json); g_free (tmp_files_json); if (rc < 0) { @@ -1114,29 +1352,34 @@ upload_ajax_cb(evhtp_request_t *req, void *arg) } g_free (ret_json); - // send_success_reply (req); - set_content_length_header (req); + if (fsm->rstart >= 0 && fsm->rend == fsm->fsize - 1) { + // File upload success, try to remove tmp file from WebUploadTmpFile table + char *abs_path; - const char *accept = evhtp_kv_find (req->headers_in, "Accept"); - if (accept && strstr (accept, "application/json") != NULL) { - evhtp_headers_add_header ( - req->headers_out, - evhtp_header_new("Content-Type", "application/json; charset=utf-8", 1, 1)); - } else { - evhtp_headers_add_header ( - req->headers_out, - evhtp_header_new("Content-Type", "text/plain", 1, 1)); + if (new_parent_dir[strlen(new_parent_dir) - 1] == '/') { + abs_path = g_strconcat (new_parent_dir, (char *)fsm->filenames->data, NULL); + } else { + abs_path = g_strconcat (new_parent_dir, "/", (char *)fsm->filenames->data, NULL); + } + + seaf_repo_manager_del_upload_tmp_file (seaf->repo_mgr, fsm->repo_id, abs_path, NULL); + g_free (abs_path); } - evhtp_send_reply (req, EVHTP_RES_OK); + + send_success_reply_ie8_compatible (req, EVHTP_RES_OK); char *oper = "web-file-upload"; if (g_strcmp0(fsm->token_type, "upload-link") == 0) oper = "link-file-upload"; send_statistic_msg(fsm->repo_id, fsm->user, oper, (guint64)content_len); +out: + g_free (new_parent_dir); + return; error: + g_free(new_parent_dir); switch (error_code) { case ERROR_FILENAME: send_error_reply (req, SEAF_HTTP_RES_BADFILENAME, "Invalid filename."); @@ -1783,6 +2026,10 @@ upload_finish_cb (evhtp_request_t *req, void *arg) g_free (fsm->file_name); if (fsm->tmp_file) { close (fsm->fd); + // For resumable upload, in case tmp file not be deleted + if (fsm->rstart >= 0) { + g_unlink (fsm->tmp_file); + } } g_free (fsm->tmp_file); @@ -1827,8 +2074,41 @@ get_mime_header_param_value (const char *param) return value; } +static char * +parse_file_name_from_header (evhtp_request_t *req) +{ + const char *dispose = NULL; + char **p; + char **params; + char *dec_file_name = NULL; + + dispose = evhtp_kv_find (req->headers_in, "Content-Disposition"); + if (!dispose) + return NULL; + + params = g_strsplit (dispose, ";", 2); + for (p = params; *p != NULL; ++p) + *p = g_strstrip (*p); + + if (g_strv_length (params) != 2 || + strcasecmp (params[0], "attachment") != 0 || + strncasecmp (params[1], "filename", strlen("filename")) != 0) { + seaf_warning ("[upload] Invalid Content-Disposition header.\n"); + g_strfreev (params); + return NULL; + } + + char *file_name = get_mime_header_param_value (params[1]); + if (file_name) + dec_file_name = g_uri_unescape_string (file_name, NULL); + g_free (file_name); + g_strfreev (params); + + return dec_file_name; +} + static int -parse_mime_header (char *header, RecvFSM *fsm) +parse_mime_header (evhtp_request_t *req, char *header, RecvFSM *fsm) { char *colon; char **params, **p; @@ -1872,7 +2152,11 @@ parse_mime_header (char *header, RecvFSM *fsm) char *file_name; for (p = params; *p != NULL; ++p) { if (strncasecmp (*p, "filename", strlen("filename")) == 0) { - file_name = get_mime_header_param_value (*p); + if (fsm->rstart >= 0) { + file_name = parse_file_name_from_header (req); + } else { + file_name = get_mime_header_param_value (*p); + } if (file_name) { fsm->file_name = normalize_utf8_path (file_name); if (!fsm->file_name) @@ -1906,13 +2190,16 @@ open_temp_file (RecvFSM *fsm) fsm->fd = g_mkstemp (temp_file->str); if (fsm->fd < 0) { + seaf_warning("[upload] Failed to open temp file: %s.\n", strerror(errno)); g_string_free (temp_file, TRUE); return -1; } fsm->tmp_file = g_string_free (temp_file, FALSE); /* For clean up later. */ - fsm->tmp_files = g_list_prepend (fsm->tmp_files, g_strdup(fsm->tmp_file)); + if (fsm->rstart < 0) { + fsm->tmp_files = g_list_prepend (fsm->tmp_files, g_strdup(fsm->tmp_file)); + } return 0; } @@ -1954,16 +2241,25 @@ recv_form_field (RecvFSM *fsm, gboolean *no_line) static void add_uploaded_file (RecvFSM *fsm) { - fsm->filenames = g_list_prepend (fsm->filenames, - get_basename(fsm->file_name)); - fsm->files = g_list_prepend (fsm->files, g_strdup(fsm->tmp_file)); + if (fsm->rstart < 0) { + // Non breakpoint transfer, same as original + fsm->filenames = g_list_prepend (fsm->filenames, + get_basename(fsm->file_name)); + fsm->files = g_list_prepend (fsm->files, g_strdup(fsm->tmp_file)); - g_free (fsm->file_name); - g_free (fsm->tmp_file); - close (fsm->fd); - fsm->file_name = NULL; - fsm->tmp_file = NULL; - fsm->recved_crlf = FALSE; + g_free (fsm->file_name); + g_free (fsm->tmp_file); + close (fsm->fd); + fsm->file_name = NULL; + fsm->tmp_file = NULL; + fsm->recved_crlf = FALSE; + } else { + fsm->filenames = g_list_prepend (fsm->filenames, + get_basename(fsm->file_name)); + g_free (fsm->file_name); + fsm->file_name = NULL; + fsm->recved_crlf = FALSE; + } } static evhtp_res @@ -2111,7 +2407,7 @@ upload_read_cb (evhtp_request_t *req, evbuf_t *buf, void *arg) } seaf_debug ("[upload] Start to recv %s.\n", fsm->input_name); fsm->state = RECV_CONTENT; - } else if (parse_mime_header (line, fsm) < 0) { + } else if (parse_mime_header (req, line, fsm) < 0) { free (line); res = EVHTP_RES_BADREQ; goto out; @@ -2252,6 +2548,56 @@ check_access_token (const char *token, return 0; } +static gboolean +parse_range_val (evhtp_headers_t *hdr, gint64 *rstart, + gint64 *rend, gint64 *rfsize) +{ + const char *tmp = evhtp_kv_find (hdr, "Content-Range"); + if (!tmp) + return TRUE; + + char *next = NULL; + gint64 start; + gint64 end; + gint64 fsize; + + if (strstr (tmp, "bytes") != tmp) { + return FALSE; + } + + tmp += strlen("bytes"); + while (tmp && *tmp == ' ') { + tmp++; + } + + start = strtoll (tmp, &next, 10); + if ((start == 0 && next == tmp) || *next != '-') { + return FALSE; + } + + tmp = next + 1; + end = strtoll (tmp, &next, 10); + if ((end == 0 && next == tmp) || *next != '/') { + return FALSE; + } + + tmp = next + 1; + fsize = strtoll (tmp, &next, 10); + if ((fsize == 0 && next == tmp) || *next != '\0') { + return FALSE; + } + + if (start > end || end >= fsize) { + return FALSE; + } + + *rstart = start; + *rend = end; + *rfsize = fsize; + + return TRUE; +} + static int get_progress_info (evhtp_request_t *req, evhtp_headers_t *hdr, @@ -2332,11 +2678,23 @@ upload_headers_cb (evhtp_request_t *req, evhtp_headers_t *hdr, void *arg) pthread_mutex_unlock (&pg_lock); } + gint64 rstart = -1; + gint64 rend = -1; + gint64 fsize = -1; + if (!parse_range_val (hdr, &rstart, &rend, &fsize)) { + seaf_warning ("Invalid Seafile-Content-Range value.\n"); + err_msg = "Invalid Seafile-Content-Range"; + goto err; + } + fsm = g_new0 (RecvFSM, 1); fsm->boundary = boundary; fsm->repo_id = repo_id; fsm->user = user; fsm->token_type = token_type; + fsm->rstart = rstart; + fsm->rend = rend; + fsm->fsize = fsize; fsm->line = evbuffer_new (); fsm->form_kvs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); @@ -2464,6 +2822,15 @@ upload_file_init (evhtp_t *htp, const char *http_temp_dir) return -1; } + char *cluster_shared_dir = g_strdup_printf ("%s/cluster-shared", http_temp_dir); + if (g_mkdir_with_parents (cluster_shared_dir, 0777) < 0) { + seaf_warning ("Failed to create cluster shared dir %s.\n", + cluster_shared_dir); + g_free (cluster_shared_dir); + return -1; + } + g_free (cluster_shared_dir); + cb = evhtp_set_regex_cb (htp, "^/upload/.*", upload_cb, NULL); /* upload_headers_cb() will be called after evhtp parsed all http headers. */ evhtp_set_hook(&cb->hooks, evhtp_hook_on_headers, upload_headers_cb, NULL);