diff --git a/ci/run.py b/ci/run.py index 9d67c69..48fed17 100755 --- a/ci/run.py +++ b/ci/run.py @@ -237,10 +237,7 @@ def main(): args = parse_args() if on_github_actions() and not args.test_only: fetch_and_build() - if on_github_actions(): - dbs = ('sqlite3', 'mysql') - else: - dbs = ('sqlite3',) + dbs = ('mysql',) for db in dbs: start_and_test_with_db(db) diff --git a/fileserver/fileop.go b/fileserver/fileop.go index 00a10f1..08e3718 100644 --- a/fileserver/fileop.go +++ b/fileserver/fileop.go @@ -1684,6 +1684,12 @@ func postMultiFiles(rsp http.ResponseWriter, r *http.Request, repoID, parentDir, cryptKey = key } + gcID, err := repomgr.GetCurrentGCID(repo.StoreID) + if err != nil { + err := fmt.Errorf("failed to get current gc id for repo %s: %v", repoID, err) + return &appError{err, "", http.StatusInternalServerError} + } + var ids []string var sizes []int64 if fsm.rstart >= 0 { @@ -1714,12 +1720,6 @@ func postMultiFiles(rsp http.ResponseWriter, r *http.Request, repoID, parentDir, } } - gcID, err := repomgr.GetCurrentGCID(repo.StoreID) - if err != nil { - err := fmt.Errorf("failed to get current gc id for repo %s: %v", repoID, err) - return &appError{err, "", http.StatusInternalServerError} - } - retStr, err := postFilesAndGenCommit(fileNames, repo.ID, user, canonPath, replace, ids, sizes, lastModify, gcID) if err != nil { err := fmt.Errorf("failed to post files and gen commit: %v", err) @@ -3160,6 +3160,12 @@ func putFile(rsp http.ResponseWriter, r *http.Request, repoID, parentDir, user, cryptKey = key } + gcID, err := repomgr.GetCurrentGCID(repo.StoreID) + if err != nil { + err := fmt.Errorf("failed to get current gc id: %v", err) + return &appError{err, "", http.StatusInternalServerError} + } + var fileID string var size int64 if fsm.rstart >= 0 { @@ -3218,11 +3224,6 @@ func putFile(rsp http.ResponseWriter, r *http.Request, repoID, parentDir, user, return &appError{err, "", http.StatusInternalServerError} } - gcID, err := repomgr.GetCurrentGCID(repo.StoreID) - if err != nil { - err := fmt.Errorf("failed to get current gc id: %v", err) - return &appError{err, "", http.StatusInternalServerError} - } desc := fmt.Sprintf("Modified \"%s\"", fileName) _, err = genNewCommit(repo, headCommit, rootID, user, desc, true, gcID, true) if err != nil { @@ -3437,6 +3438,12 @@ func commitFileBlocks(repoID, parentDir, fileName, blockIDsJSON, user string, fi return "", appErr } + gcID, err := repomgr.GetCurrentGCID(repo.StoreID) + if err != nil { + err := fmt.Errorf("failed to get current gc id: %v", err) + return "", &appError{err, "", http.StatusInternalServerError} + } + fileID, appErr := indexExistedFileBlocks(repoID, repo.Version, blkIDs, fileSize) if appErr != nil { return "", appErr @@ -3455,11 +3462,6 @@ func commitFileBlocks(repoID, parentDir, fileName, blockIDsJSON, user string, fi return "", &appError{err, "", http.StatusInternalServerError} } - gcID, err := repomgr.GetCurrentGCID(repo.StoreID) - if err != nil { - err := fmt.Errorf("failed to get current gc id: %v", err) - return "", &appError{err, "", http.StatusInternalServerError} - } desc := fmt.Sprintf("Added \"%s\"", fileName) _, err = genNewCommit(repo, headCommit, rootID, user, desc, true, gcID, true) if err != nil { diff --git a/tests/test_gc/test_gc.py b/tests/test_gc/test_gc.py index db0cb21..ca086ba 100644 --- a/tests/test_gc/test_gc.py +++ b/tests/test_gc/test_gc.py @@ -5,6 +5,8 @@ import time from subprocess import run from tests.config import USER, USER2 from seaserv import seafile_api as api +from concurrent.futures import ThreadPoolExecutor +from requests_toolbelt import MultipartEncoder file_name = 'file.txt' first_name = 'first.txt' @@ -30,6 +32,18 @@ def create_test_file(): fp.write(third_content) fp.close() +large_file_name = 'large.txt' +large_file_size = 2*1024*1024*1024 +large_file_path = os.getcwd() + '/' + large_file_name + +def create_large_file(): + fp = open(large_file_path, 'wb') + fp.write(os.urandom(large_file_size)) + fp.close() + +def del_large_file(): + os.remove(large_file_path) + def del_local_files(): os.remove(first_path) os.remove(second_path) @@ -131,3 +145,52 @@ def test_gc_partial_history(repo, rm_fs): run_gc(repo.id, '', '--check') del_local_files() + +def upload_file(url, m): + start = 0 + end = large_file_size - 1 + total = large_file_size + response = requests.post(url, + data = m, headers = {'Content-Type': m.content_type, + 'Content-Range': f"bytes {start}-{end}/{total}", + 'Content-Disposition': 'attachment; filename="large.txt"'}) + return response.status_code, response.text + + +@pytest.mark.parametrize('rm_fs', ['', '--rm-fs']) +def test_gc_on_upload(repo, rm_fs): + create_large_file() + api.set_repo_valid_since (repo.id, 0) + + obj_id = '{"parent_dir":"/"}' + token = api.get_fileserver_access_token(repo.id, obj_id, 'upload', USER, False) + upload_url_base = 'http://127.0.0.1:8082/upload-aj/'+ token + m = MultipartEncoder( + fields={ + 'parent_dir': '/', + 'file': (large_file_name, open(large_file_path, 'rb'), 'application/octet-stream') + }) + + + status_code = 200 + executor = ThreadPoolExecutor() + future = executor.submit(upload_file, upload_url_base, m) + + while True: + offset = api.get_upload_tmp_file_offset(repo.id, "/" + large_file_name) + if offset == large_file_size: + break + time.sleep (0.5) + time.sleep (1) + run_gc(repo.id, rm_fs, '') + + while not future.done(): + time.sleep(0.5) + + status_code = future.result()[0] + assert status_code == 500 + + api.set_repo_valid_since (repo.id, 0) + run_gc(repo.id, '', '--check') + + del_large_file ()