Get gc id before index blocks (#718)

* Get gc id before index blocks

* Add test gc online

* Add test gc online

* Adjust time of run gc

---------

Co-authored-by: 杨赫然 <heran.yang@seafile.com>

ci/run.py

@@ -237,10 +237,7 @@ def main():
     args = parse_args()
     if on_github_actions() and not args.test_only:
         fetch_and_build()
-    if on_github_actions():
+    dbs = ('mysql',)
-        dbs = ('sqlite3', 'mysql')
-    else:
-        dbs = ('sqlite3',)
     for db in dbs:
         start_and_test_with_db(db)
 

fileserver/fileop.go

@@ -1684,6 +1684,12 @@ func postMultiFiles(rsp http.ResponseWriter, r *http.Request, repoID, parentDir,
 		cryptKey = key
 	}
 
+	gcID, err := repomgr.GetCurrentGCID(repo.StoreID)
+	if err != nil {
+		err := fmt.Errorf("failed to get current gc id for repo %s: %v", repoID, err)
+		return &appError{err, "", http.StatusInternalServerError}
+	}
+
 	var ids []string
 	var sizes []int64
 	if fsm.rstart >= 0 {
@@ -1714,12 +1720,6 @@ func postMultiFiles(rsp http.ResponseWriter, r *http.Request, repoID, parentDir,
 		}
 	}
 
-	gcID, err := repomgr.GetCurrentGCID(repo.StoreID)
-	if err != nil {
-		err := fmt.Errorf("failed to get current gc id for repo %s: %v", repoID, err)
-		return &appError{err, "", http.StatusInternalServerError}
-	}
-
 	retStr, err := postFilesAndGenCommit(fileNames, repo.ID, user, canonPath, replace, ids, sizes, lastModify, gcID)
 	if err != nil {
 		err := fmt.Errorf("failed to post files and gen commit: %v", err)
@@ -3160,6 +3160,12 @@ func putFile(rsp http.ResponseWriter, r *http.Request, repoID, parentDir, user,
 		cryptKey = key
 	}
 
+	gcID, err := repomgr.GetCurrentGCID(repo.StoreID)
+	if err != nil {
+		err := fmt.Errorf("failed to get current gc id: %v", err)
+		return &appError{err, "", http.StatusInternalServerError}
+	}
+
 	var fileID string
 	var size int64
 	if fsm.rstart >= 0 {
@@ -3218,11 +3224,6 @@ func putFile(rsp http.ResponseWriter, r *http.Request, repoID, parentDir, user,
 		return &appError{err, "", http.StatusInternalServerError}
 	}
 
-	gcID, err := repomgr.GetCurrentGCID(repo.StoreID)
-	if err != nil {
-		err := fmt.Errorf("failed to get current gc id: %v", err)
-		return &appError{err, "", http.StatusInternalServerError}
-	}
 	desc := fmt.Sprintf("Modified \"%s\"", fileName)
 	_, err = genNewCommit(repo, headCommit, rootID, user, desc, true, gcID, true)
 	if err != nil {
@@ -3437,6 +3438,12 @@ func commitFileBlocks(repoID, parentDir, fileName, blockIDsJSON, user string, fi
 		return "", appErr
 	}
 
+	gcID, err := repomgr.GetCurrentGCID(repo.StoreID)
+	if err != nil {
+		err := fmt.Errorf("failed to get current gc id: %v", err)
+		return "", &appError{err, "", http.StatusInternalServerError}
+	}
+
 	fileID, appErr := indexExistedFileBlocks(repoID, repo.Version, blkIDs, fileSize)
 	if appErr != nil {
 		return "", appErr
@@ -3455,11 +3462,6 @@ func commitFileBlocks(repoID, parentDir, fileName, blockIDsJSON, user string, fi
 		return "", &appError{err, "", http.StatusInternalServerError}
 	}
 
-	gcID, err := repomgr.GetCurrentGCID(repo.StoreID)
-	if err != nil {
-		err := fmt.Errorf("failed to get current gc id: %v", err)
-		return "", &appError{err, "", http.StatusInternalServerError}
-	}
 	desc := fmt.Sprintf("Added \"%s\"", fileName)
 	_, err = genNewCommit(repo, headCommit, rootID, user, desc, true, gcID, true)
 	if err != nil {

tests/test_gc/test_gc.py

@@ -5,6 +5,8 @@ import time
 from subprocess import run
 from tests.config import USER, USER2
 from seaserv import seafile_api as api
+from concurrent.futures import ThreadPoolExecutor
+from requests_toolbelt import MultipartEncoder
 
 file_name = 'file.txt'
 first_name = 'first.txt'
@@ -30,6 +32,18 @@ def create_test_file():
     fp.write(third_content)
     fp.close()
 
+large_file_name = 'large.txt'
+large_file_size = 2*1024*1024*1024
+large_file_path = os.getcwd() + '/' + large_file_name
+
+def create_large_file():
+    fp = open(large_file_path, 'wb')
+    fp.write(os.urandom(large_file_size))
+    fp.close()
+
+def del_large_file():
+    os.remove(large_file_path)
+
 def del_local_files():
     os.remove(first_path)
     os.remove(second_path)
@@ -131,3 +145,52 @@ def test_gc_partial_history(repo, rm_fs):
     run_gc(repo.id, '', '--check')
 
     del_local_files()
+
+def upload_file(url, m):
+    start = 0
+    end = large_file_size - 1
+    total = large_file_size
+    response = requests.post(url,
+            data = m, headers = {'Content-Type': m.content_type, 
+                'Content-Range': f"bytes {start}-{end}/{total}", 
+                'Content-Disposition': 'attachment; filename="large.txt"'})
+    return response.status_code, response.text
+
+
+@pytest.mark.parametrize('rm_fs', ['', '--rm-fs'])
+def test_gc_on_upload(repo, rm_fs):
+    create_large_file()
+    api.set_repo_valid_since (repo.id, 0)
+
+    obj_id = '{"parent_dir":"/"}'
+    token = api.get_fileserver_access_token(repo.id, obj_id, 'upload', USER, False)
+    upload_url_base = 'http://127.0.0.1:8082/upload-aj/'+ token
+    m = MultipartEncoder(
+            fields={
+                    'parent_dir': '/',
+                    'file': (large_file_name, open(large_file_path, 'rb'), 'application/octet-stream')
+            })
+
+
+    status_code = 200
+    executor = ThreadPoolExecutor()
+    future = executor.submit(upload_file, upload_url_base, m)
+
+    while True:
+        offset = api.get_upload_tmp_file_offset(repo.id, "/" + large_file_name)
+        if offset == large_file_size:
+            break
+        time.sleep (0.5)
+    time.sleep (1)
+    run_gc(repo.id, rm_fs, '')
+
+    while not future.done():
+        time.sleep(0.5)
+
+    status_code = future.result()[0]
+    assert status_code == 500
+
+    api.set_repo_valid_since (repo.id, 0)
+    run_gc(repo.id, '', '--check')
+
+    del_large_file ()