create merge virtual repo task pool (#473)

* create merge virtual repo task pool

* use for range

fileserver/fileop.go

@@ -1210,7 +1210,7 @@ func mkdirWithParents(repoID, parentDir, newDirPath, user string) error {
 		return err
 	}
 
-	go mergeVirtualRepo(repo.ID, "")
+	mergeVirtualRepoTasks <- repo.ID
 
 	return nil
 }
@@ -1529,7 +1529,7 @@ func postFilesAndGenCommit(fileNames []string, repo *repomgr.Repo, user, canonPa
 		return "", err
 	}
 
-	go mergeVirtualRepo(repo.ID, "")
+	mergeVirtualRepoTasks <- repo.ID
 
 	retJSON, err := formatJSONRet(names, ids, sizes)
 	if err != nil {
@@ -2742,7 +2742,7 @@ func putFile(rsp http.ResponseWriter, r *http.Request, repoID, parentDir, user,
 		rsp.Write([]byte(fileID))
 	}
 
-	go mergeVirtualRepo(repo.ID, "")
+	mergeVirtualRepoTasks <- repo.ID
 
 	return nil
 }

fileserver/fileserver.go

@@ -408,6 +408,8 @@ func main() {
 
 	sizeSchedulerInit()
 
+	virtualRepoInit()
+
 	initUpload()
 
 	router := newHTTPRouter()

fileserver/sync_api.go

@@ -923,7 +923,7 @@ func putUpdateBranchCB(rsp http.ResponseWriter, r *http.Request) *appError {
 		return &appError{err, "", http.StatusInternalServerError}
 	}
 
-	mergeVirtualRepo(repoID, "")
+	mergeVirtualRepoTasks <- repoID
 
 	go updateRepoSize(repoID)
 

fileserver/virtual_repo.go

@@ -15,6 +15,26 @@ import (
 	"github.com/haiwen/seafile-server/fileserver/repomgr"
 )
 
+const mergeVirtualRepoWorkerNumber = 5
+
+var mergeVirtualRepoTasks = make(chan string, 100)
+
+func virtualRepoInit() {
+	go createMergeVirtualRepoTaskPool(mergeVirtualRepoWorkerNumber)
+}
+
+func createMergeVirtualRepoTaskPool(n int) {
+	for i := 0; i < n; i++ {
+		go mergeVirtualRepoWorker()
+	}
+}
+
+func mergeVirtualRepoWorker() {
+	for repoID := range mergeVirtualRepoTasks {
+		mergeVirtualRepo(repoID, "")
+	}
+}
+
 func mergeVirtualRepo(repoID, excludeRepo string) {
 	virtual, err := repomgr.IsVirtualRepo(repoID)
 	if err != nil {