A few optimizations for SQL queries related to repos.

- Support paging for listing owned repos
- Reduce SQL queries for getting and listing repos.

common/rpc-service.c

@@ -77,6 +77,7 @@ convert_repo (SeafRepo *r)
     g_object_set (repo, "store_id", r->store_id,
                   "repaired", r->repaired,
                   "size", r->size, "file_count", r->file_count, NULL);
+    g_object_set (repo, "is_corrupted", r->is_corrupted, NULL);
 #endif
 
 #ifndef SEAFILE_SERVER
@@ -1973,32 +1974,32 @@ seafile_get_orphan_repo_list(GError **error)
 }
 
 GList *
-seafile_list_owned_repos (const char *email, int ret_corrupted, GError **error)
+seafile_list_owned_repos (const char *email, int ret_corrupted,
+                          int start, int limit, GError **error)
 {
     GList *ret = NULL;
     GList *repos, *ptr;
-    char *repo_id = NULL;
-    int is_shared;
 
-    repos = seaf_repo_manager_get_repos_by_owner (seaf->repo_mgr, email, ret_corrupted);
+    repos = seaf_repo_manager_get_repos_by_owner (seaf->repo_mgr, email, ret_corrupted,
+                                                  start, limit);
     ret = convert_repo_list (repos);
 
-    for (ptr = ret; ptr; ptr = ptr->next) {
+    /* for (ptr = ret; ptr; ptr = ptr->next) { */
-        g_object_get (ptr->data, "repo_id", &repo_id, NULL);
+    /*     g_object_get (ptr->data, "repo_id", &repo_id, NULL); */
-        is_shared = seaf_share_manager_is_repo_shared (seaf->share_mgr, repo_id);
+    /*     is_shared = seaf_share_manager_is_repo_shared (seaf->share_mgr, repo_id); */
-        if (is_shared < 0) {
+    /*     if (is_shared < 0) { */
-            g_free (repo_id);
+    /*         g_free (repo_id); */
-            break;
+    /*         break; */
-        } else {
+    /*     } else { */
-            g_object_set (ptr->data, "is_shared", is_shared, NULL);
+    /*         g_object_set (ptr->data, "is_shared", is_shared, NULL); */
-            g_free (repo_id);
+    /*         g_free (repo_id); */
-        }
+    /*     } */
-    }
+    /* } */
 
-    while (ptr) {
+    /* while (ptr) { */
-        g_object_set (ptr->data, "is_shared", FALSE, NULL);
+    /*     g_object_set (ptr->data, "is_shared", FALSE, NULL); */
-        ptr = ptr->prev;
+    /*     ptr = ptr->prev; */
-    }
+    /* } */
 
     for(ptr = repos; ptr; ptr = ptr->next) {
         seaf_repo_unref ((SeafRepo *)ptr->data);

include/seafile-rpc.h

@@ -446,7 +446,8 @@ GList *
 seafile_get_orphan_repo_list(GError **error);
 
 GList *
-seafile_list_owned_repos (const char *email, int ret_corrupted, GError **error);
+seafile_list_owned_repos (const char *email, int ret_corrupted, int start, int limit,
+                          GError **error);
 
 /**
  * seafile_add_chunk_server:

lib/rpc_table.py

@@ -72,6 +72,7 @@ func_table = [
     [ "objlist", ["string"] ],        
     [ "objlist", ["string", "int"] ],
     [ "objlist", ["string", "int", "int"] ],
+    [ "objlist", ["string", "int", "int", "int"] ],
     [ "objlist", ["string", "int", "string"] ],
     [ "objlist", ["string", "string"] ],        
     [ "objlist", ["string", "string", "string"] ],

python/seafile/rpcclient.py

@@ -375,8 +375,8 @@ class SeafServerThreadedRpcClient(ccnet.RpcClientBase):
         pass
     get_orphan_repo_list = seafile_get_orphan_repo_list
 
-    @searpc_func("objlist", ["string", "int"])
+    @searpc_func("objlist", ["string", "int", "int", "int"])
-    def seafile_list_owned_repos(user_id, ret_corrupted):
+    def seafile_list_owned_repos(user_id, ret_corrupted, start, limit):
         pass
     list_owned_repos = seafile_list_owned_repos
 

python/seaserv/api.py

@@ -107,12 +107,13 @@ class SeafileAPI(object):
         """
         return seafserv_threaded_rpc.get_repo_owner(repo_id)
 
-    def get_owned_repo_list(self, username, ret_corrupted=False):
+    def get_owned_repo_list(self, username, ret_corrupted=False, start=-1, limit=-1):
         """
         Return: a list of Repo objects
         """
         return seafserv_threaded_rpc.list_owned_repos(username,
-                                                      1 if ret_corrupted else 0)
+                                                      1 if ret_corrupted else 0,
+                                                      start, limit)
 
     def get_orphan_repo_list(self):
         return seafserv_threaded_rpc.get_orphan_repo_list()

server/repo-mgr.c

@@ -635,15 +635,24 @@ gboolean
 create_repo_fill_size (SeafDBRow *row, void *data)
 {
     SeafRepo **repo = data;
+    SeafBranch *head;
 
     const char *repo_id = seaf_db_row_get_column_text (row, 0);
     gint64 size = seaf_db_row_get_column_int64 (row, 1);
+    const char *commit_id = seaf_db_row_get_column_text (row, 2);
 
     *repo = seaf_repo_new (repo_id, NULL, NULL);
     if (!*repo)
         return FALSE;
 
+    if (!commit_id) {
+        (*repo)->is_corrupted = TRUE;
+        return FALSE;
+    }
+
     (*repo)->size = size;
+    head = seaf_branch_new ("master", repo_id, commit_id);
+    (*repo)->head = head;
 
     return TRUE;
 }
@@ -652,8 +661,18 @@ static SeafRepo*
 get_repo_from_db (SeafRepoManager *mgr, const char *id, gboolean *db_err)
 {
     SeafRepo *repo = NULL;
-    const char *sql = "SELECT r.repo_id, s.size FROM Repo r left join RepoSize s "
+    const char *sql;
-                      "ON r.repo_id = s.repo_id WHERE r.repo_id = ?";
+
+    if (seaf_db_type(mgr->seaf->db) != SEAF_DB_TYPE_PGSQL)
+        sql = "SELECT r.repo_id, s.size, b.commit_id FROM "
+            "Repo r LEFT JOIN Branch b ON r.repo_id = b.repo_id "
+            "LEFT JOIN RepoSize s ON r.repo_id = s.repo_id "
+            "WHERE r.repo_id = ? AND b.name = 'master'";
+    else
+        sql = "SELECT r.repo_id, s.\"size\", b.commit_id FROM "
+            "Repo r LEFT JOIN Branch b ON r.repo_id = b.repo_id "
+            "LEFT JOIN RepoSize s ON r.repo_id = s.repo_id "
+            "WHERE r.repo_id = ? AND b.name = 'master'";
 
     int ret = seaf_db_statement_foreach_row (mgr->seaf->db, sql,
                                              create_repo_fill_size, &repo,
@@ -668,15 +687,20 @@ SeafRepo*
 seaf_repo_manager_get_repo (SeafRepoManager *manager, const gchar *id)
 {
     int len = strlen(id);
-    gboolean db_err = FALSE;
     SeafRepo *repo = NULL;
+    gboolean has_err = FALSE;
 
     if (len >= 37)
         return NULL;
 
-    repo = get_repo_from_db (manager, id, &db_err);
+    repo = get_repo_from_db (manager, id, &has_err);
 
     if (repo) {
+        if (repo->is_corrupted) {
+            seaf_repo_unref (repo);
+            return NULL;
+        }
+
         load_repo (manager, repo);
         if (repo->is_corrupted) {
             seaf_repo_unref (repo);
@@ -691,20 +715,24 @@ SeafRepo*
 seaf_repo_manager_get_repo_ex (SeafRepoManager *manager, const gchar *id)
 {
     int len = strlen(id);
-    gboolean db_err = FALSE;
+    gboolean has_err = FALSE;
     SeafRepo *ret = NULL;
 
     if (len >= 37)
         return NULL;
 
-    ret = get_repo_from_db (manager, id, &db_err);
+    ret = get_repo_from_db (manager, id, &has_err);
-    if (db_err) {
+    if (has_err) {
         ret = seaf_repo_new(id, NULL, NULL);
         ret->is_corrupted = TRUE;
         return ret;
     }
 
     if (ret) {
+        if (ret->is_corrupted) {
+            return ret;
+        }
+
         load_repo (manager, ret);
     }
 
@@ -765,21 +793,19 @@ seaf_repo_manager_branch_repo_unmap (SeafRepoManager *manager, SeafBranch *branc
 
 static void
 load_repo_commit (SeafRepoManager *manager,
-                  SeafRepo *repo,
+                  SeafRepo *repo)
-                  SeafBranch *branch)
 {
     SeafCommit *commit;
 
     commit = seaf_commit_manager_get_commit_compatible (manager->seaf->commit_mgr,
                                                         repo->id,
-                                                        branch->commit_id);
+                                                        repo->head->commit_id);
     if (!commit) {
-        seaf_warning ("Commit %s:%s is missing\n", repo->id, branch->commit_id);
+        seaf_warning ("Commit %s:%s is missing\n", repo->id, repo->head->commit_id);
         repo->is_corrupted = TRUE;
         return;
     }
 
-    set_head_common (repo, branch);
     seaf_repo_from_commit (repo, commit);
 
     seaf_commit_unref (commit);
@@ -788,18 +814,9 @@ load_repo_commit (SeafRepoManager *manager,
 static void
 load_repo (SeafRepoManager *manager, SeafRepo *repo)
 {
-    SeafBranch *branch;
-
     repo->manager = manager;
 
-    branch = seaf_branch_manager_get_branch (seaf->branch_mgr, repo->id, "master");
+    load_repo_commit (manager, repo);
-    if (!branch) {
-        seaf_warning ("Failed to get master branch of repo %.8s.\n", repo->id);
-        repo->is_corrupted = TRUE;
-    } else {
-        load_repo_commit (manager, repo, branch);
-        seaf_branch_unref (branch);
-    }
 
     if (repo->is_corrupted) {
         return;
@@ -2032,36 +2049,115 @@ seaf_repo_manager_get_orphan_repo_list (SeafRepoManager *mgr)
     return ret;
 }
 
+gboolean
+collect_repos_fill_size_commit (SeafDBRow *row, void *data)
+{
+    GList **prepos = data;
+    SeafRepo *repo;
+    SeafBranch *head;
+
+    const char *repo_id = seaf_db_row_get_column_text (row, 0);
+    gint64 size = seaf_db_row_get_column_int64 (row, 1);
+    const char *commit_id = seaf_db_row_get_column_text (row, 2);
+
+    repo = seaf_repo_new (repo_id, NULL, NULL);
+    if (!repo)
+        return TRUE;
+
+    if (!commit_id) {
+        repo->is_corrupted = TRUE;
+        goto out;
+    }
+
+    repo->size = size;
+    head = seaf_branch_new ("master", repo_id, commit_id);
+    repo->head = head;
+
+out:
+    *prepos = g_list_prepend (*prepos, repo);
+
+    return TRUE;
+}
+
 GList *
 seaf_repo_manager_get_repos_by_owner (SeafRepoManager *mgr,
                                       const char *email,
-                                      int ret_corrupted)
+                                      int ret_corrupted,
+                                      int start,
+                                      int limit)
 {
-    GList *id_list = NULL, *ptr;
+    GList *repo_list = NULL, *ptr;
     GList *ret = NULL;
     char *sql;
     SeafRepo *repo = NULL;
+    int db_type = seaf_db_type(mgr->seaf->db);
 
-    sql = "SELECT repo_id FROM RepoOwner WHERE owner_id=?";
+    if (start == -1 && limit == -1) {
+        if (db_type != SEAF_DB_TYPE_PGSQL)
+            sql = "SELECT o.repo_id, s.size, b.commit_id FROM "
+                "RepoOwner o LEFT JOIN RepoSize s ON o.repo_id = s.repo_id "
+                "LEFT JOIN Branch b ON o.repo_id = b.repo_id "
+                "WHERE owner_id=? AND "
+                "o.repo_id NOT IN (SELECT v.repo_id FROM VirtualRepo v)";
+        else
+            sql = "SELECT o.repo_id, s.\"size\", b.commit_id FROM "
+                "RepoOwner o LEFT JOIN RepoSize s ON o.repo_id = s.repo_id "
+                "LEFT JOIN Branch b ON o.repo_id = b.repo_id "
+                "WHERE owner_id=? AND "
+                "o.repo_id NOT IN (SELECT v.repo_id FROM VirtualRepo v)";
 
-    if (seaf_db_statement_foreach_row (mgr->seaf->db, sql, 
+        if (seaf_db_statement_foreach_row (mgr->seaf->db, sql, 
-                                       collect_repo_id, &id_list,
+                                           collect_repos_fill_size_commit, &repo_list,
-                                       1, "string", email) < 0)
+                                           1, "string", email) < 0)
-        return NULL;
+            return NULL;
+    } else {
+        if (db_type != SEAF_DB_TYPE_PGSQL)
+            sql = "SELECT o.repo_id, s.size, b.commit_id FROM "
+                "RepoOwner o LEFT JOIN RepoSize s ON o.repo_id = s.repo_id "
+                "LEFT JOIN Branch b ON o.repo_id = b.repo_id "
+                "WHERE owner_id=? AND "
+                "o.repo_id NOT IN (SELECT v.repo_id FROM VirtualRepo v) "
+                "LIMIT ? OFFSET ?";
+        else
+            sql = "SELECT o.repo_id, s.\"size\", b.commit_id FROM "
+                "RepoOwner o LEFT JOIN RepoSize s ON o.repo_id = s.repo_id "
+                "LEFT JOIN Branch b ON o.repo_id = b.repo_id "
+                "WHERE owner_id=? AND "
+                "o.repo_id NOT IN (SELECT v.repo_id FROM VirtualRepo v) "
+                "LIMIT ? OFFSET ?";
 
-    for (ptr = id_list; ptr; ptr = ptr->next) {
+        if (seaf_db_statement_foreach_row (mgr->seaf->db, sql, 
-        char *repo_id = ptr->data;
+                                           collect_repos_fill_size_commit,
+                                           &repo_list,
+                                           3, "string", email,
+                                           "int", limit,
+                                           "int", start) < 0) {
+            return NULL;
+        }
+    }
+
+    for (ptr = repo_list; ptr; ptr = ptr->next) {
+        repo = ptr->data;
         if (ret_corrupted) {
-            repo = seaf_repo_manager_get_repo_ex (mgr, repo_id);
+            if (!repo->is_corrupted)
+                load_repo (mgr, repo);
         } else {
-            repo = seaf_repo_manager_get_repo (mgr, repo_id);
+            if (repo->is_corrupted) {
+                seaf_repo_unref (repo);
+                continue;
+            }
+
+            load_repo (mgr, repo);
+
+            if (repo->is_corrupted) {
+                seaf_repo_unref (repo);
+                continue;
+            }
         }
         if (repo != NULL)
             ret = g_list_prepend (ret, repo);
     }
 
-    string_list_free (id_list);
-
     return ret;
 }
 

server/repo-mgr.h

@@ -552,7 +552,9 @@ seaf_repo_manager_get_orphan_repo_list (SeafRepoManager *mgr);
 GList *
 seaf_repo_manager_get_repos_by_owner (SeafRepoManager *mgr,
                                       const char *email,
-                                      int ret_corrupted);
+                                      int ret_corrupted,
+                                      int start,
+                                      int limit);
 
 GList *
 seaf_repo_manager_get_repo_ids_by_owner (SeafRepoManager *mgr,

server/seaf-server.c

@@ -174,7 +174,7 @@ static void start_rpc_service (CcnetClient *client, int cloud_mode)
     searpc_server_register_function ("seafserv-threaded-rpcserver",
                                      seafile_list_owned_repos,
                                      "seafile_list_owned_repos",
-                                     searpc_signature_objlist__string_int());
+                                     searpc_signature_objlist__string_int_int_int());
 
     searpc_server_register_function ("seafserv-threaded-rpcserver",
                                      seafile_server_repo_size,