Add repair-repo to fix corrupt fs (#657)

Co-authored-by: 杨赫然 <heran.yang@seafile.com>

common/branch-mgr.c

@@ -356,6 +356,8 @@ notify_repo_update (const char *repo_id, const char *commit_id)
 static void
 on_branch_updated (SeafBranchManager *mgr, SeafBranch *branch)
 {
+    if (seaf->is_repair)
+        return;
     seaf_repo_manager_update_repo_info (seaf->repo_mgr, branch->repo_id, branch->commit_id);
 
     notify_repo_update(branch->repo_id, branch->commit_id);

common/merge-new.c

@@ -81,15 +81,12 @@ out:
     return conflict_name;
 }
 
-static int
-merge_entries (const char *store_id, int version,
-               int n, SeafDirent *dents[],
-               const char *basedir,
-               GList **dents_out,
-               MergeOptions *opt)
+int twoway_merge(const char *store_id, int version, const char *basedir,
+                 SeafDirent *dents[], GList **dents_out, struct MergeOptions *opt)
 {
-    SeafDirent *files[3];
+    SeafDirent *files[2];
     int i;
+    int n = opt->n_ways;
 
     memset (files, 0, sizeof(files[0])*n);
     for (i = 0; i < n; ++i) {
@@ -97,15 +94,66 @@ merge_entries (const char *store_id, int version,
             files[i] = dents[i];
     }
 
-    /* If we're running 2-way merge, or the caller requires not to
-     * actually merge contents, just call the callback function.
-     */
-    if (n == 2 || !opt->do_merge)
-        return opt->callback (basedir, files, opt);
+    SeafDirent *head, *remote;
+    char *conflict_name;
 
-    /* Otherwise, we're doing a real 3-way merge of the trees.
-     * It means merge files and handle any conflicts.
-     */
+    head = files[0];
+    remote = files[1];
+
+    if (head && remote) {
+        if (strcmp (head->id, remote->id) == 0) {
+            // file match
+            seaf_debug ("%s%s: files match\n", basedir, head->name);
+            *dents_out = g_list_prepend (*dents_out, seaf_dirent_dup(head));
+        } else {
+            // file content conflict
+            seaf_debug ("%s%s: files conflict\n", basedir, head->name);
+            conflict_name = merge_conflict_filename(store_id, version,
+                                                    opt,
+                                                    basedir,
+                                                    head->name);
+            if (!conflict_name)
+                return -1;
+
+            g_free (remote->name);
+            remote->name = conflict_name;
+            remote->name_len = strlen (remote->name);
+
+            *dents_out = g_list_prepend (*dents_out, seaf_dirent_dup(head));
+            *dents_out = g_list_prepend (*dents_out, seaf_dirent_dup(remote));
+
+            opt->conflict = TRUE;
+        }
+    } else if (!head && remote) {
+        // file not in head, but in remote
+        seaf_debug ("%s%s: added in remote\n", basedir, remote->name);
+        *dents_out = g_list_prepend (*dents_out, seaf_dirent_dup(remote));
+    } else if (head && !remote) {
+        // file in head, but not in remote
+        seaf_debug ("%s%s: added in head\n", basedir, head->name);
+        *dents_out = g_list_prepend (*dents_out, seaf_dirent_dup(head));
+    }
+
+    return 0;
+}
+
+static int
+threeway_merge (const char *store_id, int version,
+                 SeafDirent *dents[],
+                 const char *basedir,
+                 GList **dents_out,
+                 MergeOptions *opt)
+{
+    SeafDirent *files[3];
+    int i;
+    gint64 curr_time;
+    int n = opt->n_ways;
+
+    memset (files, 0, sizeof(files[0])*n);
+    for (i = 0; i < n; ++i) {
+        if (dents[i] && S_ISREG(dents[i]->mode))
+            files[i] = dents[i];
+    }
 
     SeafDirent *base, *head, *remote;
     char *conflict_name;
@@ -323,6 +371,25 @@ merge_entries (const char *store_id, int version,
     return 0;
 }
 
+static int
+merge_entries (const char *store_id, int version,
+               int n, SeafDirent *dents[],
+               const char *basedir,
+               GList **dents_out,
+               MergeOptions *opt)
+{
+    /* If we're running 2-way merge, it means merge files base on head and remote.
+     */
+    if (n == 2)
+        return twoway_merge (store_id, version, basedir, dents, dents_out, opt);
+
+    /* Otherwise, we're doing a real 3-way merge of the trees.
+     * It means merge files and handle any conflicts.
+     */
+
+    return threeway_merge (store_id, version, dents, basedir, dents_out, opt);
+}
+
 static int
 merge_directories (const char *store_id, int version,
                    int n, SeafDirent *dents[],
@@ -345,7 +412,7 @@ merge_directories (const char *store_id, int version,
 
     seaf_debug ("dir_mask = %d\n", dir_mask);
 
-    if (n == 3 && opt->do_merge) {
+    if (n == 3) {
         switch (dir_mask) {
         case 0:
             g_return_val_if_reached (-1);
@@ -407,6 +474,33 @@ merge_directories (const char *store_id, int version,
         default:
             g_return_val_if_reached (-1);
         }
+    } else if (n == 2) {
+        switch (dir_mask) {
+        case 0:
+            g_return_val_if_reached (-1);
+        case 1:
+            /*head is dir, remote is not dir*/
+            seaf_debug ("%s%s: only head is dir\n", basedir, dents[0]->name);
+            *dents_out = g_list_prepend (*dents_out, seaf_dirent_dup(dents[0]));
+            return 0;
+        case 2:
+            /*head is not dir, remote is dir*/
+            seaf_debug ("%s%s: only remote is dir\n", basedir, dents[1]->name);
+            *dents_out = g_list_prepend (*dents_out, seaf_dirent_dup(dents[1]));
+            return 0;
+        case 3:
+            if (strcmp (dents[0]->id, dents[1]->id) == 0) {
+                seaf_debug ("%s%s: dir is the same in head and remote\n",
+                            basedir, dents[0]->name);
+                *dents_out = g_list_prepend (*dents_out, seaf_dirent_dup(dents[1]));
+                return 0;
+            }
+            seaf_debug ("%s%s: dir is changed in head and remote, merge recursively\n",
+                        basedir, dents[0]->name);
+            break;
+        default:
+            g_return_val_if_reached (-1);
+        }
     }
 
     memset (sub_dirs, 0, sizeof(sub_dirs[0])*n);
@@ -433,7 +527,7 @@ merge_directories (const char *store_id, int version,
 
     g_free (new_basedir);
 
-    if (n == 3 && opt->do_merge) {
+    if (n == 3) {
         if (dir_mask == 3 || dir_mask == 6 || dir_mask == 7) {
             merged_dent = seaf_dirent_dup (dents[1]);
             memcpy (merged_dent->id, opt->merged_tree_root, 40);
@@ -443,6 +537,12 @@ merge_directories (const char *store_id, int version,
             memcpy (merged_dent->id, opt->merged_tree_root, 40);
             *dents_out = g_list_prepend (*dents_out, merged_dent);
         }
+    } else if (n == 2) {
+        if (dir_mask == 3) {
+            merged_dent = seaf_dirent_dup (dents[1]);
+            memcpy (merged_dent->id, opt->merged_tree_root, 40);
+            *dents_out = g_list_prepend (*dents_out, merged_dent);
+        }
     }
 
 free_sub_dirs:
@@ -539,7 +639,7 @@ merge_trees_recursive (const char *store_id, int version,
         }
     }
 
-    if (n == 3 && opt->do_merge) {
+    if (n == 3) {
         merged_dents = g_list_sort (merged_dents, compare_dirents);
         merged_tree = seaf_dir_new (NULL, merged_dents,
                                     dir_version_from_repo_version(version));
@@ -556,6 +656,23 @@ merge_trees_recursive (const char *store_id, int version,
                 seaf_warning ("Failed to save merged tree %s:%s.\n", store_id, basedir);
             }
         }
+    } else if (n == 2) {
+        merged_dents = g_list_sort (merged_dents, compare_dirents);
+        merged_tree = seaf_dir_new (NULL, merged_dents,
+                                    dir_version_from_repo_version(version));
+
+        memcpy (opt->merged_tree_root, merged_tree->dir_id, 40);
+
+        if ((trees[0] && strcmp (trees[0]->dir_id, merged_tree->dir_id) == 0) || 
+            (trees[1] && strcmp (trees[1]->dir_id, merged_tree->dir_id) == 0)) {
+            seaf_dir_free (merged_tree);
+        } else {
+            ret = seaf_dir_save (seaf->fs_mgr, store_id, version, merged_tree);
+            seaf_dir_free (merged_tree);
+            if (ret < 0) {
+                seaf_warning ("Failed to save merged tree %s:%s.\n", store_id, basedir);
+            }
+        }
     }
 
     return ret;

server/repo-mgr.h

@@ -923,4 +923,7 @@ seaf_repo_manager_set_repo_status(SeafRepoManager *mgr,
 int
 seaf_repo_manager_get_repo_status(SeafRepoManager *mgr,
                                   const char *repo_id);
+
+int
+seaf_repo_manager_repair_virtual_repo (char *repo_id);
 #endif

server/seaf-server.c

@@ -32,7 +32,7 @@ SeafileSession *seaf;
 
 char *pidfile = NULL;
 
-static const char *short_options = "hvc:d:l:fP:D:F:p:t";
+static const char *short_options = "hvc:d:l:fP:D:F:p:tr:";
 static struct option long_options[] = {
     { "help", no_argument, NULL, 'h', },
     { "version", no_argument, NULL, 'v', },
@@ -45,6 +45,7 @@ static struct option long_options[] = {
     { "pidfile", required_argument, NULL, 'P' },
     { "rpc-pipe-path", required_argument, NULL, 'p' },
     { "test-config", no_argument, NULL, 't' },
+    { "repair-repo", required_argument, NULL, 'r' },
     { NULL, 0, NULL, 0, },
 };
 
@@ -1211,6 +1212,7 @@ main (int argc, char **argv)
     const char *debug_str = NULL;
     int daemon_mode = 1;
     gboolean test_config = FALSE;
+    char *repo_id = NULL;
 
 #ifdef WIN32
     argv = get_argv_utf8 (&argc);
@@ -1253,6 +1255,9 @@ main (int argc, char **argv)
         case 't':
             test_config = TRUE;
             break;
+        case 'r':
+            repo_id = g_strdup (optarg);
+            break;
         default:
             usage ();
             exit (1);
@@ -1315,6 +1320,16 @@ main (int argc, char **argv)
 
     event_init ();
 
+    if (repo_id) {
+        seaf = seafile_repair_session_new (central_config_dir, seafile_dir, ccnet_dir);
+        if (!seaf) {
+            seaf_warning ("Failed to create repair seafile session.\n");
+            exit (1);
+        }
+        seaf_repo_manager_repair_virtual_repo (repo_id);
+        exit (0);
+    }
+
     seaf = seafile_session_new (central_config_dir, seafile_dir, ccnet_dir);
     if (!seaf) {
         seaf_warning ("Failed to create seafile session.\n");

server/seafile-session.c

@@ -313,6 +313,113 @@ onerror:
     return NULL;    
 }
 
+SeafileSession *
+seafile_repair_session_new(const char *central_config_dir,
+                           const char *seafile_dir,
+                           const char *ccnet_dir)
+{
+    char *abs_central_config_dir = NULL;
+    char *abs_seafile_dir;
+    char *abs_ccnet_dir = NULL;
+    char *tmp_file_dir;
+    char *config_file_path;
+    char *config_file_ccnet;
+    GKeyFile *config;
+    GKeyFile *ccnet_config;
+    SeafileSession *session = NULL;
+    gboolean notif_enabled = FALSE;
+    int notif_port = 8083;
+    gboolean cluster_mode;
+    gboolean use_block_cache;
+    int block_cache_size_limit;
+    char **block_cache_file_types;
+    gint64 repo_file_number_limit = -1;
+
+    abs_ccnet_dir = ccnet_expand_path (ccnet_dir);
+    abs_seafile_dir = ccnet_expand_path (seafile_dir);
+    tmp_file_dir = g_build_filename (abs_seafile_dir, "tmpfiles", NULL);
+    if (central_config_dir) {
+        abs_central_config_dir = ccnet_expand_path (central_config_dir);
+    }
+
+    config_file_path = g_build_filename(
+        abs_central_config_dir ? abs_central_config_dir : abs_seafile_dir,
+        "seafile.conf", NULL);
+
+    config_file_ccnet = g_build_filename(
+        abs_central_config_dir ? abs_central_config_dir : abs_ccnet_dir,
+        "ccnet.conf", NULL);
+
+    GError *error = NULL;
+    config = g_key_file_new ();
+    if (!g_key_file_load_from_file (config, config_file_path, 
+                                    G_KEY_FILE_NONE, &error)) {
+        seaf_warning ("Failed to load config file.\n");
+        g_key_file_free (config);
+        g_free (config_file_path);
+        goto onerror;
+    }
+    ccnet_config = g_key_file_new ();
+    g_key_file_set_list_separator (ccnet_config, ',');
+    if (!g_key_file_load_from_file (ccnet_config, config_file_ccnet,
+                                    G_KEY_FILE_KEEP_COMMENTS, NULL))
+    {
+        seaf_warning ("Can't load ccnet config file %s.\n", config_file_ccnet);
+        g_key_file_free (ccnet_config);
+        g_free (config_file_ccnet);
+        goto onerror;
+    }
+    g_free (config_file_path);
+    g_free (config_file_ccnet);
+
+    session = g_new0(SeafileSession, 1);
+    session->seaf_dir = abs_seafile_dir;
+    session->ccnet_dir = abs_ccnet_dir;
+    session->tmp_file_dir = tmp_file_dir;
+    session->config = config;
+    session->ccnet_config = ccnet_config;
+    session->is_repair = TRUE;
+
+    if (load_database_config (session) < 0) {
+        seaf_warning ("Failed to load database config.\n");
+        goto onerror;
+    }
+
+    if (load_ccnet_database_config (session) < 0) {
+        seaf_warning ("Failed to load ccnet database config.\n");
+        goto onerror;
+    }
+
+    session->fs_mgr = seaf_fs_manager_new (session, abs_seafile_dir);
+    if (!session->fs_mgr)
+        goto onerror;
+    session->block_mgr = seaf_block_manager_new (session, abs_seafile_dir);
+    if (!session->block_mgr)
+        goto onerror;
+    session->commit_mgr = seaf_commit_manager_new (session);
+    if (!session->commit_mgr)
+        goto onerror;
+    session->repo_mgr = seaf_repo_manager_new (session);
+    if (!session->repo_mgr)
+        goto onerror;
+    session->branch_mgr = seaf_branch_manager_new (session);
+    if (!session->branch_mgr)
+        goto onerror;
+
+    session->job_mgr = ccnet_job_manager_new (DEFAULT_THREAD_POOL_SIZE);
+
+    session->size_sched = size_scheduler_new (session);
+
+    return session;
+
+onerror:
+    free (abs_seafile_dir);
+    free (abs_ccnet_dir);
+    g_free (tmp_file_dir);
+    g_free (session);
+    return NULL;
+}
+
 int
 seafile_session_init (SeafileSession *session)
 {

server/seafile-session.h

@@ -90,6 +90,8 @@ struct _SeafileSession {
     // For notification server
     NotifManager *notif_mgr;
     char         *private_key;
+
+    gboolean is_repair;
 };
 
 extern SeafileSession *seaf;
@@ -98,6 +100,12 @@ SeafileSession *
 seafile_session_new(const char *central_config_dir, 
                     const char *seafile_dir,
                     const char *ccnet_dir);
+
+SeafileSession *
+seafile_repair_session_new(const char *central_config_dir,
+                           const char *seafile_dir,
+                           const char *ccnet_dir);
+
 int
 seafile_session_init (SeafileSession *session);
 

server/virtual-repo.c

@@ -1123,3 +1123,153 @@ seaf_repo_manager_init_merge_scheduler ()
                                         SCHEDULE_INTERVAL);
     return 0;
 }
+
+int
+seaf_repo_manager_repair_virtual_repo (char *repo_id)
+{
+    SeafRepoManager *mgr = seaf->repo_mgr;
+    SeafVirtRepo *vinfo = NULL;
+    SeafRepo *repo = NULL, *orig_repo = NULL;
+    SeafCommit *head = NULL, *orig_head = NULL;
+    char *root = NULL, *orig_root = NULL;
+    char new_base_commit[41] = {0};
+    int ret = 0;
+    GError *error = NULL;
+
+    /* repos */
+    repo = seaf_repo_manager_get_repo (mgr, repo_id);
+    if (!repo) {
+        seaf_warning ("Failed to get virt repo %.10s.\n", repo_id);
+        ret = -1;
+        goto out;
+    }
+
+    if (!repo->virtual_info) {
+        seaf_warning ("Repo %.10s is not a virtual repo.\n", repo_id);
+        ret = -1;
+        goto out;
+    }
+
+    vinfo = seaf_repo_manager_get_virtual_repo_info (mgr, repo_id);
+    if (!vinfo) {
+        seaf_warning ("Failed to get virt repo info %.10s.\n", repo_id);
+        ret = -1;
+        goto out;
+    }
+
+    orig_repo = seaf_repo_manager_get_repo (mgr, vinfo->origin_repo_id);
+    if (!orig_repo) {
+        seaf_warning ("Failed to get orig repo %.10s.\n", vinfo->origin_repo_id);
+        ret = -1;
+        goto out;
+    }
+
+    /* commits */
+    head = seaf_commit_manager_get_commit (seaf->commit_mgr,
+                                           repo->id, repo->version,
+                                           repo->head->commit_id);
+    if (!head) {
+        seaf_warning ("Failed to get virtual repo commit %s:%.8s.\n",
+                      repo->id, repo->head->commit_id);
+        ret = -1;
+        goto out;
+    }
+
+    orig_head = seaf_commit_manager_get_commit (seaf->commit_mgr,
+                                                orig_repo->id, orig_repo->version,
+                                                orig_repo->head->commit_id);
+    if (!orig_head) {
+        seaf_warning ("Failed to get origin repo commit %s:%.8s.\n",
+                      orig_repo->id, orig_repo->head->commit_id);
+        ret = -1;
+        goto out;
+    }
+
+    orig_root = seaf_fs_manager_get_seafdir_id_by_path (seaf->fs_mgr,
+                                                        orig_repo->store_id,
+                                                        orig_repo->version,
+                                                        orig_head->root_id,
+                                                        vinfo->path,
+                                                        &error);
+    if (error &&
+        !g_error_matches(error,
+                         SEAFILE_DOMAIN,
+                         SEAF_ERR_PATH_NO_EXIST)) {
+        seaf_warning ("Failed to get seafdir id by path in origin repo %.10s: %s.\n", orig_repo->store_id, error->message);
+        ret = -1;
+        goto out;
+    }
+    if (!orig_root) {
+        seaf_message("Path %s not found in origin repo %.8s, delete or rename virtual repo %.8s\n",
+                    vinfo->path, vinfo->origin_repo_id, repo_id);
+
+        goto out;
+    }
+
+    /* fs roots */
+    root = head->root_id;
+
+    MergeOptions opt;
+    const char *roots[2];
+
+    memset (&opt, 0, sizeof(opt));
+    opt.n_ways = 2;
+    memcpy (opt.remote_repo_id, repo_id, 36);
+    memcpy (opt.remote_head, head->commit_id, 40);
+
+    roots[0] = orig_root;
+    roots[1] = root;
+
+    /* Merge virtual into origin */
+    if (seaf_merge_trees (orig_repo->store_id, orig_repo->version,
+                          2, roots, &opt) < 0) {
+        seaf_warning ("Failed to merge virtual repo %.10s.\n", repo_id);
+        ret = -1;
+        goto out;
+    }
+
+    seaf_debug ("Number of dirs visted in merge: %d.\n", opt.visit_dirs);
+
+    /* Update virtual repo root. */
+    ret = seaf_repo_manager_update_dir (mgr,
+                                        repo_id,
+                                        "/",
+                                        opt.merged_tree_root,
+                                        orig_head->creator_name,
+                                        head->commit_id,
+                                        NULL,
+                                        NULL);
+    if (ret < 0) {
+        seaf_warning ("Failed to update root of virtual repo %.10s.\n",
+                      repo_id);
+        goto out;
+    }
+
+    /* Update origin repo path. */
+    ret = seaf_repo_manager_update_dir (mgr,
+                                        vinfo->origin_repo_id,
+                                        vinfo->path,
+                                        opt.merged_tree_root,
+                                        head->creator_name,
+                                        orig_head->commit_id,
+                                        new_base_commit,
+                                        NULL);
+    if (ret < 0) {
+        seaf_warning ("Failed to update origin repo %.10s path %s.\n",
+                      vinfo->origin_repo_id, vinfo->path);
+        goto out;
+    }
+
+    set_virtual_repo_base_commit_path (repo->id, new_base_commit, vinfo->path);
+
+out:
+    if (error)
+        g_clear_error (&error);
+    seaf_virtual_repo_info_free (vinfo);
+    seaf_repo_unref (repo);
+    seaf_repo_unref (orig_repo);
+    seaf_commit_unref (head);
+    seaf_commit_unref (orig_head);
+    g_free (orig_root);
+    return ret;
+}