Initial commit of Seafile server core.

scripts/upgrade/win32/py/add_collate.py

@@ -0,0 +1,180 @@
+# coding: UTF-8
+
+'''
+Database Upgrade scripts for seafile windows server 2.0.2
+'''
+
+import os
+import sys
+import re
+import sqlite3
+import logging
+import shutil
+
+from upgrade_common import seafserv_dir, ccnet_dir, seafile_dir
+
+# seafserv_dir = '/tmp/haiwen'
+# ccnet_dir = os.path.join(seafserv_dir, 'ccnet')
+# seafile_dir = os.path.join(seafserv_dir, 'seafile-data')
+
+def error_exit(msg):
+    print 'Error: %s' % msg
+    sys.exit(1)
+
+class Pattern(object):
+    def __init__(self, old, new):
+        self.old = old
+        self.new = new
+
+class AbstractDBUpdater(object):
+    '''Base class to update a database'''
+
+    name = ''
+    patterns = []
+
+    def __init__(self, db_path):
+        self.db_path = db_path
+        self.lines = []
+        self.tmp = self.db_path + '.tmp'
+
+        try:
+            if os.path.exists(self.tmp):
+                os.remove(self.tmp)
+        except:
+            logging.exception('Error when delete temporary database %s' % self.tmp)
+            sys.exit(1)
+
+    def do_update(self):
+        print 'updating %s' % self.name
+        self.dump_db()
+        self.update_schema()
+        self.write_db()
+
+    def dump_db(self):
+        '''Dump all the schema and data'''
+        with sqlite3.connect(self.db_path) as conn:
+            for line in conn.iterdump():
+                self.lines.append(line.replace('\n', ' '))
+
+    def update_schema(self):
+        '''Update schema of tables in this database to add "collate nocase"'''
+        new_lines = []
+        for line in self.lines:
+            new_line = line
+            if line.lower().startswith("create table"):
+                for pattern in self.patterns:
+                    new_line = re.sub(pattern.old, pattern.new, new_line)
+            new_lines.append(new_line)
+
+        self.lines = new_lines
+
+    def write_db(self):
+        with sqlite3.connect(self.tmp) as conn:
+            cursor = conn.cursor()
+            for line in self.lines:
+                if line.lower().strip().strip(';') in ('begin transaction', 'commit'):
+                    continue
+                cursor.execute(line)
+
+        shutil.copy(self.tmp, self.db_path)
+
+        try:
+            if os.path.exists(self.tmp):
+                os.remove(self.tmp)
+        except:
+            pass
+
+class CcnetUserDBUpdater(AbstractDBUpdater):
+    name = 'user database'
+    patterns = [
+        Pattern(r'(CREATE TABLE EmailUser.*)email TEXT,(.*)',
+                r'\1email TEXT COLLATE NOCASE,\2'),
+
+        Pattern(r'(CREATE TABLE Binding.*)email TEXT,(.*)',
+                r'\1email TEXT COLLATE NOCASE,\2'),
+    ]
+
+    def __init__(self, user_db):
+        AbstractDBUpdater.__init__(self, user_db)
+
+class CcnetGroupDBUpdater(AbstractDBUpdater):
+    name = 'group database'
+    patterns = [
+        Pattern(r'(CREATE TABLE `Group`.*)`creator_name` VARCHAR\(255\),(.*)',
+                r'\1`creator_name` VARCHAR(255) COLLATE NOCASE,\2'),
+        Pattern(r'(CREATE TABLE `GroupUser`.*)`user_name` VARCHAR\(255\),(.*)',
+                r'\1`user_name` VARCHAR(255) COLLATE NOCASE,\2'),
+    ]
+
+    def __init__(self, group_db):
+        AbstractDBUpdater.__init__(self, group_db)
+
+class SeafileDBUpdater(AbstractDBUpdater):
+    name = 'seafile database'
+    patterns = [
+        Pattern(r'(CREATE TABLE RepoOwner.*)owner_id TEXT(.*)',
+                r'\1owner_id TEXT COLLATE NOCASE\2'),
+
+        Pattern(r'(CREATE TABLE RepoGroup.*)user_name TEXT,(.*)',
+                r'\1user_name TEXT COLLATE NOCASE,\2'),
+
+        Pattern(r'(CREATE TABLE RepoUserToken.*)email VARCHAR\(255\),(.*)',
+                r'\1email VARCHAR(255) COLLATE NOCASE,\2'),
+
+        Pattern(r'(CREATE TABLE UserQuota.*)user VARCHAR\(255\),(.*)',
+                r'\1user VARCHAR(255) COLLATE NOCASE,\2' ),
+
+        Pattern(r'(CREATE TABLE SharedRepo.*)from_email VARCHAR\(512\), to_email VARCHAR\(512\),(.*)',
+                r'\1from_email VARCHAR(512), to_email VARCHAR(512) COLLATE NOCASE,\2'),
+    ]
+
+    def __init__(self, seafile_db):
+        AbstractDBUpdater.__init__(self, seafile_db)
+
+class SeahubDBUpdater(AbstractDBUpdater):
+    name = 'seahub database'
+    patterns = [
+        Pattern(r'(CREATE TABLE "notifications_usernotification".*)"to_user" varchar\(255\) NOT NULL,(.*)',
+                r'\1"to_user" varchar(255) NOT NULL COLLATE NOCASE,\2'),
+
+        Pattern(r'(CREATE TABLE "profile_profile".*)"user" varchar\(75\) NOT NULL UNIQUE,(.*)',
+                r'\1"user" varchar(75) NOT NULL UNIQUE COLLATE NOCASE,\2'),
+
+        Pattern(r'(CREATE TABLE "share_fileshare".*)"username" varchar\(255\) NOT NULL,(.*)',
+                r'\1"username" varchar(255) NOT NULL COLLATE NOCASE,\2'),
+
+        Pattern(r'(CREATE TABLE "api2_token".*)"user" varchar\(255\) NOT NULL UNIQUE,(.*)',
+                r'\1"user" varchar(255) NOT NULL UNIQUE COLLATE NOCASE,\2'),
+
+        Pattern(r'(CREATE TABLE "wiki_personalwiki".*)"username" varchar\(255\) NOT NULL UNIQUE,(.*)',
+                r'\1"username" varchar(255) NOT NULL UNIQUE COLLATE NOCASE,\2'),
+
+        Pattern(r'(CREATE TABLE "message_usermessage".*)"from_email" varchar\(75\) NOT NULL,\s*"to_email" varchar\(75\) NOT NULL,(.*)',
+                r'\1"from_email" varchar(75) NOT NULL COLLATE NOCASE, "to_email" varchar(75) NOT NULL COLLATE NOCASE,\2'),
+
+        Pattern(r'(CREATE TABLE "avatar_avatar".*)"emailuser" varchar\(255\) NOT NULL,(.*)',
+                r'\1"emailuser" varchar(255) NOT NULL COLLATE NOCASE,\2'),
+    ]
+
+    def __init__(self, seahub_db):
+        AbstractDBUpdater.__init__(self, seahub_db)
+
+def upgrade_collate():
+    '''Update database schema to add "COLLATE NOCASE" of email field'''
+    user_db = os.path.join(ccnet_dir, 'PeerMgr', 'usermgr.db')
+    group_db = os.path.join(ccnet_dir, 'GroupMgr', 'groupmgr.db')
+    seafile_db = os.path.join(seafile_dir, 'seafile.db')
+    seahub_db = os.path.join(seafserv_dir, 'seahub.db')
+    updaters = [
+        CcnetUserDBUpdater(user_db),
+        CcnetGroupDBUpdater(group_db),
+        SeafileDBUpdater(seafile_db),
+        SeahubDBUpdater(seahub_db),
+    ]
+
+    for updater in updaters:
+        updater.do_update()
+
+
+if __name__ == '__main__':
+    upgrade_collate()

scripts/upgrade/win32/py/gc.py

@@ -0,0 +1,40 @@
+# coding: UTF-8
+
+import os
+import sys
+import traceback
+import ccnet
+
+from upgrade_common import install_path, seafile_dir, ccnet_dir, run_argv, ensure_server_not_running, central_config_dir
+
+
+def call_seafserv_gc():
+    args = [
+        os.path.join(install_path, 'seafile', 'bin', 'seafserv-gc.exe'),
+        '-c',
+        ccnet_dir,
+        '-d',
+        seafile_dir,
+        '-F',
+        central_config_dir,
+    ]
+
+    print 'Starting gc...\n'
+    run_argv(args)
+
+
+def main():
+    try:
+        ensure_server_not_running()
+        call_seafserv_gc()
+    except Exception, e:
+        print 'Error:\n', e
+    else:
+        print '\ndone\n'
+    finally:
+        print '\nprint ENTER to exit\n'
+        raw_input()
+
+
+if __name__ == '__main__':
+    main()

scripts/upgrade/win32/py/upgrade_1.7_1.8.py

@@ -0,0 +1,17 @@
+# coding: UTF-8
+
+from upgrade_common import upgrade_db
+
+def main():
+    try:
+        upgrade_db('1.8.0')
+    except Exception, e:
+        print 'Error:\n', e
+    else:
+        print '\ndone\n'
+    finally:
+        print '\nprint ENTER to exit\n'
+        raw_input()
+
+if __name__ == '__main__':
+    main()

scripts/upgrade/win32/py/upgrade_1.8_2.0.py

@@ -0,0 +1,19 @@
+# coding: UTF-8
+
+from upgrade_common import upgrade_db
+from add_collate import upgrade_collate
+
+def main():
+    try:
+        upgrade_db('2.0.0')
+        upgrade_collate()
+    except Exception, e:
+        print 'Error:\n', e
+    else:
+        print '\ndone\n'
+    finally:
+        print '\nprint ENTER to exit\n'
+        raw_input()
+
+if __name__ == '__main__':
+    main()

scripts/upgrade/win32/py/upgrade_2.0_2.1.py

@@ -0,0 +1,31 @@
+# coding: UTF-8
+
+import os
+import glob
+import shutil
+
+from upgrade_common import install_path, seafile_dir, upgrade_db
+
+def copy_template_library():
+    src_docs_dir = os.path.join(install_path, 'seafile', 'docs')
+    library_template_dir= os.path.join(seafile_dir, 'library-template')
+    if not os.path.exists(library_template_dir):
+        os.mkdir(library_template_dir)
+
+    for fn in glob.glob(os.path.join(src_docs_dir, '*.doc')):
+        shutil.copy(fn, library_template_dir)
+
+def main():
+    try:
+        upgrade_db('2.1.0')
+        copy_template_library()
+    except Exception, e:
+        print 'Error:\n', e
+    else:
+        print '\ndone\n'
+    finally:
+        print '\nprint ENTER to exit\n'
+        raw_input()
+
+if __name__ == '__main__':
+    main()

scripts/upgrade/win32/py/upgrade_2.1_3.0.py

@@ -0,0 +1,32 @@
+# coding: UTF-8
+
+import os
+
+from upgrade_common import install_path, ccnet_dir, seafile_dir, upgrade_db, run_argv
+
+def do_migrate_storage():
+    '''use seaf-migrate to migrate objects from the 2.1 layout to 3.0 layout'''
+    args = [
+        os.path.join(install_path, 'seafile', 'bin', 'seaf-migrate.exe'),
+        '-c', ccnet_dir,
+        '-d', seafile_dir,
+    ]
+
+    print 'Starting migrate your data...\n'
+    if run_argv(args) != 0:
+        raise Exception('failed to migrate seafile data to 3.0 format')
+
+def main():
+    try:
+        upgrade_db('3.0.0')
+        do_migrate_storage()
+    except Exception, e:
+        print 'Error:\n', e
+    else:
+        print '\ndone\n'
+    finally:
+        print '\nprint ENTER to exit\n'
+        raw_input()
+
+if __name__ == '__main__':
+    main()

scripts/upgrade/win32/py/upgrade_3.0_3.1.py

@@ -0,0 +1,19 @@
+# coding: UTF-8
+
+import os
+
+from upgrade_common import install_path, ccnet_dir, seafile_dir, upgrade_db, run_argv
+
+def main():
+    try:
+        upgrade_db('3.1.0')
+    except Exception, e:
+        print 'Error:\n', e
+    else:
+        print '\ndone\n'
+    finally:
+        print '\nprint ENTER to exit\n'
+        raw_input()
+
+if __name__ == '__main__':
+    main()

scripts/upgrade/win32/py/upgrade_3.1_4.0.py

@@ -0,0 +1,19 @@
+# coding: UTF-8
+
+import os
+
+from upgrade_common import install_path, ccnet_dir, seafile_dir, upgrade_db, run_argv
+
+def main():
+    try:
+        upgrade_db('4.0.0')
+    except Exception, e:
+        print 'Error:\n', e
+    else:
+        print '\ndone\n'
+    finally:
+        print '\nprint ENTER to exit\n'
+        raw_input()
+
+if __name__ == '__main__':
+    main()

scripts/upgrade/win32/py/upgrade_4.0_4.1.py

@@ -0,0 +1,19 @@
+# coding: UTF-8
+
+import os
+
+from upgrade_common import install_path, ccnet_dir, seafile_dir, upgrade_db, run_argv
+
+def main():
+    try:
+        upgrade_db('4.1.0')
+    except Exception, e:
+        print 'Error:\n', e
+    else:
+        print '\ndone\n'
+    finally:
+        print '\nprint ENTER to exit\n'
+        raw_input()
+
+if __name__ == '__main__':
+    main()

scripts/upgrade/win32/py/upgrade_4.1_4.2.py

@@ -0,0 +1,19 @@
+# coding: UTF-8
+
+import os
+
+from upgrade_common import install_path, ccnet_dir, seafile_dir, upgrade_db, run_argv
+
+def main():
+    try:
+        upgrade_db('4.2.0')
+    except Exception, e:
+        print 'Error:\n', e
+    else:
+        print '\ndone\n'
+    finally:
+        print '\nprint ENTER to exit\n'
+        raw_input()
+
+if __name__ == '__main__':
+    main()

scripts/upgrade/win32/py/upgrade_4.2_4.3.py

@@ -0,0 +1,19 @@
+# coding: UTF-8
+
+import os
+
+from upgrade_common import install_path, ccnet_dir, seafile_dir, upgrade_db, run_argv
+
+def main():
+    try:
+        upgrade_db('4.3.0')
+    except Exception, e:
+        print 'Error:\n', e
+    else:
+        print '\ndone\n'
+    finally:
+        print '\nprint ENTER to exit\n'
+        raw_input()
+
+if __name__ == '__main__':
+    main()

scripts/upgrade/win32/py/upgrade_4.3_5.0.py

@@ -0,0 +1,49 @@
+# coding: UTF-8
+
+import shutil
+import os
+import traceback
+from os.path import abspath, basename, exists, dirname, join
+from upgrade_common import (install_path, seafserv_dir, ccnet_dir, seafile_dir,
+                            upgrade_db, run_argv)
+
+
+def move_all_conf_to_central_config_dir():
+    central_config_dir = join(seafserv_dir, 'conf')
+    if not exists(central_config_dir):
+        os.mkdir(central_config_dir)
+    files = [
+        join(ccnet_dir, 'ccnet.conf'),
+        join(seafile_dir, 'seafile.conf'),
+        join(seafserv_dir, 'seahub_settings.py'),
+    ]
+    for fn in files:
+        if not exists(fn):
+            raise RuntimeError('file %s does not exist' % fn)
+    for fn in files:
+        with open(fn, 'r') as fp:
+            if 'This file has been moved' in fp.read():
+                return
+        dstfile = join(central_config_dir, basename(fn))
+        shutil.copyfile(fn, dstfile)
+        with open(fn, 'w') as fp:
+            content = '# This file has been moved to %s in seafile 5.0.0' % dstfile
+            fp.write(content)
+
+
+def main():
+    try:
+        upgrade_db('5.0.0')
+        move_all_conf_to_central_config_dir()
+    except Exception, e:
+        traceback.print_exc()
+        print 'Error:\n', e
+    else:
+        print '\ndone\n'
+    finally:
+        print '\nprint ENTER to exit\n'
+        raw_input()
+
+
+if __name__ == '__main__':
+    main()

scripts/upgrade/win32/py/upgrade_5.0_5.1.py

@@ -0,0 +1,24 @@
+# coding: UTF-8
+
+import shutil
+import os
+import traceback
+from os.path import abspath, basename, exists, dirname, join
+from upgrade_common import (install_path, seafserv_dir, ccnet_dir, seafile_dir,
+                            upgrade_db, run_argv)
+
+def main():
+    try:
+        upgrade_db('5.1.0')
+    except Exception, e:
+        traceback.print_exc()
+        print 'Error:\n', e
+    else:
+        print '\ndone\n'
+    finally:
+        print '\nprint ENTER to exit\n'
+        raw_input()
+
+
+if __name__ == '__main__':
+    main()

scripts/upgrade/win32/py/upgrade_common.py

@@ -0,0 +1,155 @@
+# coding: UTF-8
+
+import os
+import sys
+import sqlite3
+import subprocess
+import ccnet
+import glob
+
+
+# Directory layout:
+#
+# - SeafileProgram/
+#   - seafserv.ini
+#   - seafile-server-1.7.0/
+#   - seafile-server-1.8.0/
+#   - seafile-server-1.9.0/
+#     - upgrade/
+#       - sql/
+#         - 1.8.0/
+#           - sqlite3
+#             - ccnet.sql
+#             - seafile.sql
+#             - seahub.sql
+#       - upgrade_1.7_1.8.bat
+#       - upgrade_1.8_1.9.bat
+#       - py/
+#         - upgrade_1.7_1.8.py
+#         - upgrade_1.8_1.9.py
+
+pyscript_dir = os.path.dirname(os.path.abspath(__file__))
+upgrade_dir = os.path.dirname(pyscript_dir)
+sql_dir = os.path.join(upgrade_dir, 'sql')
+install_path = os.path.dirname(upgrade_dir)
+program_top_dir = os.path.dirname(install_path)
+
+seafserv_dir = ''
+ccnet_dir = ''
+seafile_dir = ''
+central_config_dir = ''
+
+def run_argv(argv, cwd=None, env=None, suppress_stdout=False, suppress_stderr=False):
+    '''Run a program and wait it to finish, and return its exit code. The
+    standard output of this program is supressed.
+
+    '''
+    with open(os.devnull, 'w') as devnull:
+        if suppress_stdout:
+            stdout = devnull
+        else:
+            stdout = sys.stdout
+
+        if suppress_stderr:
+            stderr = devnull
+        else:
+            stderr = sys.stderr
+
+        proc = subprocess.Popen(argv,
+                                cwd=cwd,
+                                stdout=stdout,
+                                stderr=stderr,
+                                env=env)
+        return proc.wait()
+
+def error(message):
+    print message
+    sys.exit(1)
+
+def read_seafserv_dir():
+    global seafserv_dir, ccnet_dir, seafile_dir, central_config_dir
+    seafserv_ini = os.path.join(program_top_dir, 'seafserv.ini')
+    if not os.path.exists(seafserv_ini):
+        error('%s not found' % seafserv_ini)
+
+    with open(seafserv_ini, 'r') as fp:
+        seafserv_dir = fp.read().strip()
+
+    ccnet_dir = os.path.join(seafserv_dir, 'ccnet')
+    seafile_dir = os.path.join(seafserv_dir, 'seafile-data')
+    central_config_dir = os.path.join(seafserv_dir, 'conf')
+
+def apply_sqls(db_path, sql_path):
+    with open(sql_path, 'r') as fp:
+        lines = fp.read().split(';')
+
+    with sqlite3.connect(db_path) as conn:
+        for line in lines:
+            line = line.strip()
+            if not line:
+                continue
+            else:
+                conn.execute(line)
+
+def _get_ccnet_db(ccnet_dir, dbname):
+    dbs = (
+        'ccnet.db',
+        'GroupMgr/groupmgr.db',
+        'misc/config.db',
+        'OrgMgr/orgmgr.db',
+    )
+    for db in dbs:
+        if os.path.splitext(os.path.basename(db))[0] == dbname:
+            return os.path.join(ccnet_dir, db)
+
+def _handle_ccnet_sqls(version):
+    for sql_path in glob.glob(os.path.join(sql_dir, version, 'sqlite3', 'ccnet', '*.sql')):
+        dbname = os.path.splitext(os.path.basename(sql_path))[0]
+        apply_sqls(_get_ccnet_db(ccnet_dir, dbname), sql_path)
+
+def upgrade_db(version):
+    ensure_server_not_running()
+    print 'upgrading databases ...'
+    ccnet_db = os.path.join(ccnet_dir, 'ccnet.db')
+    seafile_db = os.path.join(seafile_dir, 'seafile.db')
+    seahub_db = os.path.join(seafserv_dir, 'seahub.db')
+
+    def get_sql(prog):
+        ret = os.path.join(sql_dir, version, 'sqlite3', '%s.sql' % prog)
+        return ret
+
+    ccnet_sql = get_sql('ccnet')
+    seafile_sql = get_sql('seafile')
+    seahub_sql = get_sql('seahub')
+
+    if os.path.exists(ccnet_sql):
+        print '    upgrading ccnet databases ...'
+        apply_sqls(ccnet_db, ccnet_sql)
+    _handle_ccnet_sqls(version)
+
+    if os.path.exists(seafile_sql):
+        print '    upgrading seafile databases ...'
+        apply_sqls(seafile_db, seafile_sql)
+
+    if os.path.exists(seahub_sql):
+        print '    upgrading seahub databases ...'
+        apply_sqls(seahub_db, seahub_sql)
+
+def get_current_version():
+    return os.path.basename(install_path).split('-')[-1]
+
+def ensure_server_not_running():
+    if os.path.exists(os.path.join(central_config_dir, 'ccnet.conf')):
+        client = ccnet.SyncClient(ccnet_dir,
+                                  central_config_dir=central_config_dir)
+    else:
+        client = ccnet.SyncClient(ccnet_dir)
+    try:
+        client.connect_daemon()
+    except ccnet.NetworkError:
+        pass
+    else:
+        raise Exception('Seafile server is running! You must turn it off before running this script!')
+
+
+read_seafserv_dir()