Add functional tests (#120)

* functional tests setup

* add a test case

.gitignore

@@ -93,3 +93,5 @@ tests/conf/PeerMgr
 /test-driver
 *.dmp
 /symbols
+__pycache__/
+.cache/

.travis.yml

@@ -4,7 +4,6 @@ sudo: false
 language: python
 compiler:
  - gcc
- - clang
 addons:
   apt:
     packages:
@@ -21,11 +20,10 @@ cache:
   directories:
     - $HOME/.cache/pip
     - $HOME/.ccache
-    - $HOME/downloads
 before_install:
   - ccache -s
   - export PATH=/usr/lib/ccache:${PATH}
 install:
- - ./integration-tests/install-deps.sh
+ - ./ci/install-deps.sh
 script:
- - ./integration-tests/run.py
+ - ./ci/run.py

ci/install-deps.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e -x
+
+SCRIPT=${BASH_SOURCE[0]}
+TESTS_DIR=$(dirname "${SCRIPT}")/..
+SETUP_DIR=${TESTS_DIR}/ci
+
+cd $SETUP_DIR
+
+pip install -r requirements.txt
+
+# download precompiled libevhtp
+# TODO(lins05): we should consider build from source with https://github.com/criticalstack/libevhtp in the future
+libevhtp_bin=libevhtp-bin_1.2.0.tar.gz
+wget https://dl.bintray.com/lins05/generic/libevhtp-bin/$libevhtp_bin
+# tar xvf $libevhtp_bin --strip-components=3 -C /usr
+tar xf $libevhtp_bin -C $HOME

ci/requirements.txt

@@ -0,0 +1,6 @@
+termcolor>=1.1.0
+requests>=2.8.0
+httpie>=0.9.9
+pytest>=3.3.2
+backports.functools_lru_cache>=1.4
+tenacity>=4.8.0

ci/run.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+"""
+Install dir: ~/opt/local
+Data dir: /tmp/haiwen
+"""
+
+import argparse
+import glob
+import json
+import logging
+import os
+import re
+import sys
+from os.path import abspath, basename, exists, expanduser, join
+
+import requests
+import termcolor
+
+from serverctl import MYSQL_ROOT_PASSWD, ServerCtl
+from utils import (
+    cd, chdir, debug, green, info, lru_cache, mkdirs, on_travis, red,
+    setup_logging, shell, warning
+)
+
+logger = logging.getLogger(__name__)
+
+TOPDIR = abspath(join(os.getcwd(), '..'))
+if on_travis():
+    PREFIX = expanduser('~/opt/local')
+else:
+    PREFIX = os.environ.get('SEAFILE_INSTALL_PREFIX', '/usr/local')
+INSTALLDIR = '/tmp/seafile-tests'
+
+
+def num_jobs():
+    return int(os.environ.get('NUM_JOBS', 2))
+
+
+@lru_cache()
+def make_build_env():
+    env = dict(os.environ)
+    libsearpc_dir = abspath(join(TOPDIR, 'libsearpc'))
+    ccnet_dir = abspath(join(TOPDIR, 'ccnet-server'))
+
+    def _env_add(*a, **kw):
+        kw['env'] = env
+        return prepend_env_value(*a, **kw)
+
+    _env_add('CPPFLAGS', '-I%s' % join(PREFIX, 'include'), seperator=' ')
+
+    _env_add('LDFLAGS', '-L%s' % join(PREFIX, 'lib'), seperator=' ')
+
+    _env_add('LDFLAGS', '-L%s' % join(PREFIX, 'lib64'), seperator=' ')
+
+    _env_add('PATH', join(PREFIX, 'bin'))
+    _env_add('PYTHONPATH', join(PREFIX, 'lib/python2.7/site-packages'))
+    _env_add('PKG_CONFIG_PATH', join(PREFIX, 'lib', 'pkgconfig'))
+    _env_add('PKG_CONFIG_PATH', join(PREFIX, 'lib64', 'pkgconfig'))
+    _env_add('PKG_CONFIG_PATH', libsearpc_dir)
+    _env_add('PKG_CONFIG_PATH', ccnet_dir)
+    _env_add('LD_LIBRARY_PATH', join(PREFIX, 'lib'))
+
+    for key in ('PATH', 'PKG_CONFIG_PATH', 'CPPFLAGS', 'LDFLAGS', 'PYTHONPATH'):
+        info('%s: %s', key, env.get(key, ''))
+    return env
+
+
+def prepend_env_value(name, value, seperator=':', env=None):
+    '''append a new value to a list'''
+    env = env or os.environ
+    current_value = env.get(name, '')
+    new_value = value
+    if current_value:
+        new_value += seperator + current_value
+
+    env[name] = new_value
+    return env
+
+
+@lru_cache()
+def get_branch_json_file():
+    url = 'https://raw.githubusercontent.com/haiwen/seafile-test-deploy/master/branches.json'
+    return requests.get(url).json()
+
+
+def get_project_branch(project, default_branch='master'):
+    travis_branch = os.environ.get('TRAVIS_BRANCH', 'master')
+    if project.name == 'seafile-server':
+        return travis_branch
+    conf = get_branch_json_file()
+    return conf.get(travis_branch, {}).get(project.name, default_branch)
+
+
+class Project(object):
+    def __init__(self, name):
+        self.name = name
+        self.version = ''
+
+    @property
+    def url(self):
+        return 'https://www.github.com/haiwen/{}.git'.format(self.name)
+
+    @property
+    def projectdir(self):
+        return join(TOPDIR, self.name)
+
+    @property
+    def branch(self):
+        return get_project_branch(self)
+
+    def clone(self):
+        if exists(self.name):
+            with cd(self.name):
+                shell('git fetch origin --tags')
+        else:
+            shell(
+                'git clone --depth=1 --branch {} {}'.
+                format(self.branch, self.url)
+            )
+
+    @chdir
+    def compile_and_install(self):
+        cmds = [
+            './autogen.sh',
+            './configure --prefix={}'.format(PREFIX),
+            'make -j{}'.format(num_jobs()),
+            'make install',
+        ]
+        for cmd in cmds:
+            shell(cmd)
+
+    @chdir
+    def use_branch(self, branch):
+        shell('git checkout {}'.format(branch))
+
+
+class Libsearpc(Project):
+    def __init__(self):
+        super(Libsearpc, self).__init__('libsearpc')
+
+
+class CcnetServer(Project):
+    def __init__(self):
+        super(CcnetServer, self).__init__('ccnet-server')
+
+
+class SeafileServer(Project):
+    def __init__(self):
+        super(SeafileServer, self).__init__('seafile-server')
+
+
+def fetch_and_build():
+    libsearpc = Project('libsearpc')
+    ccnet = CcnetServer()
+    seafile = SeafileServer()
+
+    libsearpc.clone()
+    libsearpc.compile_and_install()
+
+    ccnet.clone()
+    ccnet.compile_and_install()
+
+    seafile.compile_and_install()
+
+
+def parse_args():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('-v', '--verbose', action='store_true')
+    ap.add_argument('-t', '--test-only', action='store_true')
+
+    return ap.parse_args()
+
+
+def main():
+    mkdirs(INSTALLDIR)
+    os.environ.update(make_build_env())
+    args = parse_args()
+    if on_travis() and not args.test_only:
+        fetch_and_build()
+    # for db in ('sqlite3', 'mysql'):
+    for db in ('sqlite3', ):
+        shell('rm -rf {}/*'.format(INSTALLDIR))
+        start_and_test_with_db(db)
+
+
+def start_and_test_with_db(db):
+    info('Setting up seafile server with %s database', db)
+    server = ServerCtl(INSTALLDIR, db)
+    server.setup()
+    with server.run():
+        info('Testing with %s database', db)
+        with cd(SeafileServer().projectdir):
+            shell('py.test', env=server.get_seaserv_envs())
+
+
+if __name__ == '__main__':
+    os.chdir(TOPDIR)
+    setup_logging()
+    main()

ci/serverctl.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python
+#coding: UTF-8
+
+import argparse
+import glob
+import logging
+import os
+import re
+import sys
+from collections import namedtuple
+from contextlib import contextmanager
+from os.path import abspath, basename, dirname, exists, join
+
+import requests
+from tenacity import TryAgain, retry, stop_after_attempt, wait_fixed
+
+from utils import (
+    cd, chdir, debug, green, info, mkdirs, red, setup_logging, shell, warning
+)
+
+logger = logging.getLogger(__name__)
+
+MYSQL_ROOT_PASSWD = 's123'
+
+
+class ServerCtl(object):
+    def __init__(self, datadir, db='sqlite3'):
+        self.db = db
+        self.datadir = datadir
+        self.central_conf_dir = join(datadir, 'conf')
+        self.seafile_conf_dir = join(datadir, 'seafile-data')
+        self.ccnet_conf_dir = join(datadir, 'ccnet')
+
+        self.log_dir = join(datadir, 'logs')
+        mkdirs(self.log_dir)
+        self.ccnet_log = join(self.log_dir, 'ccnet.log')
+        self.seafile_log = join(self.log_dir, 'seafile.log')
+
+        self.ccnet_proc = None
+        self.seafile_proc = None
+
+    def setup(self):
+        if self.db == 'mysql':
+            create_mysql_dbs()
+
+        self.init_ccnet()
+        self.init_seafile()
+
+    def init_ccnet(self):
+        cmd = [
+            'ccnet-init',
+            '-F',
+            self.central_conf_dir,
+            '-c',
+            self.ccnet_conf_dir,
+            '--name',
+            'test',
+            '--host',
+            'test.seafile.com',
+        ]
+        shell(cmd)
+
+    def init_seafile(self):
+        cmd = [
+            'seaf-server-init',
+            '--central-config-dir',
+            self.central_conf_dir,
+            '--seafile-dir',
+            self.seafile_conf_dir,
+            '--fileserver-port',
+            '8082',
+        ]
+
+        shell(cmd)
+
+    @contextmanager
+    def run(self):
+        try:
+            self.start()
+            yield self
+        except:
+            self.print_logs()
+            raise
+        finally:
+            self.stop()
+
+    def print_logs(self):
+        for logfile in self.ccnet_log, self.seafile_log:
+            if exists(logfile):
+                shell('cat {0}'.format(logfile))
+
+    @retry(wait=wait_fixed(1), stop=stop_after_attempt(10))
+    def wait_ccnet_ready(self):
+        if not exists(join(self.ccnet_conf_dir, 'ccnet.sock')):
+            raise TryAgain
+
+    def start(self):
+        logger.info('Starting ccnet server')
+        self.start_ccnet()
+        self.wait_ccnet_ready()
+        logger.info('Starting seafile server')
+        self.start_seafile()
+
+    def start_ccnet(self):
+        cmd = [
+            "ccnet-server",
+            "-F",
+            self.central_conf_dir,
+            "-c",
+            self.ccnet_conf_dir,
+            "-f",
+            self.ccnet_log,
+        ]
+        self.ccnet_proc = shell(cmd, wait=False)
+
+    def start_seafile(self):
+        cmd = [
+            "seaf-server",
+            "-F",
+            self.central_conf_dir,
+            "-c",
+            self.ccnet_conf_dir,
+            "-d",
+            self.seafile_conf_dir,
+            "-l",
+            self.seafile_log,
+        ]
+        self.seafile_proc = shell(cmd, wait=False)
+
+    def stop(self):
+        if self.ccnet_proc:
+            logger.info('Stopping ccnet server')
+            self.ccnet_proc.terminate()
+        if self.seafile_proc:
+            logger.info('Stopping seafile server')
+            self.seafile_proc.terminate()
+
+    def get_seaserv_envs(self):
+        envs = dict(os.environ)
+        envs.update({
+            'SEAFILE_CENTRAL_CONF_DIR': self.central_conf_dir,
+            'CCNET_CONF_DIR': self.ccnet_conf_dir,
+            'SEAFILE_CONF_DIR': self.seafile_conf_dir,
+        })
+        return envs
+
+
+def create_mysql_dbs():
+    shell('mysqladmin -u root password %s' % MYSQL_ROOT_PASSWD)
+    sql = '''\
+create database `ccnet-existing` character set = 'utf8';
+create database `seafile-existing` character set = 'utf8';
+create database `seahub-existing` character set = 'utf8';
+
+create user 'seafile'@'localhost' identified by 'seafile';
+
+GRANT ALL PRIVILEGES ON `ccnet-existing`.* to `seafile`@localhost;
+GRANT ALL PRIVILEGES ON `seafile-existing`.* to `seafile`@localhost;
+GRANT ALL PRIVILEGES ON `seahub-existing`.* to `seafile`@localhost;
+    '''
+
+    shell('mysql -u root -p%s' % MYSQL_ROOT_PASSWD, inputdata=sql)

ci/utils.py

@@ -1,18 +1,23 @@
 #coding: UTF-8
 
-import os
-from os.path import abspath, basename, exists, expanduser, join
-import sys
-import re
 import logging
+import os
+import re
+import sys
 from contextlib import contextmanager
-from subprocess import Popen, PIPE, CalledProcessError
+from os.path import abspath, basename, exists, expanduser, join
+from subprocess import PIPE, CalledProcessError, Popen
 
-import termcolor
 import requests
-from pexpect import spawn
+import termcolor
+
+try:
+    from functools import lru_cache
+except ImportError:
+    from backports.functools_lru_cache import lru_cache
+
+logger = logging.getLogger(__name__)
 
-logger = logging.getLogger(__file__)
 
 def _color(s, color):
     return s if not os.isatty(sys.stdout.fileno()) \
@@ -39,16 +44,19 @@ def warning(fmt, *a):
     logger.warn(red(fmt), *a)
 
 
-def shell(cmd, inputdata=None, **kw):
+def shell(cmd, inputdata=None, wait=True, **kw):
     info('calling "%s" in %s', cmd, kw.get('cwd', os.getcwd()))
     kw['shell'] = not isinstance(cmd, list)
     kw['stdin'] = PIPE if inputdata else None
     p = Popen(cmd, **kw)
     if inputdata:
         p.communicate(inputdata)
-    p.wait()
+    if wait:
-    if p.returncode:
+        p.wait()
-        raise CalledProcessError(p.returncode, cmd)
+        if p.returncode:
+            raise CalledProcessError(p.returncode, cmd)
+    else:
+        return p
 
 
 @contextmanager
@@ -68,6 +76,7 @@ def chdir(func):
 
     return wrapped
 
+
 def setup_logging():
     kw = {
         'format': '[%(asctime)s][%(module)s]: %(message)s',
@@ -77,5 +86,24 @@ def setup_logging():
     }
 
     logging.basicConfig(**kw)
-    logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(
+    logging.getLogger('requests.packages.urllib3.connectionpool'
-        logging.WARNING)
+                      ).setLevel(logging.WARNING)
+
+
+def mkdirs(*paths):
+    for path in paths:
+        if not exists(path):
+            os.mkdir(path)
+
+def on_travis():
+    return 'TRAVIS_BUILD_NUMBER' in os.environ
+
+@contextmanager
+def cd(path):
+    path = expanduser(path)
+    olddir = os.getcwd()
+    os.chdir(path)
+    try:
+        yield
+    finally:
+        os.chdir(olddir)

integration-tests/README.md

@@ -1,3 +0,0 @@
-### Seafile Integration Tests
-
-The purpose of integration tests is to build a seafile release package and run tests against it.

integration-tests/autosetup.py

@@ -1,259 +0,0 @@
-#!/usr/bin/env python
-#coding: UTF-8
-
-import os
-from os.path import abspath, basename, exists, dirname, join
-import sys
-import argparse
-import re
-from collections import namedtuple
-
-import requests
-from pexpect import spawn
-
-from utils import green, red, debug, info, warning, cd, shell, chdir, setup_logging
-
-USERNAME = 'test@seafiletest.com'
-PASSWORD = 'testtest'
-ADMIN_USERNAME = 'admin@seafiletest.com'
-ADMIN_PASSWORD = 'adminadmin'
-MYSQL_ROOT_PASSWD = 's123'
-
-ServerConfig = namedtuple('ServerConfig', [
-    'installdir',
-    'tarball',
-    'version',
-    'initmode',
-])
-
-
-def setup_server(cfg, db):
-    '''Setup seafile server with the setup-seafile.sh script. We use pexpect to
-    interactive with the setup process of the script.
-    '''
-    info('uncompressing server tarball')
-    shell('tar xf seafile-server_{}_x86-64.tar.gz -C {}'
-          .format(cfg.version, cfg.installdir))
-    if db == 'mysql':
-        autosetup_mysql(cfg)
-    else:
-        autosetup_sqlite3(cfg)
-
-    with open(join(cfg.installdir, 'conf/seahub_settings.py'), 'a') as fp:
-        fp.write('\n')
-        fp.write('DEBUG = True')
-        fp.write('\n')
-        fp.write('''\
-REST_FRAMEWORK = {
-    'DEFAULT_THROTTLE_RATES': {
-        'ping': '600/minute',
-        'anon': '1000/minute',
-        'user': '1000/minute',
-    },
-}''')
-        fp.write('\n')
-
-
-def autosetup_sqlite3(cfg):
-    setup_script = get_script(cfg, 'setup-seafile.sh')
-    shell('''sed -i -e '/^check_root;.*/d' "{}"'''.format(setup_script))
-
-    if cfg.initmode == 'prompt':
-        setup_sqlite3_prompt(setup_script)
-    else:
-        setup_sqlite3_auto(setup_script)
-
-def setup_sqlite3_prompt(setup_script):
-    info('setting up seafile server with pexepct, script %s', setup_script)
-    answers = [
-        ('ENTER', ''),
-        # server name
-        ('server name', 'my-seafile'),
-        # ip or domain
-        ('ip or domain', '127.0.0.1'),
-        # seafile data dir
-        ('seafile-data', ''),
-        # fileserver port
-        ('seafile fileserver', ''),
-        ('ENTER', ''),
-        ('ENTER', ''),
-    ]
-    _answer_questions(setup_script, answers)
-
-def setup_sqlite3_auto(setup_script):
-    info('setting up seafile server in auto mode, script %s', setup_script)
-    env = os.environ.copy()
-    env['SERVER_IP'] = '127.0.0.1'
-    shell('%s auto -n my-seafile' % setup_script, env=env)
-
-def createdbs():
-    sql = '''\
-create database `ccnet-existing` character set = 'utf8';
-create database `seafile-existing` character set = 'utf8';
-create database `seahub-existing` character set = 'utf8';
-
-create user 'seafile'@'localhost' identified by 'seafile';
-
-GRANT ALL PRIVILEGES ON `ccnet-existing`.* to `seafile`@localhost;
-GRANT ALL PRIVILEGES ON `seafile-existing`.* to `seafile`@localhost;
-GRANT ALL PRIVILEGES ON `seahub-existing`.* to `seafile`@localhost;
-    '''
-
-    shell('mysql -u root -p%s' % MYSQL_ROOT_PASSWD, inputdata=sql)
-
-
-def autosetup_mysql(cfg):
-    setup_script = get_script(cfg, 'setup-seafile-mysql.sh')
-    if not exists(setup_script):
-        print 'please specify seafile script path'
-
-    if cfg.initmode == 'prompt':
-        createdbs()
-        setup_mysql_prompt(setup_script)
-    else :
-        # in auto mode, test create new db
-        setup_mysql_auto(setup_script)
-
-def setup_mysql_prompt(setup_script):
-    info('setting up seafile server with pexepct, script %s', setup_script)
-    answers = [
-        ('ENTER', ''),
-        # server name
-        ('server name', 'my-seafile'),
-        # ip or domain
-        ('ip or domain', '127.0.0.1'),
-        # seafile data dir
-        ('seafile-data', ''),
-        # fileserver port
-        ('seafile fileserver', ''),
-        # use existing
-        ('choose a way to initialize seafile databases', '2'),
-        ('host of mysql server', ''),
-        ('port of mysql server', ''),
-        ('Which mysql user', 'seafile'),
-        ('password for mysql user', 'seafile'),
-        ('ccnet database', 'ccnet-existing'),
-        ('seafile database', 'seafile-existing'),
-        ('seahub database', 'seahub-existing'),
-        ('ENTER', ''),
-    ]
-    _answer_questions(abspath(setup_script), answers)
-
-def setup_mysql_auto(setup_script):
-    info('setting up seafile server in auto mode, script %s', setup_script)
-    env = os.environ.copy()
-    env['MYSQL_USER'] = 'seafile-new'
-    env['MYSQL_USER_PASSWD'] = 'seafile'
-    env['MYSQL_ROOT_PASSWD']= MYSQL_ROOT_PASSWD
-    env['CCNET_DB'] = 'ccnet-new'
-    env['SEAFILE_DB'] = 'seafile-new'
-    env['SEAHUB_DB'] = 'seahub-new'
-    shell('%s auto -n my-seafile -e 0' % setup_script, env=env)
-
-def start_server(cfg):
-    with cd(cfg.installdir):
-        shell('find . -maxdepth 2 | sort | xargs ls -lhd')
-    seafile_sh = get_script(cfg, 'seafile.sh')
-    shell('{} start'.format(seafile_sh))
-
-    info('starting seahub')
-    seahub_sh = get_script(cfg, 'seahub.sh')
-    answers = [
-        # admin email/pass
-        ('admin email', ADMIN_USERNAME),
-        ('admin password', ADMIN_PASSWORD),
-        ('admin password again', ADMIN_PASSWORD),
-    ]
-    _answer_questions('{} start'.format(abspath(seahub_sh)), answers)
-    with cd(cfg.installdir):
-        shell('find . -maxdepth 2 | sort | xargs ls -lhd')
-    # shell('sqlite3 ccnet/PeerMgr/usermgr.db "select * from EmailUser"', cwd=INSTALLDIR)
-    shell('http -v localhost:8000/api2/server-info/ || true')
-    # shell('http -v -f POST localhost:8000/api2/auth-token/ username=admin@seafiletest.com password=adminadmin || true')
-    shell('netstat -nltp')
-
-
-def _answer_questions(cmd, answers):
-    info('expect: spawing %s', cmd)
-    child = spawn(cmd)
-    child.logfile = sys.stdout
-
-    def autofill(pattern, line):
-        child.expect(pattern)
-        child.sendline(line)
-
-    for k, v in answers:
-        autofill(k, v)
-    child.sendline('')
-    child.logfile = None
-    child.interact()
-
-
-def get_script(cfg, path):
-    """
-    :type cfg: ServerConfig
-    """
-    return join(server_dir(cfg), path)
-
-
-def server_dir(cfg):
-    """
-    :type cfg: ServerConfig
-    """
-    return join(cfg.installdir, 'seafile-server-{}'.format(cfg.version))
-
-
-def apiurl(path):
-    path = path.lstrip('/')
-    root = os.environ.get('SEAFILE_SERVER', 'http://127.0.0.1:8000')
-    return '{}/api2/{}'.format(root, path)
-
-
-def create_test_user(cfg):
-    data = {'username': ADMIN_USERNAME, 'password': ADMIN_PASSWORD, }
-    res = requests.post(apiurl('/auth-token/'), data=data)
-    debug('%s %s', res.status_code, res.text)
-    token = res.json()['token']
-    data = {'password': PASSWORD, }
-    headers = {'Authorization': 'Token ' + token}
-    res = requests.put(
-        apiurl('/accounts/{}/'.format(USERNAME)),
-        data=data,
-        headers=headers)
-    assert res.status_code == 201
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument('-v', '--verbose', action='store_true')
-    ap.add_argument('--db', choices=('sqlite3', 'mysql'), default='sqlite3')
-    ap.add_argument('installdir')
-    ap.add_argument('tarball')
-    args = ap.parse_args()
-
-    if not exists(args.installdir):
-        print 'directory {} does not exist'.format(args.installdir)
-        sys.exit(1)
-
-    if os.listdir(args.installdir):
-        print 'directory {} is not empty'.format(args.installdir)
-        sys.exit(1)
-
-    if not exists(args.tarball):
-        print 'file {} does not exist'.format(args.tarball)
-        sys.exit(1)
-
-    m = re.match(r'^.*?_([\d\.]+).*?\.tar\.gz$', basename(args.tarball))
-    version = m.group(1)
-
-    cfg = ServerConfig(installdir=args.installdir,
-                       tarball=args.tarball,
-                       version=version)
-    setup_server(cfg, args.db)
-    start_server(cfg)
-    create_test_user(cfg)
-
-
-if __name__ == '__main__':
-    setup_logging()
-    main()

integration-tests/install-deps.sh

@@ -1,56 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-pip install -r ./integration-tests/requirements.txt
-
-pushd $HOME
-
-# download precompiled libevhtp
-libevhtp_bin=libevhtp-bin_1.2.0.tar.gz
-wget https://dl.bintray.com/lins05/generic/libevhtp-bin/$libevhtp_bin
-tar xf $libevhtp_bin
-find $HOME/opt
-
-# download seahub thirdpart python libs
-WGET="wget --no-check-certificate"
-downloads=$HOME/downloads
-thirdpart=$HOME/thirdpart
-
-mkdir -p $downloads $thirdpart
-cd $thirdpart
-save_pythonpath=$PYTHONPATH
-export PYTHONPATH=.
-urls=(
-    https://pypi.python.org/packages/source/p/pytz/pytz-2016.1.tar.gz
-    https://www.djangoproject.com/m/releases/1.8/Django-1.8.10.tar.gz
-    https://pypi.python.org/packages/source/d/django-statici18n/django-statici18n-1.1.3.tar.gz
-    https://pypi.python.org/packages/source/d/djangorestframework/djangorestframework-3.3.2.tar.gz
-    https://pypi.python.org/packages/source/d/django_compressor/django_compressor-1.4.tar.gz
-
-    https://pypi.python.org/packages/source/j/jsonfield/jsonfield-1.0.3.tar.gz
-    https://pypi.python.org/packages/source/d/django-post_office/django-post_office-2.0.6.tar.gz
-
-    http://pypi.python.org/packages/source/g/gunicorn/gunicorn-19.4.5.tar.gz
-    http://pypi.python.org/packages/source/f/flup/flup-1.0.2.tar.gz
-    https://pypi.python.org/packages/source/c/chardet/chardet-2.3.0.tar.gz
-    https://labix.org/download/python-dateutil/python-dateutil-1.5.tar.gz
-    https://pypi.python.org/packages/source/s/six/six-1.9.0.tar.gz
-
-    https://pypi.python.org/packages/source/d/django-picklefield/django-picklefield-0.3.2.tar.gz
-    https://pypi.python.org/packages/source/d/django-constance/django-constance-1.0.1.tar.gz
-
-    https://pypi.python.org/packages/source/j/jdcal/jdcal-1.2.tar.gz
-    https://pypi.python.org/packages/source/e/et_xmlfile/et_xmlfile-1.0.1.tar.gz
-    https://pypi.python.org/packages/source/o/openpyxl/openpyxl-2.3.0.tar.gz
-)
-for url in ${urls[*]}; do
-    path="${downloads}/$(basename $url)"
-    if [[ ! -e $path ]]; then
-        $WGET -O $path $url
-    fi
-    easy_install -d . $path
-done
-export PYTHONPATH=$save_pythonpath
-
-popd

integration-tests/requirements.txt

@@ -1,8 +0,0 @@
-Pillow==4.1.0
-termcolor==1.1.0
-prettytable==0.7.2
-pexpect==4.0
-requests==2.8.0
-httpie
-django-constance[database]
-MySQL-python==1.2.5

integration-tests/run.py

@@ -1,307 +0,0 @@
-#!/usr/bin/env python
-
-import os
-from os.path import abspath, basename, exists, expanduser, join
-import sys
-import re
-import glob
-import json
-import logging
-import requests
-
-import termcolor
-from pexpect import spawn
-from utils import green, red, debug, info, warning, cd, shell, chdir, setup_logging
-from autosetup import (setup_server, ServerConfig, get_script, server_dir,
-                       start_server, create_test_user, MYSQL_ROOT_PASSWD)
-
-TOPDIR = abspath(join(os.getcwd(), '..'))
-PREFIX = expanduser('~/opt/local')
-SRCDIR = '/tmp/src'
-INSTALLDIR = '/tmp/haiwen'
-THIRDPARTDIR = expanduser('~/thirdpart')
-
-logger = logging.getLogger(__file__)
-seafile_version = ''
-
-TRAVIS_BRANCH = os.environ.get('TRAVIS_BRANCH', 'master')
-
-
-def make_build_env():
-    env = dict(os.environ)
-    libsearpc_dir = abspath(join(TOPDIR, 'libsearpc'))
-    ccnet_dir = abspath(join(TOPDIR, 'ccnet-server'))
-
-    def _env_add(*a, **kw):
-        kw['env'] = env
-        return prepend_env_value(*a, **kw)
-
-    _env_add('CPPFLAGS', '-I%s' % join(PREFIX, 'include'), seperator=' ')
-
-    _env_add('LDFLAGS', '-L%s' % os.path.join(PREFIX, 'lib'), seperator=' ')
-
-    _env_add('LDFLAGS', '-L%s' % os.path.join(PREFIX, 'lib64'), seperator=' ')
-
-    _env_add('PATH', os.path.join(PREFIX, 'bin'))
-    _env_add('PATH', THIRDPARTDIR)
-    _env_add('PKG_CONFIG_PATH', os.path.join(PREFIX, 'lib', 'pkgconfig'))
-    _env_add('PKG_CONFIG_PATH', os.path.join(PREFIX, 'lib64', 'pkgconfig'))
-    _env_add('PKG_CONFIG_PATH', libsearpc_dir)
-    _env_add('PKG_CONFIG_PATH', ccnet_dir)
-
-    for key in ('PATH', 'PKG_CONFIG_PATH', 'CPPFLAGS', 'LDFLAGS',
-                'PYTHONPATH'):
-        info('%s: %s', key, env.get(key, ''))
-    return env
-
-
-def prepend_env_value(name, value, seperator=':', env=None):
-    '''append a new value to a list'''
-    env = env or os.environ
-    current_value = env.get(name, '')
-    new_value = value
-    if current_value:
-        new_value += seperator + current_value
-
-    env[name] = new_value
-    return env
-
-
-def get_project_branch(project, default_branch='master'):
-    if project.name == 'seafile-server':
-        return TRAVIS_BRANCH
-    conf = json.loads(requests.get(
-        'https://raw.githubusercontent.com/haiwen/seafile-test-deploy/master/branches.json').text)
-    return conf.get(TRAVIS_BRANCH, {}).get(project.name,
-                                           default_branch)
-
-
-class Project(object):
-    configure_cmd = './configure'
-
-    def __init__(self, name):
-        self.name = name
-        self.version = ''
-
-    @property
-    def url(self):
-        return 'https://www.github.com/haiwen/{}.git'.format(self.name)
-
-    @property
-    def projectdir(self):
-        return join(TOPDIR, self.name)
-
-    @property
-    def branch(self):
-        return get_project_branch(self)
-
-    def clone(self):
-        if exists(self.name):
-            with cd(self.name):
-                shell('git fetch origin --tags')
-        else:
-            shell('git clone --depth=1 --branch {} {}'.format(self.branch,
-                                                              self.url))
-
-    @chdir
-    def make_dist(self):
-        info('making tarball for %s', self.name)
-        if exists('./autogen.sh'):
-            shell('./autogen.sh')
-            shell(self.configure_cmd, env=make_build_env())
-        shell('make dist')
-
-    @chdir
-    def copy_dist(self):
-        self.make_dist()
-        tarball = glob.glob('*.tar.gz')[0]
-        info('copying %s to %s', tarball, SRCDIR)
-        shell('cp {} {}'.format(tarball, SRCDIR))
-        if self.name == 'seafile-server':
-            name = 'seafile'
-        elif self.name == 'ccnet-server':
-            name = 'ccnet'
-        else:
-            name = self.name
-        m = re.match('{}-(.*).tar.gz'.format(name), basename(tarball))
-        if m:
-            self.version = m.group(1)
-
-    @chdir
-    def use_branch(self, branch):
-        shell('git checkout {}'.format(branch))
-
-
-class CcnetServer(Project):
-    def __init__(self):
-        super(CcnetServer, self).__init__('ccnet-server')
-
-
-class SeafileServer(Project):
-    configure_cmd = './configure'
-
-    def __init__(self):
-        super(SeafileServer, self).__init__('seafile-server')
-
-    @chdir
-    def copy_dist(self):
-        super(SeafileServer, self).copy_dist()
-        global seafile_version
-        seafile_version = self.version
-
-
-class Seahub(Project):
-    def __init__(self):
-        super(Seahub, self).__init__('seahub')
-
-    @chdir
-    def make_dist(self):
-        cmds = [
-            # 'git add -f media/css/*.css',
-            # 'git commit -a -m "%s"' % msg,
-            './tools/gen-tarball.py --version={} --branch=HEAD >/dev/null'
-            .format(seafile_version),
-        ]
-        for cmd in cmds:
-            shell(cmd, env=make_build_env())
-
-
-class SeafDAV(Project):
-    def __init__(self):
-        super(SeafDAV, self).__init__('seafdav')
-
-    @chdir
-    def make_dist(self):
-        shell('make')
-
-
-class SeafObj(Project):
-    def __init__(self):
-        super(SeafObj, self).__init__('seafobj')
-
-    @chdir
-    def make_dist(self):
-        shell('make dist')
-
-
-def build_server(libsearpc, ccnet, seafile):
-    cmd = [
-        'python',
-        join(TOPDIR, 'seafile-server/scripts/build/build-server.py'),
-        '--yes',
-        '--version=%s' % seafile.version,
-        '--libsearpc_version=%s' % libsearpc.version,
-        '--ccnet_version=%s' % ccnet.version,
-        '--seafile_version=%s' % seafile.version,
-        '--thirdpartdir=%s' % THIRDPARTDIR,
-        '--srcdir=%s' % SRCDIR,
-        '--jobs=4',
-    ]
-    shell(cmd, shell=False, env=make_build_env())
-
-
-def fetch_and_build():
-    libsearpc = Project('libsearpc')
-    ccnet = CcnetServer()
-    seafile = SeafileServer()
-    seahub = Seahub()
-    seafobj = SeafObj()
-    seafdav = SeafDAV()
-
-    for project in (libsearpc, ccnet, seafile, seahub, seafdav, seafobj):
-        if project.name != 'seafile-server':
-            project.clone()
-        project.copy_dist()
-
-    build_server(libsearpc, ccnet, seafile)
-
-
-def run_tests(cfg):
-    # run_python_seafile_tests()
-    # run_seafdav_tests(cfg)
-    # must stop seafile server before running seaf-gc
-    shell('{} stop'.format(get_script(cfg, 'seafile.sh')))
-    shell('{} stop'.format(get_script(cfg, 'seahub.sh')))
-    shell('{} --verbose --rm-deleted'.format(get_script(cfg, 'seaf-gc.sh')))
-
-
-def run_python_seafile_tests():
-    python_seafile = Project('python-seafile')
-    if not exists(python_seafile.projectdir):
-        python_seafile.clone()
-        shell('pip install -r {}/requirements.txt'.format(
-            python_seafile.projectdir))
-
-    with cd(python_seafile.projectdir):
-        # install python-seafile because seafdav tests needs it
-        shell('python setup.py install')
-        shell('py.test')
-
-
-def _seafdav_env(cfg):
-    env = dict(os.environ)
-    env['CCNET_CONF_DIR'] = join(INSTALLDIR, 'ccnet')
-    env['SEAFILE_CONF_DIR'] = join(INSTALLDIR, 'seafile-data')
-    env['SEAFILE_CENTRAL_CONF_DIR'] = join(INSTALLDIR, 'conf')
-    for path in glob.glob(join(
-            server_dir(cfg), 'seafile/lib*/python*/*-packages')):
-        prepend_env_value('PYTHONPATH', path, env=env)
-    return env
-
-
-def run_seafdav_tests(cfg):
-    seafdav = SeafDAV()
-    shell('pip install -r {}/test-requirements.txt'.format(seafdav.projectdir))
-    with cd(seafdav.projectdir):
-        shell('nosetests -v -s', env=_seafdav_env(cfg))
-
-
-def _mkdirs(*paths):
-    for path in paths:
-        if not exists(path):
-            os.mkdir(path)
-
-
-def main():
-    _mkdirs(SRCDIR, INSTALLDIR)
-    setup_logging()
-    fetch_and_build()
-    for db in ('sqlite3', 'mysql'):
-        if db == 'mysql':
-            shell('mysqladmin -u root password %s' % MYSQL_ROOT_PASSWD)
-        for i in ('prompt', 'auto'):
-            shell('rm -rf {}/*'.format(INSTALLDIR))
-            setup_and_test(db, i)
-
-
-def setup_and_test(db, initmode):
-    cfg = ServerConfig(
-        installdir=INSTALLDIR,
-        tarball=join(TOPDIR, 'seafile-server_{}_x86-64.tar.gz'.format(
-            seafile_version)),
-        version=seafile_version,
-        initmode=initmode)
-    info('Setting up seafile server with %s database', db)
-    setup_server(cfg, db)
-    # enable webdav, we're going to seafdav tests later
-    shell('''sed -i -e "s/enabled = false/enabled = true/g" {}'''
-          .format(join(INSTALLDIR, 'conf/seafdav.conf')))
-    try:
-        start_server(cfg)
-        info('Testing seafile server with %s database', db)
-        create_test_user(cfg)
-        run_tests(cfg)
-    except:
-        for logfile in glob.glob('{}/logs/*.log'.format(INSTALLDIR)):
-            shell('echo {0}; cat {0}'.format(logfile))
-        for logfile in glob.glob('{}/seafile-server-{}/runtime/*.log'.format(
-                INSTALLDIR, seafile_version)):
-            shell('echo {0}; cat {0}'.format(logfile))
-        raise
-
-
-if __name__ == '__main__':
-    os.chdir(TOPDIR)
-    # Add the location where libevhtp is installed so ldd can know it.
-    prepend_env_value('LD_LIBRARY_PATH', os.path.expanduser('~/opt/local/lib'))
-    main()

pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+addopts = -vv -s
+log_format = %(asctime)s:%(name)s:%(levelname)s:%(message)s
+log_date_format = %Y-%m-%d %H:%M:%S
+# log_cli_level = info

run_tests.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT=${BASH_SOURCE[0]}
+PROJECT_DIR=$(dirname "${SCRIPT}")
+
+cd $PROJECT_DIR
+
+export PYTHONPATH=$PROJECT_DIR:$PYTHONPATH
+
+ci/run.py --test-only

tests/conf/ccnet.conf

@@ -1,11 +0,0 @@
-[General]
-USER_NAME = server
-ID = 8e4b13b49ca79f35732d9f44a0804940d985627c
-NAME = server
-SERVICE_URL = http://127.0.0.1
-
-[Network]
-PORT = 10002
-
-[Client]
-PORT = 9999

tests/conf/mykey.peer

@@ -1,27 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEAuZFwgxkKQGaqYyFMxIUz1JHnZPaOgEQ+fX/jRVYbGMiHkSbX
-K9X3XUHUGEjUt8b3zW6UZJGjgyV5S08YuaN0eE5z6Q6bnuWEhkTmgZgXaybc9Hiu
-y2WAHpKj+qbXcmewE0WEys/Ov9AIe0TRXmvL6r1793VcLSzgb/aIQA2WFg97DfEA
-hGAHo5BesKRfEEvXL6ZB9cGxXP9qIy0ObTvLXlOgbYchfV4rrXJk0u9xWjRyXABv
-2Myv3fgxmGmTR+TAw2G5GCKeh9IoIuWVMGPyjSlERGMqQYymNz3NgyWFayyZ5HQS
-tihCnflOGEiMHRkOwIczB16YZhan2YqKpsjHGwIBIwKCAQEArvbXzBBLfoyvR4XM
-Cb9rYgXozOh3usQAZ7MYHM2HQ0C6VahHN/WgFhl+1RF4Gv1tTKoW4nqwHJEL9oxn
-xPkzTNxBZrYAcT7NaKdc/diLG+LQVDdFuHWkrxyL+vUUR0vR5kjcSjGlrYmhmMvb
-WQaNEIbFVwhA92TTnMPfjNmcI2wRKI1K9NEKDAMIPSwW/sgkls2h4KW3Y7DooJ0k
-l0apjN/rlaR4ohZp6oMVifW8GFY43Xau+4dIrYTnvvSyvGvtB+8cWuhqqvWHRZdM
-rFjgOJoZH5l0zxt2dYW2WFiqgT7xXsvu6L+nylXktEMxC33rehYdPrd427J409A6
-caO5cwKBgQDyrBQ8UXu7cDAktiKTwH7+pA0wNyTvKsGYw0RcFILccpxty2r5gYhI
-eLFPVyjoYxwauW6vX3cSAYLKR+2PlYvkPpEvBQIJbaurx++ejez/KxYD65ZeFTfs
-Kb9A08hgMxCvJmnRvojhez1OZmmmWYPT57XeZXnCiNoyJWKA0mMNvwKBgQDDwn02
-o5n7ugetXIlV1PiStVogPPTBobh9jsXooQFh4fB+lsrO082hapMlbVVNG1gLzvTY
-V0oDM/AzdnC6feZlAEdM+IcruinVnMnbnhiwPVDInCJIhvmJ/XScvkTsgHwRiAss
-Tlf8wH/uGXiaeVV/KMlkKRK6h54znTPq37/VpQKBgQDkziG1NuJgRTS05j3bxB/3
-Z3omJV1Wh2YTsMtswuHIiVGpWWTcnrOyC2VZb2+2iVUDQR83oycfmwZJsYg27BYu
-+SnNPzxvSiWEtTJiS00rGf7QfwoeMUNbAspEb+jPux5b/6WZ34hfkXRRO/02cagu
-Mj3DDzhJtDtxG+8pAOEM9QKBgQC+KqWFiPv72UlJUpQKPJmzFpIQsD44cTbgXs7h
-+32viwbhX0irqS4nxp2SEnAfBJ6sYqS05xSyp3uftOKJRxpTfJ0I8W1drYe5kP6a
-1Bf7qUcpRzc/JAhaKWn3Wb9MJQrPM7MVGOfCVJmINgAhCCcrEa2xwX/oZnxsp1cB
-a6RpIwKBgQDW15IebNwVOExTqtfh6UvIjMSrk9OoHDyjoPLI3eyPt3ujKdXFJ8qF
-CWg9ianQyE5Y8vfDI+x1YRCOwq2WapeXzkSO8CzVFHgz5kFqJQolr4+o6wr5mLLC
-+6iW9u81/X3bMAWshtNfsWbRSFLT1WNVTKRg+xO7YG/3wcyeIeqigA==
------END RSA PRIVATE KEY-----

tests/config.py

@@ -0,0 +1,9 @@
+USER = 'testuser@test.seafile.com'
+PASSWORD = 'testuser'
+USER2 = 'testuser2@test.seafile.com'
+PASSWORD2 = 'testuser2'
+ADMIN_USER = 'adminuser@test.seafile.com'
+ADMIN_PASSWORD = 'adminuser'
+
+INACTIVE_USER = 'inactiveuser@test.seafile.com'
+INACTIVE_PASSWORD = 'inactiveuser'

tests/conftest.py

@@ -0,0 +1,51 @@
+#coding: UTF-8
+
+import logging
+import os
+
+import pytest
+from tenacity import retry, stop_after_attempt, wait_fixed
+from tests.config import (
+    ADMIN_PASSWORD, ADMIN_USER, INACTIVE_PASSWORD, INACTIVE_USER, PASSWORD,
+    PASSWORD2, USER, USER2
+)
+from tests.utils import create_and_get_repo, randstring
+
+from seaserv import ccnet_api, seafile_api
+
+logger = logging.getLogger(__name__)
+
+
+@retry(wait=wait_fixed(2), stop=stop_after_attempt(10))
+def wait_for_server():
+    seafile_api.get_repo_list(0, 1)
+
+
+@pytest.fixture(scope='session', autouse=True)
+def create_users():
+    """
+    Create an admin user and a normal user
+    """
+    wait_for_server()
+    logger.info('preparing users for testing')
+    ccnet_api.add_emailuser(USER, PASSWORD, is_staff=False, is_active=True)
+    ccnet_api.add_emailuser(USER2, PASSWORD2, is_staff=False, is_active=True)
+    ccnet_api.add_emailuser(
+        INACTIVE_USER, INACTIVE_PASSWORD, is_staff=False, is_active=False
+    )
+    ccnet_api.add_emailuser(
+        ADMIN_USER, ADMIN_PASSWORD, is_staff=True, is_active=True
+    )
+
+
+@pytest.yield_fixture(scope='function')
+def repo():
+    repo = create_and_get_repo(
+        'testrepo测试-{}'.format(randstring(10)), '', USER, passwd=None
+    )
+    try:
+        yield repo
+    finally:
+        if seafile_api.get_repo(repo.id):
+            # The repo may be deleted in the test case
+            seafile_api.remove_repo(repo.id)

tests/test_sharing.py

@@ -0,0 +1,19 @@
+import pytest
+from seaserv import seafile_api as api
+from seaserv import ccnet_api
+
+from tests.config import ADMIN_USER, USER, USER2
+
+
+@pytest.mark.parametrize('permission', ['r', 'rw'])
+def test_share_repo(repo, permission):
+    assert api.check_permission(repo.id, USER2) is None
+
+    api.share_repo(repo.id, USER, USER2, permission)
+    assert api.check_permission(repo.id, USER2) == permission
+
+    repos = api.get_share_in_repo_list(USER2, 0, 1)
+    assert len(repos) == 1
+    r = repos[0]
+    assert r.id == repo.id
+    assert r.permission == permission

tests/utils.py

@@ -0,0 +1,15 @@
+import os
+import random
+import string
+
+from seaserv import ccnet_api, seafile_api
+
+
+def create_and_get_repo(*a, **kw):
+    repo_id = seafile_api.create_repo(*a, **kw)
+    repo = seafile_api.get_repo(repo_id)
+    return repo
+
+
+def randstring(length=12):
+    return ''.join(random.choice(string.lowercase) for i in range(length))