[api] Add users in bulk from excel, export batch add user help file, add (#1792)

* [api] Add users in bulk from excel, export batch add user help file, add
test

* remove csv feature

* review

* column width, del tip, add head

* del write_xls_sample, modify head

* review

* download text

* option->optional, trans

seahub/forms.py

@@ -173,7 +173,7 @@ class RepoSettingForm(forms.Form):
 
 class BatchAddUserForm(forms.Form):
     """
-    Form for importing users from CSV file.
+    Form for importing users from XLSX file.
     """
     file = forms.FileField()
 

seahub/templates/sysadmin/sys_useradmin.html

@@ -25,7 +25,7 @@
     </ul>
 
     <div class="js-op-for-all fright">
-        <button id="import-users-btn">{% trans "Import users" %}</button>
+        <button id="import-users-from-excel-btn">{% trans "Import users" %}</button>
         <button id="add-user-btn">{% trans "Add user" %}</button>
         <button id="export-excel">{% trans "Export Excel" %}</button>
     </div>
@@ -70,14 +70,13 @@
     <button type="submit" class="submit">{% trans "Submit" %}</button>
 </form>
 
-<form id="upload-csv-form" class="hide" enctype="multipart/form-data" method="post" action="{% url 'batch_add_user' %}">{% csrf_token %}
+<form id="upload-excel-form" class="hide" enctype="multipart/form-data" method="post" action="{% url 'batch_add_user' %}">{% csrf_token %}
-    <h3>{% trans "Import users from a CSV file" %}</h3>
+    <h3>{% trans "Import users from a .xlsx file" %}</h3>
+    <a href="#" id="excel-example">{% trans "Download an example file" %}</a>
+    <br />
     <input type="file" name="file" />
-    <p class="tip">
+    <br />
-        {% trans "File format: user@mail.com,password,name,department,role,quota"%}<br />
+    <p class="error hide">{% trans "Please choose a .xlsx file." %}</p>
-        {% trans "Name, department, role and quota are optional." %}
-    </p>
-    <p class="error hide">{% trans "Please choose a CSV file" %}</p>
     <button type="submit" class="submit">{% trans "Submit" %}</button>
 </form>
 
@@ -145,7 +144,6 @@ $(function(){
     });
     {% endif %}
 });
-
 $('#add-user-btn').click(function() {
     $('#add-user-form').modal();
     $('#simplemodal-container').css({'width':'auto', 'height':'auto'});
@@ -237,11 +235,15 @@ $('#add-user-form').submit(function() {
     });
     return false;
 });
-$('#import-users-btn').click(function () {
+$('#import-users-from-excel-btn').click(function () {
-    $('#upload-csv-form').modal();
+    $('#upload-excel-form').modal();
     $('#simplemodal-container').css({'width':'auto', 'height':'auto'});
 });
-$('#upload-csv-form').submit(function() {
+$("#excel-example").click(function() {
+    location.href = "{% url 'batch_add_user_example' %}";
+    return false;
+});
+$('#upload-excel-form').submit(function() {
     var form = $(this);
     if (!$('[name=file]', form).val()) {
         $('.error', form).removeClass('hide');

seahub/urls.py

@@ -435,6 +435,7 @@ urlpatterns = patterns(
     url(r'^useradmin/password/reset/(?P<email>[^/]+)/$', user_reset, name='user_reset'),
     url(r'^useradmin/batchmakeadmin/$', batch_user_make_admin, name='batch_user_make_admin'),
     url(r'^useradmin/batchadduser/$', batch_add_user, name='batch_add_user'),
+    url(r'^useradmin/batchadduser/example/$', batch_add_user_example, name='batch_add_user_example'),
 
     url(r'^client-login/$', client_token_login, name='client_token_login'),
 )

seahub/views/sysadmin.py

@@ -2,14 +2,15 @@
 # encoding: utf-8
 
 import os
+from io import BytesIO
 from types import FunctionType
 import logging
 import json
 import re
 import datetime
-import csv, chardet, StringIO
 import time
 from constance import config
+from openpyxl import load_workbook
 
 from django.db.models import Q
 from django.conf import settings as dj_settings
@@ -1905,10 +1906,41 @@ def batch_user_make_admin(request):
 
     return HttpResponse(json.dumps({'success': True,}), content_type=content_type)
 
+@login_required
+@sys_staff_required
+def batch_add_user_example(request):
+    """ get example file.
+    """
+    next = request.META.get('HTTP_REFERER', None)
+    if not next:
+        next = SITE_ROOT
+    data_list = []
+    head = [_('email'), _('password'), _('name')+ '(' + _('optional') + ')', 
+            _('department')+ '(' + _('optional') + ')', _('role')+
+            '(' + _('optional') + ')', _('quota') + '(MB, ' + _('optional') + ')']
+    for i in xrange(5):
+        username = "test" + str(i) +"@example.com"
+        password = "123456"
+        name = "test" + str(i)
+        department = "department" + str(i)
+        role = "default"
+        quota = "1000"
+        data_list.append([username, password, name, department, role, quota])
+
+    wb = write_xls('sample', head, data_list)
+    if not wb:
+        messages.error(request, _(u'Failed to export Excel'))
+        return HttpResponseRedirect(next)
+
+    response = HttpResponse(content_type='application/ms-excel')
+    response['Content-Disposition'] = 'attachment; filename=users.xlsx'
+    wb.save(response)
+    return response
+
 @login_required
 @sys_staff_required
 def batch_add_user(request):
-    """Batch add users. Import users from CSV file.
+    """  Batch add users. Import users from XLSX file.
     """
     if request.method != 'POST':
         raise Http404
@@ -1918,25 +1950,30 @@ def batch_add_user(request):
     form = BatchAddUserForm(request.POST, request.FILES)
     if form.is_valid():
         content = request.FILES['file'].read()
-        encoding = chardet.detect(content)['encoding']
+        if str(request.FILES['file']).split('.')[-1].lower() != 'xlsx':
-        if encoding != 'utf-8':
+            messages.error(request, _(u'Please choose a .xlsx file.'))
-            content = content.decode(encoding, 'replace').encode('utf-8')
+            return HttpResponseRedirect(next)
 
-        filestream = StringIO.StringIO(content)
+        try:
-        reader = csv.reader(filestream)
+            fs = BytesIO(content)
-        new_users_count = len(list(reader))
+            wb = load_workbook(filename=fs, read_only=True)
-        if user_number_over_limit(new_users=new_users_count):
+        except Exception as e:
+            logger.error(e)
+            messages.error(request, _('Internal Server Error'))
+            return HttpResponseRedirect(next)
+
+        rows = wb.worksheets[0].rows
+        records = []
+        # remove first row(head field).
+        next(rows)
+        for row in rows:
+            records.append([c.value for c in row])
+
+        if user_number_over_limit(new_users=len(records)):
             messages.error(request, _(u'The number of users exceeds the limit.'))
             return HttpResponseRedirect(next)
 
-        # return to the top of the file
+        for row in records:
-        filestream.seek(0)
-        reader = csv.reader(filestream)
-        for row in reader:
-
-            if not row:
-                continue
-
             try:
                 username = row[0].strip()
                 password = row[1].strip()
@@ -1978,8 +2015,7 @@ def batch_add_user(request):
                     logger.error(e)
 
                 try:
-                    space_quota_mb = row[5].strip()
+                    space_quota_mb = int(row[5])
-                    space_quota_mb = int(space_quota_mb)
                     if space_quota_mb >= 0:
                         space_quota = int(space_quota_mb) * get_file_size_unit('MB')
                         seafile_api.set_user_quota(username, space_quota)
@@ -2001,10 +2037,9 @@ def batch_add_user(request):
                 }
                 admin_operation.send(sender=None, admin_name=request.user.username,
                                      operation=USER_ADD, detail=admin_op_detail)
-
         messages.success(request, _('Import succeeded'))
     else:
-        messages.error(request, _(u'Please select a csv file first.'))
+        messages.error(request, _(u'Please choose a .xlsx file.'))
 
     return HttpResponseRedirect(next)
 

tests/seahub/views/sysadmin/test_sysadmin.py

@@ -1,11 +1,12 @@
 import os
+import openpyxl
+from io import BytesIO
 from mock import patch
 from django.core.urlresolvers import reverse
 from post_office.models import Email
 
 from seahub.base.accounts import User
-from seahub.options.models import (UserOptions, KEY_FORCE_PASSWD_CHANGE,
+from seahub.options.models import (UserOptions, KEY_FORCE_PASSWD_CHANGE)
-                                   VAL_FORCE_PASSWD_CHANGE)
 from seahub.test_utils import BaseTestCase
 from seahub.utils.ms_excel import write_xls as real_write_xls
 
@@ -278,3 +279,162 @@ class BatchAddUserTest(BaseTestCase):
         assert self.new_users[0] == email.to[0]
         assert "Email: %s" % self.new_users[0] in email.html_message
         assert email.status == 2
+
+
+class BatchAddUserUsingExcelTest(BaseTestCase):
+    def setUp(self):
+        self.clear_cache()
+        self.login_as(self.admin)
+
+        self.new_users = []
+        self.excel_file = os.path.join(os.getcwd(), 'tests/seahub/views/sysadmin/batch_add_user.xlsx')
+        data_list = []
+        for i in xrange(20):
+            username = "username@test" + str(i) +".com"
+            password = "password"
+            name = "name_test" + str(i)
+            department = "department_test" + str(i)
+            if i < 10:
+                role = "guest"
+            else:
+                role = "default"
+            quota = "999"
+            data_list.append([username, password, name, department, role, quota])
+            self.new_users.append(username)
+        wb = real_write_xls('test', data_list[0], data_list[1:])
+        wb.save(self.excel_file)
+
+    def tearDown(self):
+        for u in self.new_users:
+            self.remove_user(u)
+
+    def test_can_batch_add(self):
+        for e in self.new_users:
+            try:
+                r = User.objects.get(e)
+            except User.DoesNotExist:
+                r = None
+            assert r is None
+
+        with open(self.excel_file) as f:
+            resp = self.client.post(reverse('batch_add_user_using_excel'), {
+                'file': f
+            })
+
+        self.assertEqual(302, resp.status_code)
+        assert 'Import succeeded' in resp.cookies['messages'].value
+        for e in self.new_users:
+            assert User.objects.get(e) is not None
+
+    def test_can_batch_add_when_pwd_change_required(self):
+        config.FORCE_PASSWORD_CHANGE = 1
+
+        for e in self.new_users:
+            assert len(UserOptions.objects.filter(
+                email=e, option_key=KEY_FORCE_PASSWD_CHANGE)) == 0
+
+        for e in self.new_users:
+            try:
+                r = User.objects.get(e)
+            except User.DoesNotExist:
+                r = None
+            assert r is None
+
+        with open(self.excel_file) as f:
+            resp = self.client.post(reverse('batch_add_user_using_excel'), {
+                'file': f
+            })
+
+        self.assertEqual(302, resp.status_code)
+        assert 'Import succeeded' in resp.cookies['messages'].value
+        for e in self.new_users:
+            assert User.objects.get(e) is not None
+            assert UserOptions.objects.passwd_change_required(e)
+
+    def test_can_batch_add_when_pwd_change_not_required(self):
+        config.FORCE_PASSWORD_CHANGE = 0
+
+        for e in self.new_users:
+            assert len(UserOptions.objects.filter(
+                email=e, option_key=KEY_FORCE_PASSWD_CHANGE)) == 0
+
+        for e in self.new_users:
+            try:
+                r = User.objects.get(e)
+            except User.DoesNotExist:
+                r = None
+            assert r is None
+
+        with open(self.excel_file) as f:
+            resp = self.client.post(reverse('batch_add_user_using_excel'), {
+                'file': f
+            })
+
+        self.assertEqual(302, resp.status_code)
+        assert 'Import succeeded' in resp.cookies['messages'].value
+        for e in self.new_users:
+            assert User.objects.get(e) is not None
+            assert not UserOptions.objects.passwd_change_required(e)
+
+    @patch('seahub.views.sysadmin.user_number_over_limit')
+    def test_can_not_batch_add_if_user_over_limit(self, mock_user_number_over_limit):
+
+        mock_user_number_over_limit.return_value = True
+
+        for e in self.new_users:
+            try:
+                r = User.objects.get(e)
+            except User.DoesNotExist:
+                r = None
+            assert r is None
+
+        with open(self.excel_file) as f:
+            resp = self.client.post(reverse('batch_add_user_using_excel'), {
+                'file': f
+            })
+
+        self.assertEqual(302, resp.status_code)
+        assert 'users exceeds the limit' in resp.cookies['messages'].value
+
+    def test_can_send_email(self):
+        self.assertEqual(0, len(Email.objects.all()))
+
+        with open(self.excel_file) as f:
+            resp = self.client.post(reverse('batch_add_user_using_excel'), {
+                'file': f
+            })
+
+        self.assertEqual(302, resp.status_code)
+        self.assertNotEqual(0, len(Email.objects.all()))
+
+        email = Email.objects.all()[0]
+        assert self.new_users[0] == email.to[0]
+        assert "Email: %s" % self.new_users[0] in email.html_message
+        assert email.status == 2
+
+
+class BatchAddUserUsingExcelHelpTest(BaseTestCase):
+    def setUp(self):
+        self.login_as(self.admin)
+
+    def test_can_get_excel(self):
+        resp = self.client.get(reverse('batch_add_user_example')+"?type=xlsx")
+        assert resp.status_code == 200
+
+    def test_validate_excel(self):
+        resp = self.client.get(reverse('batch_add_user_example')+"?type=xlsx")
+        wb = openpyxl.load_workbook(filename=BytesIO(resp.content), read_only=True)
+        assert wb.sheetnames[0] == 'sample'
+        rows = wb.worksheets[0].rows
+        i = 0
+        for r in rows:
+            assert r[0].value == 'username@test' + str(i) + '.com'
+            assert r[1].value == 'password'
+            assert r[2].value == 'name_test' + str(i)
+            assert r[3].value == 'department_test' + str(i)
+            if i < 10:
+                assert r[4].value == 'guest'
+            else:
+                assert r[4].value == 'default'
+            assert r[5].value == '999'
+            i += 1