Added database storage for avatars

seahub/avatar/models.py

@@ -23,7 +23,7 @@ try:
 except ImportError:
     import Image
 
-from seahub.avatar.util import invalidate_cache
+from seahub.avatar.util import invalidate_cache, get_avatar_file_storage
 from seahub.avatar.settings import (AVATAR_STORAGE_DIR, AVATAR_RESIZE_METHOD,
                              AVATAR_MAX_AVATARS_PER_USER, AVATAR_THUMB_FORMAT,
                              AVATAR_HASH_USERDIRNAMES, AVATAR_HASH_FILENAMES,
@@ -127,11 +127,14 @@ class AvatarBase(object):
             size=size,
             ext=ext
         )
-    
+
 class Avatar(models.Model, AvatarBase):
     emailuser = LowerCaseCharField(max_length=255)
     primary = models.BooleanField(default=False)
-    avatar = models.ImageField(max_length=1024, upload_to=avatar_file_path, blank=True)
+    avatar = models.ImageField(max_length=1024,
+                               upload_to=avatar_file_path,
+                               storage=get_avatar_file_storage(),
+                               blank=True)
     date_uploaded = models.DateTimeField(default=datetime.datetime.now)
     
     def __unicode__(self):
@@ -156,7 +159,10 @@ class Avatar(models.Model, AvatarBase):
 
 class GroupAvatar(models.Model, AvatarBase):
     group_id = models.CharField(max_length=255)
-    avatar = models.ImageField(max_length=1024, upload_to=avatar_file_path, blank=True)
+    avatar = models.ImageField(max_length=1024,
+                               upload_to=avatar_file_path,
+                               storage=get_avatar_file_storage(),
+                               blank=True)
     date_uploaded = models.DateTimeField(default=datetime.datetime.now)
     
     def __unicode__(self):

seahub/avatar/settings.py

@@ -20,6 +20,7 @@ GROUP_AVATAR_DEFAULT_URL = getattr(settings, 'GROUP_AVATAR_DEFAULT_URL', 'avatar
 AUTO_GENERATE_GROUP_AVATAR_SIZES = getattr(settings, 'AUTO_GENERATE_GROUP_AVATAR_SIZES', (GROUP_AVATAR_DEFAULT_SIZE,))
 
 ### Common settings ###
+AVATAR_FILE_STORAGE = getattr(settings, 'AVATAR_FILE_STORAGE', '')
 AVATAR_RESIZE_METHOD = getattr(settings, 'AVATAR_RESIZE_METHOD', Image.ANTIALIAS)
 AVATAR_GRAVATAR_BACKUP = getattr(settings, 'AVATAR_GRAVATAR_BACKUP', True)
 AVATAR_GRAVATAR_DEFAULT = getattr(settings, 'AVATAR_GRAVATAR_DEFAULT', None)

seahub/avatar/sql/uploaded_file.sql

@@ -0,0 +1 @@
+CREATE TABLE `avatar_uploaded` (`filename` TEXT NOT NULL, `filename_md5` CHAR(32) NOT NULL PRIMARY KEY, `data` MEDIUMTEXT NOT NULL, `size` INTEGER NOT NULL, `mtime` datetime NOT NULL);

seahub/avatar/util.py

@@ -1,12 +1,13 @@
 from django.conf import settings
 from django.core.cache import cache
+from django.core.files.storage import default_storage, get_storage_class
 from django.utils.http import urlquote
 
 from seahub.base.accounts import User
-from seahub.avatar.settings import (AVATAR_DEFAULT_URL, AVATAR_CACHE_TIMEOUT,
-                             AUTO_GENERATE_AVATAR_SIZES, AVATAR_DEFAULT_SIZE,
-                             AVATAR_DEFAULT_NON_REGISTERED_URL,
-                             AUTO_GENERATE_GROUP_AVATAR_SIZES)
+from seahub.avatar.settings import AVATAR_DEFAULT_URL, AVATAR_CACHE_TIMEOUT,\
+    AUTO_GENERATE_AVATAR_SIZES, AVATAR_DEFAULT_SIZE, \
+    AVATAR_DEFAULT_NON_REGISTERED_URL, AUTO_GENERATE_GROUP_AVATAR_SIZES, \
+    AVATAR_FILE_STORAGE
 
 cached_funcs = set()
 
@@ -114,3 +115,20 @@ def get_primary_avatar(user, size=AVATAR_DEFAULT_SIZE):
         if not avatar.thumbnail_exists(size):
             avatar.create_thumbnail(size)
     return avatar
+
+def get_avatar_file_storage():
+    """Get avatar file storage, defaults to file system storage.
+    """
+    if not AVATAR_FILE_STORAGE:
+        return default_storage
+    else:
+        dbs_options = {
+            'table': 'avatar_uploaded',
+            'base_url': '/image-view/',
+            'name_column': 'filename',
+            'data_column': 'data',
+            'size_column': 'size',
+            }
+        return get_storage_class(AVATAR_FILE_STORAGE)(options=dbs_options)
+    
+    

seahub/base/database_storage/__init__.py

@@ -0,0 +1,4 @@
+# Allow users to: from database_storage import DatabaseStorage
+# (reduce redundancy a little bit)
+
+from database_storage import *

seahub/base/database_storage/database_storage.py

@@ -0,0 +1,240 @@
+# DatabaseStorage for django.
+# 2011 (c) Mike Mueller <mike@subfocal.net>
+# 2009 (c) GameKeeper Gambling Ltd, Ivanov E.
+
+from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist
+from django.core.files.storage import Storage
+from django.core.files import File
+from django.db import connection, transaction
+
+import base64
+import hashlib
+import StringIO
+import urlparse
+from datetime import datetime
+
+from seahub.utils.time import value_to_db_datetime
+
+class DatabaseStorage(Storage):
+    """
+    Implements the Django Storage API for storing files in the database,
+    rather than on the filesystem.  Uses the Django database layer, so any
+    database supported by Django should theoretically work.
+
+    Usage: Create an instance of DatabaseStorage and pass it as the storage
+    parameter of your FileField, ImageField, etc.::
+
+        image = models.ImageField(
+            null=True,
+            blank=True,
+            upload_to='attachments/',
+            storage=DatabaseStorage(options=DBS_OPTIONS),
+        )
+
+    Files submitted using this field will be saved into the default Django
+    database, using the options specified in the constructor.  The upload_to
+    path will be prepended to uploads, so that the file 'bar.png' would be
+    retrieved later as 'attachments/bar.png' in this example.
+
+    Uses the default get_available_name strategy, so duplicate filenames will
+    be silently renamed to foo_1.jpg, foo_2.jpg, etc.
+
+    You are responsible for creating a table in the database with the
+    following columns:
+
+        filename VARCHAR(256) NOT NULL PRIMARY KEY,
+        data TEXT NOT NULL,
+        size INTEGER NOT NULL,
+
+    The best place to do this is probably in your_app/sql/foo.sql, which will
+    run during syncdb.  The length 256 is up to you, you can also pass a
+    max_length parameter to FileFields to be consistent with your column here.
+    On SQL Server, you should probably use nvarchar to support unicode.
+
+    Remember, this is not designed for huge objects.  It is probably best used
+    on files under 1MB in size.  All files are base64-encoded before being
+    stored, so they will use 1.33x the storage of the original file.
+
+    Here's an example view to serve files stored in the database.
+
+      def image_view(request, filename):
+          # Read file from database
+          storage = DatabaseStorage(options=DBS_OPTIONS)
+          image_file = storage.open(filename, 'rb')
+          if not image_file:
+              raise Http404
+          file_content = image_file.read()
+
+          # Prepare response
+          content_type, content_encoding = mimetypes.guess_type(filename)
+          response = HttpResponse(content=file_content, mimetype=content_type)
+          response['Content-Disposition'] = 'inline; filename=%s' % filename
+          if content_encoding:
+              response['Content-Encoding'] = content_encoding
+          return response
+    """
+
+    def __init__(self, options):
+        """
+        Create a DatabaseStorage object with the specified options dictionary.
+
+        Required options:
+
+            'table': The name of the database table for file storage.
+            'base_url': The base URL where database files should be found.
+                        This is used to construct URLs for FileFields and
+                        you will need to define a view that handles requests
+                        at this location (example given above).
+
+        Allowed options:
+
+            'name_column': Name of the filename column (default: 'filename')
+            'data_column': Name of the data column (default: 'data')
+            'size_column': Name of the size column (default: 'size')
+
+                      'data_column', 'size_column', 'base_url' keys.
+        """
+
+        required_keys = [
+            'table',
+            'base_url',
+        ]
+        allowed_keys = [
+            'name_column',
+            'name_md5_column',
+            'data_column',
+            'size_column',
+            'mtime_column',
+        ]
+        for key in required_keys:
+            if key not in options:
+                raise ImproperlyConfigured(
+                    'DatabaseStorage missing required option: ' + key)
+        for key in options:
+            if key not in required_keys and key not in allowed_keys:
+                raise ImproperlyConfigured(
+                    'Unrecognized DatabaseStorage option: ' + key)
+
+        # Note: These fields are used as keys in string substitutions
+        # throughout this class.  If you change a name here, be sure to update
+        # all the affected format strings.
+        self.table = options['table']
+        self.base_url = options['base_url']
+        self.name_column = options.get('name_column', 'filename')
+        self.name_md5_column = options.get('name_md5_column', 'filename_md5')
+        self.data_column = options.get('data_column', 'data')
+        self.size_column = options.get('size_column', 'size')
+        self.mtime_column = options.get('mtime_column', 'mtime')
+
+    def _open(self, name, mode='rb'):
+        """
+        Open a file stored in the database.  name should be the full name of
+        the file, including the upload_to path that may have been used.
+        Path separator should always be '/'.  mode should always be 'rb'.
+
+        Returns a Django File object if found, otherwise None.
+        """
+        assert mode == 'rb', "DatabaseStorage open mode must be 'rb'."
+
+        name_md5 = hashlib.md5(name).hexdigest()
+
+        query = 'SELECT %(data_column)s FROM %(table)s ' + \
+                'WHERE %(name_md5_column)s = %%s'
+        query %= self.__dict__
+        cursor = connection.cursor()
+        cursor.execute(query, [name_md5])
+        row = cursor.fetchone()
+        if row is None:
+            return None
+
+        inMemFile = StringIO.StringIO(base64.b64decode(row[0]))
+        inMemFile.name = name
+        inMemFile.mode = mode
+
+        return File(inMemFile)
+
+    def _save(self, name, content):
+        """
+        Save the given content as file with the specified name.  Backslashes
+        in the name will be converted to forward '/'.
+        """
+        name = name.replace('\\', '/')
+        name_md5 = hashlib.md5(name).hexdigest()
+        binary = content.read()
+
+        size = len(binary)
+        encoded = base64.b64encode(binary)
+        mtime = value_to_db_datetime(datetime.today())
+
+        cursor = connection.cursor()
+
+        if self.exists(name):
+            query = 'UPDATE %(table)s SET %(data_column)s = %%s, ' + \
+                    '%(size_column)s = %%s, %(mtime_column)s = %%s ' + \
+                    'WHERE %(name_md5_column)s = %%s'
+            query %= self.__dict__
+            cursor.execute(query, [encoded, size, mtime, name])
+        else:
+            query = 'INSERT INTO %(table)s (%(name_column)s, ' + \
+                '%(name_md5_column)s, %(data_column)s, %(size_column)s, '+ \
+                '%(mtime_column)s) VALUES (%%s, %%s, %%s, %%s, %%s)'
+            query %= self.__dict__
+            cursor.execute(query, (name, name_md5, encoded, size, mtime))
+        transaction.commit_unless_managed(using='default')
+        return name
+
+    def exists(self, name):
+        name_md5 = hashlib.md5(name).hexdigest()
+        query = 'SELECT COUNT(*) FROM %(table)s WHERE %(name_md5_column)s = %%s'
+        query %= self.__dict__
+        cursor = connection.cursor()
+        cursor.execute(query, [name_md5])
+        row = cursor.fetchone()
+        return int(row[0]) > 0
+
+    def delete(self, name):
+        if self.exists(name):
+            name_md5 = hashlib.md5(name).hexdigest()
+            query = 'DELETE FROM %(table)s WHERE %(name_md5_column)s = %%s'
+            query %= self.__dict__
+            connection.cursor().execute(query, [name_md5])
+            transaction.commit_unless_managed(using='default')
+
+    def path(self, name):
+        raise NotImplementedError('DatabaseStorage does not support path().')
+
+    def url(self, name):
+        if self.base_url is None:
+            raise ValueError("This file is not accessible via a URL.")
+        result = urlparse.urljoin(self.base_url, name).replace('\\', '/')
+        return result
+
+    def size(self, name):
+        "Get the size of the given filename or raise ObjectDoesNotExist."
+        name_md5 = hashlib.md5(name).hexdigest()
+        query = 'SELECT %(size_column)s FROM %(table)s ' + \
+                'WHERE %(name_md5_column)s = %%s'
+        query %= self.__dict__
+        cursor = connection.cursor()
+        cursor.execute(query, [name_md5])
+        row = cursor.fetchone()
+        if not row:
+            raise ObjectDoesNotExist(
+                "DatabaseStorage file not found: %s" % name)
+        return int(row[0])
+
+    def modified_time(self, name):
+        "Get the modified time of the given filename or raise ObjectDoesNotExist."
+        name_md5 = hashlib.md5(name).hexdigest()
+        query = 'SELECT %(mtime_column)s FROM %(table)s ' + \
+                'WHERE %(name_md5_column)s = %%s'
+        query %= self.__dict__
+        cursor = connection.cursor()
+        cursor.execute(query, [name_md5])
+        row = cursor.fetchone()
+        if not row:
+            raise ObjectDoesNotExist(
+                "DatabaseStorage file not found: %s" % name)
+        
+        return row[0]
+    

seahub/settings.py

@@ -207,6 +207,9 @@ USE_PDFJS = True
 FILE_ENCODING_LIST = ['auto', 'utf-8', 'gbk', 'ISO-8859-1', 'ISO-8859-5']
 FILE_ENCODING_TRY_LIST = ['utf-8', 'gbk']
 
+# Common settings(file extension, storage) for avatar and group avatar.
+AVATAR_FILE_STORAGE = '' # Replace with 'seahub.base.database_storage.DatabaseStorage' if save avatar files to database
+AVATAR_ALLOWED_FILE_EXTS = ('.jpg', '.png', '.jpeg', '.gif')
 # Avatar
 AVATAR_STORAGE_DIR = 'avatars'
 AVATAR_GRAVATAR_BACKUP = False
@@ -214,7 +217,6 @@ AVATAR_DEFAULT_URL = '/avatars/default.jpg'
 AVATAR_DEFAULT_NON_REGISTERED_URL = '/avatars/default-non-register.jpg'
 AVATAR_MAX_AVATARS_PER_USER = 1
 AVATAR_CACHE_TIMEOUT = 14 * 24 * 60 * 60
-AVATAR_ALLOWED_FILE_EXTS = ('.jpg', '.png', '.jpeg', '.gif')
 AUTO_GENERATE_AVATAR_SIZES = (16, 20, 28, 36, 40, 48, 60, 80)
 # Group avatar
 GROUP_AVATAR_STORAGE_DIR = 'avatars/groups'

seahub/urls.py

@@ -96,6 +96,7 @@ urlpatterns = patterns('',
     url(r'^u/d/(?P<token>[a-f0-9]{10})/$', view_shared_upload_link, name='view_shared_upload_link'),
 
     ### Misc ###
+    url(r'^image-view/(?P<filename>.*)$', image_view, name='image_view'),
    (r'^file_upload_progress_page/$', file_upload_progress_page),
     url(r'^activities/$', activities, name='activities'),
     url(r'^starred/$', starred, name='starred'),

seahub/utils/time.py

@@ -1,4 +1,7 @@
 import datetime
+from django.conf import settings
+from django.utils import six
+from django.utils import timezone
 
 def dt(value):
     """Convert 32/64 bits timestamp to datetime object.
@@ -9,3 +12,18 @@ def dt(value):
         # TODO: need a better way to handle 64 bits timestamp.
         return datetime.datetime.utcfromtimestamp(value/1000000)
     
+
+def value_to_db_datetime(value):
+    if value is None:
+        return None
+
+    # MySQL doesn't support tz-aware datetimes
+    if timezone.is_aware(value):
+        if settings.USE_TZ:
+            value = value.astimezone(timezone.utc).replace(tzinfo=None)
+        else:
+            raise ValueError("MySQL backend does not support timezone-aware datetimes when USE_TZ is False.")
+
+    # MySQL doesn't support microseconds
+    return six.text_type(value.replace(microsecond=0))
+    

seahub/views/__init__.py

@@ -1,7 +1,9 @@
 # encoding: utf-8
+import hashlib
 import os
 import stat
 import simplejson as json
+import mimetypes
 import re
 import sys
 import urllib
@@ -9,6 +11,7 @@ import urllib2
 import logging
 import chardet
 from types import FunctionType
+import datetime as dt
 from datetime import datetime
 from math import ceil
 from urllib import quote
@@ -21,13 +24,14 @@ from django.contrib.sites.models import Site, RequestSite
 from django.db import IntegrityError
 from django.db.models import F
 from django.http import HttpResponse, HttpResponseBadRequest, Http404, \
-    HttpResponseRedirect
+    HttpResponseRedirect, HttpResponseNotModified
 from django.shortcuts import render_to_response, redirect
 from django.template import Context, loader, RequestContext
 from django.template.loader import render_to_string
 from django.utils.translation import ugettext as _
 from django.utils import timezone
 from django.utils.http import urlquote
+from django.views.decorators.http import condition
 
 import seaserv
 from seaserv import ccnet_rpc, ccnet_threaded_rpc, get_repos, get_emailusers, \
@@ -49,6 +53,7 @@ from seaserv import ccnet_rpc, ccnet_threaded_rpc, get_repos, get_emailusers, \
 from seaserv import seafile_api
 from pysearpc import SearpcError
 
+from seahub.avatar.util import get_avatar_file_storage
 from seahub.auth.decorators import login_required
 from seahub.auth import login as auth_login
 from seahub.auth import authenticate, get_backends
@@ -89,8 +94,9 @@ if HAS_OFFICE_CONVERTER:
     from seahub.utils import prepare_converted_html, OFFICE_PREVIEW_MAX_SIZE, OFFICE_PREVIEW_MAX_PAGES
 
 import seahub.settings as settings
-from seahub.settings import FILE_PREVIEW_MAX_SIZE, INIT_PASSWD, USE_PDFJS, FILE_ENCODING_LIST, \
-    FILE_ENCODING_TRY_LIST, SEND_EMAIL_ON_ADDING_SYSTEM_MEMBER, SEND_EMAIL_ON_RESETTING_USER_PASSWD, \
+from seahub.settings import FILE_PREVIEW_MAX_SIZE, INIT_PASSWD, USE_PDFJS, \
+    FILE_ENCODING_LIST, FILE_ENCODING_TRY_LIST, AVATAR_FILE_STORAGE, \
+    SEND_EMAIL_ON_ADDING_SYSTEM_MEMBER, SEND_EMAIL_ON_RESETTING_USER_PASSWD, \
     ENABLE_SUB_LIBRARY
 
 # Get an instance of a logger
@@ -2068,3 +2074,32 @@ def toggle_modules(request):
 
     return HttpResponseRedirect(next)
 
+
+storage = get_avatar_file_storage()
+def latest_entry(request, filename):
+    return storage.modified_time(filename)
+
+@condition(last_modified_func=latest_entry)
+def image_view(request, filename):
+    if AVATAR_FILE_STORAGE is None:
+        raise Http404
+
+    # read file from cache, if hit
+    filename_md5 = hashlib.md5(filename).hexdigest()
+    cache_key = 'image_view__%s' % filename_md5
+    file_content = cache.get(cache_key)
+    if file_content is None:
+        # otherwise, read file from database and update cache
+        image_file = storage.open(filename, 'rb')
+        if not image_file:
+            raise Http404
+        file_content = image_file.read()
+        cache.set(cache_key, file_content, 365 * 24 * 60 * 60)
+
+    # Prepare response
+    content_type, content_encoding = mimetypes.guess_type(filename)
+    response = HttpResponse(content=file_content, mimetype=content_type)
+    response['Content-Disposition'] = 'inline; filename=%s' % filename
+    if content_encoding:
+        response['Content-Encoding'] = content_encoding
+    return response