From 614ea152d922c2134989a1f67bb7fc8916d8b020 Mon Sep 17 00:00:00 2001 From: zhengxie Date: Thu, 2 Jan 2014 11:24:25 +0800 Subject: [PATCH] Added database storage for avatars --- seahub/avatar/models.py | 14 +- seahub/avatar/settings.py | 1 + seahub/avatar/sql/uploaded_file.sql | 1 + seahub/avatar/util.py | 26 +- seahub/base/database_storage/__init__.py | 4 + .../base/database_storage/database_storage.py | 240 ++++++++++++++++++ seahub/settings.py | 4 +- seahub/urls.py | 1 + seahub/utils/time.py | 18 ++ seahub/views/__init__.py | 41 ++- 10 files changed, 338 insertions(+), 12 deletions(-) create mode 100644 seahub/avatar/sql/uploaded_file.sql create mode 100644 seahub/base/database_storage/__init__.py create mode 100644 seahub/base/database_storage/database_storage.py diff --git a/seahub/avatar/models.py b/seahub/avatar/models.py index 582e4720f6..d4357c51e2 100644 --- a/seahub/avatar/models.py +++ b/seahub/avatar/models.py @@ -23,7 +23,7 @@ try: except ImportError: import Image -from seahub.avatar.util import invalidate_cache +from seahub.avatar.util import invalidate_cache, get_avatar_file_storage from seahub.avatar.settings import (AVATAR_STORAGE_DIR, AVATAR_RESIZE_METHOD, AVATAR_MAX_AVATARS_PER_USER, AVATAR_THUMB_FORMAT, AVATAR_HASH_USERDIRNAMES, AVATAR_HASH_FILENAMES, @@ -127,11 +127,14 @@ class AvatarBase(object): size=size, ext=ext ) - + class Avatar(models.Model, AvatarBase): emailuser = LowerCaseCharField(max_length=255) primary = models.BooleanField(default=False) - avatar = models.ImageField(max_length=1024, upload_to=avatar_file_path, blank=True) + avatar = models.ImageField(max_length=1024, + upload_to=avatar_file_path, + storage=get_avatar_file_storage(), + blank=True) date_uploaded = models.DateTimeField(default=datetime.datetime.now) def __unicode__(self): @@ -156,7 +159,10 @@ class Avatar(models.Model, AvatarBase): class GroupAvatar(models.Model, AvatarBase): group_id = models.CharField(max_length=255) - avatar = models.ImageField(max_length=1024, upload_to=avatar_file_path, blank=True) + avatar = models.ImageField(max_length=1024, + upload_to=avatar_file_path, + storage=get_avatar_file_storage(), + blank=True) date_uploaded = models.DateTimeField(default=datetime.datetime.now) def __unicode__(self): diff --git a/seahub/avatar/settings.py b/seahub/avatar/settings.py index d9da14e478..a6fd045920 100644 --- a/seahub/avatar/settings.py +++ b/seahub/avatar/settings.py @@ -20,6 +20,7 @@ GROUP_AVATAR_DEFAULT_URL = getattr(settings, 'GROUP_AVATAR_DEFAULT_URL', 'avatar AUTO_GENERATE_GROUP_AVATAR_SIZES = getattr(settings, 'AUTO_GENERATE_GROUP_AVATAR_SIZES', (GROUP_AVATAR_DEFAULT_SIZE,)) ### Common settings ### +AVATAR_FILE_STORAGE = getattr(settings, 'AVATAR_FILE_STORAGE', '') AVATAR_RESIZE_METHOD = getattr(settings, 'AVATAR_RESIZE_METHOD', Image.ANTIALIAS) AVATAR_GRAVATAR_BACKUP = getattr(settings, 'AVATAR_GRAVATAR_BACKUP', True) AVATAR_GRAVATAR_DEFAULT = getattr(settings, 'AVATAR_GRAVATAR_DEFAULT', None) diff --git a/seahub/avatar/sql/uploaded_file.sql b/seahub/avatar/sql/uploaded_file.sql new file mode 100644 index 0000000000..dea417d0e5 --- /dev/null +++ b/seahub/avatar/sql/uploaded_file.sql @@ -0,0 +1 @@ +CREATE TABLE `avatar_uploaded` (`filename` TEXT NOT NULL, `filename_md5` CHAR(32) NOT NULL PRIMARY KEY, `data` MEDIUMTEXT NOT NULL, `size` INTEGER NOT NULL, `mtime` datetime NOT NULL); diff --git a/seahub/avatar/util.py b/seahub/avatar/util.py index 9a316bd9e5..287e0ff32d 100644 --- a/seahub/avatar/util.py +++ b/seahub/avatar/util.py @@ -1,12 +1,13 @@ from django.conf import settings from django.core.cache import cache +from django.core.files.storage import default_storage, get_storage_class from django.utils.http import urlquote from seahub.base.accounts import User -from seahub.avatar.settings import (AVATAR_DEFAULT_URL, AVATAR_CACHE_TIMEOUT, - AUTO_GENERATE_AVATAR_SIZES, AVATAR_DEFAULT_SIZE, - AVATAR_DEFAULT_NON_REGISTERED_URL, - AUTO_GENERATE_GROUP_AVATAR_SIZES) +from seahub.avatar.settings import AVATAR_DEFAULT_URL, AVATAR_CACHE_TIMEOUT,\ + AUTO_GENERATE_AVATAR_SIZES, AVATAR_DEFAULT_SIZE, \ + AVATAR_DEFAULT_NON_REGISTERED_URL, AUTO_GENERATE_GROUP_AVATAR_SIZES, \ + AVATAR_FILE_STORAGE cached_funcs = set() @@ -114,3 +115,20 @@ def get_primary_avatar(user, size=AVATAR_DEFAULT_SIZE): if not avatar.thumbnail_exists(size): avatar.create_thumbnail(size) return avatar + +def get_avatar_file_storage(): + """Get avatar file storage, defaults to file system storage. + """ + if not AVATAR_FILE_STORAGE: + return default_storage + else: + dbs_options = { + 'table': 'avatar_uploaded', + 'base_url': '/image-view/', + 'name_column': 'filename', + 'data_column': 'data', + 'size_column': 'size', + } + return get_storage_class(AVATAR_FILE_STORAGE)(options=dbs_options) + + diff --git a/seahub/base/database_storage/__init__.py b/seahub/base/database_storage/__init__.py new file mode 100644 index 0000000000..2c59e78865 --- /dev/null +++ b/seahub/base/database_storage/__init__.py @@ -0,0 +1,4 @@ +# Allow users to: from database_storage import DatabaseStorage +# (reduce redundancy a little bit) + +from database_storage import * diff --git a/seahub/base/database_storage/database_storage.py b/seahub/base/database_storage/database_storage.py new file mode 100644 index 0000000000..bb3eadc1b8 --- /dev/null +++ b/seahub/base/database_storage/database_storage.py @@ -0,0 +1,240 @@ +# DatabaseStorage for django. +# 2011 (c) Mike Mueller +# 2009 (c) GameKeeper Gambling Ltd, Ivanov E. + +from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist +from django.core.files.storage import Storage +from django.core.files import File +from django.db import connection, transaction + +import base64 +import hashlib +import StringIO +import urlparse +from datetime import datetime + +from seahub.utils.time import value_to_db_datetime + +class DatabaseStorage(Storage): + """ + Implements the Django Storage API for storing files in the database, + rather than on the filesystem. Uses the Django database layer, so any + database supported by Django should theoretically work. + + Usage: Create an instance of DatabaseStorage and pass it as the storage + parameter of your FileField, ImageField, etc.:: + + image = models.ImageField( + null=True, + blank=True, + upload_to='attachments/', + storage=DatabaseStorage(options=DBS_OPTIONS), + ) + + Files submitted using this field will be saved into the default Django + database, using the options specified in the constructor. The upload_to + path will be prepended to uploads, so that the file 'bar.png' would be + retrieved later as 'attachments/bar.png' in this example. + + Uses the default get_available_name strategy, so duplicate filenames will + be silently renamed to foo_1.jpg, foo_2.jpg, etc. + + You are responsible for creating a table in the database with the + following columns: + + filename VARCHAR(256) NOT NULL PRIMARY KEY, + data TEXT NOT NULL, + size INTEGER NOT NULL, + + The best place to do this is probably in your_app/sql/foo.sql, which will + run during syncdb. The length 256 is up to you, you can also pass a + max_length parameter to FileFields to be consistent with your column here. + On SQL Server, you should probably use nvarchar to support unicode. + + Remember, this is not designed for huge objects. It is probably best used + on files under 1MB in size. All files are base64-encoded before being + stored, so they will use 1.33x the storage of the original file. + + Here's an example view to serve files stored in the database. + + def image_view(request, filename): + # Read file from database + storage = DatabaseStorage(options=DBS_OPTIONS) + image_file = storage.open(filename, 'rb') + if not image_file: + raise Http404 + file_content = image_file.read() + + # Prepare response + content_type, content_encoding = mimetypes.guess_type(filename) + response = HttpResponse(content=file_content, mimetype=content_type) + response['Content-Disposition'] = 'inline; filename=%s' % filename + if content_encoding: + response['Content-Encoding'] = content_encoding + return response + """ + + def __init__(self, options): + """ + Create a DatabaseStorage object with the specified options dictionary. + + Required options: + + 'table': The name of the database table for file storage. + 'base_url': The base URL where database files should be found. + This is used to construct URLs for FileFields and + you will need to define a view that handles requests + at this location (example given above). + + Allowed options: + + 'name_column': Name of the filename column (default: 'filename') + 'data_column': Name of the data column (default: 'data') + 'size_column': Name of the size column (default: 'size') + + 'data_column', 'size_column', 'base_url' keys. + """ + + required_keys = [ + 'table', + 'base_url', + ] + allowed_keys = [ + 'name_column', + 'name_md5_column', + 'data_column', + 'size_column', + 'mtime_column', + ] + for key in required_keys: + if key not in options: + raise ImproperlyConfigured( + 'DatabaseStorage missing required option: ' + key) + for key in options: + if key not in required_keys and key not in allowed_keys: + raise ImproperlyConfigured( + 'Unrecognized DatabaseStorage option: ' + key) + + # Note: These fields are used as keys in string substitutions + # throughout this class. If you change a name here, be sure to update + # all the affected format strings. + self.table = options['table'] + self.base_url = options['base_url'] + self.name_column = options.get('name_column', 'filename') + self.name_md5_column = options.get('name_md5_column', 'filename_md5') + self.data_column = options.get('data_column', 'data') + self.size_column = options.get('size_column', 'size') + self.mtime_column = options.get('mtime_column', 'mtime') + + def _open(self, name, mode='rb'): + """ + Open a file stored in the database. name should be the full name of + the file, including the upload_to path that may have been used. + Path separator should always be '/'. mode should always be 'rb'. + + Returns a Django File object if found, otherwise None. + """ + assert mode == 'rb', "DatabaseStorage open mode must be 'rb'." + + name_md5 = hashlib.md5(name).hexdigest() + + query = 'SELECT %(data_column)s FROM %(table)s ' + \ + 'WHERE %(name_md5_column)s = %%s' + query %= self.__dict__ + cursor = connection.cursor() + cursor.execute(query, [name_md5]) + row = cursor.fetchone() + if row is None: + return None + + inMemFile = StringIO.StringIO(base64.b64decode(row[0])) + inMemFile.name = name + inMemFile.mode = mode + + return File(inMemFile) + + def _save(self, name, content): + """ + Save the given content as file with the specified name. Backslashes + in the name will be converted to forward '/'. + """ + name = name.replace('\\', '/') + name_md5 = hashlib.md5(name).hexdigest() + binary = content.read() + + size = len(binary) + encoded = base64.b64encode(binary) + mtime = value_to_db_datetime(datetime.today()) + + cursor = connection.cursor() + + if self.exists(name): + query = 'UPDATE %(table)s SET %(data_column)s = %%s, ' + \ + '%(size_column)s = %%s, %(mtime_column)s = %%s ' + \ + 'WHERE %(name_md5_column)s = %%s' + query %= self.__dict__ + cursor.execute(query, [encoded, size, mtime, name]) + else: + query = 'INSERT INTO %(table)s (%(name_column)s, ' + \ + '%(name_md5_column)s, %(data_column)s, %(size_column)s, '+ \ + '%(mtime_column)s) VALUES (%%s, %%s, %%s, %%s, %%s)' + query %= self.__dict__ + cursor.execute(query, (name, name_md5, encoded, size, mtime)) + transaction.commit_unless_managed(using='default') + return name + + def exists(self, name): + name_md5 = hashlib.md5(name).hexdigest() + query = 'SELECT COUNT(*) FROM %(table)s WHERE %(name_md5_column)s = %%s' + query %= self.__dict__ + cursor = connection.cursor() + cursor.execute(query, [name_md5]) + row = cursor.fetchone() + return int(row[0]) > 0 + + def delete(self, name): + if self.exists(name): + name_md5 = hashlib.md5(name).hexdigest() + query = 'DELETE FROM %(table)s WHERE %(name_md5_column)s = %%s' + query %= self.__dict__ + connection.cursor().execute(query, [name_md5]) + transaction.commit_unless_managed(using='default') + + def path(self, name): + raise NotImplementedError('DatabaseStorage does not support path().') + + def url(self, name): + if self.base_url is None: + raise ValueError("This file is not accessible via a URL.") + result = urlparse.urljoin(self.base_url, name).replace('\\', '/') + return result + + def size(self, name): + "Get the size of the given filename or raise ObjectDoesNotExist." + name_md5 = hashlib.md5(name).hexdigest() + query = 'SELECT %(size_column)s FROM %(table)s ' + \ + 'WHERE %(name_md5_column)s = %%s' + query %= self.__dict__ + cursor = connection.cursor() + cursor.execute(query, [name_md5]) + row = cursor.fetchone() + if not row: + raise ObjectDoesNotExist( + "DatabaseStorage file not found: %s" % name) + return int(row[0]) + + def modified_time(self, name): + "Get the modified time of the given filename or raise ObjectDoesNotExist." + name_md5 = hashlib.md5(name).hexdigest() + query = 'SELECT %(mtime_column)s FROM %(table)s ' + \ + 'WHERE %(name_md5_column)s = %%s' + query %= self.__dict__ + cursor = connection.cursor() + cursor.execute(query, [name_md5]) + row = cursor.fetchone() + if not row: + raise ObjectDoesNotExist( + "DatabaseStorage file not found: %s" % name) + + return row[0] + diff --git a/seahub/settings.py b/seahub/settings.py index 3880599e97..f57793a92f 100644 --- a/seahub/settings.py +++ b/seahub/settings.py @@ -207,6 +207,9 @@ USE_PDFJS = True FILE_ENCODING_LIST = ['auto', 'utf-8', 'gbk', 'ISO-8859-1', 'ISO-8859-5'] FILE_ENCODING_TRY_LIST = ['utf-8', 'gbk'] +# Common settings(file extension, storage) for avatar and group avatar. +AVATAR_FILE_STORAGE = '' # Replace with 'seahub.base.database_storage.DatabaseStorage' if save avatar files to database +AVATAR_ALLOWED_FILE_EXTS = ('.jpg', '.png', '.jpeg', '.gif') # Avatar AVATAR_STORAGE_DIR = 'avatars' AVATAR_GRAVATAR_BACKUP = False @@ -214,7 +217,6 @@ AVATAR_DEFAULT_URL = '/avatars/default.jpg' AVATAR_DEFAULT_NON_REGISTERED_URL = '/avatars/default-non-register.jpg' AVATAR_MAX_AVATARS_PER_USER = 1 AVATAR_CACHE_TIMEOUT = 14 * 24 * 60 * 60 -AVATAR_ALLOWED_FILE_EXTS = ('.jpg', '.png', '.jpeg', '.gif') AUTO_GENERATE_AVATAR_SIZES = (16, 20, 28, 36, 40, 48, 60, 80) # Group avatar GROUP_AVATAR_STORAGE_DIR = 'avatars/groups' diff --git a/seahub/urls.py b/seahub/urls.py index 5870c938a9..6e5b7cafda 100644 --- a/seahub/urls.py +++ b/seahub/urls.py @@ -96,6 +96,7 @@ urlpatterns = patterns('', url(r'^u/d/(?P[a-f0-9]{10})/$', view_shared_upload_link, name='view_shared_upload_link'), ### Misc ### + url(r'^image-view/(?P.*)$', image_view, name='image_view'), (r'^file_upload_progress_page/$', file_upload_progress_page), url(r'^activities/$', activities, name='activities'), url(r'^starred/$', starred, name='starred'), diff --git a/seahub/utils/time.py b/seahub/utils/time.py index 6d1b2eee04..8ce0f950a3 100644 --- a/seahub/utils/time.py +++ b/seahub/utils/time.py @@ -1,4 +1,7 @@ import datetime +from django.conf import settings +from django.utils import six +from django.utils import timezone def dt(value): """Convert 32/64 bits timestamp to datetime object. @@ -9,3 +12,18 @@ def dt(value): # TODO: need a better way to handle 64 bits timestamp. return datetime.datetime.utcfromtimestamp(value/1000000) + +def value_to_db_datetime(value): + if value is None: + return None + + # MySQL doesn't support tz-aware datetimes + if timezone.is_aware(value): + if settings.USE_TZ: + value = value.astimezone(timezone.utc).replace(tzinfo=None) + else: + raise ValueError("MySQL backend does not support timezone-aware datetimes when USE_TZ is False.") + + # MySQL doesn't support microseconds + return six.text_type(value.replace(microsecond=0)) + diff --git a/seahub/views/__init__.py b/seahub/views/__init__.py index 416c64bf7e..2bfc861984 100644 --- a/seahub/views/__init__.py +++ b/seahub/views/__init__.py @@ -1,7 +1,9 @@ # encoding: utf-8 +import hashlib import os import stat import simplejson as json +import mimetypes import re import sys import urllib @@ -9,6 +11,7 @@ import urllib2 import logging import chardet from types import FunctionType +import datetime as dt from datetime import datetime from math import ceil from urllib import quote @@ -21,13 +24,14 @@ from django.contrib.sites.models import Site, RequestSite from django.db import IntegrityError from django.db.models import F from django.http import HttpResponse, HttpResponseBadRequest, Http404, \ - HttpResponseRedirect + HttpResponseRedirect, HttpResponseNotModified from django.shortcuts import render_to_response, redirect from django.template import Context, loader, RequestContext from django.template.loader import render_to_string from django.utils.translation import ugettext as _ from django.utils import timezone from django.utils.http import urlquote +from django.views.decorators.http import condition import seaserv from seaserv import ccnet_rpc, ccnet_threaded_rpc, get_repos, get_emailusers, \ @@ -49,6 +53,7 @@ from seaserv import ccnet_rpc, ccnet_threaded_rpc, get_repos, get_emailusers, \ from seaserv import seafile_api from pysearpc import SearpcError +from seahub.avatar.util import get_avatar_file_storage from seahub.auth.decorators import login_required from seahub.auth import login as auth_login from seahub.auth import authenticate, get_backends @@ -89,8 +94,9 @@ if HAS_OFFICE_CONVERTER: from seahub.utils import prepare_converted_html, OFFICE_PREVIEW_MAX_SIZE, OFFICE_PREVIEW_MAX_PAGES import seahub.settings as settings -from seahub.settings import FILE_PREVIEW_MAX_SIZE, INIT_PASSWD, USE_PDFJS, FILE_ENCODING_LIST, \ - FILE_ENCODING_TRY_LIST, SEND_EMAIL_ON_ADDING_SYSTEM_MEMBER, SEND_EMAIL_ON_RESETTING_USER_PASSWD, \ +from seahub.settings import FILE_PREVIEW_MAX_SIZE, INIT_PASSWD, USE_PDFJS, \ + FILE_ENCODING_LIST, FILE_ENCODING_TRY_LIST, AVATAR_FILE_STORAGE, \ + SEND_EMAIL_ON_ADDING_SYSTEM_MEMBER, SEND_EMAIL_ON_RESETTING_USER_PASSWD, \ ENABLE_SUB_LIBRARY # Get an instance of a logger @@ -2068,3 +2074,32 @@ def toggle_modules(request): return HttpResponseRedirect(next) + +storage = get_avatar_file_storage() +def latest_entry(request, filename): + return storage.modified_time(filename) + +@condition(last_modified_func=latest_entry) +def image_view(request, filename): + if AVATAR_FILE_STORAGE is None: + raise Http404 + + # read file from cache, if hit + filename_md5 = hashlib.md5(filename).hexdigest() + cache_key = 'image_view__%s' % filename_md5 + file_content = cache.get(cache_key) + if file_content is None: + # otherwise, read file from database and update cache + image_file = storage.open(filename, 'rb') + if not image_file: + raise Http404 + file_content = image_file.read() + cache.set(cache_key, file_content, 365 * 24 * 60 * 60) + + # Prepare response + content_type, content_encoding = mimetypes.guess_type(filename) + response = HttpResponse(content=file_content, mimetype=content_type) + response['Content-Disposition'] = 'inline; filename=%s' % filename + if content_encoding: + response['Content-Encoding'] = content_encoding + return response