Source code for kotti.filedepot

import uuid
from datetime import datetime

from sqlalchemy import Column
from sqlalchemy import DateTime
from sqlalchemy import Integer
from sqlalchemy import LargeBinary
from sqlalchemy import String
from sqlalchemy import Unicode
from sqlalchemy import event
from sqlalchemy.orm import deferred

from import FileStorage

from kotti import Base
from kotti import DBSession
from kotti.util import camel_case_to_name
from kotti.util import command

_marker = object()

[docs]class DBStoredFile(Base): """ :class:`` implementation that stores file data in SQL database. Can be used together with :class:`kotti.filedepot.DBFileStorage` to implement blobs storage in the database. """ __tablename__ = "blobs" #: Primary key column in the DB #: (:class:`sqlalchemy.types.Integer`) id = Column(Integer(), primary_key=True) #: Unique file id given to this blob #: (:class:`sqlalchemy.types.String`) file_id = Column(String(36), index=True) #: The original filename it had when it was uploaded. #: (:class:`sqlalchemy.types.String`) filename = Column(Unicode(100)) #: MIME type of the blob #: (:class:`sqlalchemy.types.String`) content_type = Column(String(100)) #: Size of the blob in bytes #: (:class:`sqlalchemy.types.Integer`) content_length = Column(Integer()) #: Date / time the blob was created or last modified #: (:class:`sqlalchemy.types.DateTime`) last_modified = Column(DateTime()) #: The binary data itself #: (:class:`sqlalchemy.types.LargeBinary`) data = deferred(Column('data', LargeBinary())) _cursor = 0 _data = _marker def __init__(self, file_id, filename=None, content_type=None, last_modified=None, content_length=None, **kwds): self.file_id = file_id self.filename = filename self.content_type = content_type self.last_modified = last_modified or self.content_length = content_length for k, v in kwds.items(): setattr(self, k, v)
[docs] def read(self, n=-1): """Reads ``n`` bytes from the file. If ``n`` is not specified or is ``-1`` the whole file content is read in memory and returned """ if self._data is _marker: file_id = DBSession.merge(self).file_id self._data = DBSession.query(\ filter_by(file_id=file_id).scalar() if n == -1: result = self._data[self._cursor:] else: result = self._data[self._cursor:self._cursor + n] self._cursor += len(result) return result
[docs] def close(self, *args, **kwargs): """Implement :meth:`StoredFile.close`. :class:`DBStoredFile` never closes. """ return
[docs] def closed(self): """Implement :meth:`StoredFile.closed`. """ return False
[docs] def writable(self): """Implement :meth:`StoredFile.writable`. """ return False
[docs] def seekable(self): """Implement :meth:`StoredFile.seekable`. """ return True
[docs] def seek(self, n): """ Move the file cursor to position `n` :param n: Position for the cursor :type n: int """ self._cursor = n
[docs] def tell(self): """ Returns current position of file cursor :result: Current file cursor position. :rtype: int """ return self._cursor
@property def name(self): """Implement :meth:``. :result: the filename of the saved file :rtype: string """ return self.filename @property def public_url(self): """ Integration with :class:`depot.middleware.DepotMiddleware` When supported by the storage this will provide the public url to which the file content can be accessed. In case this returns ``None`` it means that the file can only be served by the :class:`depot.middleware.DepotMiddleware` itself. """ return None @classmethod def __declare_last__(cls): """ Executed by SQLAlchemy as part of mapper configuration When the data changes, we want to reset the cursor position of target instance, to allow proper streaming of data. """ event.listen(, 'set', handle_change_data)
def handle_change_data(target, value, oldvalue, initiator): target._cursor = 0 target._data = _marker
[docs]def set_metadata(event): """Set DBStoredFile metadata based on data :param event: event that trigerred this handler. :type event: :class:`ObjectInsert` or :class:`ObjectUpdate` """ obj = event.object obj.content_length = and len( or 0 obj.last_modified =
[docs]class DBFileStorage(FileStorage): """Implementation of :class:``, Uses `kotti.filedepot.DBStoredFile` to store blob data in an SQL database. """
[docs] def get(self, file_id): """Returns the file given by the file_id :param file_id: the unique id associated to the file :type file_id: string :result: a :class:`kotti.filedepot.DBStoredFile` instance :rtype: :class:`kotti.filedepot.DBStoredFile` """ f = DBSession.query(DBStoredFile).filter_by(file_id=file_id).first() if f is None: raise IOError return f
[docs] def create(self, content, filename=None, content_type=None): """Saves a new file and returns the file id :param content: can either be ``bytes``, another ``file object`` or a :class:`cgi.FieldStorage`. When ``filename`` and ``content_type`` parameters are not provided they are deducted from the content itself. :param filename: filename for this file :type filename: string :param content_type: Mimetype of this file :type content_type: string :return: the unique ``file_id`` associated to this file :rtype: string """ new_file_id = str(uuid.uuid1()) content, filename, content_type = self.fileinfo( content, filename, content_type) if hasattr(content, 'read'): content = fstore = DBStoredFile(data=content, file_id=new_file_id, filename=filename, content_type=content_type, ) DBSession.add(fstore) return new_file_id
[docs] def replace(self, file_or_id, content, filename=None, content_type=None): """Replaces an existing file, an ``IOError`` is raised if the file didn't already exist. Given a :class:`StoredFile` or its ID it will replace the current content with the provided ``content`` value. If ``filename`` and ``content_type`` are provided or can be deducted by the ``content`` itself they will also replace the previous values, otherwise the current values are kept. :param file_or_id: can be either ``DBStoredFile`` or a ``file_id`` :param content: can either be ``bytes``, another ``file object`` or a :class:`cgi.FieldStorage`. When ``filename`` and ``content_type`` parameters are not provided they are deducted from the content itself. :param filename: filename for this file :type filename: string :param content_type: Mimetype of this file :type content_type: string """ file_id = self._get_file_id(file_or_id) content, filename, content_type = self.fileinfo( content, filename, content_type) fstore = self.get(file_id) if filename is not None: fstore.filename = filename if content_type is not None: fstore.content_type = content_type if hasattr(content, 'read'): content = = content
[docs] def delete(self, file_or_id): """Deletes a file. If the file didn't exist it will just do nothing. :param file_or_id: can be either ``DBStoredFile`` or a ``file_id`` """ file_id = self._get_file_id(file_or_id) DBSession.query(DBStoredFile).filter_by(file_id=file_id).delete()
[docs] def exists(self, file_or_id): """Returns if a file or its ID still exist. :return: Returns if a file or its ID still exist. :rtype: bool """ file_id = self._get_file_id(file_or_id) return bool( DBSession.query(DBStoredFile).filter_by(file_id=file_id).count())
def _get_file_id(self, file_or_id): if hasattr(file_or_id, 'file_id'): return file_or_id.file_id return file_or_id
def configure_filedepot(settings): from kotti.util import extract_depot_settings from depot.manager import DepotManager config = extract_depot_settings('kotti.depot.', settings) for conf in config: name = conf.pop('name') if name not in DepotManager._depots: DepotManager.configure(name, conf, prefix='') def migrate_storage(from_storage, to_storage): from depot.fields.sqlalchemy import _SQLAMutationTracker from depot.manager import DepotManager from kotti.util import _to_fieldstorage import logging log = logging.getLogger(__name__) old_default = DepotManager._default_depot DepotManager._default_depot = to_storage for klass, props in _SQLAMutationTracker.mapped_entities.items():"Migrating %r", klass) mapper = klass._sa_class_manager.mapper # use type column to avoid polymorphism issues, getting the same # Node item multiple times. type_ = camel_case_to_name(klass.__name__) for instance in DBSession.query(klass).filter_by(type=type_): for prop in props: uf = getattr(instance, prop) if not uf: continue pk = mapper.primary_key_from_instance(instance)"Migrating %s for %r with pk %r", prop, klass, pk) filename = uf['filename'] content_type = uf['content_type'] data = _to_fieldstorage(fp=uf.file, filename=filename, mimetype=content_type, size=uf.file.content_length) setattr(instance, prop, data) DepotManager._default_depot = old_default def migrate_storages_command(): # pragma: no cover __doc__ = """ Migrate blobs between two configured filedepot storages Usage: kotti-migrate-storage <config_uri> --from-storage <name> --to-storage <name> Options: -h --help Show this screen. --from-storage <name> The storage name that has blob data to migrate --to-storage <name> The storage name where we want to put the blobs """ return command( lambda args: migrate_storage( from_storage=args['--from-storage'], to_storage=args['--to-storage'], ), __doc__, ) def adjust_for_engine(conn, branch): # adjust for engine type if == 'mysql': # pragma: no cover from sqlalchemy.dialects.mysql.base import LONGBLOB = LONGBLOB() # sqlite's Unicode columns return a buffer which can't be encoded by # a json encoder. We have to convert to a unicode string so that the value # can be saved corectly by # :class:`depot.fields.sqlalchemy.upload.UploadedFile` def patched_processed_result_value(self, value, dialect): if not value: return None return self._upload_type.decode(unicode(value)) if == 'sqlite': # pragma: no cover from depot.fields.sqlalchemy import UploadedFileField UploadedFileField.process_result_value = patched_processed_result_value
[docs]def includeme(config): """ Pyramid includeme hook. :param config: app config :type config: :class:`pyramid.config.Configurator` """ from import objectevent_listeners from import ObjectInsert from import ObjectUpdate from depot.fields.sqlalchemy import _SQLAMutationTracker from sqlalchemy.event import listen from sqlalchemy.engine import Engine listen(Engine, 'engine_connect', adjust_for_engine) configure_filedepot(config.get_settings()) # Update file metadata on change of blob data objectevent_listeners[ (ObjectInsert, DBStoredFile)].append(set_metadata) objectevent_listeners[ (ObjectUpdate, DBStoredFile)].append(set_metadata) # depot's _SQLAMutationTracker._session_committed is executed on # after_commit, that's too late for DBFileStorage to interact with the # session event.listen(DBSession, 'before_commit', _SQLAMutationTracker._session_committed)