From 07106ed8dc38088215deb8e4c482d8ed13114efe Mon Sep 17 00:00:00 2001 From: Stefan Rijnhart Date: Wed, 18 Feb 2015 18:10:25 +0100 Subject: [PATCH 1/6] [ADD] Module to provide file types for attachments --- attachment_file_type/__init__.py | 1 + attachment_file_type/__openerp__.py | 79 ++++++++++++++++++ attachment_file_type/data/ir_cron.xml | 18 +++++ attachment_file_type/models/__init__.py | 1 + attachment_file_type/models/ir_attachment.py | 85 ++++++++++++++++++++ 5 files changed, 184 insertions(+) create mode 100644 attachment_file_type/__init__.py create mode 100644 attachment_file_type/__openerp__.py create mode 100644 attachment_file_type/data/ir_cron.xml create mode 100644 attachment_file_type/models/__init__.py create mode 100644 attachment_file_type/models/ir_attachment.py diff --git a/attachment_file_type/__init__.py b/attachment_file_type/__init__.py new file mode 100644 index 00000000..0650744f --- /dev/null +++ b/attachment_file_type/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/attachment_file_type/__openerp__.py b/attachment_file_type/__openerp__.py new file mode 100644 index 00000000..95ba49ca --- /dev/null +++ b/attachment_file_type/__openerp__.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# This module copyright (C) 2015 Therp BV (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +{ + "name": "File types for attachments", + "version": "1.0", + "author": "Therp BV", + "license": "AGPL-3", + "description": """ +File types for attachments +========================== +Adds a content type to attachments. This is useful for instance in combination +with http://bazaar.launchpad.net/~ocb/ocb-web/6.1/revision/2524, which makes +Odoo offer downloads with the correct file type based on this field. + +The document module makes this field available as well. This module does not +interfere with its functionality when both are installed. + +Installation +============ +To install this module, you need to install python-magic in your environment +first. + +Configuration +============= +This module comes with a daily scheduled task to provide content types for +attachments without one in small batches (instead of trying to do this for all +attachments at installation time). Depending on the number of attachments in +your system you may deactivate this task after some time. You can follow the +progress in the logs at the time that the cron job is fired. + +Credits +======= +Contributors +------------ + +* Stefan Rijnhart +* Holger Brunn + +Maintainer +---------- + +.. image:: http://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: http://odoo-community.org + +This module is maintained by the OCA. + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +To contribute to this module, please visit http://odoo-community.org. + """, + "category": "Knowledge Management", + "depends": ['base'], + "data": ['data/ir_cron.xml'], + "license": 'AGPL-3', + "external_dependencies": { + 'python': ['magic'], + }, +} diff --git a/attachment_file_type/data/ir_cron.xml b/attachment_file_type/data/ir_cron.xml new file mode 100644 index 00000000..4911b9f1 --- /dev/null +++ b/attachment_file_type/data/ir_cron.xml @@ -0,0 +1,18 @@ + + + + + + Provide file types for attachments without one + 24 + hours + -1 + True + + ir.attachment + update_file_type_from_cron + (1000,) + + + + diff --git a/attachment_file_type/models/__init__.py b/attachment_file_type/models/__init__.py new file mode 100644 index 00000000..aaf38a16 --- /dev/null +++ b/attachment_file_type/models/__init__.py @@ -0,0 +1 @@ +from . import ir_attachment diff --git a/attachment_file_type/models/ir_attachment.py b/attachment_file_type/models/ir_attachment.py new file mode 100644 index 00000000..c4347499 --- /dev/null +++ b/attachment_file_type/models/ir_attachment.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# This module copyright (C) 2015 Therp BV (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +import logging +from openerp.osv import orm, fields +from base64 import b64decode + + +class Attachment(orm.Model): + _inherit = 'ir.attachment' + + def update_file_type_from_cron(self, cr, uid, count=1000, context=None): + ids = self.search( + cr, uid, [('file_type', '=', False)], context=context) + logging.getLogger('openerp.addons.attachment_file_type').info( + 'Found %s attachments without file type in the database.', len(ids)) + self.update_file_type(cr, uid, ids, force=True, context=None) + + def update_file_type(self, cr, uid, ids, force=False, context=None): + """ Write the file types for these attachments. If the document module + is installed, don't do anything unless in force mode. """ + if not ids: + return True + if not force: + cr.execute( + "SELECT id from ir_module_module WHERE name = 'document' " + "AND state = 'installed'") + if cr.fetchone(): + return True + if isinstance(ids, (int, long)): + ids = [ids] + + logger = logging.getLogger('openerp.addons.attachment_file_type') + import magic + ms = magic.open( + # MAGIC_MIME gives additional encoding, but old versions of the + # 'file' package come with py_magic.c that lack MAGIC_MIME_TYPE + magic.MAGIC_MIME_TYPE if hasattr(magic, 'MAGIC_MIME_TYPE') + else magic.MAGIC_MIME) + ms.load() + for attachment in self.browse(cr, uid, ids, context=context): + file_type = ms.buffer( + b64decode(attachment.datas)).split(';')[0] + logger.debug( + 'Found file type %s for attachment with id %s', + file_type, attachment.id) + attachment.write({'file_type': file_type}) + return True + + def write(self, cr, uid, ids, vals, context=None): + """ Update the mime type when the contents are overwritten """ + res = super(Attachment, self).write( + cr, uid, ids, vals, context=context) + if vals.get('datas'): + self.update_file_type(cr, uid, ids, context=context) + return res + + def create(self, cr, uid, vals, context=None): + """ Determine the mime type when the attachment is created """ + res_id = super(Attachment, self).create( + cr, uid, vals, context=context) + if vals.get('datas'): + self.update_file_type(cr, uid, [res_id], context=context) + return res_id + + _columns = { + 'file_type': fields.char('Content Type', size=128), + } From cae6f3a9e1cf0ffe3018af4e3e2223455ef0c4b6 Mon Sep 17 00:00:00 2001 From: Stefan Rijnhart Date: Wed, 18 Feb 2015 19:02:07 +0100 Subject: [PATCH 2/6] [FIX] Performance with browse large amounts of attachments [FIX] Style --- attachment_file_type/models/ir_attachment.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/attachment_file_type/models/ir_attachment.py b/attachment_file_type/models/ir_attachment.py index c4347499..76eb8011 100644 --- a/attachment_file_type/models/ir_attachment.py +++ b/attachment_file_type/models/ir_attachment.py @@ -30,7 +30,8 @@ class Attachment(orm.Model): ids = self.search( cr, uid, [('file_type', '=', False)], context=context) logging.getLogger('openerp.addons.attachment_file_type').info( - 'Found %s attachments without file type in the database.', len(ids)) + 'Found %s attachments without file type in the database.', + len(ids)) self.update_file_type(cr, uid, ids, force=True, context=None) def update_file_type(self, cr, uid, ids, force=False, context=None): @@ -55,7 +56,8 @@ class Attachment(orm.Model): magic.MAGIC_MIME_TYPE if hasattr(magic, 'MAGIC_MIME_TYPE') else magic.MAGIC_MIME) ms.load() - for attachment in self.browse(cr, uid, ids, context=context): + for attachment_id in ids: + attachment = self.browse(cr, uid, attachment_id, context=context) file_type = ms.buffer( b64decode(attachment.datas)).split(';')[0] logger.debug( From eb46f3a2a710425ad0f6fe0a1a4c72d1c81a8bbe Mon Sep 17 00:00:00 2001 From: Stefan Rijnhart Date: Wed, 18 Feb 2015 19:15:14 +0100 Subject: [PATCH 3/6] [FIX] Avoid write trigger --- attachment_file_type/models/ir_attachment.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/attachment_file_type/models/ir_attachment.py b/attachment_file_type/models/ir_attachment.py index 76eb8011..289f0b60 100644 --- a/attachment_file_type/models/ir_attachment.py +++ b/attachment_file_type/models/ir_attachment.py @@ -63,7 +63,12 @@ class Attachment(orm.Model): logger.debug( 'Found file type %s for attachment with id %s', file_type, attachment.id) - attachment.write({'file_type': file_type}) + # Use SQL, not ORM, to prevent write triggers which are harmful + # in 6.1 on attachments of which the model or the record no longer + # exists + cr.execute( + "UPDATE ir_attachment SET file_type = %s WHERE id = %s", + (file_type, attachment_id)) return True def write(self, cr, uid, ids, vals, context=None): From 3901852379ada27608ce00559bc9b02373f04b93 Mon Sep 17 00:00:00 2001 From: Stefan Rijnhart Date: Wed, 18 Feb 2015 19:18:37 +0100 Subject: [PATCH 4/6] [FIX] Take empty attachments into account --- attachment_file_type/models/ir_attachment.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/attachment_file_type/models/ir_attachment.py b/attachment_file_type/models/ir_attachment.py index 289f0b60..26ec2bf9 100644 --- a/attachment_file_type/models/ir_attachment.py +++ b/attachment_file_type/models/ir_attachment.py @@ -58,6 +58,8 @@ class Attachment(orm.Model): ms.load() for attachment_id in ids: attachment = self.browse(cr, uid, attachment_id, context=context) + if not attachment.datas: + continue file_type = ms.buffer( b64decode(attachment.datas)).split(';')[0] logger.debug( From beb84c7c5943ad46a49107f3ee546ecb04d21f1b Mon Sep 17 00:00:00 2001 From: Stefan Rijnhart Date: Wed, 18 Feb 2015 19:23:25 +0100 Subject: [PATCH 5/6] [IMP] Scale up number of processed attachments a little bit [FIX] Actually truncate the list of ids to length of --- attachment_file_type/data/ir_cron.xml | 2 +- attachment_file_type/models/ir_attachment.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/attachment_file_type/data/ir_cron.xml b/attachment_file_type/data/ir_cron.xml index 4911b9f1..dccde8c5 100644 --- a/attachment_file_type/data/ir_cron.xml +++ b/attachment_file_type/data/ir_cron.xml @@ -11,7 +11,7 @@ ir.attachment update_file_type_from_cron - (1000,) + (10000,) diff --git a/attachment_file_type/models/ir_attachment.py b/attachment_file_type/models/ir_attachment.py index 26ec2bf9..4488da9a 100644 --- a/attachment_file_type/models/ir_attachment.py +++ b/attachment_file_type/models/ir_attachment.py @@ -26,13 +26,19 @@ from base64 import b64decode class Attachment(orm.Model): _inherit = 'ir.attachment' - def update_file_type_from_cron(self, cr, uid, count=1000, context=None): + def update_file_type_from_cron(self, cr, uid, count=10000, context=None): ids = self.search( cr, uid, [('file_type', '=', False)], context=context) logging.getLogger('openerp.addons.attachment_file_type').info( 'Found %s attachments without file type in the database.', len(ids)) - self.update_file_type(cr, uid, ids, force=True, context=None) + if ids: + logging.getLogger('openerp.addons.attachment_file_type').info( + 'Providing file types for a maximum of %s of them', + count) + return self.update_file_type( + cr, uid, ids[:count], force=True, context=None) + return False def update_file_type(self, cr, uid, ids, force=False, context=None): """ Write the file types for these attachments. If the document module From 4431630f7a90d729ebd59b0526afbeec95d68025 Mon Sep 17 00:00:00 2001 From: Stefan Rijnhart Date: Wed, 18 Feb 2015 20:08:23 +0100 Subject: [PATCH 6/6] [ADD] magic dependency to travis conf --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index cdb0b715..a6b4ca39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ virtualenv: system_site_packages: true install: + - pip install python-magic - git clone https://github.com/OCA/maintainer-quality-tools.git ${HOME}/maintainer-quality-tools - export PATH=${HOME}/maintainer-quality-tools/travis:${PATH} - travis_install_nightly ${VERSION}