From c4f104ae73ac71df244ef0b74f22fdbe352427ea Mon Sep 17 00:00:00 2001 From: Holger Brunn Date: Tue, 9 May 2017 10:19:07 +0200 Subject: [PATCH] [FIX] CI [ADD] tests for attachments_to_filesystem --- attachments_to_filesystem/__openerp__.py | 3 ++ .../demo/ir_attachment.xml | 14 +++++++ .../models/ir_attachment.py | 23 +++++------ attachments_to_filesystem/tests/__init__.py | 4 ++ .../tests/test_attachments_to_filesystem.py | 22 ++++++++++ document_multiple_records/__openerp__.py | 1 - document_ocr/models/ir_attachment.py | 2 +- document_ocr/tests/test_document_ocr.py | 40 ++++++++++++------- document_page_multi_company/__openerp__.py | 1 - .../document_page_multi_company.py | 1 - document_page_tags/hooks.py | 1 + .../security/ir.model.access.csv | 2 +- .../tests/test_document_page_tags.py | 6 ++- 13 files changed, 86 insertions(+), 34 deletions(-) create mode 100644 attachments_to_filesystem/demo/ir_attachment.xml create mode 100644 attachments_to_filesystem/tests/__init__.py create mode 100644 attachments_to_filesystem/tests/test_attachments_to_filesystem.py diff --git a/attachments_to_filesystem/__openerp__.py b/attachments_to_filesystem/__openerp__.py index d94af6cd..42dc908c 100644 --- a/attachments_to_filesystem/__openerp__.py +++ b/attachments_to_filesystem/__openerp__.py @@ -28,6 +28,9 @@ "depends": [ 'base', ], + "demo": [ + "demo/ir_attachment.xml", + ], "data": [ "data/ir_cron.xml", "data/init.xml", diff --git a/attachments_to_filesystem/demo/ir_attachment.xml b/attachments_to_filesystem/demo/ir_attachment.xml new file mode 100644 index 00000000..280ebfa9 --- /dev/null +++ b/attachments_to_filesystem/demo/ir_attachment.xml @@ -0,0 +1,14 @@ + + + + + Test attachment + aGVsbG8gd29ybGQ= + + + Test attachment + aGVsbG8gd29ybGQ= + unknown.unknown + + + diff --git a/attachments_to_filesystem/models/ir_attachment.py b/attachments_to_filesystem/models/ir_attachment.py index 880b4c57..e88b6c8d 100644 --- a/attachments_to_filesystem/models/ir_attachment.py +++ b/attachments_to_filesystem/models/ir_attachment.py @@ -61,7 +61,7 @@ class IrAttachment(models.Model): # otherwise, configure our cronjob to run next night user = self.env.user next_night = datetime.now() + relativedelta( - hour=01, minute=42, second=0) + hour=1, minute=42, second=0) user_tz = user.tz or 'UTC' next_night = pytz.timezone(user_tz).localize(next_night).astimezone( pytz.utc).replace(tzinfo=None) @@ -79,28 +79,27 @@ class IrAttachment(models.Model): """Do the actual moving""" limit = int( self.env['ir.config_parameter'].get_param( - 'attachments_to_filesystem.limit', '0')) or limit - ir_attachment = self.env['ir.attachment'] - attachments = ir_attachment.search( + 'attachments_to_filesystem.limit', '0') + ) or limit + attachments = self.env['ir.attachment'].search( [('db_datas', '!=', False)], limit=limit) logging.info('moving %d attachments to filestore', len(attachments)) # attachments can be big, so we read every attachment on its own - for counter, attachment_id in enumerate(attachments.ids, start=1): - attachment_data = self.pool['ir.attachment'].read( - self._cr, self._uid, [attachment_id], ['datas', 'res_model'] - )[0] + for counter, attachment in enumerate(attachments, start=1): + attachment_data = attachment.read(['datas', 'res_model'])[0] if attachment_data['res_model'] and not self.env.registry.get( - attachment_data['res_model']): + attachment_data['res_model'] + ): logging.warning( 'not moving attachment %d because it links to unknown ' - 'model %s', attachment_id, attachment_data['res_model']) + 'model %s', attachment.id, attachment_data['res_model']) continue try: - ir_attachment.browse(attachment_id).write({ + attachment.write({ 'datas': attachment_data['datas'], 'db_datas': False, }) except Exception: - logging.exception('Error moving attachment #%d', attachment_id) + logging.exception('Error moving attachment #%d', attachment.id) if not counter % (len(attachments) / 100 or limit): logging.info('moving attachments: %d done', counter) diff --git a/attachments_to_filesystem/tests/__init__.py b/attachments_to_filesystem/tests/__init__.py new file mode 100644 index 00000000..c122603c --- /dev/null +++ b/attachments_to_filesystem/tests/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# © 2017 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from . import test_attachments_to_filesystem diff --git a/attachments_to_filesystem/tests/test_attachments_to_filesystem.py b/attachments_to_filesystem/tests/test_attachments_to_filesystem.py new file mode 100644 index 00000000..2dc929f7 --- /dev/null +++ b/attachments_to_filesystem/tests/test_attachments_to_filesystem.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# © 2017 Therp BV +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). +from openerp.tests.common import TransactionCase + + +class TestAttachmentsToFilesystem(TransactionCase): + def test_attachments_to_filesystem(self): + # given the init function was run for the asynchronous behavior + # already, we just need to test synchronous behavior + self.env['ir.config_parameter'].set_param( + 'attachments_to_filesystem.move_during_init', 'yes' + ) + self.env['ir.attachment']._attachments_to_filesystem_cron() + self.assertFalse( + self.env.ref('attachments_to_filesystem.test_attachment').db_datas + ) + self.assertTrue( + self.env.ref( + 'attachments_to_filesystem.test_attachment_unknown_model' + ).db_datas + ) diff --git a/document_multiple_records/__openerp__.py b/document_multiple_records/__openerp__.py index db647f9f..a83af872 100644 --- a/document_multiple_records/__openerp__.py +++ b/document_multiple_records/__openerp__.py @@ -56,4 +56,3 @@ Contributors 'installable': False, 'auto_install': False, } - diff --git a/document_ocr/models/ir_attachment.py b/document_ocr/models/ir_attachment.py index ec161712..da9fff31 100644 --- a/document_ocr/models/ir_attachment.py +++ b/document_ocr/models/ir_attachment.py @@ -53,7 +53,7 @@ class IrAttachment(models.Model): stderr=subprocess.PIPE, ) stdout, stderr = process.communicate(image_data.getvalue()) - if stderr: + if process.returncode: _logger.error('Error during OCR: %s', stderr) return stdout diff --git a/document_ocr/tests/test_document_ocr.py b/document_ocr/tests/test_document_ocr.py index 7dccb672..d4545ac2 100644 --- a/document_ocr/tests/test_document_ocr.py +++ b/document_ocr/tests/test_document_ocr.py @@ -4,7 +4,8 @@ from PIL import Image, ImageDraw, ImageFont from StringIO import StringIO from openerp.tests.common import TransactionCase -from openerp.addons.document_ocr.models.ir_attachment import _MARKER_PHRASE +from ..models.ir_attachment import _MARKER_PHRASE +from openerp.tools.misc import mute_logger class TestDocumentOcr(TransactionCase): @@ -21,12 +22,15 @@ class TestDocumentOcr(TransactionCase): result = self.env['ir.attachment']._index( data.getvalue(), 'test.png', None) self.assertEqual(result[1].strip(), 'Hello world') - # should also work for pdfs - data = StringIO() - test_image.save(data, 'pdf', resolution=300) - result = self.env['ir.attachment']._index( - data.getvalue(), 'test.pdf', None) - self.assertEqual(result[1].strip(), 'Hello world') + # should also work for pdfs if supported, protect against + # ancient pillows + if hasattr(Image, 'registered_extensions') and\ + 'PDF' in Image.registered_extensions().values(): + data = StringIO() + test_image.save(data, 'pdf', resolution=300) + result = self.env['ir.attachment']._index( + data.getvalue(), 'test.pdf', None) + self.assertEqual(result[1].strip(), 'Hello world') # check cron self.env['ir.config_parameter'].set_param( 'document_ocr.synchronous', 'False') @@ -38,11 +42,17 @@ class TestDocumentOcr(TransactionCase): attachment._ocr_cron() self.assertEqual(attachment.index_content.strip(), 'Hello world') # and for an unreadable image, we expect an error - self.env['ir.config_parameter'].set_param( - 'document_ocr.synchronous', 'True') - data = StringIO() - test_image = Image.new('1', (200, 30)) - test_image.save(data, 'Palm') - result = self.env['ir.attachment']._index( - data.getvalue(), 'test.palm', None) - self.assertEqual(result[1], None) + if hasattr(Image, 'registered_extensions') and\ + 'PALM' in Image.registered_extensions().values(): + self.env['ir.config_parameter'].set_param( + 'document_ocr.synchronous', 'True') + data = StringIO() + test_image = Image.new('1', (200, 30)) + test_image.save(data, 'Palm') + with mute_logger( + 'openerp.addons.document_ocr.models.ir_attachment' + ): + result = self.env['ir.attachment']._index( + data.getvalue(), 'test.palm', None + ) + self.assertEqual(result[1], None) diff --git a/document_page_multi_company/__openerp__.py b/document_page_multi_company/__openerp__.py index 78acc024..f378fd3a 100644 --- a/document_page_multi_company/__openerp__.py +++ b/document_page_multi_company/__openerp__.py @@ -40,4 +40,3 @@ This module adds a company field to document page and the multi-company rule. 'auto_install': False, 'images': [], } - diff --git a/document_page_multi_company/document_page_multi_company.py b/document_page_multi_company/document_page_multi_company.py index 372f8ccf..f120e149 100644 --- a/document_page_multi_company/document_page_multi_company.py +++ b/document_page_multi_company/document_page_multi_company.py @@ -44,4 +44,3 @@ class document_page(orm.Model): 'company_id': lambda self, cr, uid, c: self.pool.get('res.company') ._company_default_get(cr, uid, 'document_page', context=c) } - diff --git a/document_page_tags/hooks.py b/document_page_tags/hooks.py index a68dfe54..fdd8fcbb 100644 --- a/document_page_tags/hooks.py +++ b/document_page_tags/hooks.py @@ -37,6 +37,7 @@ def populate_tags(cr, column): env = Environment(cr, SUPERUSER_ID, {}) document_page_tag = env['document.page.tag'] document_page = env['document.page'] + # pylint: disable=E8103 cr.execute( 'SELECT %s, ARRAY_AGG(id) from %s WHERE %s IS NOT NULL GROUP BY %s' % ( column, env['document.page']._table, column, column)) diff --git a/document_page_tags/security/ir.model.access.csv b/document_page_tags/security/ir.model.access.csv index f9dce761..0a0d36a0 100644 --- a/document_page_tags/security/ir.model.access.csv +++ b/document_page_tags/security/ir.model.access.csv @@ -1,3 +1,3 @@ "id","name","model_id:id","group_id:id","perm_read","perm_write","perm_create","perm_unlink" -access_document_page_tag,access_document_page_tag,model_document_page_tag,,1,0,0,0 +access_document_page_tag_all,access_document_page_tag,model_document_page_tag,,1,0,0,0 access_document_page_tag,access_document_page_tag,model_document_page_tag,base.group_user,1,1,1,1 diff --git a/document_page_tags/tests/test_document_page_tags.py b/document_page_tags/tests/test_document_page_tags.py index 5a1c2752..3b6251b9 100644 --- a/document_page_tags/tests/test_document_page_tags.py +++ b/document_page_tags/tests/test_document_page_tags.py @@ -20,6 +20,7 @@ from psycopg2 import IntegrityError from openerp.tests.common import TransactionCase from ..hooks import post_init_hook +from openerp.tools.misc import mute_logger class TestDocumentPageTags(TransactionCase): @@ -28,5 +29,6 @@ class TestDocumentPageTags(TransactionCase): post_init_hook(self.cr, self.registry) # check we can't create nonunique tags with self.assertRaises(IntegrityError): - self.env['document.page.tag'].name_create('test') - self.env['document.page.tag'].name_create('test') + with mute_logger('openerp.sql_db'): + self.env['document.page.tag'].name_create('test') + self.env['document.page.tag'].name_create('test')