Merge pull request #53 from hbrunn/8.0-document_reindex

[ADD] document_reindex

document_reindex/README.rst

@@ -0,0 +1,35 @@
+Reindex documents
+=================
+
+This module allows you to reindex documents in case they were uploaded when the right configuration for indexation was missing.
+
+Usage
+=====
+
+To reindex a single document, open its form and click the `Reindex document` button.
+
+To reindex all documents, go to Settings / Configuration / Knowledge, check the box `Reindex all documents` or `Reindex all unindexed documents` and click apply. Those are done in the background, so watch your logs for the process to finish.
+
+In a migration context, you might want to reindex all or unindexed documents. To achieve this, create a config parameter `document_reindex.reindex_unindexed_on_init` or `document_reindex.reindex_all_on_init` to have the reindexation run synchronously during installation of this module.
+
+Credits
+=======
+
+Contributors
+------------
+
+* Holger Brunn <hbrunn@therp.nl>
+* Icon courtesy of http://www.picol.org (refresh.svg) and https://github.com/odoo/odoo/blob/8.0/addons/knowledge/static/description/icon.png
+
+Maintainer
+----------
+
+.. image:: http://odoo-community.org/logo.png
+    :alt: Odoo Community Association
+    :target: http://odoo-community.org
+
+This module is maintained by the OCA.
+
+OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.
+
+To contribute to this module, please visit http://odoo-community.org.

document_reindex/__init__.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    This module copyright (C) 2015 Therp BV <http://therp.nl>.
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+from . import models
+from .hooks import post_init_hook

document_reindex/__openerp__.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    This module copyright (C) 2015 Therp BV <http://therp.nl>.
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+{
+    "name": "Reindex documents",
+    "version": "1.0",
+    "author": "Therp BV, Odoo Community Association (OCA)",
+    "license": "AGPL-3",
+    "category": "Knowledge Management",
+    "summary": "Reindex your already uploaded documents",
+    "depends": [
+        'document',
+    ],
+    "data": [
+        "views/knowledge_config_settings.xml",
+        "views/ir_attachment.xml",
+    ],
+    "post_init_hook": 'post_init_hook',
+    "auto_install": False,
+    "installable": True,
+    "application": False,
+    "external_dependencies": {
+        'python': [],
+    },
+}

document_reindex/hooks.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    This module copyright (C) 2015 Therp BV <http://therp.nl>.
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+from openerp import SUPERUSER_ID, api
+
+
+def post_init_hook(cr, pool):
+    env = api.Environment(cr, SUPERUSER_ID, {})
+    with env.manage():
+        func = None
+        if env['ir.config_parameter'].get_param(
+                'document_reindex.reindex_all_on_init'):
+            func = 'document_reindex_all'
+        if env['ir.config_parameter'].get_param(
+                'document_reindex.reindex_unindexed_on_init'):
+            func = 'document_reindex_unindexed'
+        if func:
+            getattr(env['ir.attachment'], func)()

document_reindex/models/__init__.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    This module copyright (C) 2015 Therp BV <http://therp.nl>.
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+from . import ir_attachment
+from . import knowledge_config_settings

document_reindex/models/ir_attachment.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    This module copyright (C) 2015 Therp BV (<http://therp.nl>).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+import logging
+from openerp import models, api
+_logger = logging.getLogger(__name__)
+
+
+class IrAttachment(models.Model):
+    _inherit = 'ir.attachment'
+
+    @api.multi
+    def document_reindex(self):
+        for this in self:
+            if not this.datas:
+                continue
+            try:
+                mimetype, indexed_content = this._index(
+                    this.datas.decode('base64'),
+                    this.datas_fname, this.file_type)
+                this.write({
+                    'file_type': mimetype,
+                    'index_content': indexed_content,
+                })
+            except:
+                self.env.clear()
+                self.env.clear_recompute_old()
+                _logger.exception('ignoring attachment id %d', this.id)
+                continue
+
+    @api.model
+    def document_reindex_domain(self, domain, limit=100):
+        offset = 0
+        counter = 0
+        limit = int(
+            self.env['ir.config_parameter'].get_param(
+                'document_reindex.limit', '0')) or limit
+        logging.info(
+            'reindexing %d attachments', self.search(domain, count=True))
+        while True:
+            attachments = self.search(domain, limit=limit, offset=offset)
+            if not attachments:
+                return
+            attachments.document_reindex()
+            logging.info('%d done', counter * limit + len(attachments))
+            offset += len(attachments)
+            counter += 1
+
+    @api.model
+    def document_reindex_all(self):
+        return self.document_reindex_domain([('type', '=', 'binary')])
+
+    @api.model
+    def document_reindex_unindexed(self):
+        return self.document_reindex_domain([
+            ('type', '=', 'binary'),
+            '|',
+            ('index_content', '=', False),
+            ('index_content', '=', ''),
+        ])

document_reindex/models/knowledge_config_settings.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    This module copyright (C) 2015 Therp BV (<http://therp.nl>).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+from openerp import models, fields, api
+
+
+class KnowledgeConfigSettings(models.TransientModel):
+    _inherit = 'knowledge.config.settings'
+
+    document_reindex_all = fields.Boolean('Reindex all documents')
+    document_reindex_unindexed = fields.Boolean('Reindex unindexed documents')
+
+    @api.one
+    def set_document_reindex(self):
+        if self.document_reindex_all:
+            self.document_reindex_create_cronjob('document_reindex_all')
+        if self.document_reindex_unindexed:
+            self.document_reindex_create_cronjob('document_reindex_unindexed')
+
+    @api.multi
+    def document_reindex_create_cronjob(self, function):
+        self.env['ir.cron'].create({
+            'name': function,
+            'model': 'ir.attachment',
+            'function': function,
+        })

document_reindex/tests/__init__.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    This module copyright (C) 2015 Therp BV <http://therp.nl>.
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+from . import test_reindex

document_reindex/tests/test_reindex.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    This module copyright (C) 2015 Therp BV (<http://therp.nl>).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+import base64
+from openerp.tests.common import TransactionCase
+
+
+class TestReindex(TransactionCase):
+    def test_reindex(self):
+        '''test if the indexer indexes what we want to index and only that'''
+        # we do this to avoid error messages about word files in demo data
+        self.env['ir.attachment'].search([]).unlink()
+        att1 = self.env['ir.attachment'].create({
+            'name': 'helloworld1.txt',
+            'datas_fname': 'helloworld1.txt',
+            'datas': base64.b64encode('hello'),
+        })
+        att2 = self.env['ir.attachment'].create({
+            'name': 'helloworld2.txt',
+            'datas_fname': 'helloworld2.txt',
+            'datas': base64.b64encode('world'),
+        })
+        self.assertEqual(att1.file_type, 'text/plain')
+        self.assertEqual(att2.file_type, 'text/plain')
+        att1.write({'index_content': False})
+        att2.write({'index_content': 'hello'})
+        self.env['ir.attachment'].document_reindex_unindexed()
+        self.assertEqual(att1.index_content, 'hello')
+        self.assertEqual(att2.index_content, 'hello')
+        self.env['ir.attachment'].document_reindex_all()
+        self.assertEqual(att2.index_content, 'world')

document_reindex/views/ir_attachment.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<openerp>
+    <data>
+        <record id="view_attachment_form" model="ir.ui.view">
+            <field name="model">ir.attachment</field>
+            <field name="inherit_id" ref="base.view_attachment_form" />
+            <field name="arch" type="xml">
+                <sheet position="before">
+                    <header>
+                        <button type="object" name="document_reindex" string="Reindex document" attrs="{'invisible': [('datas', '=', False)]}" class="oe_highlight" groups="base.group_document_user" />
+                    </header>
+                </sheet>
+            </field>
+        </record>
+    </data>
+</openerp>

document_reindex/views/knowledge_config_settings.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<openerp>
+    <data>
+        <record id="view_knowledge_configuration" model="ir.ui.view">
+            <field name="model">knowledge.config.settings</field>
+            <field name="inherit_id" ref="knowledge.view_knowledge_configuration" />
+            <field name="arch" type="xml">
+                <xpath expr="//group/div" position="inside">
+                    <div attrs="{'invisible': [('document_reindex_unindexed', '=', True)]}">
+                        <field name="document_reindex_all" class="oe_inline" />
+                        <label for="document_reindex_all" />
+                    </div>
+                    <div attrs="{'invisible': [('document_reindex_all', '=', True)]}">
+                        <field name="document_reindex_unindexed" class="oe_inline" />
+                        <label for="document_reindex_unindexed" />
+                    </div>
+                </xpath>
+            </field>
+        </record>
+    </data>
+</openerp>