Merge pull request #53 from hbrunn/8.0-document_reindex

[ADD] document_reindex
This commit is contained in:
Maxime Chambreuil - http://www.savoirfairelinux.com 2015-09-05 11:51:34 -04:00
commit a1cd9985eb
12 changed files with 382 additions and 0 deletions

View File

@ -0,0 +1,35 @@
Reindex documents
=================
This module allows you to reindex documents in case they were uploaded when the right configuration for indexation was missing.
Usage
=====
To reindex a single document, open its form and click the `Reindex document` button.
To reindex all documents, go to Settings / Configuration / Knowledge, check the box `Reindex all documents` or `Reindex all unindexed documents` and click apply. Those are done in the background, so watch your logs for the process to finish.
In a migration context, you might want to reindex all or unindexed documents. To achieve this, create a config parameter `document_reindex.reindex_unindexed_on_init` or `document_reindex.reindex_all_on_init` to have the reindexation run synchronously during installation of this module.
Credits
=======
Contributors
------------
* Holger Brunn <hbrunn@therp.nl>
* Icon courtesy of http://www.picol.org (refresh.svg) and https://github.com/odoo/odoo/blob/8.0/addons/knowledge/static/description/icon.png
Maintainer
----------
.. image:: http://odoo-community.org/logo.png
:alt: Odoo Community Association
:target: http://odoo-community.org
This module is maintained by the OCA.
OCA, or the Odoo Community Association, is a nonprofit organization whose mission is to support the collaborative development of Odoo features and promote its widespread use.
To contribute to this module, please visit http://odoo-community.org.

View File

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# This module copyright (C) 2015 Therp BV <http://therp.nl>.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from . import models
from .hooks import post_init_hook

View File

@ -0,0 +1,42 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# This module copyright (C) 2015 Therp BV <http://therp.nl>.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
"name": "Reindex documents",
"version": "1.0",
"author": "Therp BV, Odoo Community Association (OCA)",
"license": "AGPL-3",
"category": "Knowledge Management",
"summary": "Reindex your already uploaded documents",
"depends": [
'document',
],
"data": [
"views/knowledge_config_settings.xml",
"views/ir_attachment.xml",
],
"post_init_hook": 'post_init_hook',
"auto_install": False,
"installable": True,
"application": False,
"external_dependencies": {
'python': [],
},
}

35
document_reindex/hooks.py Normal file
View File

@ -0,0 +1,35 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# This module copyright (C) 2015 Therp BV <http://therp.nl>.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp import SUPERUSER_ID, api
def post_init_hook(cr, pool):
env = api.Environment(cr, SUPERUSER_ID, {})
with env.manage():
func = None
if env['ir.config_parameter'].get_param(
'document_reindex.reindex_all_on_init'):
func = 'document_reindex_all'
if env['ir.config_parameter'].get_param(
'document_reindex.reindex_unindexed_on_init'):
func = 'document_reindex_unindexed'
if func:
getattr(env['ir.attachment'], func)()

View File

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# This module copyright (C) 2015 Therp BV <http://therp.nl>.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from . import ir_attachment
from . import knowledge_config_settings

View File

@ -0,0 +1,77 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# This module copyright (C) 2015 Therp BV (<http://therp.nl>).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import logging
from openerp import models, api
_logger = logging.getLogger(__name__)
class IrAttachment(models.Model):
_inherit = 'ir.attachment'
@api.multi
def document_reindex(self):
for this in self:
if not this.datas:
continue
try:
mimetype, indexed_content = this._index(
this.datas.decode('base64'),
this.datas_fname, this.file_type)
this.write({
'file_type': mimetype,
'index_content': indexed_content,
})
except:
self.env.clear()
self.env.clear_recompute_old()
_logger.exception('ignoring attachment id %d', this.id)
continue
@api.model
def document_reindex_domain(self, domain, limit=100):
offset = 0
counter = 0
limit = int(
self.env['ir.config_parameter'].get_param(
'document_reindex.limit', '0')) or limit
logging.info(
'reindexing %d attachments', self.search(domain, count=True))
while True:
attachments = self.search(domain, limit=limit, offset=offset)
if not attachments:
return
attachments.document_reindex()
logging.info('%d done', counter * limit + len(attachments))
offset += len(attachments)
counter += 1
@api.model
def document_reindex_all(self):
return self.document_reindex_domain([('type', '=', 'binary')])
@api.model
def document_reindex_unindexed(self):
return self.document_reindex_domain([
('type', '=', 'binary'),
'|',
('index_content', '=', False),
('index_content', '=', ''),
])

View File

@ -0,0 +1,43 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# This module copyright (C) 2015 Therp BV (<http://therp.nl>).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp import models, fields, api
class KnowledgeConfigSettings(models.TransientModel):
_inherit = 'knowledge.config.settings'
document_reindex_all = fields.Boolean('Reindex all documents')
document_reindex_unindexed = fields.Boolean('Reindex unindexed documents')
@api.one
def set_document_reindex(self):
if self.document_reindex_all:
self.document_reindex_create_cronjob('document_reindex_all')
if self.document_reindex_unindexed:
self.document_reindex_create_cronjob('document_reindex_unindexed')
@api.multi
def document_reindex_create_cronjob(self, function):
self.env['ir.cron'].create({
'name': function,
'model': 'ir.attachment',
'function': function,
})

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

View File

@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# This module copyright (C) 2015 Therp BV <http://therp.nl>.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from . import test_reindex

View File

@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# This module copyright (C) 2015 Therp BV (<http://therp.nl>).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import base64
from openerp.tests.common import TransactionCase
class TestReindex(TransactionCase):
def test_reindex(self):
'''test if the indexer indexes what we want to index and only that'''
# we do this to avoid error messages about word files in demo data
self.env['ir.attachment'].search([]).unlink()
att1 = self.env['ir.attachment'].create({
'name': 'helloworld1.txt',
'datas_fname': 'helloworld1.txt',
'datas': base64.b64encode('hello'),
})
att2 = self.env['ir.attachment'].create({
'name': 'helloworld2.txt',
'datas_fname': 'helloworld2.txt',
'datas': base64.b64encode('world'),
})
self.assertEqual(att1.file_type, 'text/plain')
self.assertEqual(att2.file_type, 'text/plain')
att1.write({'index_content': False})
att2.write({'index_content': 'hello'})
self.env['ir.attachment'].document_reindex_unindexed()
self.assertEqual(att1.index_content, 'hello')
self.assertEqual(att2.index_content, 'hello')
self.env['ir.attachment'].document_reindex_all()
self.assertEqual(att2.index_content, 'world')

View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<openerp>
<data>
<record id="view_attachment_form" model="ir.ui.view">
<field name="model">ir.attachment</field>
<field name="inherit_id" ref="base.view_attachment_form" />
<field name="arch" type="xml">
<sheet position="before">
<header>
<button type="object" name="document_reindex" string="Reindex document" attrs="{'invisible': [('datas', '=', False)]}" class="oe_highlight" groups="base.group_document_user" />
</header>
</sheet>
</field>
</record>
</data>
</openerp>

View File

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<openerp>
<data>
<record id="view_knowledge_configuration" model="ir.ui.view">
<field name="model">knowledge.config.settings</field>
<field name="inherit_id" ref="knowledge.view_knowledge_configuration" />
<field name="arch" type="xml">
<xpath expr="//group/div" position="inside">
<div attrs="{'invisible': [('document_reindex_unindexed', '=', True)]}">
<field name="document_reindex_all" class="oe_inline" />
<label for="document_reindex_all" />
</div>
<div attrs="{'invisible': [('document_reindex_all', '=', True)]}">
<field name="document_reindex_unindexed" class="oe_inline" />
<label for="document_reindex_unindexed" />
</div>
</xpath>
</field>
</record>
</data>
</openerp>