NeoAnd/knowledge
1
0
Fork 0
mirror of https://github.com/OCA/knowledge.git synced 2025-07-25 18:08:42 -06:00
Code Issues Actions Packages Projects Releases Wiki Activity

FTP import working with subclass discovery

Browse Source
This commit is contained in:
Valentin Chemiere 2015-02-24 18:11:58 +01:00
parent af6f43c6ef
commit 193465c4ea
18 changed files with 173 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
external_file_location/__init__.py
View File

@ -20,7 +20,8 @@
#
###############################################################################
import attachment
import location
import task
from . import attachment
from . import location
from . import task
from . import tasks

23
external_file_location/abstract_task.py Executable file
View File

@ -0,0 +1,23 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from base64 import b64encode
class AbstractTask(object):
def create_file(self, filename, data):
ir_attachment_id = self.env['ir.attachment'].create(
{
'name': filename,
'datas': b64encode(data),
'datas_fname': filename
}
)
return ir_attachment_id
# def load_file(self, file_id):
# f = self.session.browse('impexp.file', file_id)
# if not f.attachment_id.datas:
# return None
# return b64decode(f.attachment_id.datas)

7
external_file_location/attachment.py
View File

@ -20,12 +20,10 @@
#
###############################################################################
from openerp import models, fields, api, _
from openerp.exceptions import Warning
import hashlib
from openerp import models, fields
class AttachmentMetadata(models.Model):
class IrAttachment(models.Model):
_inherit = 'ir.attachment'
sync_date = fields.Datetime()
@ -34,4 +32,3 @@ class AttachmentMetadata(models.Model):
('failed', 'Failed'),
('done', 'Done'),
], readonly=True, required=True, default='pending')

38
external_file_location/backend.py
View File

@ -1,38 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#class AbstractConnection(object):
#
# def __init__(self, host, user, pwd, port=None, allow_dir_creation=False):
# self.host = host
# self.user = user
# self.pwd = pwd
# self.port = port
# self.allow_dir_creation = allow_dir_creation
# self.connection = None
#
# def connect(self):
# return NotImplemented
#
# def close(self):
# return NotImplemented
#
# def get(self, filename, path=None):
# return NotImplemented
#
# def put(self, fileobject, filename, path=None):
# return NotImplemented
#
# def search(self, filename, path=None):
# return NotImplemented
#
# def move(self, filename, oldpath, newpath):
# return NotImplemented
#
# def rename(self, oldfilename, newfilename, path=None):
# return NotImplemented
class AbstractTask():

60
external_file_location/helper.py Normal file
View File

@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (C) 2014 initOS GmbH & Co. KG (<http://www.initos.com>).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
def itersubclasses(cls, _seen=None):
"""
itersubclasses(cls)
Generator over all subclasses of a given class, in depth first order.
>>> list(itersubclasses(int)) == [bool]
True
>>> class A(object): pass
>>> class B(A): pass
>>> class C(A): pass
>>> class D(B,C): pass
>>> class E(D): pass
>>>
>>> for cls in itersubclasses(A):
... print(cls.__name__)
B
D
E
C
>>> # get ALL (new-style) classes currently defined
>>> [cls.__name__ for cls in itersubclasses(object)] #doctest: +ELLIPSIS
['type', ...'tuple', ...]
"""
#import pdb; pdb.set_trace()
if not isinstance(cls, type):
raise TypeError('itersubclasses must be called with '
'new-style classes, not %.100r' % cls
)
if _seen is None:
_seen = set()
try:
subs = cls.__subclasses__()
except TypeError: # fails only when cls is type
subs = cls.__subclasses__(cls)
for sub in subs:
if sub not in _seen:
_seen.add(sub)
yield sub
for sub in itersubclasses(sub, _seen):
yield sub

44
external_file_location/location.py
View File

@ -21,10 +21,11 @@
###############################################################################
from openerp import models, fields
from backend import AbstractTask
from abstract_task import AbstractTask
from helper import itersubclasses
class Location(models.Model):
_name = 'ir.location'
_name = 'external.file.location'
_description = 'Description'
name = fields.Char(string='Name')
@ -43,42 +44,3 @@ class Location(models.Model):
res.append(cls_info)
return res
def itersubclasses(cls, _seen=None):
"""
itersubclasses(cls)
Generator over all subclasses of a given class, in depth first order.
>>> list(itersubclasses(int)) == [bool]
True
>>> class A(object): pass
>>> class B(A): pass
>>> class C(A): pass
>>> class D(B,C): pass
>>> class E(D): pass
>>>
>>> for cls in itersubclasses(A):
... print(cls.__name__)
B
D
E
C
>>> # get ALL (new-style) classes currently defined
>>> [cls.__name__ for cls in itersubclasses(object)] #doctest: +ELLIPSIS
['type', ...'tuple', ...]
"""
if not isinstance(cls, type):
raise TypeError('itersubclasses must be called with '
'new-style classes, not %.100r' % cls
)
if _seen is None:
_seen = set()
try:
subs = cls.__subclasses__()
except TypeError: # fails only when cls is type
subs = cls.__subclasses__(cls)
for sub in subs:
if sub not in _seen:
_seen.add(sub)
yield sub
for sub in itersubclasses(sub, _seen):
yield sub

6
external_file_location/location_view.xml
View File

@ -3,7 +3,7 @@
<data>
<record id="view_location_form" model="ir.ui.view">
<field name="model">ir.location</field>
<field name="model">external.file.location</field>
<field name="arch" type="xml">
<form string="Location" version="7.0">
<sheet>
@ -22,7 +22,7 @@
</record>
<record id="view_location_tree" model="ir.ui.view">
<field name="model">ir.location</field>
<field name="model">external.file.location</field>
<field name="arch" type="xml">
<tree string="Location">
<field name="name" select="1"/>
@ -38,7 +38,7 @@
<record id="action_location" model="ir.actions.act_window">
<field name="name">Locations</field>
<field name="type">ir.actions.act_window</field>
<field name="res_model">ir.location</field>
<field name="res_model">external.file.location</field>
<field name="view_type">form</field>
<field name="view_id" eval="False"/>
</record>

78
external_file_location/task.py
View File

@ -2,8 +2,8 @@
###############################################################################
#
# Module for OpenERP
# Copyright (C) 2014 Akretion (http://www.akretion.com).
# @author Sébastien BEAU <sebastien.beau@akretion.com>
# Copyright (C) 2015 Akretion (http://www.akretion.com).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@ -20,87 +20,45 @@
#
###############################################################################
import sys
from openerp import models, fields
from openerp import models, fields, api
from helper import itersubclasses
from abstract_task import AbstractTask
class Task(models.Model):
_name = 'ir.location.task'
_description = 'Description'
name = fields.Char()
# method = fields.Selection([
# ('ftp_import', 'FTP import'),
# ('ftp_export', 'FTP export'),
# ('sftp_import', 'SFTP import'),
# ('sftp_export', 'SFTP export'),
# ('filestore_import', 'Filestore import'),
# ('filestore_export', 'Filestore export'),
# ])
method = fields.Selection(selection='_get_method')
filename = fields.Char()
filepath = fields.Char()
location_id = fields.Many2one('ir.location', string='Location')
location_id = fields.Many2one('external.file.location', string='Location')
def _get_method(self):
res = []
for cls in itersubclasses(AbstractTask):
if cls._synchronize_type:
cls_info = (cls._key + cls._synchronize_type, cls._name + cls._synchronize_type)
cls_info = (cls._key + '_' + cls._synchronize_type,
cls._name + ' ' + cls._synchronize_type)
res.append(cls_info)
return res
@api.multi
def run(self):
connection_class = ...
method_class = getattr(sys.modules[__name__], self.method)
for cls in itersubclasses(AbstractTask):
if cls._synchronize_type and \
cls._key + '_' + cls._synchronize_type == self.method:
method_class = cls
config = {
'host': self.location_id.address,
'user': self.location_id.login,
'pwd': self.location_id.password,
'port': self.location_id.port,
'allow_dir_creation': False,
'filename': self.filename,
'file_name': self.filename,
'path': self.filepath
}
conn = method_class(config)
conn.run()
conn = method_class(self.env, config)
file_id = conn.run()
def itersubclasses(cls, _seen=None):
"""
itersubclasses(cls)
Generator over all subclasses of a given class, in depth first order.
>>> list(itersubclasses(int)) == [bool]
True
>>> class A(object): pass
>>> class B(A): pass
>>> class C(A): pass
>>> class D(B,C): pass
>>> class E(D): pass
>>>
>>> for cls in itersubclasses(A):
... print(cls.__name__)
B
D
E
C
>>> # get ALL (new-style) classes currently defined
>>> [cls.__name__ for cls in itersubclasses(object)] #doctest: +ELLIPSIS
['type', ...'tuple', ...]
"""
if not isinstance(cls, type):
raise TypeError('itersubclasses must be called with '
'new-style classes, not %.100r' % cls
)
if _seen is None:
_seen = set()
try:
subs = cls.__subclasses__()
except TypeError: # fails only when cls is type
subs = cls.__subclasses__(cls)
for sub in subs:
if sub not in _seen:
_seen.add(sub)
yield sub
for sub in itersubclasses(sub, _seen):
yield sub

2
external_file_location/task_view.xml
View File

@ -10,8 +10,10 @@
<group col="4">
<field name="name" select="1" colspan="4"/>
<field name="method" colspan="4"/>
<field name="location_id" colspan="4"/>
<field name="filename" colspan="4"/>
<field name="filepath" colspan="4"/>
<button name="run" type="object" string="Run"/>
</group>
</sheet>
</form>

0
external_file_location/backends/.filestore.py.swp → external_file_location/tasks/.filestore.py.swp
View File

23
external_file_location/tasks/__init__.py Normal file
View File

@ -0,0 +1,23 @@
# -*- coding: utf-8 -*-
###############################################################################
#
# Module for OpenERP
# Copyright (C) 2015 Akretion (http://www.akretion.com).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
from . import ftp

0
external_file_location/backends/filestore.py → external_file_location/tasks/filestore.py
View File

31
external_file_location/backends/ftp.py → external_file_location/tasks/ftp.py
View File

@ -18,16 +18,20 @@
#
##############################################################################
from openerp.osv import orm, fields
from ..AbstractTask import AbstractTask
from ..abstract_task import AbstractTask
from base64 import b64decode
import ftputil
import ftputil.session
import logging
_logger = logging.getLogger(__name__)
class FtpTask(AbstractTask):
_key = 'ftp'
_name = 'FTP'
_synchronize_type = None
def __init__(self, env, config):
self.env = env
self.host = config.get('host', '')
@ -52,6 +56,8 @@ class FtpImportTask(FtpTask):
after download.
"""
_synchronize_type = 'import'
def _handle_new_source(self, ftp_conn, download_directory, file_name,
move_directory):
"""open and read given file into create_file method,
@ -82,16 +88,17 @@ class FtpImportTask(FtpTask):
self.pwd,
session_factory=port_session_factory) as ftp_conn:
file_list = ftp_conn.listdir(path)
file_list = ftp_conn.listdir(self.path)
downloaded_files = []
for ftpfile in file_list:
if ftp_conn.path.isfile(self._source_name(self.path,
self.file_name)):
file_id = self._handle_new_source(ftp_conn,
self.path,
self.file_name,
self.move_path)
self.run_successor_tasks(file_id=file_id, async=async)
source_name = self._source_name(self.path, self.file_name)
if ftp_conn.path.isfile(source_name) and \
ftpfile == self.file_name:
self._handle_new_source(
ftp_conn,
self.path,
self.file_name,
self.move_path)
downloaded_files.append(self.file_name)
# Move/delete files only after all files have been processed.
@ -110,7 +117,6 @@ class FtpImportTask(FtpTask):
self._source_name(self.move_path, ftpfile))
class FtpExportTask(FtpTask):
"""FTP Configuration options:
- host, user, password, port
@ -118,6 +124,8 @@ class FtpExportTask(FtpTask):
uploaded to
"""
_synchronize_type = 'export'
def _handle_existing_target(self, ftp_conn, target_name, filedata):
raise Exception("%s already exists" % target_name)
@ -125,6 +133,7 @@ class FtpExportTask(FtpTask):
with ftp_conn.open(target_name, mode='wb') as fileobj:
fileobj.write(filedata)
_logger.info('wrote %s, size %d', target_name, len(filedata))
return file_id
def _target_name(self, ftp_conn, upload_directory, filename):
return upload_directory + '/' + filename

0
external_file_location/backends/ftp_backup.py → external_file_location/tasks/ftp_backup.py
View File

0
external_file_location/backends/sftp.py → external_file_location/tasks/sftp.py
View File

7
ir_attachment_metadata/__init__.py
View File

@ -2,8 +2,8 @@
###############################################################################
#
# Module for OpenERP
# Copyright (C) 2014 Akretion (http://www.akretion.com).
# @author Sébastien BEAU <sebastien.beau@akretion.com>
# Copyright (C) 2015 Akretion (http://www.akretion.com).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@ -20,5 +20,4 @@
#
###############################################################################
import attachment
from . import attachment

9
ir_attachment_metadata/__openerp__.py
View File

@ -2,8 +2,8 @@
###############################################################################
#
# Module for OpenERP
# Copyright (C) 2014 Akretion (http://www.akretion.com).
# @author Sébastien BEAU <sebastien.beau@akretion.com>
# Copyright (C) 2015 Akretion (http://www.akretion.com).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@ -27,7 +27,8 @@
'license': 'AGPL-3',
'category': 'Generic Modules',
'description': """
Add some useful field to ir.attachment object like:
internal and external hash for coherence verification
""",
'depends': [
'base',
@ -36,7 +37,7 @@
'attachment_view.xml',
],
'installable': True,
'application': True,
'application': False,
}

15
ir_attachment_metadata/attachment.py
View File

@ -2,8 +2,8 @@
###############################################################################
#
# Module for OpenERP
# Copyright (C) 2014 Akretion (http://www.akretion.com).
# @author Sébastien BEAU <sebastien.beau@akretion.com>
# Copyright (C) 2015 Akretion (http://www.akretion.com).
# @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@ -25,18 +25,17 @@ from openerp.exceptions import Warning
import hashlib
class AttachmentMetadata(models.Model):
class IrAttachment(models.Model):
_inherit = 'ir.attachment'
internal_hash = fields.Char(store=True, compute='_compute_hash')
external_hash = fields.Char()
@api.depends('datas')
@api.depends('datas', 'external_hash')
def _compute_hash(self):
if self.datas:
print hashlib.md5(self.datas).hexdigest()
self.internal_hash = hashlib.md5(self.datas).hexdigest()
if self.external_hash and self.internal_hash != self.external_hash:
raise Warning(_('File corrupted'), _("Something was wrong with the retreived file, please relaunch the task."))
raise Warning(_('File corrupted'),
_("Something was wrong with the retreived file, "
"please relaunch the task."))