FTP import working with subclass discovery

external_file_location/__init__.py

@@ -20,7 +20,8 @@
 #
 ###############################################################################
 
-import attachment
+from . import attachment
-import location
+from . import location
-import task
+from . import task
+from . import tasks
 

external_file_location/abstract_task.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from base64 import b64encode
+
+
+class AbstractTask(object):
+
+    def create_file(self, filename, data):
+        ir_attachment_id = self.env['ir.attachment'].create(
+                {
+                    'name': filename,
+                    'datas': b64encode(data),
+                    'datas_fname': filename
+                }
+                )
+        return ir_attachment_id
+
+    # def load_file(self, file_id):
+    #     f = self.session.browse('impexp.file', file_id)
+    #     if not f.attachment_id.datas:
+    #         return None
+    #     return b64decode(f.attachment_id.datas)

external_file_location/attachment.py

@@ -20,12 +20,10 @@
 #
 ###############################################################################
 
-from openerp import models, fields, api, _
+from openerp import models, fields
-from openerp.exceptions import Warning
-import hashlib
 
 
-class AttachmentMetadata(models.Model):
+class IrAttachment(models.Model):
     _inherit = 'ir.attachment'
 
     sync_date = fields.Datetime()
@@ -34,4 +32,3 @@ class AttachmentMetadata(models.Model):
         ('failed', 'Failed'),
         ('done', 'Done'),
         ], readonly=True, required=True, default='pending')
-

external_file_location/backend.py

@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-
-#class AbstractConnection(object):
-#
-#    def __init__(self, host, user, pwd, port=None, allow_dir_creation=False):
-#        self.host = host
-#        self.user = user
-#        self.pwd = pwd
-#        self.port = port
-#        self.allow_dir_creation = allow_dir_creation
-#        self.connection = None
-#
-#    def connect(self):
-#        return NotImplemented
-#
-#    def close(self):
-#        return NotImplemented
-#
-#    def get(self, filename, path=None):
-#        return NotImplemented
-#
-#    def put(self, fileobject, filename, path=None):
-#        return NotImplemented
-#
-#    def search(self, filename, path=None):
-#        return NotImplemented
-#
-#    def move(self, filename, oldpath, newpath):
-#        return NotImplemented
-#
-#    def rename(self, oldfilename, newfilename, path=None):
-#        return NotImplemented
-
-class AbstractTask():
-
-    

external_file_location/helper.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    Copyright (C) 2014 initOS GmbH & Co. KG (<http://www.initos.com>).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+
+def itersubclasses(cls, _seen=None):
+    """
+    itersubclasses(cls)
+    Generator over all subclasses of a given class, in depth first order.
+    >>> list(itersubclasses(int)) == [bool]
+    True
+    >>> class A(object): pass
+    >>> class B(A): pass
+    >>> class C(A): pass
+    >>> class D(B,C): pass
+    >>> class E(D): pass
+    >>>
+    >>> for cls in itersubclasses(A):
+    ... print(cls.__name__)
+    B
+    D
+    E
+    C
+    >>> # get ALL (new-style) classes currently defined
+    >>> [cls.__name__ for cls in itersubclasses(object)] #doctest: +ELLIPSIS
+    ['type', ...'tuple', ...]
+    """
+    #import pdb; pdb.set_trace()
+    if not isinstance(cls, type):
+        raise TypeError('itersubclasses must be called with '
+               'new-style classes, not %.100r' % cls
+               )
+    if _seen is None:
+        _seen = set()
+    try:
+        subs = cls.__subclasses__()
+    except TypeError: # fails only when cls is type
+        subs = cls.__subclasses__(cls)
+    for sub in subs:
+        if sub not in _seen:
+            _seen.add(sub)
+            yield sub
+            for sub in itersubclasses(sub, _seen):
+                yield sub

external_file_location/location.py

@@ -21,10 +21,11 @@
 ###############################################################################
 
 from openerp import models, fields
-from backend import AbstractTask
+from abstract_task import AbstractTask
+from helper import itersubclasses
 
 class Location(models.Model):
-    _name = 'ir.location'
+    _name = 'external.file.location'
     _description = 'Description'
 
     name = fields.Char(string='Name')
@@ -43,42 +44,3 @@ class Location(models.Model):
                 res.append(cls_info)
         return res
 
-
-def itersubclasses(cls, _seen=None):
-    """
-    itersubclasses(cls)
-    Generator over all subclasses of a given class, in depth first order.
-    >>> list(itersubclasses(int)) == [bool]
-    True
-    >>> class A(object): pass
-    >>> class B(A): pass
-    >>> class C(A): pass
-    >>> class D(B,C): pass
-    >>> class E(D): pass
-    >>>
-    >>> for cls in itersubclasses(A):
-    ... print(cls.__name__)
-    B
-    D
-    E
-    C
-    >>> # get ALL (new-style) classes currently defined
-    >>> [cls.__name__ for cls in itersubclasses(object)] #doctest: +ELLIPSIS
-    ['type', ...'tuple', ...]
-    """
-    if not isinstance(cls, type):
-        raise TypeError('itersubclasses must be called with '
-               'new-style classes, not %.100r' % cls
-               )
-    if _seen is None:
-        _seen = set()
-    try:
-        subs = cls.__subclasses__()
-    except TypeError: # fails only when cls is type
-        subs = cls.__subclasses__(cls)
-    for sub in subs:
-        if sub not in _seen:
-            _seen.add(sub)
-            yield sub
-            for sub in itersubclasses(sub, _seen):
-                yield sub

external_file_location/location_view.xml

@@ -3,7 +3,7 @@
     <data>
 
         <record id="view_location_form" model="ir.ui.view">
-            <field name="model">ir.location</field>
+            <field name="model">external.file.location</field>
             <field name="arch" type="xml">
                 <form string="Location" version="7.0">
                     <sheet>
@@ -22,7 +22,7 @@
         </record>
 
         <record id="view_location_tree" model="ir.ui.view">
-            <field name="model">ir.location</field>
+            <field name="model">external.file.location</field>
             <field name="arch" type="xml">
                 <tree string="Location">
                     <field name="name" select="1"/>
@@ -38,7 +38,7 @@
         <record id="action_location" model="ir.actions.act_window">
             <field name="name">Locations</field>
             <field name="type">ir.actions.act_window</field>
-            <field name="res_model">ir.location</field>
+            <field name="res_model">external.file.location</field>
             <field name="view_type">form</field>
             <field name="view_id" eval="False"/>
         </record>

external_file_location/task.py

@@ -2,8 +2,8 @@
 ###############################################################################
 #
 #   Module for OpenERP
-#   Copyright (C) 2014 Akretion (http://www.akretion.com).
+#   Copyright (C) 2015 Akretion (http://www.akretion.com).
-#   @author Sébastien BEAU <sebastien.beau@akretion.com>
+#   @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
 #
 #   This program is free software: you can redistribute it and/or modify
 #   it under the terms of the GNU Affero General Public License as
@@ -20,87 +20,45 @@
 #
 ###############################################################################
 
-import sys
+from openerp import models, fields, api
-from openerp import models, fields
+from helper import itersubclasses
+from abstract_task import AbstractTask
+
 
 class Task(models.Model):
     _name = 'ir.location.task'
     _description = 'Description'
-    
+
     name = fields.Char()
-    # method = fields.Selection([
-    #     ('ftp_import', 'FTP import'),
-    #     ('ftp_export', 'FTP export'),
-    #     ('sftp_import', 'SFTP import'),
-    #     ('sftp_export', 'SFTP export'),
-    #     ('filestore_import', 'Filestore import'),
-    #     ('filestore_export', 'Filestore export'),
-    #     ])
     method = fields.Selection(selection='_get_method')
     filename = fields.Char()
     filepath = fields.Char()
-    location_id = fields.Many2one('ir.location', string='Location')
+    location_id = fields.Many2one('external.file.location', string='Location')
 
     def _get_method(self):
         res = []
         for cls in itersubclasses(AbstractTask):
             if cls._synchronize_type:
-                cls_info = (cls._key + cls._synchronize_type, cls._name + cls._synchronize_type)
+                cls_info = (cls._key + '_' + cls._synchronize_type,
+                            cls._name + ' ' + cls._synchronize_type)
                 res.append(cls_info)
         return res
 
+    @api.multi
     def run(self):
-        connection_class = ...
+        for cls in itersubclasses(AbstractTask):
-
+            if cls._synchronize_type and \
-        method_class = getattr(sys.modules[__name__], self.method)
+              cls._key + '_' + cls._synchronize_type == self.method:
+                method_class = cls
         config = {
                 'host': self.location_id.address,
                 'user': self.location_id.login,
                 'pwd': self.location_id.password,
                 'port': self.location_id.port,
                 'allow_dir_creation': False,
-                'filename': self.filename,
+                'file_name': self.filename,
                 'path': self.filepath
                 }
-        conn = method_class(config)
+        conn = method_class(self.env, config)
-        conn.run()
+        file_id = conn.run()
 
-
-def itersubclasses(cls, _seen=None):
-    """
-    itersubclasses(cls)
-    Generator over all subclasses of a given class, in depth first order.
-    >>> list(itersubclasses(int)) == [bool]
-    True
-    >>> class A(object): pass
-    >>> class B(A): pass
-    >>> class C(A): pass
-    >>> class D(B,C): pass
-    >>> class E(D): pass
-    >>>
-    >>> for cls in itersubclasses(A):
-    ... print(cls.__name__)
-    B
-    D
-    E
-    C
-    >>> # get ALL (new-style) classes currently defined
-    >>> [cls.__name__ for cls in itersubclasses(object)] #doctest: +ELLIPSIS
-    ['type', ...'tuple', ...]
-    """
-    if not isinstance(cls, type):
-        raise TypeError('itersubclasses must be called with '
-               'new-style classes, not %.100r' % cls
-               )
-    if _seen is None:
-        _seen = set()
-    try:
-        subs = cls.__subclasses__()
-    except TypeError: # fails only when cls is type
-        subs = cls.__subclasses__(cls)
-    for sub in subs:
-        if sub not in _seen:
-            _seen.add(sub)
-            yield sub
-            for sub in itersubclasses(sub, _seen):
-                yield sub

external_file_location/task_view.xml

@@ -10,8 +10,10 @@
                         <group col="4">
                             <field name="name" select="1" colspan="4"/>
                             <field name="method" colspan="4"/>
+                            <field name="location_id" colspan="4"/>
                             <field name="filename" colspan="4"/>
                             <field name="filepath" colspan="4"/>
+                            <button name="run" type="object" string="Run"/>
                         </group>
                     </sheet>
                 </form>

external_file_location/tasks/__init__.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+###############################################################################
+#
+#   Module for OpenERP
+#   Copyright (C) 2015 Akretion (http://www.akretion.com).
+#   @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from . import ftp

external_file_location/tasks/ftp.py

@@ -18,16 +18,20 @@
 #
 ##############################################################################
 
-from openerp.osv import orm, fields
+from ..abstract_task import AbstractTask
-from ..AbstractTask import AbstractTask
 from base64 import b64decode
 import ftputil
 import ftputil.session
 import logging
 _logger = logging.getLogger(__name__)
 
+
 class FtpTask(AbstractTask):
 
+    _key = 'ftp'
+    _name = 'FTP'
+    _synchronize_type = None
+
     def __init__(self, env, config):
         self.env = env
         self.host = config.get('host', '')
@@ -52,6 +56,8 @@ class FtpImportTask(FtpTask):
                       after download.
     """
 
+    _synchronize_type = 'import'
+
     def _handle_new_source(self, ftp_conn, download_directory, file_name,
                            move_directory):
         """open and read given file into create_file method,
@@ -82,16 +88,17 @@ class FtpImportTask(FtpTask):
                              self.pwd,
                              session_factory=port_session_factory) as ftp_conn:
 
-            file_list = ftp_conn.listdir(path)
+            file_list = ftp_conn.listdir(self.path)
             downloaded_files = []
             for ftpfile in file_list:
-                if ftp_conn.path.isfile(self._source_name(self.path,
+                source_name = self._source_name(self.path, self.file_name)
-                                                          self.file_name)):
+                if ftp_conn.path.isfile(source_name) and \
-                    file_id = self._handle_new_source(ftp_conn,
+                        ftpfile == self.file_name:
-                                                      self.path,
+                    self._handle_new_source(
-                                                      self.file_name,
+                            ftp_conn,
-                                                      self.move_path)
+                            self.path,
-                    self.run_successor_tasks(file_id=file_id, async=async)
+                            self.file_name,
+                            self.move_path)
                     downloaded_files.append(self.file_name)
 
             # Move/delete files only after all files have been processed.
@@ -110,7 +117,6 @@ class FtpImportTask(FtpTask):
                         self._source_name(self.move_path, ftpfile))
 
 
-
 class FtpExportTask(FtpTask):
     """FTP Configuration options:
      - host, user, password, port
@@ -118,6 +124,8 @@ class FtpExportTask(FtpTask):
                           uploaded to
     """
 
+    _synchronize_type = 'export'
+
     def _handle_existing_target(self, ftp_conn, target_name, filedata):
         raise Exception("%s already exists" % target_name)
 
@@ -125,6 +133,7 @@ class FtpExportTask(FtpTask):
         with ftp_conn.open(target_name, mode='wb') as fileobj:
             fileobj.write(filedata)
             _logger.info('wrote %s, size %d', target_name, len(filedata))
+        return file_id
 
     def _target_name(self, ftp_conn, upload_directory, filename):
         return upload_directory + '/' + filename

ir_attachment_metadata/__init__.py

@@ -2,8 +2,8 @@
 ###############################################################################
 #
 #   Module for OpenERP
-#   Copyright (C) 2014 Akretion (http://www.akretion.com).
+#   Copyright (C) 2015 Akretion (http://www.akretion.com).
-#   @author Sébastien BEAU <sebastien.beau@akretion.com>
+#   @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
 #
 #   This program is free software: you can redistribute it and/or modify
 #   it under the terms of the GNU Affero General Public License as
@@ -20,5 +20,4 @@
 #
 ###############################################################################
 
-import attachment
+from . import attachment
-

ir_attachment_metadata/__openerp__.py

@@ -2,8 +2,8 @@
 ###############################################################################
 #
 #   Module for OpenERP
-#   Copyright (C) 2014 Akretion (http://www.akretion.com).
+#   Copyright (C) 2015 Akretion (http://www.akretion.com).
-#   @author Sébastien BEAU <sebastien.beau@akretion.com>
+#   @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
 #
 #   This program is free software: you can redistribute it and/or modify
 #   it under the terms of the GNU Affero General Public License as
@@ -27,7 +27,8 @@
  'license': 'AGPL-3',
  'category': 'Generic Modules',
  'description': """
- 
+ Add some useful field to ir.attachment object like:
+ internal and external hash for coherence verification
  """,
  'depends': [
      'base',
@@ -36,7 +37,7 @@
      'attachment_view.xml',
  ],
  'installable': True,
- 'application': True,
+ 'application': False,
 }
 
 

ir_attachment_metadata/attachment.py

@@ -2,8 +2,8 @@
 ###############################################################################
 #
 #   Module for OpenERP
-#   Copyright (C) 2014 Akretion (http://www.akretion.com).
+#   Copyright (C) 2015 Akretion (http://www.akretion.com).
-#   @author Sébastien BEAU <sebastien.beau@akretion.com>
+#   @author Valentin CHEMIERE <valentin.chemiere@akretion.com>
 #
 #   This program is free software: you can redistribute it and/or modify
 #   it under the terms of the GNU Affero General Public License as
@@ -25,18 +25,17 @@ from openerp.exceptions import Warning
 import hashlib
 
 
-class AttachmentMetadata(models.Model):
+class IrAttachment(models.Model):
     _inherit = 'ir.attachment'
 
     internal_hash = fields.Char(store=True, compute='_compute_hash')
     external_hash = fields.Char()
 
-
+    @api.depends('datas', 'external_hash')
-    @api.depends('datas')
     def _compute_hash(self):
         if self.datas:
-            print hashlib.md5(self.datas).hexdigest()
             self.internal_hash = hashlib.md5(self.datas).hexdigest()
         if self.external_hash and self.internal_hash != self.external_hash:
-            raise Warning(_('File corrupted'), _("Something was wrong with the retreived file, please relaunch the task."))
+            raise Warning(_('File corrupted'),
-
+                          _("Something was wrong with the retreived file, "
+                              "please relaunch the task."))