diff --git a/external_file_location/.location.py.swp b/external_file_location/.location.py.swp deleted file mode 100644 index 5edb09b8..00000000 Binary files a/external_file_location/.location.py.swp and /dev/null differ diff --git a/external_file_location/.task.py.swp b/external_file_location/.task.py.swp deleted file mode 100644 index 5b9a1957..00000000 Binary files a/external_file_location/.task.py.swp and /dev/null differ diff --git a/external_file_location/backends/.filestore.py.swo b/external_file_location/backends/.filestore.py.swo deleted file mode 100644 index 65c54ea1..00000000 Binary files a/external_file_location/backends/.filestore.py.swo and /dev/null differ diff --git a/external_file_location/backends/ftp.py b/external_file_location/backends/ftp.py old mode 100755 new mode 100644 index 7316b81f..86536a44 --- a/external_file_location/backends/ftp.py +++ b/external_file_location/backends/ftp.py @@ -1,58 +1,155 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- +############################################################################## +# +# Copyright (C) 2014 initOS GmbH & Co. KG (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +from openerp.osv import orm, fields +from ..AbstractTask import AbstractTask +from base64 import b64decode +import ftputil +import ftputil.session +import logging +_logger = logging.getLogger(__name__) + +class FtpTask(AbstractTask): + + def __init__(self, env, config): + self.env = env + self.host = config.get('host', '') + self.user = config.get('user', '') + self.pwd = config.get('pwd', '') + self.port = config.get('port', '') + self.allow_dir_creation = config.get('allow_dir_creation', '') + self.file_name = config.get('file_name', '') + self.path = config.get('path', '') + self.move_path = config.get('move_path', '') + self.delete_file = config.get('delete_file', False) -import sys -import os -from tempfile import TemporaryFile -from ftplib import FTP +class FtpImportTask(FtpTask): + """FTP Configuration options: + - host, user, password, port + - download_directory: directory on the FTP server where files are + downloaded from + - move_directory: If present, files will be moved to this directory + on the FTP server after download. + - delete_files: If true, files will be deleted on the FTP server + after download. + """ -class FTPConnection(object): + def _handle_new_source(self, ftp_conn, download_directory, file_name, + move_directory): + """open and read given file into create_file method, + move file if move_directory is given""" + with ftp_conn.open(self._source_name(download_directory, file_name), + "rb") as fileobj: + data = fileobj.read() + return self.create_file(file_name, data) - def __init__(self, host, user, pwd, port=None, allow_dir_creation=False): - super(FTPConnection, self).__init__(host, user, pwd, port, allow_dir_creation) - if not port: - self.port = 21 - self.protocol = "FTP" + def _source_name(self, download_directory, file_name): + """helper to get the full name""" + return download_directory + '/' + file_name - def connect(self): - self.connection = FTP(self.location, self.port) - self.connection.login(self.user, self.pwd) + def _move_file(self, ftp_conn, source, target): + """Moves a file on the FTP server""" + _logger.info('Moving file %s %s' % (source, target)) + ftp_conn.rename(source, target) - def close(self): - self.connection.close() + def _delete_file(self, ftp_conn, source): + """Deletes a file from the FTP server""" + _logger.info('Deleting file %s' % source) + ftp_conn.remove(source) - def get(self, filename, path=None): - if path: - filepath = "{}/{}".format(path, filename) - else: - filepath = filename - outfile = TemporaryFile('w+b') - self.connection.retrbinary('RETR ' + filepath, outfile.write) - return outfile + def run(self): + port_session_factory = ftputil.session.session_factory( + port=self.port) + with ftputil.FTPHost(self.host, self.user, + self.pwd, + session_factory=port_session_factory) as ftp_conn: - def put(self, fileobject, filename, path=None): - if path: - filepath = "{}/{}".format(path, filename) - else: - filepath = filename - self.connection.storbinary('STOR ' + filepath, fileobject) - return True + file_list = ftp_conn.listdir(path) + downloaded_files = [] + for ftpfile in file_list: + if ftp_conn.path.isfile(self._source_name(self.path, + self.file_name)): + file_id = self._handle_new_source(ftp_conn, + self.path, + self.file_name, + self.move_path) + self.run_successor_tasks(file_id=file_id, async=async) + downloaded_files.append(self.file_name) - def search(self, filename, path=None): - if path: - filepath = "{}/{}".format(path, filename) - else: - filepath = filename - connection_list_result = self.connection.nlst() - return [x for x in connection_list_result if filename in x] + # Move/delete files only after all files have been processed. + if self.delete_file: + for ftpfile in downloaded_files: + self._delete_file(ftp_conn, + self._source_name(self.path, + ftpfile)) + elif self.move_path: + if not ftp_conn.path.exists(self.move_path): + ftp_conn.mkdir(self.move_path) + for ftpfile in downloaded_files: + self._move_file( + ftp_conn, + self._source_name(self.path, ftpfile), + self._source_name(self.move_path, ftpfile)) - def move(self, filename, oldpath, newpath): - self.connection.rename( - os.path.join(oldpath, filename), - os.path.join(newpath, filename) - ) - def rename(self, oldfilename, newfilename, path=None): - return NotImplemented +class FtpExportTask(FtpTask): + """FTP Configuration options: + - host, user, password, port + - upload_directory: directory on the FTP server where files are + uploaded to + """ + + def _handle_existing_target(self, ftp_conn, target_name, filedata): + raise Exception("%s already exists" % target_name) + + def _handle_new_target(self, ftp_conn, target_name, filedata): + with ftp_conn.open(target_name, mode='wb') as fileobj: + fileobj.write(filedata) + _logger.info('wrote %s, size %d', target_name, len(filedata)) + + def _target_name(self, ftp_conn, upload_directory, filename): + return upload_directory + '/' + filename + + def _upload_file(self, config, filename, filedata): + ftp_config = config['ftp'] + upload_directory = ftp_config.get('upload_directory', '') + port_session_factory = ftputil.session.session_factory( + port=int(ftp_config.get('port', 21)) + ) + with ftputil.FTPHost(ftp_config['host'], ftp_config['user'], + ftp_config['password'], + session_factory=port_session_factory) as ftp_conn: + target_name = self._target_name(ftp_conn, + upload_directory, + filename) + if ftp_conn.path.isfile(target_name): + self._handle_existing_target(ftp_conn, target_name, filedata) + else: + self._handle_new_target(ftp_conn, target_name, filedata) + + def run(self, config=None, file_id=None, async=True): + #TODO change when object was made + f = self.env.get('impexp.file') \ + .browse(self.env.cr, self.env.uid, file_id) + self._upload_file(config, f.attachment_id.datas_fname, + b64decode(f.attachment_id.datas)) + diff --git a/external_file_location/backends/ftp_backup.py b/external_file_location/backends/ftp_backup.py new file mode 100755 index 00000000..7316b81f --- /dev/null +++ b/external_file_location/backends/ftp_backup.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + + +import sys +import os +from tempfile import TemporaryFile +from ftplib import FTP + +class FTPConnection(object): + + def __init__(self, host, user, pwd, port=None, allow_dir_creation=False): + super(FTPConnection, self).__init__(host, user, pwd, port, allow_dir_creation) + if not port: + self.port = 21 + self.protocol = "FTP" + + def connect(self): + self.connection = FTP(self.location, self.port) + self.connection.login(self.user, self.pwd) + + def close(self): + self.connection.close() + + def get(self, filename, path=None): + if path: + filepath = "{}/{}".format(path, filename) + else: + filepath = filename + outfile = TemporaryFile('w+b') + self.connection.retrbinary('RETR ' + filepath, outfile.write) + return outfile + + def put(self, fileobject, filename, path=None): + if path: + filepath = "{}/{}".format(path, filename) + else: + filepath = filename + self.connection.storbinary('STOR ' + filepath, fileobject) + return True + + def search(self, filename, path=None): + if path: + filepath = "{}/{}".format(path, filename) + else: + filepath = filename + connection_list_result = self.connection.nlst() + return [x for x in connection_list_result if filename in x] + + + def move(self, filename, oldpath, newpath): + self.connection.rename( + os.path.join(oldpath, filename), + os.path.join(newpath, filename) + ) + + def rename(self, oldfilename, newfilename, path=None): + return NotImplemented