From af6f43c6efdcf2510e05775d11096ebab83c03ae Mon Sep 17 00:00:00 2001 From: Valentin Chemiere Date: Fri, 20 Feb 2015 16:31:14 +0100 Subject: [PATCH] WIP reprise du fichier ftp du projet connector_flow https://github.com/initOS/connector-interfaces --- external_file_location/.location.py.swp | Bin 12288 -> 0 bytes external_file_location/.task.py.swp | Bin 12288 -> 0 bytes .../backends/.filestore.py.swo | Bin 12288 -> 0 bytes external_file_location/backends/ftp.py | 187 +++++++++++++----- external_file_location/backends/ftp_backup.py | 58 ++++++ 5 files changed, 200 insertions(+), 45 deletions(-) delete mode 100644 external_file_location/.location.py.swp delete mode 100644 external_file_location/.task.py.swp delete mode 100644 external_file_location/backends/.filestore.py.swo mode change 100755 => 100644 external_file_location/backends/ftp.py create mode 100755 external_file_location/backends/ftp_backup.py diff --git a/external_file_location/.location.py.swp b/external_file_location/.location.py.swp deleted file mode 100644 index 5edb09b8df114ac745293cb315961e388119fa77..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHNO>88^6|Nk)q+Mf-PJSA_Efd1 zs%;O7!Wof60D-uWDBN-)N{$;ThvXC~Iph*?MSzfy$RVdl5#+1tw#PQ!EJ9pXbW7iK zch##`?|tv5+j0*Vp6PDTW5FW~$Nh|b`r$|YzufUW`(?t|ZZcBZjo;$%a`XF{xrN-% zy0~G@m0~G@m0~G@m0~G@m0~G_` zA_i=BhP@1b?c-H$^e}> zrin-+GWn35nD{!kH&=H0Osurqni5F~zG%W_m~vx;spGOq10#fNEFQD$bVXh=;8Elj zHVzjgu~#pZbz69$LfeszvB1o>aCfvsVQPZG05_#QgFzibg<^b#B{EC>X(l>aE4@f_ z6!RoCq+~j!y|IvPJtc-LWDbqFx#~i%)zg_B-c}f5gsm6T@TC}p-c2RNOty^F%TkMG@M@3p5w0&%I>VG@_cg_E<_+MB|nNFm2&K+7W)tmtN!F2NeVrGH>5NGeW9!GGU5BHM1}!bowV_gJ5uc}p z6q;MaMC}Sq$OBhhWonU%iPI?A_1O=Xny%y`RLYp*L?d^Uj)V@przbW8qXm&JPY{fg z%>eUdGgxw&4?dY!LpZdsxt2K1GT9Bg17iWtG9e=uvq2(b^`#nd>E&loYei(Qi)qu1 z20jUoFDNtJ&zh+W$6CqcB{6VWJ*&t6lL|v)H4kl{o9jpV8Vtqu*9w-M{{!UhAk)gK zP^EPv()iG4mo0ms5({G2x=pY;=DO?=sTzUw(`GWI?QEhetr&@eRcM>Az?pDVBP>ni zXsOmNW@`l#2S&|=dm=?8;`38yXK}SuJLW*S)YbHCnYa32k4;QLin*u_B4I+EWKJc- zYiqOHqTTqoxS2!;2VcC_SW!{y<)^UqiHZbbxZzHYyI89U4S9GS*+9v})6qT>jY2>n zS%l6G1PUe1k`{a#_{|<{j?KYgo+>#i`l%hGxr5@7=97S^>lj;+lWTdlpl zyd zE?wECHf^=H``y)@_4YPx?QCyd>2=_24^F~5@5nH>^lZ0(`N~e8+M8GD+4lA}6kdJ8 z)1)vfT%U2yAeL$3o=043F0FIYtLPu0(mjeJG&!9>F=SCAZs{1|#s$aAfrVnUGd$jM zBjP-5mK2nPG@TE12u@Xw?&ou}L0nEL<#=x)v_pe*E!@u>mu8H8hEu=KB@NelDz>mu z&?S}2$bYZhB!P$MY&kt#M#wt=n;?i;XDT+C;Qit&#*er;m4nzBVoF06oilivyWTk? z$5Fv$)Pldr{v@83L_EcFJH}w|&%YgVV-v8y(rNF|lSY*H!BFu0DSjOd*-;Uy$+9vuyOv!^Qeyq N-ENWm$G~oc{SOeWqp<)0 diff --git a/external_file_location/.task.py.swp b/external_file_location/.task.py.swp deleted file mode 100644 index 5b9a195722af86955c6466f72bdac5ab8c8dc829..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHNO>87b6|T(!34!pFT!;|T+qG6RlO6YXH%o#{Z0*<{XD70~mS;?~GES?fyLzUx zJzbruYI`REBBZ@=KthUy;KBjHfn3N5DdG?$PMi>kBR3KOXOJKQ0bf=3{Ma6|IphNB zmcE(ks#mYx``&w1<=yHneWkrk&pJ04p3g9*-+Qln^w3w?cOu4)qP~!({Q-}aTfCR5 z2grTH3N}O_MB_-PKK*>PKK*>PK zK*>PKK*_-WJp($uz+Q)!4;7xiTzp?R^6Vj0&UuL3Tx1AGDa+as_Ucm;R~I0Rk*t^!X1e|(s+-vK`aBybb> zEb#8f7<(Id1NbIz2)qEafm^@=@Oj`O@b*U;`!;YN2!U@UD?fH#1bfNkJe;1cjT;IH^cnO^|k2krxXU<-H#xCq#oebAhv zQ_m>kaX>2TvDuDLNYh49LJ=-s;HMvZN^w=i&pNq^^Q5+XV#s_&R?_i;zznQS21KD% zp9pQHutK@EnTat#O{Xw#v_gKY9M{E5VUO!p(UdF3g%O@=>W)(0l2XWJTAki5X9kA`O{|G3Tq!^=dhq+F@;lA&Tp2-VI;!M(EuU37^Uqj85Gy8Y&4tUQ~J% zbJ$~Pt&^eRigRsgNml5JF>_*`^diNlUR5h6BSOPWY3f0h7souH<46yvVn@R=%dm

LB%jqsgvV46`iGV@%zyBuCA`q0fvJ4#2a$gB@-q~<(d2t zy_dPKYlpND2w!uhm+5M2YpcE6X?G^d7O2lPHMh2?I-#OA=7Ri8N?ZU!$O0Zl2@mYJ zb^F<}pBwgb&3?jSN@bL2S|dypCqIAZun=S7v!)8e`%}>FUX7SK(-_Mk_0H&=VrgZarZRLX$DNFsNH47hZxr>7&2^4@BTv`>GHNqyCu^p|4LAnUUtHq@EsBmQi_e=$NZu z=AGHDnU9L;FJ$}niti`6{gR7gG6IlhDDMuB76=jcw4 zgSpZOz%dA)Ga^%5&KnYoew56S9|$$qAS||$Bg_+jbBaOyYm=tg5TFCqh>Lf!sz%Bg ziXe-*IvCgbVLfL&A5uFRV3azp;^m=)9@L_p!la4`y$O{`3MwASbtV&nd6TATmvX!` zx52nhi{VpHTRkIQ~t{Nr&K#C-KmAhs!tLM!o^yi73oB*rkTt(CM z{J$(EJ*B1R>n`pj7ORSJ435IogS<)&DK40Hb0$xaGm}sl5Sl$%Ion<`x{k8KiZ7k9 z1lDLw6(#+Z%2sg`vXc8TE{Umv^?@gg@*7vh+|JyfX;+8Gd0>r=`LC7AjOoRM3pRF# z`Rw^i)s~;aQp9n8)0Rw~Dw`-t+eSrOEljd( zm7K5hq{{O(8o8-LBX?Tg`HUVg!n=ey^p%gP5=2+z-M!fTQk*`>+LO;_byE z>F0+OJ-~@6m7*`bp+nTRrxml>aC&Z_b-LVW93LM${UmdQ>^I`b=LwFWjaA2Lx+5}D z1CinQI`ED-ip}TPzDP%A-_rw5&+qJ0GYq*Dl<$)`7vX z%*N)a&9O2!W4pDteyh3DZLYPq+TA$cbTx0-vjyT7-4 zyVHWR9XJW=tRuZl(~Iryt=s!uYVO>j7n^%~Pb@4B`_<=8)$}=_T4Y zX;pM*s5Ix>04E0-K`~^p+nb>SL>@mpoVYa<N;T z)38QZj41Z-#{l7sp^%xm`>Ekl4KPn{WartGaIGUk4I4RaiYy82)>h+bnVCK=+;{sH4ho|?-+Xbe$>A{-O|~a8wX+)5~x~ zk<|I+PkNrx5!hdAHTUT`#fx{R$Gz;PcNaUFp~4r#RcpjLGEYYRfu`zujjk_UyFtxD th^jM5Jv*{iB$C~DZI<0ek+$}BjWr+c0~;5gT10&W*g=-bUIsQT>_33pg*^ZO diff --git a/external_file_location/backends/.filestore.py.swo b/external_file_location/backends/.filestore.py.swo deleted file mode 100644 index 65c54ea1df472a615683a515122d86b055c4d486..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2&ub(_6vt~ftpdhc65 zhLGx?esbk8esA|KhH?XAFaQ42>L*8^VXt|NZA5EQQTsl{+AWS#y^r0C*03i-DSI2z zK2l#9-HVjx8{-GVkw}au#=+1})F6{{LV8{r}7LjC}>Z1n+^jz#Cu#WWa#);6ZRJm;%SZ&&L^i6f`| z!`SOUfMsw$Xo0K2Rp8g_82bWz4n73$fB|?4EP>--2K;#~V?Tno!6ooKcof_N?gqDj zDe&1eV;_J^;5l$JI0a6EKc*P_4txu~0iS~R!7D%m4_p8bfK%WkxCu;y%h2d+@G&60 z-UJ~Cz~f*R{01$*2k!z`oB~b(r+`!7a4EoxqD4ID2C>wly}))hvR$=ClWv-sw(Wp! zxh4Kcpo@dLHP|;a)q<;yd{BqI_E(xMN7>ukPA&iWtyQl{M(P+PpB|YcA~1ypp%`$L zCGCY}tXQ(~^dt#}N+!_@!ZVXp@H4z6#;H`M#e&$^nm=M|XE94p&|SoK^^H2AKBy>R zGL>+eOJ*DJ(YQRw0%O>BYhv#bD4ef2_Gq*-_7z*JBHkT)6=)Is(_=o6ijR4e@F15B z*D<{~(#BIlXR+bYz@|vcm8{0kR(<@9!x=7hmwE^z)u!B2-wd}m3uRljma0&l2B^9g zu@=i!t>)6YxYfJ3)#@VhkzdG3V{~$f%g(lkL<&}1GRk|UMWKbSf}!1z4b!@+vh8qW zX0YfGtxQEy<<#Nxo45Dpx5uf)X~!(1sgn6OQY&{zLtga^)PFOBa-TG0a84Sm@mSEs zVi8(hX*%;mnKVCuPjOPzy{grMDaR@wL~C6NXr30(Jf?leobQ_Cnt3_*O)h>M%S{hk z#0!+Lm!NCTdkvbpp`BlIq^3sG&9H8{88#VaTBw>OH8q(IrLGT8O*PXwuCr7qdlcEr z-49nsO}=Lrx}Fy$k@37Xxpz2?*Wu}Ah!3RG;d|A{w;j!VR`orR2o( zGnucVE|sq)P). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +from openerp.osv import orm, fields +from ..AbstractTask import AbstractTask +from base64 import b64decode +import ftputil +import ftputil.session +import logging +_logger = logging.getLogger(__name__) + +class FtpTask(AbstractTask): + + def __init__(self, env, config): + self.env = env + self.host = config.get('host', '') + self.user = config.get('user', '') + self.pwd = config.get('pwd', '') + self.port = config.get('port', '') + self.allow_dir_creation = config.get('allow_dir_creation', '') + self.file_name = config.get('file_name', '') + self.path = config.get('path', '') + self.move_path = config.get('move_path', '') + self.delete_file = config.get('delete_file', False) -import sys -import os -from tempfile import TemporaryFile -from ftplib import FTP +class FtpImportTask(FtpTask): + """FTP Configuration options: + - host, user, password, port + - download_directory: directory on the FTP server where files are + downloaded from + - move_directory: If present, files will be moved to this directory + on the FTP server after download. + - delete_files: If true, files will be deleted on the FTP server + after download. + """ -class FTPConnection(object): + def _handle_new_source(self, ftp_conn, download_directory, file_name, + move_directory): + """open and read given file into create_file method, + move file if move_directory is given""" + with ftp_conn.open(self._source_name(download_directory, file_name), + "rb") as fileobj: + data = fileobj.read() + return self.create_file(file_name, data) - def __init__(self, host, user, pwd, port=None, allow_dir_creation=False): - super(FTPConnection, self).__init__(host, user, pwd, port, allow_dir_creation) - if not port: - self.port = 21 - self.protocol = "FTP" + def _source_name(self, download_directory, file_name): + """helper to get the full name""" + return download_directory + '/' + file_name - def connect(self): - self.connection = FTP(self.location, self.port) - self.connection.login(self.user, self.pwd) + def _move_file(self, ftp_conn, source, target): + """Moves a file on the FTP server""" + _logger.info('Moving file %s %s' % (source, target)) + ftp_conn.rename(source, target) - def close(self): - self.connection.close() + def _delete_file(self, ftp_conn, source): + """Deletes a file from the FTP server""" + _logger.info('Deleting file %s' % source) + ftp_conn.remove(source) - def get(self, filename, path=None): - if path: - filepath = "{}/{}".format(path, filename) - else: - filepath = filename - outfile = TemporaryFile('w+b') - self.connection.retrbinary('RETR ' + filepath, outfile.write) - return outfile + def run(self): + port_session_factory = ftputil.session.session_factory( + port=self.port) + with ftputil.FTPHost(self.host, self.user, + self.pwd, + session_factory=port_session_factory) as ftp_conn: - def put(self, fileobject, filename, path=None): - if path: - filepath = "{}/{}".format(path, filename) - else: - filepath = filename - self.connection.storbinary('STOR ' + filepath, fileobject) - return True + file_list = ftp_conn.listdir(path) + downloaded_files = [] + for ftpfile in file_list: + if ftp_conn.path.isfile(self._source_name(self.path, + self.file_name)): + file_id = self._handle_new_source(ftp_conn, + self.path, + self.file_name, + self.move_path) + self.run_successor_tasks(file_id=file_id, async=async) + downloaded_files.append(self.file_name) - def search(self, filename, path=None): - if path: - filepath = "{}/{}".format(path, filename) - else: - filepath = filename - connection_list_result = self.connection.nlst() - return [x for x in connection_list_result if filename in x] + # Move/delete files only after all files have been processed. + if self.delete_file: + for ftpfile in downloaded_files: + self._delete_file(ftp_conn, + self._source_name(self.path, + ftpfile)) + elif self.move_path: + if not ftp_conn.path.exists(self.move_path): + ftp_conn.mkdir(self.move_path) + for ftpfile in downloaded_files: + self._move_file( + ftp_conn, + self._source_name(self.path, ftpfile), + self._source_name(self.move_path, ftpfile)) - def move(self, filename, oldpath, newpath): - self.connection.rename( - os.path.join(oldpath, filename), - os.path.join(newpath, filename) - ) - def rename(self, oldfilename, newfilename, path=None): - return NotImplemented +class FtpExportTask(FtpTask): + """FTP Configuration options: + - host, user, password, port + - upload_directory: directory on the FTP server where files are + uploaded to + """ + + def _handle_existing_target(self, ftp_conn, target_name, filedata): + raise Exception("%s already exists" % target_name) + + def _handle_new_target(self, ftp_conn, target_name, filedata): + with ftp_conn.open(target_name, mode='wb') as fileobj: + fileobj.write(filedata) + _logger.info('wrote %s, size %d', target_name, len(filedata)) + + def _target_name(self, ftp_conn, upload_directory, filename): + return upload_directory + '/' + filename + + def _upload_file(self, config, filename, filedata): + ftp_config = config['ftp'] + upload_directory = ftp_config.get('upload_directory', '') + port_session_factory = ftputil.session.session_factory( + port=int(ftp_config.get('port', 21)) + ) + with ftputil.FTPHost(ftp_config['host'], ftp_config['user'], + ftp_config['password'], + session_factory=port_session_factory) as ftp_conn: + target_name = self._target_name(ftp_conn, + upload_directory, + filename) + if ftp_conn.path.isfile(target_name): + self._handle_existing_target(ftp_conn, target_name, filedata) + else: + self._handle_new_target(ftp_conn, target_name, filedata) + + def run(self, config=None, file_id=None, async=True): + #TODO change when object was made + f = self.env.get('impexp.file') \ + .browse(self.env.cr, self.env.uid, file_id) + self._upload_file(config, f.attachment_id.datas_fname, + b64decode(f.attachment_id.datas)) + diff --git a/external_file_location/backends/ftp_backup.py b/external_file_location/backends/ftp_backup.py new file mode 100755 index 00000000..7316b81f --- /dev/null +++ b/external_file_location/backends/ftp_backup.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + + +import sys +import os +from tempfile import TemporaryFile +from ftplib import FTP + +class FTPConnection(object): + + def __init__(self, host, user, pwd, port=None, allow_dir_creation=False): + super(FTPConnection, self).__init__(host, user, pwd, port, allow_dir_creation) + if not port: + self.port = 21 + self.protocol = "FTP" + + def connect(self): + self.connection = FTP(self.location, self.port) + self.connection.login(self.user, self.pwd) + + def close(self): + self.connection.close() + + def get(self, filename, path=None): + if path: + filepath = "{}/{}".format(path, filename) + else: + filepath = filename + outfile = TemporaryFile('w+b') + self.connection.retrbinary('RETR ' + filepath, outfile.write) + return outfile + + def put(self, fileobject, filename, path=None): + if path: + filepath = "{}/{}".format(path, filename) + else: + filepath = filename + self.connection.storbinary('STOR ' + filepath, fileobject) + return True + + def search(self, filename, path=None): + if path: + filepath = "{}/{}".format(path, filename) + else: + filepath = filename + connection_list_result = self.connection.nlst() + return [x for x in connection_list_result if filename in x] + + + def move(self, filename, oldpath, newpath): + self.connection.rename( + os.path.join(oldpath, filename), + os.path.join(newpath, filename) + ) + + def rename(self, oldfilename, newfilename, path=None): + return NotImplemented