From 326894ceea7cf02e4a7d04250704342481577053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Ji=C5=99=C3=ADk?= Date: Wed, 11 May 2016 15:31:47 +0200 Subject: [PATCH 01/37] pypi and conda distribution meta files --- meta.yaml | 68 +++++++++++++++++++++++++++++++++++++++++++++ setup.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 meta.yaml create mode 100644 setup.py diff --git a/meta.yaml b/meta.yaml new file mode 100644 index 0000000..89d259e --- /dev/null +++ b/meta.yaml @@ -0,0 +1,68 @@ +package: + name: sftpsync + version: "1.0.0" + +source: +# this is used for build from git hub + git_rev: 1.0.0 + git_url: https://github.com/mjirik/sftpsync-py.git + +# this is used for pypi + # fn: io3d-1.0.30.tar.gz + # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.30.tar.gz + # md5: a3ce512c4c97ac2410e6dcc96a801bd8 +# patches: + # List any patch files here + # - fix.patch + +# build: + # noarch_python: True + # preserve_egg_dir: True + # entry_points: + # Put any entry points (scripts to be generated automatically) here. The + # syntax is module:function. For example + # + # - io3d = io3d:main + # + # Would create an entry point called io3d that calls io3d.main() + + + # If this is a new build for the same version, increment the build + # number. If you do not include this key, it defaults to 0. + # number: 1 + +requirements: + build: + - python + - setuptools + - paramiko + + run: + - python + - paramiko + +test: + # Python imports + imports: + - sftpsync + + # commands: + # You can put test commands to be run here. Use this to test that the + # entry points work. + + + # You can also put a file called run_test.py in the recipe that will be run + # at test time. + + # requires: + # Put any additional test requirements here. For example + # - nose + +about: + home: https://github.com/mjirik/sftpsync-py + license: s + summary: 'Sync files and directories over SSH' + +# See +# http://docs.continuum.io/conda/build.html for +# more information about meta.yaml diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..dfc110c --- /dev/null +++ b/setup.py @@ -0,0 +1,83 @@ +# Fallowing command is used to upload to pipy +# python setup.py register sdist upload +from setuptools import setup, find_packages +# Always prefer setuptools over distutils +from os import path + +here = path.abspath(path.dirname(__file__)) +setup( + name='sftpsync', + description='Sync files and directories over SSH', + # Versions should comply with PEP440. For a discussion on single-sourcing + # the version across setup.py and the project code, see + # http://packaging.python.org/en/latest/tutorial.html#version + version='1.0.0', + url='https://github.com/mjirik/sftpsync', + author='Jerome Clerc', + author_email='', + license='', + + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + 'Development Status :: 3 - Alpha', + + # Indicate who your project is intended for + 'Intended Audience :: Developers', + # 'Topic :: Scientific/Engineering :: Bio-Informatics', + + # Pick your license as you wish (should match "license" above) + # 'License :: OSI Approved :: BSD License', + + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + # 'Programming Language :: Python :: 2', + # 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + # 'Programming Language :: Python :: 3', + # 'Programming Language :: Python :: 3.2', + # 'Programming Language :: Python :: 3.3', + # 'Programming Language :: Python :: 3.4', + ], + + # What does your project relate to? + keywords='sftp sync', + + # You can just specify the packages manually here if your project is + # simple. Or you can use find_packages(). + packages=find_packages(exclude=['dist', 'docs', 'tests*']), + + # List run-time dependencies here. These will be installed by pip when + # your project is installed. For an analysis of "install_requires" vs pip's + # requirements files see: + # https://packaging.python.org/en/latest/technical.html#install-requires-vs-requirements-files + install_requires=['paramiko'], + # 'SimpleITK'], # Removed becaouse of errors when pip is installing + dependency_links=[], + + # If there are data files included in your packages that need to be + # installed, specify them here. If using Python 2.6 or less, then these + # have to be included in MANIFEST.in as well. + # package_data={ + # 'sample': ['package_data.dat'], + # }, + + # Although 'package_data' is the preferred approach, in some case you may + # need to place data files outside of your packages. + # see + # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa + # In this case, 'data_file' will be installed into '/my_data' + # data_files=[('my_data', ['data/data_file'])], + + # To provide executable scripts, use entry points in preference to the + # "scripts" keyword. Entry points provide cross-platform support and allow + # pip to create the appropriate form of executable for the target platform. + # entry_points={ + # 'console_scripts': [ + # 'sample=sample:main', + # ], + # }, +) From 610b95367e6a63d0878ef802994fc6940223e2b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Ji=C5=99=C3=ADk?= Date: Wed, 11 May 2016 15:37:30 +0200 Subject: [PATCH 02/37] =?UTF-8?q?Bump=20version:=201.0.0=20=E2=86=92=201.0?= =?UTF-8?q?.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- meta.yaml | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/meta.yaml b/meta.yaml index 89d259e..95abdee 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,10 +1,10 @@ package: name: sftpsync - version: "1.0.0" + version: "1.0.1" source: # this is used for build from git hub - git_rev: 1.0.0 + git_rev: 1.0.1 git_url: https://github.com/mjirik/sftpsync-py.git # this is used for pypi diff --git a/setup.py b/setup.py index dfc110c..959d6c3 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='1.0.0', + version='1.0.1', url='https://github.com/mjirik/sftpsync', author='Jerome Clerc', author_email='', From a34ad1eb2fdc18bd10114e358f96f9f818430040 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Thu, 12 May 2016 22:45:07 +0200 Subject: [PATCH 03/37] callback function --- sftpsync/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index 28335d8..ee34477 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -131,11 +131,11 @@ def _validate_dst(self, file, src_stat, remote=True): def _save(self, src, dst, src_stat, remote=True): if remote: logger.info('copying %s to %s@%s:%s', src, self.username, self.host, dst) - self.sftp.put(src, dst) + self.sftp.put(src, dst, callback=self.callback) self.sftp.utime(dst, (int(src_stat.st_atime), int(src_stat.st_mtime))) else: logger.info('copying %s@%s:%s to %s', self.username, self.host, src, dst) - self.sftp.get(src, dst) + self.sftp.get(src, dst, callback=self.callback) os.utime(dst, (int(src_stat.st_atime), int(src_stat.st_mtime))) def _delete_dst(self, path, files, remote=True, dry=False): @@ -160,7 +160,7 @@ def _get_filters(self, filters): return [] return [re.compile(f) for f in filters] - def sync(self, src, dst, download=True, include=None, exclude=None, delete=False, dry=False): + def sync(self, src, dst, download=True, include=None, exclude=None, delete=False, dry=False, callback=None): '''Sync files and directories. :param src: source directory @@ -171,7 +171,11 @@ def sync(self, src, dst, download=True, include=None, exclude=None, delete=False :param exclude: list of regex patterns the source files must not match :param delete: remove destination files and directories not present at source or filtered by the include/exlude patterns + :param callback: callback function (form: func(int, int)) that accepts the + bytes transferred so far and the total bytes to be transferred ''' + + self.callback = callback include = self._get_filters(include) exclude = self._get_filters(exclude) From f985563f93b067d5bd20565846a4115f4306caff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Ji=C5=99=C3=ADk?= Date: Thu, 12 May 2016 22:46:12 +0200 Subject: [PATCH 04/37] =?UTF-8?q?Bump=20version:=201.0.1=20=E2=86=92=201.0?= =?UTF-8?q?.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- meta.yaml | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/meta.yaml b/meta.yaml index 95abdee..63b2174 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,10 +1,10 @@ package: name: sftpsync - version: "1.0.1" + version: "1.0.2" source: # this is used for build from git hub - git_rev: 1.0.1 + git_rev: 1.0.2 git_url: https://github.com/mjirik/sftpsync-py.git # this is used for pypi diff --git a/setup.py b/setup.py index 959d6c3..116cd4a 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='1.0.1', + version='1.0.2', url='https://github.com/mjirik/sftpsync', author='Jerome Clerc', author_email='', From cc8e65b595ce445ed7a0c9ca7e8fc33fafad234d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Ji=C5=99=C3=ADk?= Date: Tue, 24 May 2016 12:59:20 +0200 Subject: [PATCH 05/37] install notes --- README | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README b/README index e7ce43d..b4a82b4 100644 --- a/README +++ b/README @@ -26,3 +26,11 @@ Example: dst = '/mnt/sdcard/data/' sftp.sync(src, dst, download=False, delete=True) + + +Install: + + pip install sftpsync + +or + conda install -c mjirik sftpsync From 03c3ec517ebf338dad4d69f75911b1625d92eeb8 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Tue, 7 Jun 2016 22:04:41 +0200 Subject: [PATCH 06/37] fixed join path on windows --- sftpsync/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index ee34477..018387b 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -43,6 +43,17 @@ def __init__(self, host, username, password=None, port=22, timeout=10, if i == max_attempts - 1: raise SshError(str(e)) + def _join_remote(self, path1, path2): + """ + for remote path is separator always "/" + :param path1: + :param path2: + :return: joined paths + """ + + path1 = path1.rstrip('/') + return path1 + "/" + path2 + def _walk_remote(self, path, topdown=True): try: res = self.sftp.listdir_attr(path) @@ -50,7 +61,7 @@ def _walk_remote(self, path, topdown=True): res = [] for stat in res: - file = os.path.join(path, stat.filename) + file = self._join_remote(path, stat.filename) if not S_ISDIR(stat.st_mode): yield 'file', file, stat From 6a5e6b0f5353090cf1016404f1eed89d2dc9343e Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Tue, 7 Jun 2016 22:05:17 +0200 Subject: [PATCH 07/37] test added --- tests/sftpsync_test.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/sftpsync_test.py diff --git a/tests/sftpsync_test.py b/tests/sftpsync_test.py new file mode 100644 index 0000000..ff7af87 --- /dev/null +++ b/tests/sftpsync_test.py @@ -0,0 +1,34 @@ +import unittest + + +import os.path as op +class MyTestCase(unittest.TestCase): + def test_connection(self): + from sftpsync import Sftp + + sftp = Sftp('147.228.47.162', 'paul', 'P4ul') + # hu = sftp.sftp.listdir_attr("from_server") + dir_list = sftp.sftp.listdir_attr("from_server") + first_fn = dir_list[0].filename + self.assertEqual(first_fn, "test.txt") + + def test_sync(self): + from sftpsync import Sftp + + sftp = Sftp('147.228.47.162', 'paul', 'P4ul') + # hu = sftp.sftp.listdir_attr("from_server") + dir_list = sftp.sftp.listdir_attr("from_server") + + src = 'from_server/' + # src = 'to_server' + dst = 'test_temp/' + dst = op.expanduser(dst) + + # We don't want to backup everything + exclude = [r'^Music/', r'^Video/'] + + sftp.sync(src, dst, download=True, exclude=exclude, delete=False) + + self.assertTrue(op.exists(op.join(dst,"test.txt"))) +if __name__ == '__main__': + unittest.main() From 2f41f099cf9b8759d5b279225066536c4de6368a Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Tue, 7 Jun 2016 22:07:19 +0200 Subject: [PATCH 08/37] =?UTF-8?q?Bump=20version:=201.0.2=20=E2=86=92=201.0?= =?UTF-8?q?.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- meta.yaml | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/meta.yaml b/meta.yaml index 63b2174..f5765d8 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,10 +1,10 @@ package: name: sftpsync - version: "1.0.2" + version: "1.0.3" source: # this is used for build from git hub - git_rev: 1.0.2 + git_rev: 1.0.3 git_url: https://github.com/mjirik/sftpsync-py.git # this is used for pypi diff --git a/setup.py b/setup.py index 116cd4a..bdc9779 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='1.0.2', + version='1.0.3', url='https://github.com/mjirik/sftpsync', author='Jerome Clerc', author_email='', From 7a79daf1d9d4b9cec9c4045b7da17d3b3678de10 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Tue, 7 Jun 2016 22:22:04 +0200 Subject: [PATCH 09/37] convert os.sep to '/' --- sftpsync/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index 018387b..d28a1db 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -190,6 +190,8 @@ def sync(self, src, dst, download=True, include=None, exclude=None, delete=False include = self._get_filters(include) exclude = self._get_filters(exclude) + src = src.replace(os.sep, '/') + dst = dst.replace(os.sep, '/') if src.endswith('/') != dst.endswith('/'): dst = os.path.join(dst, os.path.basename(src.rstrip('/'))) src = src.rstrip('/') From a619cf3adad79b549f6216a51a53784e50ed50b8 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Tue, 7 Jun 2016 22:22:50 +0200 Subject: [PATCH 10/37] =?UTF-8?q?Bump=20version:=201.0.3=20=E2=86=92=201.0?= =?UTF-8?q?.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- meta.yaml | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/meta.yaml b/meta.yaml index f5765d8..cdba74a 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,15 +1,15 @@ package: name: sftpsync - version: "1.0.3" + version: "1.0.4" source: # this is used for build from git hub - git_rev: 1.0.3 + git_rev: 1.0.4 git_url: https://github.com/mjirik/sftpsync-py.git # this is used for pypi - # fn: io3d-1.0.30.tar.gz - # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.30.tar.gz + # fn: io3d-1.0.40.tar.gz + # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.40.tar.gz # md5: a3ce512c4c97ac2410e6dcc96a801bd8 # patches: # List any patch files here diff --git a/setup.py b/setup.py index bdc9779..fd69967 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='1.0.3', + version='1.0.4', url='https://github.com/mjirik/sftpsync', author='Jerome Clerc', author_email='', From 09107e11024577d3d0b9e2678e5d2128b6088799 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Tue, 7 Jun 2016 22:51:32 +0200 Subject: [PATCH 11/37] cleaning tests --- tests/sftpsync_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/sftpsync_test.py b/tests/sftpsync_test.py index ff7af87..3ca6e00 100644 --- a/tests/sftpsync_test.py +++ b/tests/sftpsync_test.py @@ -16,8 +16,6 @@ def test_sync(self): from sftpsync import Sftp sftp = Sftp('147.228.47.162', 'paul', 'P4ul') - # hu = sftp.sftp.listdir_attr("from_server") - dir_list = sftp.sftp.listdir_attr("from_server") src = 'from_server/' # src = 'to_server' From 480366d18d91af6a356b92c40a9e7a61a8732a4e Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Wed, 8 Jun 2016 09:47:05 +0200 Subject: [PATCH 12/37] test different separator --- tests/sftpsync_test.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tests/sftpsync_test.py b/tests/sftpsync_test.py index 3ca6e00..24f0d62 100644 --- a/tests/sftpsync_test.py +++ b/tests/sftpsync_test.py @@ -1,4 +1,5 @@ import unittest +import shutil import os.path as op @@ -9,24 +10,40 @@ def test_connection(self): sftp = Sftp('147.228.47.162', 'paul', 'P4ul') # hu = sftp.sftp.listdir_attr("from_server") dir_list = sftp.sftp.listdir_attr("from_server") - first_fn = dir_list[0].filename - self.assertEqual(first_fn, "test.txt") + self.assertIn(dir_list[0].filename, ["test.txt", "foo"]) + self.assertIn(dir_list[1].filename, ["test.txt", "foo"]) + dir_list2 = sftp.sftp.listdir_attr("from_server/foo") + self.assertEqual(dir_list2[0].filename, 'bar.txt') def test_sync(self): from sftpsync import Sftp - sftp = Sftp('147.228.47.162', 'paul', 'P4ul') - src = 'from_server/' - # src = 'to_server' dst = 'test_temp/' + sftp = Sftp('147.228.47.162', 'paul', 'P4ul') + dst = op.expanduser(dst) + shutil.rmtree(dst) # We don't want to backup everything exclude = [r'^Music/', r'^Video/'] - sftp.sync(src, dst, download=True, exclude=exclude, delete=False) + self.assertTrue(op.exists(op.join(dst,"test.txt"))) + + def test_sync_different_separator(self): + from sftpsync import Sftp + src = 'from_server/' + dst = 'test_temp_different_separator\\' + sftp = Sftp('147.228.47.162', 'paul', 'P4ul') + + dst = op.expanduser(dst) + shutil.rmtree(dst) + + # We don't want to backup everything + exclude = [r'^Music/', r'^Video/'] + sftp.sync(src, dst, download=True, exclude=exclude, delete=False) self.assertTrue(op.exists(op.join(dst,"test.txt"))) + if __name__ == '__main__': unittest.main() From 6e395121f87f2e4ce0f8e3e4550e6de5811b10a1 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Wed, 8 Jun 2016 09:47:53 +0200 Subject: [PATCH 13/37] =?UTF-8?q?Bump=20version:=201.0.4=20=E2=86=92=201.0?= =?UTF-8?q?.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- meta.yaml | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/meta.yaml b/meta.yaml index cdba74a..1104a3d 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,15 +1,15 @@ package: name: sftpsync - version: "1.0.4" + version: "1.0.5" source: # this is used for build from git hub - git_rev: 1.0.4 + git_rev: 1.0.5 git_url: https://github.com/mjirik/sftpsync-py.git # this is used for pypi - # fn: io3d-1.0.40.tar.gz - # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.40.tar.gz + # fn: io3d-1.0.50.tar.gz + # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.50.tar.gz # md5: a3ce512c4c97ac2410e6dcc96a801bd8 # patches: # List any patch files here diff --git a/setup.py b/setup.py index fd69967..44ff99a 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='1.0.4', + version='1.0.5', url='https://github.com/mjirik/sftpsync', author='Jerome Clerc', author_email='', From d2f2916aaee635b75b6eabed3d5c54be6bc26e5f Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Wed, 8 Jun 2016 13:05:11 +0200 Subject: [PATCH 14/37] full path test --- tests/sftpsync_test.py | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/tests/sftpsync_test.py b/tests/sftpsync_test.py index 24f0d62..847fff0 100644 --- a/tests/sftpsync_test.py +++ b/tests/sftpsync_test.py @@ -1,9 +1,13 @@ +import logging +logger = logging.getLogger(__name__) import unittest import shutil +import os.path as op +clean = False -import os.path as op class MyTestCase(unittest.TestCase): + def test_connection(self): from sftpsync import Sftp @@ -23,12 +27,15 @@ def test_sync(self): sftp = Sftp('147.228.47.162', 'paul', 'P4ul') dst = op.expanduser(dst) - shutil.rmtree(dst) + if op.exists(dst): + shutil.rmtree(dst) # We don't want to backup everything exclude = [r'^Music/', r'^Video/'] sftp.sync(src, dst, download=True, exclude=exclude, delete=False) self.assertTrue(op.exists(op.join(dst,"test.txt"))) + if clean and op.exists(dst): + shutil.rmtree(dst) def test_sync_different_separator(self): from sftpsync import Sftp @@ -38,12 +45,38 @@ def test_sync_different_separator(self): sftp = Sftp('147.228.47.162', 'paul', 'P4ul') dst = op.expanduser(dst) - shutil.rmtree(dst) + if op.exists(dst): + shutil.rmtree(dst) # We don't want to backup everything exclude = [r'^Music/', r'^Video/'] + sftp.sync(src, dst, download=True, exclude=exclude, delete=False) self.assertTrue(op.exists(op.join(dst,"test.txt"))) + if clean and op.exists(dst): + shutil.rmtree(dst) + + def test_sync_abspath(self): + from sftpsync import Sftp + + src = 'from_server/' + dst = 'test_temp_abspath\\' + sftp = Sftp('147.228.47.162', 'paul', 'P4ul') + + dst = op.abspath(dst) + dst += '\\' + if op.exists(dst): + shutil.rmtree(dst) + + # We don't want to backup everything + exclude = [r'^Music/', r'^Video/'] + logger.debug("src %s", src) + logger.debug("dst %s", dst) + sftp.sync(src, dst , download=True, exclude=exclude, delete=False) + self.assertTrue(op.exists(op.join(dst,"test.txt"))) + + if clean and op.exists(dst): + shutil.rmtree(dst) if __name__ == '__main__': unittest.main() From 106eab4889846a9614b548dc5c37ae9498e65af2 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Wed, 8 Jun 2016 13:47:29 +0200 Subject: [PATCH 15/37] logging in test --- sftpsync/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index d28a1db..0653b38 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -192,7 +192,9 @@ def sync(self, src, dst, download=True, include=None, exclude=None, delete=False src = src.replace(os.sep, '/') dst = dst.replace(os.sep, '/') + logger.debug() if src.endswith('/') != dst.endswith('/'): + dst = os.path.join(dst, os.path.basename(src.rstrip('/'))) src = src.rstrip('/') re_base = re.compile(r'^%s/' % re.escape(src)) From 3a0ffa596fec6e31433ec07c3d7c3f0183ff64ed Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Wed, 8 Jun 2016 14:48:01 +0200 Subject: [PATCH 16/37] logger message for joined paths --- sftpsync/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index 0653b38..b835f74 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -192,9 +192,8 @@ def sync(self, src, dst, download=True, include=None, exclude=None, delete=False src = src.replace(os.sep, '/') dst = dst.replace(os.sep, '/') - logger.debug() if src.endswith('/') != dst.endswith('/'): - + logger.debug("Paths ends with different symbol. Paths are joined.") dst = os.path.join(dst, os.path.basename(src.rstrip('/'))) src = src.rstrip('/') re_base = re.compile(r'^%s/' % re.escape(src)) From 65d4f54ec6e45a3caaba0e29b759354d33eed7c4 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Wed, 8 Jun 2016 14:49:55 +0200 Subject: [PATCH 17/37] =?UTF-8?q?Bump=20version:=201.0.5=20=E2=86=92=201.0?= =?UTF-8?q?.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- meta.yaml | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/meta.yaml b/meta.yaml index 1104a3d..6abfe9f 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,15 +1,15 @@ package: name: sftpsync - version: "1.0.5" + version: "1.0.6" source: # this is used for build from git hub - git_rev: 1.0.5 + git_rev: 1.0.6 git_url: https://github.com/mjirik/sftpsync-py.git # this is used for pypi - # fn: io3d-1.0.50.tar.gz - # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.50.tar.gz + # fn: io3d-1.0.60.tar.gz + # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.60.tar.gz # md5: a3ce512c4c97ac2410e6dcc96a801bd8 # patches: # List any patch files here diff --git a/setup.py b/setup.py index 44ff99a..64a0c52 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='1.0.5', + version='1.0.6', url='https://github.com/mjirik/sftpsync', author='Jerome Clerc', author_email='', From 4556c3e2a348457744cbb2718d80550947f910d8 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Wed, 8 Jun 2016 15:08:21 +0200 Subject: [PATCH 18/37] allways use slash instead of backslash --- sftpsync/__init__.py | 4 ++-- tests/sftpsync_test.py | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index b835f74..c00fcc8 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -190,8 +190,8 @@ def sync(self, src, dst, download=True, include=None, exclude=None, delete=False include = self._get_filters(include) exclude = self._get_filters(exclude) - src = src.replace(os.sep, '/') - dst = dst.replace(os.sep, '/') + src = src.replace('\\', '/') + dst = dst.replace('\\', '/') if src.endswith('/') != dst.endswith('/'): logger.debug("Paths ends with different symbol. Paths are joined.") dst = os.path.join(dst, os.path.basename(src.rstrip('/'))) diff --git a/tests/sftpsync_test.py b/tests/sftpsync_test.py index 847fff0..13edcdd 100644 --- a/tests/sftpsync_test.py +++ b/tests/sftpsync_test.py @@ -52,7 +52,7 @@ def test_sync_different_separator(self): exclude = [r'^Music/', r'^Video/'] sftp.sync(src, dst, download=True, exclude=exclude, delete=False) - self.assertTrue(op.exists(op.join(dst,"test.txt"))) + self.assertTrue(op.exists(op.join(dst.rstrip("\\"),"test.txt"))) if clean and op.exists(dst): shutil.rmtree(dst) @@ -64,7 +64,8 @@ def test_sync_abspath(self): sftp = Sftp('147.228.47.162', 'paul', 'P4ul') dst = op.abspath(dst) - dst += '\\' + if not (dst.endswith('/') or dst.endswith('\\')): + dst += '\\' if op.exists(dst): shutil.rmtree(dst) @@ -73,7 +74,9 @@ def test_sync_abspath(self): logger.debug("src %s", src) logger.debug("dst %s", dst) sftp.sync(src, dst , download=True, exclude=exclude, delete=False) - self.assertTrue(op.exists(op.join(dst,"test.txt"))) + expected_path = op.join(dst.rstrip("\\"),"test.txt") + logger.debug("Expected path: %s", expected_path) + self.assertTrue(op.exists(expected_path)) if clean and op.exists(dst): shutil.rmtree(dst) From 5d91aa13e561fb77cadf511f79fbc5ca6c87426a Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Wed, 8 Jun 2016 15:08:42 +0200 Subject: [PATCH 19/37] =?UTF-8?q?Bump=20version:=201.0.6=20=E2=86=92=201.0?= =?UTF-8?q?.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- meta.yaml | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/meta.yaml b/meta.yaml index 6abfe9f..10b7c61 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,15 +1,15 @@ package: name: sftpsync - version: "1.0.6" + version: "1.0.7" source: # this is used for build from git hub - git_rev: 1.0.6 + git_rev: 1.0.7 git_url: https://github.com/mjirik/sftpsync-py.git # this is used for pypi - # fn: io3d-1.0.60.tar.gz - # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.60.tar.gz + # fn: io3d-1.0.70.tar.gz + # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.70.tar.gz # md5: a3ce512c4c97ac2410e6dcc96a801bd8 # patches: # List any patch files here diff --git a/setup.py b/setup.py index 64a0c52..78e5864 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='1.0.6', + version='1.0.7', url='https://github.com/mjirik/sftpsync', author='Jerome Clerc', author_email='', From 0f5c182c99a563dbaaa887b891a4b2b648e8a723 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Sat, 11 Jun 2016 23:34:29 +0200 Subject: [PATCH 20/37] fix upload --- sftpsync/__init__.py | 24 ++++++++++++++++++++++-- tests/sftpsync_test.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index c00fcc8..04b9e11 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -43,6 +43,22 @@ def __init__(self, host, username, password=None, port=22, timeout=10, if i == max_attempts - 1: raise SshError(str(e)) + def _join(self, path1, path2, remote, path2_start=None): + if remote: + if not path2_start is None: + path2 = path2.replace('\\','/') + path2_start = path2_start.replace('\\','/') + if path2_start[-1] != '/': + path2_start += '/' + + path2 = path2[len(path2_start):] + + dst = self._join_remote(path1, path2) + else: + dst = os.path.join(path1, path2) + # dst = self._join_remote(path1, path2) + return dst + def _join_remote(self, path1, path2): """ for remote path is separator always "/" @@ -76,6 +92,7 @@ def _walk_remote(self, path, topdown=True): yield 'dir', file, None def _walk_local(self, path, topdown=True): + logger.debug('local walk %s', path) for path, dirs, files in os.walk(path, topdown=topdown): for file in files: file = os.path.join(path, file) @@ -194,7 +211,7 @@ def sync(self, src, dst, download=True, include=None, exclude=None, delete=False dst = dst.replace('\\', '/') if src.endswith('/') != dst.endswith('/'): logger.debug("Paths ends with different symbol. Paths are joined.") - dst = os.path.join(dst, os.path.basename(src.rstrip('/'))) + dst = self._join(dst, os.path.basename(src.rstrip('/')), remote=not download) src = src.rstrip('/') re_base = re.compile(r'^%s/' % re.escape(src)) if not src: @@ -208,11 +225,14 @@ def sync(self, src, dst, download=True, include=None, exclude=None, delete=False for type, file, stat in self._walk(src, remote=download): file_ = re_base.sub('', file) + file_ = file_.replace('\\', '/') if not self._validate_src(file_, include, exclude): logger.debug('filtered %s', file) continue - dst_file = os.path.join(dst, file_) + dst_file = self._join(dst, file_, remote=not download, path2_start=src) + logger.debug("walk %s", file) + logger.debug("walk full %s", dst_file) dst_list[type].append(dst_file) if type == 'dir': diff --git a/tests/sftpsync_test.py b/tests/sftpsync_test.py index 13edcdd..b4d133e 100644 --- a/tests/sftpsync_test.py +++ b/tests/sftpsync_test.py @@ -2,6 +2,7 @@ logger = logging.getLogger(__name__) import unittest import shutil +import os import os.path as op clean = False @@ -81,5 +82,44 @@ def test_sync_abspath(self): if clean and op.exists(dst): shutil.rmtree(dst) + def test_sync_upload(self): + from sftpsync import Sftp + src = 'to_server/' + src = 'c:/Users/mjirik/projects/sftpsync-py/to_server/' + dst = 'to_server/' + + # create test dir + srcfile = op.join(src, 'test_file.txt') + + if not op.exists(src): + logger.debug('creating dir %s', src) + os.makedirs(src) + + with open(srcfile,"a+") as f: + f.write("text\n") + + # connect to sftp + sftp = Sftp('147.228.47.162', 'paul', 'P4ul') + + # make sure that test file is not on server + dir_list = sftp.sftp.listdir_attr("to_server/") + fnames = [record.filename for record in dir_list] + if 'test_file.txt' in fnames: + sftp.sftp.remove("to_server/test_file.txt") + + # Make test: sync local directory + exclude = [r'^Music/', r'^Video/'] + sftp.sync(src, dst, download=False, exclude=exclude, delete=True) + dir_list = sftp.sftp.listdir_attr("to_server/") + # check if file is created + self.assertEqual(dir_list[0].filename, 'test_file.txt') + + # remove file and sync again + os.remove(srcfile) + sftp.sync(src, dst, download=False, exclude=exclude, delete=True) + dir_list = sftp.sftp.listdir_attr("to_server/") + # check if direcotry is empty + self.assertEqual(len(dir_list), 0) + if __name__ == '__main__': unittest.main() From 0a74e1d9ad00f1b4cca59e2e6f852deb1e10549a Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Sat, 11 Jun 2016 23:39:09 +0200 Subject: [PATCH 21/37] logger make relative --- sftpsync/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index 04b9e11..ec70c75 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -46,12 +46,16 @@ def __init__(self, host, username, password=None, port=22, timeout=10, def _join(self, path1, path2, remote, path2_start=None): if remote: if not path2_start is None: + logger.debug('make relative path to start') path2 = path2.replace('\\','/') path2_start = path2_start.replace('\\','/') + logger.debug(path2) + logger.debug(path2_start) if path2_start[-1] != '/': path2_start += '/' path2 = path2[len(path2_start):] + logger.debug(path2) dst = self._join_remote(path1, path2) else: From 948a420cadaf3d54df6adc0173b7829c6a556630 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Sat, 11 Jun 2016 23:48:09 +0200 Subject: [PATCH 22/37] check if relative is required --- sftpsync/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index ec70c75..c7e6d33 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -46,16 +46,19 @@ def __init__(self, host, username, password=None, port=22, timeout=10, def _join(self, path1, path2, remote, path2_start=None): if remote: if not path2_start is None: - logger.debug('make relative path to start') + # this is necessary on windows + # path2 is there different from the linux path2 = path2.replace('\\','/') path2_start = path2_start.replace('\\','/') - logger.debug(path2) - logger.debug(path2_start) if path2_start[-1] != '/': path2_start += '/' + if path2.startswith(path2_start): + logger.debug('make relative path to start') + logger.debug("path2 : %s", path2) + logger.debug("path2_start : %s", path2_start) path2 = path2[len(path2_start):] - logger.debug(path2) + logger.debug("path2 after : %s", path2) dst = self._join_remote(path1, path2) else: From 22bdad7ba4ac7d10b8f2bcdef1b3908e0d460da8 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Sat, 11 Jun 2016 23:49:34 +0200 Subject: [PATCH 23/37] fix ident --- sftpsync/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index c7e6d33..13ed4ca 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -55,10 +55,10 @@ def _join(self, path1, path2, remote, path2_start=None): if path2.startswith(path2_start): logger.debug('make relative path to start') - logger.debug("path2 : %s", path2) - logger.debug("path2_start : %s", path2_start) - path2 = path2[len(path2_start):] - logger.debug("path2 after : %s", path2) + logger.debug("path2 : %s", path2) + logger.debug("path2_start : %s", path2_start) + path2 = path2[len(path2_start):] + logger.debug("path2 after : %s", path2) dst = self._join_remote(path1, path2) else: From 66b00bb6844e55447405205ee28ef24c3fc3201e Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Sat, 11 Jun 2016 23:50:01 +0200 Subject: [PATCH 24/37] =?UTF-8?q?Bump=20version:=201.0.7=20=E2=86=92=201.0?= =?UTF-8?q?.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- meta.yaml | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/meta.yaml b/meta.yaml index 10b7c61..55b3d7a 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,15 +1,15 @@ package: name: sftpsync - version: "1.0.7" + version: "1.0.8" source: # this is used for build from git hub - git_rev: 1.0.7 + git_rev: 1.0.8 git_url: https://github.com/mjirik/sftpsync-py.git # this is used for pypi - # fn: io3d-1.0.70.tar.gz - # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.70.tar.gz + # fn: io3d-1.0.80.tar.gz + # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.80.tar.gz # md5: a3ce512c4c97ac2410e6dcc96a801bd8 # patches: # List any patch files here diff --git a/setup.py b/setup.py index 78e5864..b9a1b90 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='1.0.7', + version='1.0.8', url='https://github.com/mjirik/sftpsync', author='Jerome Clerc', author_email='', From 618e4a49abdfc40a96147942a866586dbceb32b0 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Wed, 31 Aug 2016 15:50:37 +0200 Subject: [PATCH 25/37] use sftpserver package to run local sftp server --- tests/sftpsync_test.py | 68 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/tests/sftpsync_test.py b/tests/sftpsync_test.py index b4d133e..ccac0c6 100644 --- a/tests/sftpsync_test.py +++ b/tests/sftpsync_test.py @@ -7,12 +7,66 @@ clean = False +def touch(fname, times=None): + with open(fname, 'a'): + os.utime(fname, times) + +def runServer(): + return host, port, process + class MyTestCase(unittest.TestCase): + + @classmethod + def setUpClass(self): + import subprocess + import time + + super(MyTestCase, self).setUpClass() + # prepare structure for test SFTP server + pth_from = "test_server/from_server/foo" + pth_to = "test_server/to_server" + if not os.path.exists(pth_from): + os.makedirs(pth_from) + if not os.path.exists(pth_to): + os.makedirs(pth_to) + touch("test_server/from_server/test.txt") + touch("test_server/from_server/foo/bar.txt") + # create RSA key + if not os.path.exists("id_rsa"): + os.system('ssh-keygen -t rsa -q -f id_rsa -P ""') + + # run test SFTP server + # port = 22 + self.port = 3373 + self.host = "localhost" + + subprocess.Popen( + "sftpserver -k ../id_rsa -p " + str(self.port), + cwd="test_server", + shell=True) + time.sleep(1) + + # Server is available for all users and any password + # sftp://user@localhost:3373/ + + @classmethod + def tearDownClass(self): + super(MyTestCase, self).setUpClass() + os.system("pkill sftpserver") + + def test_test_server(self): + import paramiko + pkey = paramiko.RSAKey.from_private_key_file('id_rsa') + transport = paramiko.Transport(('localhost', self.port)) + transport.connect(username='admin', password='admin')# , pkey=pkey) + sftp = paramiko.SFTPClient.from_transport(transport) + sftp.listdir('.') + def test_connection(self): from sftpsync import Sftp - sftp = Sftp('147.228.47.162', 'paul', 'P4ul') + sftp = Sftp(self.host, 'paul', 'P4ul', port=self.port) # hu = sftp.sftp.listdir_attr("from_server") dir_list = sftp.sftp.listdir_attr("from_server") self.assertIn(dir_list[0].filename, ["test.txt", "foo"]) @@ -25,7 +79,7 @@ def test_sync(self): src = 'from_server/' dst = 'test_temp/' - sftp = Sftp('147.228.47.162', 'paul', 'P4ul') + sftp = Sftp(self.host, 'paul', 'P4ul', port=self.port) dst = op.expanduser(dst) @@ -34,7 +88,7 @@ def test_sync(self): # We don't want to backup everything exclude = [r'^Music/', r'^Video/'] sftp.sync(src, dst, download=True, exclude=exclude, delete=False) - self.assertTrue(op.exists(op.join(dst,"test.txt"))) + self.assertTrue(op.exists(op.join(dst, "test.txt"))) if clean and op.exists(dst): shutil.rmtree(dst) @@ -43,7 +97,7 @@ def test_sync_different_separator(self): src = 'from_server/' dst = 'test_temp_different_separator\\' - sftp = Sftp('147.228.47.162', 'paul', 'P4ul') + sftp = Sftp(self.host, 'paul', 'P4ul', port=self.port) dst = op.expanduser(dst) if op.exists(dst): @@ -57,12 +111,14 @@ def test_sync_different_separator(self): if clean and op.exists(dst): shutil.rmtree(dst) + + def test_sync_abspath(self): from sftpsync import Sftp src = 'from_server/' dst = 'test_temp_abspath\\' - sftp = Sftp('147.228.47.162', 'paul', 'P4ul') + sftp = Sftp(self.host, 'paul', 'P4ul', port=self.port) dst = op.abspath(dst) if not (dst.endswith('/') or dst.endswith('\\')): @@ -99,7 +155,7 @@ def test_sync_upload(self): f.write("text\n") # connect to sftp - sftp = Sftp('147.228.47.162', 'paul', 'P4ul') + sftp = Sftp(self.host, 'paul', 'P4ul', port=self.port) # make sure that test file is not on server dir_list = sftp.sftp.listdir_attr("to_server/") From c196e0c7488a7d5a5e5ea69609210f6d8639dd4b Mon Sep 17 00:00:00 2001 From: Oscar Triano dotoscat Date: Wed, 29 Mar 2017 23:34:41 +0200 Subject: [PATCH 26/37] Fix exception SyntaxError --- sftpsync/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index 13ed4ca..ebeae3e 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -35,11 +35,11 @@ def __init__(self, host, username, password=None, port=22, timeout=10, password=password, timeout=timeout, **kwargs) self.sftp = self.client.open_sftp() return - except (paramiko.BadHostKeyException, paramiko.AuthenticationException), e: + except (paramiko.BadHostKeyException, paramiko.AuthenticationException) as e: raise AuthenticationError(str(e)) - except socket.timeout, e: + except socket.timeout as e: raise TimeoutError(str(e)) - except Exception, e: + except Exception as e: if i == max_attempts - 1: raise SshError(str(e)) @@ -184,7 +184,7 @@ def _delete_dst(self, path, files, remote=True, dry=False): if not dry: try: callables[type](file) - except Exception, e: + except Exception as e: logger.debug('failed to remove %s: %s', file, str(e)) continue From 87b33a2a752535d021ee80534294e2a5b3de6697 Mon Sep 17 00:00:00 2001 From: "Ramsay, Grant (NZ)" Date: Tue, 31 Oct 2017 01:52:31 +1300 Subject: [PATCH 27/37] Sync file permissions --- sftpsync/__init__.py | 18 ++++++-- tests/sftpsync_test.py | 97 +++++++++++++++++++++++++++++++++++------- 2 files changed, 97 insertions(+), 18 deletions(-) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index ebeae3e..579eed8 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -115,7 +115,7 @@ def _walk(self, *args, **kwargs): else: return self._walk_local(*args, **kwargs) - def _makedirs_dst(self, path, remote=True, dry=False): + def _makedirs_dst(self, path, src_stat, remote=True, dry=False): if remote: paths = [] while path not in ('/', ''): @@ -129,11 +129,15 @@ def _makedirs_dst(self, path, remote=True, dry=False): if not dry: self.sftp.mkdir(path) logger.debug('created destination directory %s', path) + if not dry: + self.sftp.chmod(path, src_stat.st_mode) else: if not os.path.exists(path): if not dry: os.makedirs(path) logger.debug('created destination directory %s', path) + if not dry: + os.chmod(path, src_stat.st_mode) def _validate_src(self, file, include, exclude): for re_ in include: @@ -161,6 +165,8 @@ def _validate_dst(self, file, src_stat, remote=True): return if dst_stat.st_size != src_stat.st_size: return + if dst_stat.st_mode != src_stat.st_mode: + return return True def _save(self, src, dst, src_stat, remote=True): @@ -168,10 +174,12 @@ def _save(self, src, dst, src_stat, remote=True): logger.info('copying %s to %s@%s:%s', src, self.username, self.host, dst) self.sftp.put(src, dst, callback=self.callback) self.sftp.utime(dst, (int(src_stat.st_atime), int(src_stat.st_mtime))) + self.sftp.chmod(dst, src_stat.st_mode) else: logger.info('copying %s@%s:%s to %s', self.username, self.host, src, dst) self.sftp.get(src, dst, callback=self.callback) os.utime(dst, (int(src_stat.st_atime), int(src_stat.st_mtime))) + os.chmod(dst, src_stat.st_mode) def _delete_dst(self, path, files, remote=True, dry=False): if remote: @@ -224,7 +232,11 @@ def sync(self, src, dst, download=True, include=None, exclude=None, delete=False if not src: src = '/' - self._makedirs_dst(dst, remote=not download, dry=dry) + if download: + stat = self.sftp.lstat(src) + else: + stat = os.stat(src) + self._makedirs_dst(dst, stat, remote=not download, dry=dry) started = datetime.utcnow() total_size = 0 @@ -243,7 +255,7 @@ def sync(self, src, dst, download=True, include=None, exclude=None, delete=False dst_list[type].append(dst_file) if type == 'dir': - self._makedirs_dst(dst_file, remote=not download, dry=dry) + self._makedirs_dst(dst_file, stat, remote=not download, dry=dry) elif type == 'file': if not self._validate_dst(dst_file, stat, remote=not download): if not dry: diff --git a/tests/sftpsync_test.py b/tests/sftpsync_test.py index ccac0c6..906ecc9 100644 --- a/tests/sftpsync_test.py +++ b/tests/sftpsync_test.py @@ -7,22 +7,19 @@ clean = False + def touch(fname, times=None): with open(fname, 'a'): os.utime(fname, times) -def runServer(): - return host, port, process - -class MyTestCase(unittest.TestCase): - +class SftpTestBase(unittest.TestCase): @classmethod - def setUpClass(self): + def setUpClass(cls): import subprocess import time - super(MyTestCase, self).setUpClass() + super(SftpTestBase, cls).setUpClass() # prepare structure for test SFTP server pth_from = "test_server/from_server/foo" pth_to = "test_server/to_server" @@ -38,11 +35,11 @@ def setUpClass(self): # run test SFTP server # port = 22 - self.port = 3373 - self.host = "localhost" + cls.port = 3373 + cls.host = "localhost" subprocess.Popen( - "sftpserver -k ../id_rsa -p " + str(self.port), + "sftpserver -k ../id_rsa -p " + str(cls.port), cwd="test_server", shell=True) time.sleep(1) @@ -51,10 +48,12 @@ def setUpClass(self): # sftp://user@localhost:3373/ @classmethod - def tearDownClass(self): - super(MyTestCase, self).setUpClass() + def tearDownClass(cls): + super(SftpTestBase, cls).setUpClass() os.system("pkill sftpserver") + +class SftpTests(SftpTestBase): def test_test_server(self): import paramiko pkey = paramiko.RSAKey.from_private_key_file('id_rsa') @@ -111,8 +110,6 @@ def test_sync_different_separator(self): if clean and op.exists(dst): shutil.rmtree(dst) - - def test_sync_abspath(self): from sftpsync import Sftp @@ -141,7 +138,6 @@ def test_sync_abspath(self): def test_sync_upload(self): from sftpsync import Sftp src = 'to_server/' - src = 'c:/Users/mjirik/projects/sftpsync-py/to_server/' dst = 'to_server/' # create test dir @@ -177,5 +173,76 @@ def test_sync_upload(self): # check if direcotry is empty self.assertEqual(len(dir_list), 0) + +class SftpTestFilePermissions(SftpTestBase): + def setUp(self): + from sftpsync import Sftp + self.sftp = Sftp(self.host, 'paul', 'P4ul', port=self.port) + + self.local_dir = 'test_local_file_permissions' + self.remote_relative_dir = 'test_remote_file_permissions' # Relative to the remote session + self.remote_dir = os.path.join('test_server', self.remote_relative_dir) + + self.test_local_sub_dir = os.path.join(self.local_dir, 'test_dir') + self.test_local_file = os.path.join(self.local_dir, 'test.txt') + self.test_remote_sub_dir = os.path.join(self.remote_dir, 'test_dir') + self.test_remote_file = os.path.join(self.remote_dir, 'test.txt') + + def tearDown(self): + shutil.rmtree(self.local_dir) + shutil.rmtree(self.remote_dir) + self.sftp.client.close() + + def test_file_permissions_download(self): + # Make remote files. + os.makedirs(self.test_remote_sub_dir) + touch(self.test_remote_file) + + # Change permissions. + os.chmod(self.test_remote_sub_dir, os.stat(self.test_remote_sub_dir).st_mode | 0o007) + os.chmod(self.test_remote_file, os.stat(self.test_remote_file).st_mode | 0o007) + + self.sftp.sync(src=self.remote_relative_dir, dst=self.local_dir, download=True) + + # Check permissions match. + self.assertEqual(os.stat(self.test_remote_sub_dir).st_mode, os.stat(self.test_local_sub_dir).st_mode) + self.assertEqual(os.stat(self.test_remote_file).st_mode, os.stat(self.test_local_file).st_mode) + + # Change permissions again (to check that files are re-syncd). + os.chmod(self.test_remote_sub_dir, os.stat(self.test_remote_sub_dir).st_mode & ~0o007) + os.chmod(self.test_remote_file, os.stat(self.test_remote_file).st_mode & ~0o007) + + self.sftp.sync(src=self.remote_relative_dir, dst=self.local_dir, download=True) + + # Check permissions match. + self.assertEqual(os.stat(self.test_remote_sub_dir).st_mode, os.stat(self.test_local_sub_dir).st_mode) + self.assertEqual(os.stat(self.test_remote_file).st_mode, os.stat(self.test_local_file).st_mode) + + def test_file_permissions_upload(self): + # Make local files. + os.makedirs(self.test_local_sub_dir) + touch(self.test_local_file) + + # Change permissions. + os.chmod(self.test_local_sub_dir, os.stat(self.test_local_sub_dir).st_mode | 0o007) + os.chmod(self.test_local_file, os.stat(self.test_local_file).st_mode | 0o007) + + self.sftp.sync(src=self.local_dir, dst=self.remote_relative_dir, download=False) + + # Check permissions match. + self.assertEqual(os.stat(self.test_local_sub_dir).st_mode, os.stat(self.test_remote_sub_dir).st_mode) + self.assertEqual(os.stat(self.test_local_file).st_mode, os.stat(self.test_remote_file).st_mode) + + # Change permissions again (to check that files are re-syncd). + os.chmod(self.test_local_sub_dir, os.stat(self.test_local_sub_dir).st_mode & ~0o007) + os.chmod(self.test_local_file, os.stat(self.test_local_file).st_mode & ~0o007) + + self.sftp.sync(src=self.local_dir, dst=self.remote_relative_dir, download=False) + + # Check permissions match. + self.assertEqual(os.stat(self.test_local_sub_dir).st_mode, os.stat(self.test_remote_sub_dir).st_mode) + self.assertEqual(os.stat(self.test_local_file).st_mode, os.stat(self.test_remote_file).st_mode) + + if __name__ == '__main__': unittest.main() From b7891848b1fcdd14b7250a2d8f79236580f3bc56 Mon Sep 17 00:00:00 2001 From: "Ramsay, Grant (NZ)" Date: Tue, 31 Oct 2017 02:09:50 +1300 Subject: [PATCH 28/37] Close connections in unit tests --- tests/sftpsync_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/sftpsync_test.py b/tests/sftpsync_test.py index 906ecc9..d3286b0 100644 --- a/tests/sftpsync_test.py +++ b/tests/sftpsync_test.py @@ -61,6 +61,7 @@ def test_test_server(self): transport.connect(username='admin', password='admin')# , pkey=pkey) sftp = paramiko.SFTPClient.from_transport(transport) sftp.listdir('.') + transport.close() def test_connection(self): from sftpsync import Sftp @@ -73,6 +74,8 @@ def test_connection(self): dir_list2 = sftp.sftp.listdir_attr("from_server/foo") self.assertEqual(dir_list2[0].filename, 'bar.txt') + sftp.client.close() + def test_sync(self): from sftpsync import Sftp @@ -91,6 +94,8 @@ def test_sync(self): if clean and op.exists(dst): shutil.rmtree(dst) + sftp.client.close() + def test_sync_different_separator(self): from sftpsync import Sftp @@ -110,6 +115,8 @@ def test_sync_different_separator(self): if clean and op.exists(dst): shutil.rmtree(dst) + sftp.client.close() + def test_sync_abspath(self): from sftpsync import Sftp @@ -135,6 +142,8 @@ def test_sync_abspath(self): if clean and op.exists(dst): shutil.rmtree(dst) + sftp.client.close() + def test_sync_upload(self): from sftpsync import Sftp src = 'to_server/' @@ -173,6 +182,8 @@ def test_sync_upload(self): # check if direcotry is empty self.assertEqual(len(dir_list), 0) + sftp.client.close() + class SftpTestFilePermissions(SftpTestBase): def setUp(self): From 30a247eb8981de33869c0ef524b7d5f9b9d508e9 Mon Sep 17 00:00:00 2001 From: "Ramsay, Grant (NZ)" Date: Thu, 2 Nov 2017 17:57:49 +1300 Subject: [PATCH 29/37] Do not set folder permissions for directories that are out of scope Previously if the folder you were syncing was /home/jim/myproject it would try set permissions for /home, /home/jim etc. Likely causing a permission error --- sftpsync/__init__.py | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index 579eed8..1648d7d 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -115,30 +115,33 @@ def _walk(self, *args, **kwargs): else: return self._walk_local(*args, **kwargs) - def _makedirs_dst(self, path, src_stat, remote=True, dry=False): + def _mkdir_dst(self, path, src_stat=None, remote=True, dry=False): if remote: - paths = [] - while path not in ('/', ''): - paths.insert(0, path) - path = os.path.dirname(path) - - for path in paths: - try: - self.sftp.lstat(path) - except Exception: - if not dry: - self.sftp.mkdir(path) - logger.debug('created destination directory %s', path) + try: + self.sftp.lstat(path) + except Exception: if not dry: - self.sftp.chmod(path, src_stat.st_mode) + self.sftp.mkdir(path) + logger.debug('created destination directory %s', path) + if not dry and src_stat: + self.sftp.chmod(path, src_stat.st_mode) else: if not os.path.exists(path): if not dry: - os.makedirs(path) + os.mkdir(path) logger.debug('created destination directory %s', path) - if not dry: + if not dry and src_stat: os.chmod(path, src_stat.st_mode) + def _makedirs_dst(self, path, remote=True, dry=False): + paths = [] + while path not in ('/', ''): + paths.insert(0, path) + path = os.path.dirname(path) + + for path in paths: + self._mkdir_dst(path, src_stat=None, remote=remote, dry=dry) + def _validate_src(self, file, include, exclude): for re_ in include: if not re_.search(file): @@ -232,11 +235,7 @@ def sync(self, src, dst, download=True, include=None, exclude=None, delete=False if not src: src = '/' - if download: - stat = self.sftp.lstat(src) - else: - stat = os.stat(src) - self._makedirs_dst(dst, stat, remote=not download, dry=dry) + self._makedirs_dst(dst, remote=not download, dry=dry) started = datetime.utcnow() total_size = 0 @@ -255,7 +254,7 @@ def sync(self, src, dst, download=True, include=None, exclude=None, delete=False dst_list[type].append(dst_file) if type == 'dir': - self._makedirs_dst(dst_file, stat, remote=not download, dry=dry) + self._mkdir_dst(dst_file, stat, remote=not download, dry=dry) elif type == 'file': if not self._validate_dst(dst_file, stat, remote=not download): if not dry: From a8b52f74a08d166a0ad628ebffe67447bbcc8612 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Sun, 3 Dec 2017 21:41:57 +0100 Subject: [PATCH 30/37] =?UTF-8?q?Bump=20version:=201.0.8=20=E2=86=92=201.0?= =?UTF-8?q?.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- meta.yaml | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/meta.yaml b/meta.yaml index 55b3d7a..811f6fd 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,15 +1,15 @@ package: name: sftpsync - version: "1.0.8" + version: "1.0.9" source: # this is used for build from git hub - git_rev: 1.0.8 + git_rev: 1.0.9 git_url: https://github.com/mjirik/sftpsync-py.git # this is used for pypi - # fn: io3d-1.0.80.tar.gz - # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.80.tar.gz + # fn: io3d-1.0.90.tar.gz + # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.90.tar.gz # md5: a3ce512c4c97ac2410e6dcc96a801bd8 # patches: # List any patch files here diff --git a/setup.py b/setup.py index b9a1b90..126e831 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='1.0.8', + version='1.0.9', url='https://github.com/mjirik/sftpsync', author='Jerome Clerc', author_email='', From 0d3639fa1aba097751871fb80c3080236d72a9ac Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Wed, 18 Apr 2018 06:47:37 +0200 Subject: [PATCH 31/37] =?UTF-8?q?Bump=20version:=201.0.9=20=E2=86=92=201.0?= =?UTF-8?q?.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- meta.yaml | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/meta.yaml b/meta.yaml index 811f6fd..09fdc9b 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,15 +1,15 @@ package: name: sftpsync - version: "1.0.9" + version: "1.0.10" source: # this is used for build from git hub - git_rev: 1.0.9 + git_rev: 1.0.10 git_url: https://github.com/mjirik/sftpsync-py.git # this is used for pypi - # fn: io3d-1.0.90.tar.gz - # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.90.tar.gz + # fn: io3d-1.0.100.tar.gz + # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.100.tar.gz # md5: a3ce512c4c97ac2410e6dcc96a801bd8 # patches: # List any patch files here diff --git a/setup.py b/setup.py index 126e831..0d216d8 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='1.0.9', + version='1.0.10', url='https://github.com/mjirik/sftpsync', author='Jerome Clerc', author_email='', From c139c274330a885d672352fa53ac8ff0be811e29 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Wed, 18 Apr 2018 11:48:51 +0200 Subject: [PATCH 32/37] fix windows path starting with 'C:/' --- sftpsync/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sftpsync/__init__.py b/sftpsync/__init__.py index 1648d7d..51c8935 100755 --- a/sftpsync/__init__.py +++ b/sftpsync/__init__.py @@ -136,6 +136,9 @@ def _mkdir_dst(self, path, src_stat=None, remote=True, dry=False): def _makedirs_dst(self, path, remote=True, dry=False): paths = [] while path not in ('/', ''): + # break also if path is like C:/ + if len(path) == 3 and path[1:] == ":/": + break paths.insert(0, path) path = os.path.dirname(path) From 77202c504d89b9d6b34c9086c7194c6f56834234 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Wed, 18 Apr 2018 11:49:02 +0200 Subject: [PATCH 33/37] =?UTF-8?q?Bump=20version:=201.0.10=20=E2=86=92=201.?= =?UTF-8?q?0.11?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- meta.yaml | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/meta.yaml b/meta.yaml index 09fdc9b..fa7003f 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,15 +1,15 @@ package: name: sftpsync - version: "1.0.10" + version: "1.0.11" source: # this is used for build from git hub - git_rev: 1.0.10 + git_rev: 1.0.11 git_url: https://github.com/mjirik/sftpsync-py.git # this is used for pypi - # fn: io3d-1.0.100.tar.gz - # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.100.tar.gz + # fn: io3d-1.0.110.tar.gz + # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.110.tar.gz # md5: a3ce512c4c97ac2410e6dcc96a801bd8 # patches: # List any patch files here diff --git a/setup.py b/setup.py index 0d216d8..4777424 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='1.0.10', + version='1.0.11', url='https://github.com/mjirik/sftpsync', author='Jerome Clerc', author_email='', From 24319164f5ea400dd597783f493e674e52a7a95b Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Thu, 3 May 2018 11:11:48 +0200 Subject: [PATCH 34/37] =?UTF-8?q?Bump=20version:=201.0.11=20=E2=86=92=201.?= =?UTF-8?q?0.12?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- meta.yaml | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/meta.yaml b/meta.yaml index fa7003f..a191eec 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,15 +1,15 @@ package: name: sftpsync - version: "1.0.11" + version: "1.0.12" source: # this is used for build from git hub - git_rev: 1.0.11 + git_rev: 1.0.12 git_url: https://github.com/mjirik/sftpsync-py.git # this is used for pypi - # fn: io3d-1.0.110.tar.gz - # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.110.tar.gz + # fn: io3d-1.0.120.tar.gz + # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.120.tar.gz # md5: a3ce512c4c97ac2410e6dcc96a801bd8 # patches: # List any patch files here diff --git a/setup.py b/setup.py index 4777424..80c4022 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='1.0.11', + version='1.0.12', url='https://github.com/mjirik/sftpsync', author='Jerome Clerc', author_email='', From 22e6bf07e85d675e29801fcd7d8333e773cddfcf Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Thu, 15 Aug 2019 07:19:59 +0200 Subject: [PATCH 35/37] moved meta.yml --- meta.yaml => conda-recipe/meta.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename meta.yaml => conda-recipe/meta.yaml (100%) diff --git a/meta.yaml b/conda-recipe/meta.yaml similarity index 100% rename from meta.yaml rename to conda-recipe/meta.yaml From 8fce205cdfc046e3dd0c6811d8407cd3ae16cb77 Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Thu, 15 Aug 2019 07:35:35 +0200 Subject: [PATCH 36/37] conda recipe update --- .condarc | 3 ++ .travis.yml | 83 +++++++++++++++++++++++++++++++++++++++++++ conda-recipe/bld.bat | 2 ++ conda-recipe/build.sh | 3 ++ setup.cfg | 16 +++++++++ 5 files changed, 107 insertions(+) create mode 100644 .condarc create mode 100644 .travis.yml create mode 100644 conda-recipe/bld.bat create mode 100644 conda-recipe/build.sh create mode 100644 setup.cfg diff --git a/.condarc b/.condarc new file mode 100644 index 0000000..d5faf1f --- /dev/null +++ b/.condarc @@ -0,0 +1,3 @@ +channels: + - default +# - mjirik \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..0e77a5c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,83 @@ +language: python +os: linux +# Ubuntu 14.04 Trusty support +sudo: required +# dist: trusty +# install new cmake +#addons: +# apt: +# packages: +# - cmake +# sources: +# - kalakris-cmake +env: + # - CONDA_PYTHON_VERSION=2.7 + - CONDA_PYTHON_VERSION=3.6 + - CONDA_PYTHON_VERSION=3.7 + +matrix: + allow_failures: + - env: CONDA_PYTHON_VERSION=2.7 + - env: CONDA_PYTHON_VERSION=3.7 + fast_finish: true +# virtualenv: +# system_site_packages: true +before_script: + # GUI + - "export DISPLAY=:99.0" +before_script: + # GUI + - "export DISPLAY=:99.0" + # - "sh -e /etc/init.d/xvfb start" + # - sleep 3 # give xvfb sume time to start + +before_install: + - sudo apt-get update + - sudo apt-get install -qq cmake libinsighttoolkit3-dev libpng12-dev libgdcm2-dev + + - wget http://home.zcu.cz/~mjirik/lisa/install/install_conda.sh && source install_conda.sh + # We do this conditionally because it saves us some downloading if the + # version is the same. + # - if [[ "$CONDA_PYTHON_VERSION" == "2.7" ]]; then + # echo "python 2" + # else + # echo "python 3" + # fi + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda config --add channels mjirik + - conda config --add channels conda-forge + - conda update -q conda + # Useful for debugging any issues with conda + - conda info -a + +# command to install dependencies +install: + + # - sudo apt-get install -qq $(< apt_requirements.txt) + - conda create --yes -n travis pip nose coveralls python=$CONDA_PYTHON_VERSION + - source activate travis +# - Install dependencies + - conda install --yes -c SimpleITK -c luispedro -c mjirik --file requirements_conda.txt + - conda install --yes pip nose coveralls pytest pytest-cov +# - pip install -r requirements_pip.txt +# - "echo $LD_LIBRARY_PATH" +# - "pip install -r requirements.txt" +# - 'mkdir build' +# - "cd build" +# - "cmake .." +# - "cmake --build ." +# - "sudo make install" +# - pip install . +# - "cd .." +# - 'echo "include /usr/local/lib" | sudo tee -a /etc/ld.so.conf' +# - 'sudo ldconfig -v' +# - conda list -e +# - python -m io3d.datasets -l 3Dircadb1.1 jatra_5mm exp_small sliver_training_001 io3d_sample_data head volumetrie +# command to run tests +# script: nosetests -v --with-coverage --cover-package=sftpsync + +script: + - python -m pytest --cov=sftpsync/ +after_success: + - coveralls diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat new file mode 100644 index 0000000..6021130 --- /dev/null +++ b/conda-recipe/bld.bat @@ -0,0 +1,2 @@ +"%PYTHON%" setup.py install +if errorlevel 1 exit 1 \ No newline at end of file diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh new file mode 100644 index 0000000..8e25a14 --- /dev/null +++ b/conda-recipe/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +$PYTHON setup.py install diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..209a0ef --- /dev/null +++ b/setup.cfg @@ -0,0 +1,16 @@ + +[bumpversion] +current_version = 1.0.12 +files = setup.py conda-recipe/meta.yaml +commit = True +tag = True +tag_name = {new_version} + +# [nosetests] +# attr = !interactive,!slow + +[tool:pytest] +addopts = -m "not interactive and not slow" +markers = + interactive: marks interactive tests + slow: marks slow tests From 0568c2aade7d337fc13e6697a84bed44b6cb323d Mon Sep 17 00:00:00 2001 From: Miroslav Jirik Date: Thu, 15 Aug 2019 07:39:23 +0200 Subject: [PATCH 37/37] =?UTF-8?q?Bump=20version:=201.0.12=20=E2=86=92=201.?= =?UTF-8?q?0.13?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conda-recipe/meta.yaml | 8 ++++---- setup.cfg | 13 +++++-------- setup.py | 2 +- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index a191eec..a777873 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -1,15 +1,15 @@ package: name: sftpsync - version: "1.0.12" + version: "1.0.13" source: # this is used for build from git hub - git_rev: 1.0.12 + git_rev: 1.0.13 git_url: https://github.com/mjirik/sftpsync-py.git # this is used for pypi - # fn: io3d-1.0.120.tar.gz - # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.120.tar.gz + # fn: io3d-1.0.130.tar.gz + # url: https://pypi.python.org/packages/source/i/io3d/io3d-1.0.130.tar.gz # md5: a3ce512c4c97ac2410e6dcc96a801bd8 # patches: # List any patch files here diff --git a/setup.cfg b/setup.cfg index 209a0ef..f0cb6e5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,16 +1,13 @@ - [bumpversion] -current_version = 1.0.12 +current_version = 1.0.13 files = setup.py conda-recipe/meta.yaml commit = True tag = True tag_name = {new_version} -# [nosetests] -# attr = !interactive,!slow - [tool:pytest] addopts = -m "not interactive and not slow" -markers = - interactive: marks interactive tests - slow: marks slow tests +markers = + interactive: marks interactive tests + slow: marks slow tests + diff --git a/setup.py b/setup.py index 80c4022..4faafad 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ # Versions should comply with PEP440. For a discussion on single-sourcing # the version across setup.py and the project code, see # http://packaging.python.org/en/latest/tutorial.html#version - version='1.0.12', + version='1.0.13', url='https://github.com/mjirik/sftpsync', author='Jerome Clerc', author_email='',