diff --git a/cassandra_mirror/backup.py b/cassandra_mirror/backup.py index 3b61a2a..839ce70 100644 --- a/cassandra_mirror/backup.py +++ b/cassandra_mirror/backup.py @@ -163,7 +163,7 @@ def upload_s3(cmd, s3_object): '-k', s3_object.key, ] - cmd | gof3r_cmd & FG + (cmd | gof3r_cmd) & FG else: with cmd.bgrun() as proc: s3_object.upload_fileobj(proc.stdout) diff --git a/cassandra_mirror/restore.py b/cassandra_mirror/restore.py index 69c906c..eb34ca2 100644 --- a/cassandra_mirror/restore.py +++ b/cassandra_mirror/restore.py @@ -65,13 +65,13 @@ def pipe(): def download_s3(cmd, s3_object): logger.info('Downloading %s', s3_object.key) if gof3r: - gof3r_cmd = s3gof3r[ + gof3r_cmd = gof3r[ 'get', '--no-md5', '-b', s3_object.bucket_name, '-k', s3_object.key, ] - gof3r_cmd | cmd & FG + (gof3r_cmd | cmd) & FG else: with cmd.bgrun(stdin=PIPE) as future: s3_object.download_fileobj(future.stdin) diff --git a/cassandra_mirror/util.py b/cassandra_mirror/util.py index b0722b3..5edca3a 100644 --- a/cassandra_mirror/util.py +++ b/cassandra_mirror/util.py @@ -6,19 +6,14 @@ from tempfile import TemporaryDirectory import os +import pkg_resources import boto3 import json import yaml -from plumbum import local +from plumbum.machines.local import LocalCommand from plumbum.commands.processes import CommandNotFound -gof3r = None -try: - # local.get raises exceptions, violating Python norms of .get() not - # raising exceptions - gof3r = local.get('gof3r') -except CommandNotFound: - pass +gof3r = LocalCommand(pkg_resources.resource_filename(__name__, "gof3r")) s3 = boto3.resource('s3') diff --git a/debian/control b/debian/control index cc0dfd2..244e5ed 100644 --- a/debian/control +++ b/debian/control @@ -14,6 +14,6 @@ Build-Depends: Package: cassandra-mirror Architecture: amd64 -Depends: ${shlibs:Depends}, ${misc:Depends}, s3gof3r +Depends: ${shlibs:Depends}, ${misc:Depends} Description: mirrors data from Cassandra into other datastores diff --git a/download.sh b/download.sh new file mode 100755 index 0000000..04147e2 --- /dev/null +++ b/download.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -o errexit +curl -LO https://github.com/rlmcpherson/s3gof3r/releases/download/v0.5.0/gof3r_0.5.0_linux_amd64.tar.gz +echo d88f199d1580d8c8cac26ba39e15dc6e2126d20e56c3894bd37a226e8b3e665c gof3r_0.5.0_linux_amd64.tar.gz | sha256sum -c +tar --strip-components 1 -C cassandra_mirror -xvf gof3r_0.5.0_linux_amd64.tar.gz diff --git a/setup.py b/setup.py index 035fe21..8f28bed 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ 'check=cassandra_mirror.check:do_check', ] ), + package_data={ 'cassandra_mirror': ['gof3r'] }, install_requires=[ 'boto3>=1.4.2', 'PyYAML>=3.12',