Satisfy testing on macOS

Author: amotl

.github/workflows/main.yml renamed to .github/workflows/nightly.yml

@@ -1,4 +1,4 @@
-name: Tests
+name: Nightly
 
 on:
   push:
@@ -14,15 +14,15 @@ jobs:
   test:
     name: "Python: ${{ matrix.python-version }}
      SQLA: ${{ matrix.sqla-version }}
-     CrateDB: ${{ matrix.crate-version }}
+     CrateDB: ${{ matrix.cratedb-version }}
      on ${{ matrix.os }}"
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        crate-version: [nightly]
         os: [ubuntu-latest]
-        sqla-version: ['1.1.18', '1.2.19', '1.3.23']
         python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
+        cratedb-version: ['nightly']
+        sqla-version: ['1.1.18', '1.2.19', '1.3.23']
       fail-fast: false
 
     steps:
@@ -45,12 +45,11 @@ jobs:
           sed -ir 's/SQLAlchemy.*/SQLAlchemy = ${{ matrix.sqla-version }}/g' versions.cfg
 
           # replace CrateDB version
-          if [ ${{ matrix.crate-version }} = "nightly" ]; then
+          if [ ${{ matrix.cratedb-version }} = "nightly" ]; then
             sed -ir 's/releases/releases\/nightly/g' base.cfg
             sed -ir 's/crate_server.*/crate_server = latest/g' versions.cfg
           else
-            sed -ir 's/crate-/crate_/g' base.cfg
-            sed -ir 's/crate_server.*/crate_server = ${{ matrix.crate-version }}/g' versions.cfg
+            sed -ir 's/crate_server.*/crate_server = ${{ matrix.cratedb-version }}/g' versions.cfg
           fi
 
           buildout -n -c base.cfg

.github/workflows/tests.yml

@@ -0,0 +1,58 @@
+name: Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  workflow_dispatch:
+
+
+jobs:
+  test:
+    name: "Python: ${{ matrix.python-version }}
+     SQLA: ${{ matrix.sqla-version }}
+     CrateDB: ${{ matrix.cratedb-version }}
+     on ${{ matrix.os }}"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
+        cratedb-version: ['4.5.0']
+        sqla-version: ['1.1.18', '1.2.19', '1.3.23']
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@master
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+
+          # Workaround needed for Python 3.5
+          python -m pip install --upgrade "setuptools>=31,<51"
+
+          pip install zc.buildout==2.13.4
+
+          # replace SQLAlchemy version
+          sed -ir 's/SQLAlchemy.*/SQLAlchemy = ${{ matrix.sqla-version }}/g' versions.cfg
+
+          # replace CrateDB version
+          if [ ${{ matrix.cratedb-version }} = "nightly" ]; then
+            sed -ir 's/releases/releases\/nightly/g' base.cfg
+            sed -ir 's/crate_server.*/crate_server = latest/g' versions.cfg
+          else
+            sed -ir 's/crate_server.*/crate_server = ${{ matrix.cratedb-version }}/g' versions.cfg
+          fi
+
+          buildout -n -c base.cfg
+
+      - name: Test
+        run: |
+          bin/flake8
+          bin/coverage run bin/test -vv1

base.cfg

@@ -19,11 +19,21 @@ eggs = crate
 recipe = zc.recipe.egg
 eggs = createcoverage
 
-[crate]
+[crate:linux]
 recipe = hexagonit.recipe.download
 url = https://cdn.crate.io/downloads/releases/crate-${versions:crate_server}.tar.gz
 strip-top-level-dir = true
 
+[crate:macosx]
+recipe = hexagonit.recipe.download
+url = https://cdn.crate.io/downloads/releases/cratedb/x64_mac/crate-${versions:crate_server}.tar.gz
+strip-top-level-dir = true
+
+[crate:windows]
+recipe = hexagonit.recipe.download
+url = https://cdn.crate.io/downloads/releases/cratedb/x64_windows/crate-${versions:crate_server}.zip
+strip-top-level-dir = true
+
 [test]
 relative-paths = true
 recipe = zc.recipe.testrunner

src/crate/testing/doctests/layer.txt

@@ -225,26 +225,9 @@ We might have to wait a moment before the cluster is finally created::
 From Uri
 --------
 
-The CrateLayer can also be created by providing a URI that points to a Crate
+The CrateLayer can also be created by providing a URI that points to a CrateDB
 tarball::
 
-    >>> import urllib.request
-    >>> import json
-    >>> with urllib.request.urlopen('http://crate.io/versions.json') as response:
-    ...     versions = json.loads(response.read().decode())
-    ...     version = versions['crate_testing']
-
-    >>> uri = 'https://cdn.crate.io/downloads/releases/crate-{}.tar.gz'.format(version)
-    >>> tmpdir = tempfile.mkdtemp()
-    >>> layer = CrateLayer.from_uri(
-    ...             uri, name='crate-uri', http_port=42203, directory=tmpdir)
-    >>> layer.setUp()
-
-    >>> work_dir = os.path.join(tmpdir, 'crate-' + version)
-    >>> os.path.exists(work_dir)
-    True
-
-    >>> layer.tearDown()
-
-    >>> os.path.exists(work_dir)
-    False
+    uri = 'https://cdn.crate.io/downloads/releases/crate-{}.tar.gz'.format(version)
+    layer = CrateLayer.from_uri(
+                uri, name='crate-uri', http_port=42203, directory=tmpdir)

src/crate/testing/layer.py

@@ -72,6 +72,8 @@ def prepend_http(host):
 
 
 def _download_and_extract(uri, directory):
+    sys.stderr.write("\nINFO:    Downloading CrateDB archive from {} into {}".format(uri, directory))
+    sys.stderr.flush()
     with io.BytesIO(urlopen(uri).read()) as tmpfile:
         with tarfile.open(fileobj=tmpfile) as t:
             t.extractall(directory)
@@ -160,7 +162,8 @@ def from_uri(uri,
         crate_home = os.path.join(directory, crate_dir)
 
         if os.path.exists(crate_home):
-            sys.stderr.write('Not extracting Crate tarball because folder already exists')
+            sys.stderr.write("\nWARNING: Not extracting Crate tarball because folder already exists")
+            sys.stderr.flush()
         else:
             _download_and_extract(uri, directory)
 

src/crate/testing/test_layer.py

@@ -18,11 +18,15 @@
 # However, if you have executed another commercial license agreement
 # with Crate these terms will supersede the license and you may use the
 # software solely pursuant to the terms of the relevant commercial agreement.
-
+import json
 import os
 import tempfile
+import urllib
+from distutils.version import LooseVersion
 from unittest import TestCase, mock
 from io import BytesIO
+
+import crate
 from .layer import CrateLayer, prepend_http, http_url_from_host_port, wait_for_http_url
 
 
@@ -58,6 +62,22 @@ def test_wait_for_http(self):
         addr = wait_for_http_url(log=log, timeout=1)
         self.assertEqual(None, addr)
 
+    @mock.patch.object(crate.testing.layer, "_download_and_extract", lambda uri, directory: None)
+    def test_layer_from_uri(self):
+        """
+        The CrateLayer can also be created by providing an URI that points to
+        a CrateDB tarball.
+        """
+        with urllib.request.urlopen("https://crate.io/versions.json") as response:
+            versions = json.loads(response.read().decode())
+            version = versions["crate_testing"]
+
+        self.assertGreaterEqual(LooseVersion(version), LooseVersion("4.5.0"))
+
+        uri = "https://cdn.crate.io/downloads/releases/crate-{}.tar.gz".format(version)
+        layer = CrateLayer.from_uri(uri, name="crate-by-uri", http_port=42203)
+        self.assertIsInstance(layer, CrateLayer)
+
     @mock.patch.dict('os.environ', {}, clear=True)
     def test_java_home_env_not_set(self):
         with tempfile.TemporaryDirectory() as tmpdir:

versions.cfg

@@ -1,5 +1,5 @@
 [versions]
-crate_server = 4.4.2
+crate_server = 4.5.0
 
 flake8 = 3.7.9
 mccabe = 0.6.1