Fetch test data from s3 instead of using git-lfs (#925)

* Fetch test data from s3 instead of using git-lfs, and save it in circleCI cache.

* Use wget in tests.sh when preparing tests data (instead of aws sync) to allow retrieval of test data from s3 without aws configure + update onnx benchmark accordingly

Author: alonre24

.circleci/config.yml

@@ -140,11 +140,18 @@ commands:
       - run:
           name: Build
           command: make -C opt all SHOW=1
+      - restore_cache:
+          keys:
+            - v1.2-tests_data
       - run:
           name: Test
           command: |
             make -C opt test SHOW=1
           no_output_timeout: 20m
+      - save_cache:
+          paths:
+            - tests/flow/test_data
+          key: v1.2-tests_data
       - run:
           name: Package
           command: |
@@ -182,6 +189,9 @@ commands:
           name: Build for valgrind
           command: |
             make -C opt all VALGRIND=1 SHOW=1
+      - restore_cache:
+          keys:
+            - v1.2-tests_data
       - run:
           name: Test with valgrind
           command: |
@@ -198,6 +208,9 @@ commands:
       - restore_cache:
           keys:
           - v1.2.5-deps-{{ checksum "get_deps.sh" }}-gpu
+      - restore_cache:
+          keys:
+            - v1.2-tests_data-gpu
       - relocate-docker-storage
       - run:
           name: Build
@@ -215,6 +228,10 @@ commands:
             mkdir -p $HOME/tests
             docker run --gpus all -v $HOME/tests:/build/tests/flow/logs -it --rm redisai-gpu:latest-test
           no_output_timeout: 40m
+      - save_cache:
+          paths:
+            - tests/flow/test_data
+          key: v1.2-tests_data-gpu
       - store_artifacts:
           path: /home/circleci/tests
 
@@ -331,6 +348,9 @@ jobs:
           name: Build for coverage
           command: |
             make -C opt all COV=1 SHOW=1
+      - restore_cache:
+          keys:
+            - v1.2-tests_data
       - run:
           name: Test with coverage
           command: |

.gitignore

@@ -137,4 +137,7 @@ dkms.conf
 .idea_modules/
 
 # docs site
-site/
+site/
+
+# test data
+tests/flow/test_data

tests/flow/onnx_benchmark.py

@@ -1,5 +1,4 @@
-import os
-
+import subprocess
 from RLTest import Env
 from includes import *
 import shutil
@@ -25,7 +24,6 @@ def handler(signum, frame):
 def _exit():
     # remove the logs that were auto generated by redis
     shutil.rmtree('logs', ignore_errors=True)
-    print("from exit\n")
     sys.exit(1)
 
 
@@ -172,6 +170,13 @@ def run_bert():
                         help='The number of parallel clients that send consecutive run requests per model')
     args = parser.parse_args()
 
+    if not os.path.isdir('tests/flow/test_data'):
+        print("Downloading tests data from s3...")
+        with open('tests/flow/test_data_files.txt') as f:
+            for resource in f.readlines():
+                subprocess.call(['wget', '-q', '-x', '-nH', '--cut-dirs=2', resource[:-1], '-P', 'tests/flow/test_data'])
+        print("Done")
+
     terminate_flag = 0
     print(f'Running ONNX benchmark on RedisAI, using {args.num_threads} working threads')
     env = Env(module='install-cpu/redisai.so',