From 13aab6f522c4082c1b6cf136838fb6bbbe308a85 Mon Sep 17 00:00:00 2001 From: ybnd Date: Sat, 15 Oct 2022 12:53:41 +0200 Subject: [PATCH 1/5] Return false on cache check if not using cache --- shapeflow/core/backend.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/shapeflow/core/backend.py b/shapeflow/core/backend.py index b27e031c..50ebac09 100644 --- a/shapeflow/core/backend.py +++ b/shapeflow/core/backend.py @@ -154,9 +154,13 @@ def _drop(self, key: str): del self._cache[key] def _is_cached(self, method, *args): + key = self._get_key(method, *args) if self._cache is None: - raise CacheAccessError - return self._get_key(method, *args) in self._cache + if settings.cache.do_cache: + raise CacheAccessError + else: + return False + return key in self._cache def cached_call(self, method, *args, **kwargs): # todo: kwargs necessary? """Call a method or get the result from the cache if available. From 00c58caa561d6f5d2b59fd25e7ee579573142d50 Mon Sep 17 00:00:00 2001 From: ybnd Date: Sat, 15 Oct 2022 13:06:18 +0200 Subject: [PATCH 2/5] Main plugin classes should match plugin module name --- docs/source/python-library.rst | 9 +++------ shapeflow/plugins/Area_mm2.py | 2 +- shapeflow/plugins/BackgroundFilter.py | 4 ++-- shapeflow/plugins/HsvRangeFilter.py | 4 ++-- shapeflow/plugins/PerspectiveTransform.py | 2 +- shapeflow/plugins/PixelSum.py | 2 +- shapeflow/plugins/Volume_uL.py | 4 ++-- test/test_config.py | 4 ++-- 8 files changed, 14 insertions(+), 17 deletions(-) diff --git a/docs/source/python-library.rst b/docs/source/python-library.rst index aad3c115..1a37375e 100644 --- a/docs/source/python-library.rst +++ b/docs/source/python-library.rst @@ -158,7 +158,7 @@ PerspectiveTransform .. automodule:: shapeflow.plugins.PerspectiveTransform :members: - :private-members: _Config, _Transform + :private-members: _Config :show-inheritance: .. _filters: @@ -171,7 +171,7 @@ HsvRangeFilter .. automodule:: shapeflow.plugins.HsvRangeFilter :members: - :private-members: _Config, _Filter + :private-members: _Config :show-inheritance: BackgroundFilter @@ -179,7 +179,7 @@ BackgroundFilter .. automodule:: shapeflow.plugins.BackgroundFilter :members: - :private-members: _Config, _Filter + :private-members: _Config :show-inheritance: .. _features: @@ -192,7 +192,6 @@ PixelSum .. automodule:: shapeflow.plugins.PixelSum :members: - :private-members: _Feature :show-inheritance: Area_mm2 @@ -200,7 +199,6 @@ Area_mm2 .. automodule:: shapeflow.plugins.Area_mm2 :members: - :private-members: _Feature :show-inheritance: Volume_uL @@ -208,7 +206,6 @@ Volume_uL .. automodule:: shapeflow.plugins.Volume_uL :members: - :private-members: _Config, _Feature :show-inheritance: diff --git a/shapeflow/plugins/Area_mm2.py b/shapeflow/plugins/Area_mm2.py index 34cf01e1..4c42781f 100644 --- a/shapeflow/plugins/Area_mm2.py +++ b/shapeflow/plugins/Area_mm2.py @@ -6,7 +6,7 @@ @extend(FeatureType, True) -class _Feature(MaskFunction): +class Area_mm2(MaskFunction): """Convert :mod:`~shapeflow.plugins.PixelSum` to an area in mm², taking into account the DPI of the design file. """ diff --git a/shapeflow/plugins/BackgroundFilter.py b/shapeflow/plugins/BackgroundFilter.py index 6ceb916a..d248229e 100644 --- a/shapeflow/plugins/BackgroundFilter.py +++ b/shapeflow/plugins/BackgroundFilter.py @@ -16,7 +16,7 @@ @extend(ConfigType, True) class _Config(FilterConfig): - """Configuration for :class:`shapeflow.plugins.BackgroundFilter._Filter` + """Configuration for :class:`shapeflow.plugins.BackgroundFilter.BackgroundFilter` """ color: HsvColor = Field(default=HsvColor()) """See :attr:`shapeflow.plugins.HsvRangeFilter._Config.color` @@ -54,7 +54,7 @@ def c1(self) -> HsvColor: @extend(FilterType, True) -class _Filter(FilterInterface): +class BackgroundFilter(FilterInterface): """Filters out colors outside of a :class:`~shapeflow.maths.colors.HsvColor` radius around a center color and inverts the resulting image. """ diff --git a/shapeflow/plugins/HsvRangeFilter.py b/shapeflow/plugins/HsvRangeFilter.py index 467a5191..8011fc63 100644 --- a/shapeflow/plugins/HsvRangeFilter.py +++ b/shapeflow/plugins/HsvRangeFilter.py @@ -14,7 +14,7 @@ @extend(ConfigType, True) class _Config(FilterConfig): - """Configuration for :class:`shapeflow.plugins.HsvRangeFilter._Filter` + """Configuration for :class:`shapeflow.plugins.HsvRangeFilter.HsvRangeFilter` """ color: HsvColor = Field(default_factory=HsvColor) """The center color. @@ -80,7 +80,7 @@ def c1(self) -> HsvColor: @extend(FilterType, True) -class _Filter(FilterInterface): +class HsvRangeFilter(FilterInterface): """Filters out colors outside of a :class:`~shapeflow.maths.colors.HsvColor` radius around a center color. """ diff --git a/shapeflow/plugins/PerspectiveTransform.py b/shapeflow/plugins/PerspectiveTransform.py index 2d42249a..1cc77064 100644 --- a/shapeflow/plugins/PerspectiveTransform.py +++ b/shapeflow/plugins/PerspectiveTransform.py @@ -18,7 +18,7 @@ class _Config(TransformConfig): # todo: not really necessary? @extend(TransformType, True) -class _Transform(TransformInterface): +class PerspectiveTransform(TransformInterface): """Wraps ``OpenCV``’s `getPerspectiveTransform `_ function to estimate the transformation matrix and `warpPerspective `_ to apply it to a video frame or a coordinate. diff --git a/shapeflow/plugins/PixelSum.py b/shapeflow/plugins/PixelSum.py index 193fc87c..06c2f794 100644 --- a/shapeflow/plugins/PixelSum.py +++ b/shapeflow/plugins/PixelSum.py @@ -7,7 +7,7 @@ @extend(FeatureType, True) -class _Feature(MaskFunction): +class PixelSum(MaskFunction): """The most basic feature: it just returns the number of ``True`` pixels the filtered frame. """ diff --git a/shapeflow/plugins/Volume_uL.py b/shapeflow/plugins/Volume_uL.py index 44657591..3fe7ca3c 100644 --- a/shapeflow/plugins/Volume_uL.py +++ b/shapeflow/plugins/Volume_uL.py @@ -8,7 +8,7 @@ @extend(ConfigType, True) class _Config(FeatureConfig): - """Configuration for :class:`shapeflow.plugins.Volume_uL._Feature` + """Configuration for :class:`shapeflow.plugins.Volume_uL.Volume_uL` """ h: float = Field(default=0.153, description='height (mm)') """The channel height of the chip. @@ -16,7 +16,7 @@ class _Config(FeatureConfig): @extend(FeatureType, True) -class _Feature(MaskFunction): +class Volume_uL(MaskFunction): """Multiply :mod:`~shapeflow.plugins.Area_mm2` by a channel height in mm to estimate the volume in µL. """ diff --git a/test/test_config.py b/test/test_config.py index 1983d30d..7ee86841 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -3,7 +3,7 @@ import yaml from pydantic import Field, validator -from shapeflow.plugins.HsvRangeFilter import _Filter +from shapeflow.plugins.HsvRangeFilter import HsvRangeFilter from shapeflow.video import * from shapeflow.core.config import Factory, BaseConfig, VERSION, CLASS from shapeflow.core import EnforcedStr @@ -32,7 +32,7 @@ def test_comparisons(self): self.assertNotEqual(ColorSpace('hsv'), ColorSpace('bgr')) def test_factory(self): - self.assertEqual(_Filter, FilterType('hsv range').get()) + self.assertEqual(HsvRangeFilter, FilterType('hsv range').get()) def test_subclassing(self): class TestEnfStr(EnforcedStr): From 86f60847fb06cab75f7c43b02994083f7f0a167d Mon Sep 17 00:00:00 2001 From: ybnd Date: Sat, 15 Oct 2022 13:29:39 +0200 Subject: [PATCH 3/5] Skip invalid analyzers when restoring state --- shapeflow/core/backend.py | 1 - shapeflow/main.py | 13 ++++++++++--- shapeflow/video.py | 8 ++++---- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/shapeflow/core/backend.py b/shapeflow/core/backend.py index 50ebac09..8ebef37b 100644 --- a/shapeflow/core/backend.py +++ b/shapeflow/core/backend.py @@ -883,7 +883,6 @@ def launch(self) -> bool: return self.launched else: - log.warning(f"{self.__class__.__qualname__} can not be launched.") # todo: try to be more verbose return False def get_name(self) -> str: diff --git a/shapeflow/main.py b/shapeflow/main.py index 33248484..6d591ce3 100644 --- a/shapeflow/main.py +++ b/shapeflow/main.py @@ -850,10 +850,17 @@ def load_state(self) -> None: analyzer._set_id(id) analyzer.set_eventstreamer(self._server.eventstreamer) - analyzer.launch() + ok = analyzer.launch() + + if ok: + self._add(analyzer) + self._history.add_analysis(analyzer, model) + else: + log.error( + f"Failed to restore analyzer '{id}' from previous application state. " + f"This probably happened because the video or design file(s) have been moved or deleted" + ) - self._add(analyzer) - self._history.add_analysis(analyzer, model) except FileNotFoundError: pass except EOFError: diff --git a/shapeflow/video.py b/shapeflow/video.py index abd3b31e..b0561c89 100644 --- a/shapeflow/video.py +++ b/shapeflow/video.py @@ -984,7 +984,7 @@ def can_launch(self) -> bool: log.warning(f"invalid video file: {self.config.video_path}") if self.config.design_path is not None: design_ok = os.path.isfile(self.config.design_path) - if not video_ok: + if not design_ok: log.warning(f"invalid design file: {self.config.design_path}") return video_ok and design_ok @@ -1710,10 +1710,10 @@ def load_config(self) -> None: self._set_config(config) self.commit() - log.info(f'config ~ database: {config}') - log.info(f'loaded as {self.config}') + log.debug(f'config ~ database: {config}') + log.debug(f'loaded as {self.config}') else: - log.warning('could not load config') + log.warning('could not load config from database') @property # todo: this was deprecated, right? From 42fac5e1b20bdae7f744368cb83d0bf72f29ff6a Mon Sep 17 00:00:00 2001 From: ybnd Date: Sat, 15 Oct 2022 14:32:15 +0200 Subject: [PATCH 4/5] Add static 404 page --- docs/source/troubleshooting.rst | 3 ++ shapeflow/server.py | 38 ++++++++++++++++++---- test/test_server.py | 57 ++++++++++++++++++++++++++++++++- ui/404.html | 40 +++++++++++++++++++++++ 4 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 ui/404.html diff --git a/docs/source/troubleshooting.rst b/docs/source/troubleshooting.rst index 8f885af1..543794cc 100644 --- a/docs/source/troubleshooting.rst +++ b/docs/source/troubleshooting.rst @@ -90,6 +90,9 @@ Application won’t run in the the ``shapeflow`` root directory. +Application runs, but the interface won't load (properly) +--------------------------------------------------------- + * ``sf.py`` runs fine, but the page says **404 not found** * Check if you have a folder ``ui/dist/`` in your ``shapeflow`` directory and diff --git a/shapeflow/server.py b/shapeflow/server.py index 9015f095..327c3964 100644 --- a/shapeflow/server.py +++ b/shapeflow/server.py @@ -5,7 +5,8 @@ from threading import Thread, Event, Lock from typing import Optional -from flask import Flask, send_from_directory, jsonify, request, Response, make_response, abort +import flask +from flask import Flask, jsonify, request, Response, make_response, abort import waitress import webbrowser @@ -19,9 +20,10 @@ log = shapeflow.get_logger(__name__) UI = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - , 'ui', 'dist' + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + 'ui' ) +DIST = os.path.join(UI, 'dist') class ServerThread(Thread, metaclass=util.Singleton): @@ -88,10 +90,31 @@ def __init__(self): app.config.from_object(__name__) app.config['JSON_SORT_KEYS'] = False - @app.route('/', defaults={'file': 'index.html'}, methods=['GET']) + @app.route('/', methods=['GET']) + def _index(): + try: + r = self.get_file('index.html') + + # Make sure the index page isn't cached so we can pick up missing ui/dist/ right away + r.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" + r.headers["Pragma"] = "no-cache" + r.headers["Expires"] = "0" + r.headers['Cache-Control'] = 'public, max-age=0' + + return r + except FileNotFoundError: + if not os.path.isdir(DIST) or not os.listdir(DIST): + log.error(f"no compiled UI in '{DIST}'") + log.error(f"follow the instructions on the 404 page to restore the application") + + return flask.send_from_directory(UI, '404.html'), 404 + @app.route('/', methods=['GET']) def _get_file(file: str): - return self.get_file(file) + try: + return self.get_file(file) + except FileNotFoundError: + abort(404) @app.route('/api/', methods=['GET', 'POST', 'PUT']) def _call_api(address: str): @@ -148,11 +171,12 @@ def get_file(self, file: str): """ self.active() - path = os.path.join(UI, *file.split("/")) + path = os.path.join(DIST, *file.split("/")) if not os.path.isfile(path): + log.error(f"no such file: '{file}'") raise FileNotFoundError log.debug(f"serving '{file}'") - return send_from_directory( + return flask.send_from_directory( os.path.dirname(path), os.path.basename(path) ) diff --git a/test/test_server.py b/test/test_server.py index 84fe77ea..952875d2 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -1,13 +1,18 @@ import os import shutil import unittest +from unittest.mock import patch from contextlib import contextmanager import time import json import subprocess +import flask +from flask.testing import FlaskClient + from shapeflow import settings, ROOTDIR, save_settings +from shapeflow.server import ShapeflowServer, UI, DIST CACHE = os.path.join(ROOTDIR, 'test_main-cache') DB = os.path.join(ROOTDIR, 'test_main-history.db') @@ -169,6 +174,56 @@ def test_set_settings_restart(self): post(api('set_settings'), data=json.dumps({'settings': first_settings})) time.sleep(10) + +@patch('os.path.isfile') +@patch('flask.send_from_directory') +@patch('os.path.isdir', lambda _: True) +@patch('os.listdir', lambda _: ['index.html']) +class FlaskTest(unittest.TestCase): + client: FlaskClient + + def setUp(self) -> None: + sfs = ShapeflowServer() + sfs._app.config['TESTING'] = True + + self.client = sfs._app.test_client() + + def test_serve_index_200(self, send_from_directory, isfile): + isfile.return_value = True + send_from_directory.return_value = flask.Response() + + r = self.client.get('/') + + send_from_directory.assert_called_with(DIST, 'index.html') + self.assertEqual(r.status_code, 200) + + def test_serve_index_404(self, send_from_directory, isfile): + isfile.return_value = False + send_from_directory.return_value = flask.Response() + + r = self.client.get('/') + + send_from_directory.assert_called_with(UI, '404.html') + self.assertEqual(r.status_code, 404) + + def test_serve_file_200(self, send_from_directory, isfile): + isfile.return_value = True + send_from_directory.return_value = flask.Response() + + r = self.client.get('/something.txt') + + send_from_directory.assert_called_with(DIST, 'something.txt') + self.assertEqual(r.status_code, 200) + + def test_serve_file_404(self, send_from_directory, isfile): + isfile.return_value = False + send_from_directory.return_value = flask.Response() + + r = self.client.get('/something.txt') + + send_from_directory.assert_not_called() + self.assertEqual(r.status_code, 404) + + if __name__ == '__main__': unittest.main() - diff --git a/ui/404.html b/ui/404.html new file mode 100644 index 00000000..a84fcd5d --- /dev/null +++ b/ui/404.html @@ -0,0 +1,40 @@ + + + + + + + + + + +

404 Not Found

+ +

If you're an end user who installed the application via a deployment script

+

...you shouldn't see this, ever, ideally.

+

+ Check out the + troubleshooting page + to see if you can solve the problem yourself, contact the maintaner if you can't. +

+ +

If you're a developer

+

You haven't compiled the UI yet, or have broken it somehow.

+

+ Fix or (re)compile it + (see here for more information) +

+ + From d763fec2aa08e6103d3326a0fb0d549d0284c4f8 Mon Sep 17 00:00:00 2001 From: ybnd Date: Sat, 15 Oct 2022 15:38:56 +0200 Subject: [PATCH 5/5] Add to changelog --- docs/source/changelog.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 11f07ea4..f7a08d51 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -1,6 +1,17 @@ Changelog ========= +0.4.5 +----- + +* Analyzers that can't be loaded properly are now skipped when restoring + application state (e.g. if the video or design file has been moved since the + last run) + +* Added a 404 page with instructions in case application is run without UI + +* Other minor fixes + 0.4.4 -----