From ff86fd05c915bbf1c5ad570dc6ff8e441206df94 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Tue, 23 Jul 2024 12:27:17 +0200 Subject: [PATCH] add quickstart sample app for analytics with hybrid tables --- data-app-with-hybrid-tables/.gitignore | 163 +++++++++++ data-app-with-hybrid-tables/LICENSE | 201 ++++++++++++++ data-app-with-hybrid-tables/Makefile | 19 ++ data-app-with-hybrid-tables/README.md | 38 +++ data-app-with-hybrid-tables/app/Controller.py | 254 ++++++++++++++++++ data-app-with-hybrid-tables/app/app.py | 195 ++++++++++++++ data-app-with-hybrid-tables/app/model.py | 90 +++++++ .../app/snowflake_connection.py | 34 +++ .../app/static/images/bug-sno-R-blue.png | Bin 0 -> 40819 bytes .../app/static/images/favicon.ico | Bin 0 -> 134888 bytes .../app/static/images/logo-sno-blue.png | Bin 0 -> 24731 bytes .../app/static/images/logo192.png | Bin 0 -> 25110 bytes .../app/static/images/logo512.png | Bin 0 -> 40819 bytes .../app/static/scripts/scripts.js | 58 ++++ .../app/static/styles/base.css | 241 +++++++++++++++++ .../app/static/styles/styles.css | 217 +++++++++++++++ .../app/templates/item.html | 118 ++++++++ .../app/templates/login.html | 38 +++ .../app/templates/macros.html | 169 ++++++++++++ .../app/templates/main.html | 122 +++++++++ data-app-with-hybrid-tables/app/viewmodel.py | 65 +++++ data-app-with-hybrid-tables/requirements.txt | 4 + data-app-with-hybrid-tables/sql/seeds.sql | 236 ++++++++++++++++ 23 files changed, 2262 insertions(+) create mode 100644 data-app-with-hybrid-tables/.gitignore create mode 100644 data-app-with-hybrid-tables/LICENSE create mode 100644 data-app-with-hybrid-tables/Makefile create mode 100644 data-app-with-hybrid-tables/README.md create mode 100644 data-app-with-hybrid-tables/app/Controller.py create mode 100644 data-app-with-hybrid-tables/app/app.py create mode 100644 data-app-with-hybrid-tables/app/model.py create mode 100644 data-app-with-hybrid-tables/app/snowflake_connection.py create mode 100644 data-app-with-hybrid-tables/app/static/images/bug-sno-R-blue.png create mode 100644 data-app-with-hybrid-tables/app/static/images/favicon.ico create mode 100644 data-app-with-hybrid-tables/app/static/images/logo-sno-blue.png create mode 100644 data-app-with-hybrid-tables/app/static/images/logo192.png create mode 100644 data-app-with-hybrid-tables/app/static/images/logo512.png create mode 100644 data-app-with-hybrid-tables/app/static/scripts/scripts.js create mode 100644 data-app-with-hybrid-tables/app/static/styles/base.css create mode 100644 data-app-with-hybrid-tables/app/static/styles/styles.css create mode 100644 data-app-with-hybrid-tables/app/templates/item.html create mode 100644 data-app-with-hybrid-tables/app/templates/login.html create mode 100644 data-app-with-hybrid-tables/app/templates/macros.html create mode 100644 data-app-with-hybrid-tables/app/templates/main.html create mode 100644 data-app-with-hybrid-tables/app/viewmodel.py create mode 100644 data-app-with-hybrid-tables/requirements.txt create mode 100644 data-app-with-hybrid-tables/sql/seeds.sql diff --git a/data-app-with-hybrid-tables/.gitignore b/data-app-with-hybrid-tables/.gitignore new file mode 100644 index 0000000..bece97d --- /dev/null +++ b/data-app-with-hybrid-tables/.gitignore @@ -0,0 +1,163 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + + +.DS_Store \ No newline at end of file diff --git a/data-app-with-hybrid-tables/LICENSE b/data-app-with-hybrid-tables/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/data-app-with-hybrid-tables/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/data-app-with-hybrid-tables/Makefile b/data-app-with-hybrid-tables/Makefile new file mode 100644 index 0000000..52cf350 --- /dev/null +++ b/data-app-with-hybrid-tables/Makefile @@ -0,0 +1,19 @@ +VENV_BIN = python3 -m venv +VENV_DIR ?= .venv +VENV_ACTIVATE = $(VENV_DIR)/bin/activate +VENV_RUN = . $(VENV_ACTIVATE) + +usage: ## Shows usage for this Makefile + @cat Makefile | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}' + +install: ## Install dependencies + test -d .venv || $(VENV_BIN) .venv + $(VENV_RUN); pip install -r requirements.txt + +start: ## Start the flask application + $(VENV_RUN); flask --app app/app run + +seed: ## Seed test data into LocalStack Snowflake + snow sql -c local -f sql/seeds.sql + +.PHONY: usage install start init diff --git a/data-app-with-hybrid-tables/README.md b/data-app-with-hybrid-tables/README.md new file mode 100644 index 0000000..2d5a295 --- /dev/null +++ b/data-app-with-hybrid-tables/README.md @@ -0,0 +1,38 @@ +# Data application with Hybrid Tables in LocalStack Snowflake + +Note: This sample application has been copied and adapted from its original version here: https://github.com/Snowflake-Labs/sfguide-build-data-application-with-hybrid-tables + +## Overview + +This quickstart will take you through building a data application that runs on Snowflake Hybrid Tables, deployed fully locally using [LocalStack for Snowflake](https://www.localstack.cloud/localstack-for-snowflake). +[Hybrid Tables](https://docs.snowflake.com/en/user-guide/tables-hybrid?_fsi=siV2rnOG) is a Snowflake table type that has been designed for transactional and operational work together with analytical workloads. +Hybrid Tables typically offers lower latency and higher throughput on row level and point reads and writes, making them a good choice for a backing source for an application that requires faster operations on individual rows and point lookup, especially when compared to standard Snowflake tables that are optimized for analytical operations. + +This sample is based on the Snowflake [QuickStart Guide](https://quickstarts.snowflake.com/guide/build_a_data_application_with_hybrid_tables) for hybrid tables. + +## Prerequisites + +- [`localstack` CLI](https://docs.localstack.cloud/getting-started/installation/#localstack-cli) with [`LOCALSTACK_AUTH_TOKEN`](https://docs.localstack.cloud/getting-started/auth-token/) environment variable set +- [LocalStack Snowflake emulator](https://snowflake.localstack.cloud/getting-started/installation/) +- [Snowpark for Python](https://docs.snowflake.com/en/developer-guide/snowpark/python/index) + +## Installing and Initializing + +First, install the dependencies for the project: +``` +make install +``` + +We then seed the test data into the LocalStack Snowflake instance: +``` +make seed +``` + +Finally, run this command to start up the Flask application: +``` +make start +``` + +## License + +The code in this project is licensed under the Apache 2.0 license. diff --git a/data-app-with-hybrid-tables/app/Controller.py b/data-app-with-hybrid-tables/app/Controller.py new file mode 100644 index 0000000..910d324 --- /dev/null +++ b/data-app-with-hybrid-tables/app/Controller.py @@ -0,0 +1,254 @@ +from model import Label, Task, User, Status +from typing import List, Dict +from snowflake.snowpark import Session +import datetime +from functools import wraps +import time + + + +class Controller: + + def measure(func): + @wraps(func) + def measure_wrapper(*args, **kwargs): + start_time = time.perf_counter() + result = func(*args, **kwargs) + end_time = time.perf_counter() + total_time = end_time - start_time + #print(f'controller: {func.__name__} latency {total_time:.4f} seconds') + return result + return measure_wrapper + + def __init__(self, session:Session) -> None: + self.session = session + + def loginUser(self, user_name:str, password:str) -> User: + sql = "SELECT id, login, name FROM HYBRID_TASK_APP_DB.DATA.USER WHERE LOWER(login) = ? AND HPASSWORD = ?;" + data_items = self.session.sql(sql, params=[user_name.lower(), password]).collect() + if len(data_items) == 1: + user = User.loadRow(data_items[0]) + return user + + @measure + def load_label_stats(self, user_id) -> Dict[str, Label]: + + data_items = self.session.sql("SELECT index, count, label, percent FROM HYBRID_TASK_APP_DB.DATA.LABEL_DETAIL WHERE owner_id = ?;", params=[user_id]).collect() + label_stats = {_.LABEL:Label.loadRow(_) for _ in data_items} + return label_stats + + @measure + def load_users(self) -> List[User]: + data_items = self.session.sql("SELECT id, login, name FROM HYBRID_TASK_APP_DB.DATA.USER;").collect() + users = [User.loadRow(_) for _ in data_items] + return users + + @measure + def load_items(self, user_id:int, label:str=None) -> List[Task]: + + if label: + sql = """ + SELECT id, item, date_due, days_due, status + FROM HYBRID_TASK_APP_DB.DATA.TASK_DETAIL t + INNER JOIN HYBRID_TASK_APP_DB.DATA.TASK_LABEL l ON t.id = l.task_id + WHERE owner_id = ? + AND l.label = LOWER(?) + AND t.complete = FALSE; + """ + data_items = self.session.sql(sql, [user_id, label]).collect() + else: + sql = """ + SELECT id, item, date_due, days_due, status + FROM HYBRID_TASK_APP_DB.DATA.TASK_DETAIL t + WHERE owner_id = ? + AND t.complete = FALSE; + """ + data_items = self.session.sql(sql, [user_id]).collect() + + items_all = [Task.loadRow(_) for _ in data_items] + + return items_all + + + @measure + def load_items_overview(self, user_id:int, label:str=None) -> List[Task]: + + if label: + sql = """ + SELECT t.id, t.item, LISTAGG(l.label,',') as labels, t.date_due, t.days_due, t.status, t.owner_id, t.index + FROM ( + SELECT t.id, t.item , t.date_due, t.days_due, t.status, owner_id + , row_number() over(PARTITION BY t.status, t.owner_id ORDER BY t.days_due ASC) as index + FROM HYBRID_TASK_APP_DB.DATA.TASK_DETAIL t + INNER JOIN HYBRID_TASK_APP_DB.DATA.TASK_LABEL l ON t.id = l.task_id + WHERE OWNER_ID = ? + AND l.label = ? + AND t.complete = FALSE + ) t + INNER JOIN HYBRID_TASK_APP_DB.DATA.TASK_LABEL l ON t.id = l.task_id + WHERE index <= 5 + GROUP BY ALL + ORDER BY index ASC; + """ + data_items = self.session.sql(sql, [user_id, label.lower()]).collect() + else: + sql = """ + SELECT t.id, t.item, LISTAGG(l.label,',') as labels, t.date_due, t.days_due, t.status, t.owner_id, t.index + FROM ( + SELECT t.id, t.item , t.date_due, t.days_due, t.status, owner_id + , row_number() over(PARTITION BY t.status, t.owner_id ORDER BY t.days_due ASC) as index + FROM HYBRID_TASK_APP_DB.DATA.TASK_DETAIL t + WHERE OWNER_ID = 1 + AND t.complete = FALSE + ) t + LEFT OUTER JOIN TASK_LABEL l ON t.id = l.task_id + WHERE index <= 5 + GROUP BY ALL + ORDER BY index ASC; + """ + data_items = self.session.sql(sql, [user_id]).collect() + + items_all = [Task.loadRow(_) for _ in data_items] + + return items_all + + @measure + def load_items_stats(self, user_id:int, label:str=None) -> Dict[str, Status]: + + if label: + sql = "SELECT status, count FROM LABEL_STATUS_DETAIL WHERE owner_id = ? AND LABEL = ?;" + data_items = self.session.sql(sql, [user_id, label]).collect() + else: + sql = "SELECT status, count FROM STATUS_DETAIL WHERE owner_id = ?;" + data_items = self.session.sql(sql, [user_id]).collect() + items_all = {_.STATUS:Status.loadRow(_) for _ in data_items} + all_stats = {**{_:Status(0, _, 0) for _ in ['overdue', 'today', 'upcoming']}, **items_all} + return all_stats + + @staticmethod + def get_tasks_by_status(tasks:List[Task]) -> tuple[List[Task], List[Task], List[Task]]: + data_today=[_ for _ in tasks if _.status=='today'] + data_overdue=[_ for _ in tasks if _.status=='overdue'] + data_upcoming=[_ for _ in tasks if _.status=='upcoming'] + return (data_overdue, data_today, data_upcoming) + + @staticmethod + def get_tasks_for_overview(tasks:tuple[List[Task], List[Task], List[Task]]) -> List[int]: + return [_[:5] for _ in tasks] + + @staticmethod + def get_task_ids_for_overview(tasks:tuple[List[Task], List[Task], List[Task]]) -> List[int]: + task_ids = [] + for tasks in tasks: + task_ids = task_ids + [_.id for _ in tasks] + return task_ids + + @measure + def load_task_labels(self, task_ids:List[int]) -> Dict[str, Label]: + + task_ids_list = ','.join(['?' for _ in task_ids]) + if len(task_ids) == 0: + return {} + sql = f""" + SELECT task_id, LISTAGG(label,',') as labels + FROM HYBRID_TASK_APP_DB.DATA.TASK_LABEL + WHERE task_id IN ({task_ids_list}) + GROUP BY task_id; + """ + data_items = self.session.sql(sql, params=task_ids).collect() + labels = {_.TASK_ID:', '.join(_.LABELS.split(',')) for _ in data_items} + return labels + + @staticmethod + def map_labels(task:Task, all_task_labels:Dict[int, str], all_labels:Dict[str, Label]): + + labels = [] + if task.id in all_task_labels: + task_labels = all_task_labels[task.id] + else: + task_labels = '' + for task_label in [_.strip().lower() for _ in task_labels.split(',')]: + if task_label in all_labels: + label = all_labels[task_label] + labels.append(label) + else: + label = Label(100, task_label, 1, 0) + labels.append(label) + + labels.sort(key=lambda l: l.index) + task.labels = labels + return task + + @measure + def load_task(self, user_id:int, task_id:int, all_labels:List[Label]) -> Task: + sql = "SELECT id, owner_id, item, date_due FROM HYBRID_TASK_APP_DB.DATA.TASK WHERE id = ?;" + data_items = self.session.sql(sql, params=[task_id]).collect() + if len(data_items) < 1: + return None + + task_data_item = data_items[0] + owner_id = task_data_item.OWNER_ID + if user_id != owner_id: + return None + + task = Task.loadRow(task_data_item) + + sql = "SELECT task_id, LISTAGG(label,',') as labels FROM HYBRID_TASK_APP_DB.DATA.TASK_LABEL WHERE task_id = ? GROUP BY task_id;" + data_items = self.session.sql(sql,params=[task.id]).collect() + if len(data_items) == 1: + task_labels:str = {int(task_id): data_items[0].LABELS} + else: + task_labels:str = {int(task_id):''} + task = Controller.map_labels(task, task_labels, all_labels) + return task + + @measure + def create_task(self, user_id:int, item:str, date_due:datetime.datetime, updated_labels:List[str]) -> None: + if len(updated_labels) == 0: + # No need for a transaction here, single statement and table execution + self.session.sql(f"INSERT INTO HYBRID_TASK_APP_DB.DATA.TASK (ITEM, DATE_DUE, OWNER_ID) VALUES (?, ?, ?);", params=[item, ("TIMESTAMP_NTZ", date_due), user_id]).collect() + return -1 + else: + try: + self.session.sql(f"BEGIN TRANSACTION;").collect() + task_id = self.session.sql(f"SELECT HYBRID_TASK_APP_DB.DATA.TASK_ID_SEQ.nextval id").collect()[0].ID + self.session.sql(f"INSERT INTO HYBRID_TASK_APP_DB.DATA.TASK (ID, ITEM, DATE_DUE, OWNER_ID) VALUES (?, ?, ?, ?);", params=[task_id, item, ("TIMESTAMP_NTZ", date_due), user_id]).collect() + for label in updated_labels: + self.session.sql(f"INSERT INTO HYBRID_TASK_APP_DB.DATA.TASK_LABEL (TASK_ID, LABEL) VALUES (?, LOWER(?));", params=[task_id, label]).collect() + self.session.sql(f"COMMIT;").collect() + return task_id + except Exception as e: + print(e) + self.session.sql(f"ROLLBACK;").collect() + + @measure + def update_task(self, task_id:int, item:str, date_due:datetime.datetime, updated_labels:List[str], existing_labels:List[str]) -> None: + try: + self.session.sql(f"BEGIN TRANSACTION;").collect() + items = self.session.sql(f"UPDATE HYBRID_TASK_APP_DB.DATA.TASK SET item=?, date_due=? WHERE id = ?;", params=[item, date_due, task_id]).collect() + for label in [_.strip().lower() for _ in updated_labels if _ not in existing_labels]: + self.session.sql(f"INSERT INTO HYBRID_TASK_APP_DB.DATA.TASK_LABEL (TASK_ID, LABEL) VALUES (?, LOWER(?));", params=[task_id, label]).collect() + for label in [_.strip().lower() for _ in existing_labels if _ not in updated_labels]: + self.session.sql(f"DELETE FROM HYBRID_TASK_APP_DB.DATA.TASK_LABEL WHERE TASK_ID = ? AND LABEL = ?;", params=[task_id, label]).collect() + self.session.sql(f"COMMIT;").collect() + except Exception as e: + print(e) + self.session.sql(f"ROLLBACK;").collect() + + @measure + def delete_task(self, task_id:int) -> None: + try: + self.session.sql(f"BEGIN TRANSACTION;").collect() + items = self.session.sql(f"DELETE FROM HYBRID_TASK_APP_DB.DATA.TASK_LABEL WHERE TASK_ID = ?;", params=[task_id]).collect() + items = self.session.sql(f"DELETE FROM HYBRID_TASK_APP_DB.DATA.TASK WHERE ID = ?;", params=[task_id]).collect() + self.session.sql(f"COMMIT;").collect() + except Exception as e: + print(e) + self.session.sql(f"ROLLBACK;").collect() + + @measure + def complete_task(self, task_id:int) -> None: + # No need for a transaction here, single statement and table execution + #session.sql(f"BEGIN TRANSACTION;").collect() + items = self.session.sql(f"UPDATE HYBRID_TASK_APP_DB.DATA.TASK SET complete=TRUE WHERE id = ?;", params=[task_id]).collect() + #session.sql(f"COMMIT;").collect() \ No newline at end of file diff --git a/data-app-with-hybrid-tables/app/app.py b/data-app-with-hybrid-tables/app/app.py new file mode 100644 index 0000000..d663529 --- /dev/null +++ b/data-app-with-hybrid-tables/app/app.py @@ -0,0 +1,195 @@ +from flask import Flask, render_template, g, request, redirect, url_for, session +from flask_caching import Cache +from functools import wraps +from typing import List, Dict +import snowflake_connection +from snowflake.snowpark import Session +import os +from dotenv import load_dotenv +import datetime + +from model import Label, Task, User +from viewmodel import HomeViewModel, ItemViewModel +from Controller import Controller + +load_dotenv(override=True) + +def get_connection()->Session: + print(f'Recreating connection') + conn = snowflake_connection.getSession() + cwd = os.getcwd() + conn.query_tag = f'{cwd}/app.py' + print(f'Connected to {conn.get_current_account()} as ROLE {conn.get_current_role()}') + return conn + +def get_controller()->Controller: + print(f'Recreating controller') + ctrl = Controller(connection) + return ctrl + +def create_app(): + app = Flask(__name__) + app.secret_key = os.getenv('APP_SECRET') + return app + +app = create_app() + +with app.app_context(): + cache = Cache(config={'CACHE_TYPE': 'SimpleCache'}) + cache.init_app(app) + +connection:Session = get_connection() +controller:Controller = get_controller() + +@cache.cached(timeout=500, key_prefix='users_list') +def get_users() -> List[User]: + + users = controller.load_users() + return users + +@cache.cached(timeout=50, key_prefix='label_stats') +def get_label_stats(user_id) -> Dict[str, Label]: + + label_stats = controller.load_label_stats(user_id) + return label_stats + +@cache.memoize(50) +def get_items_stats(user_id, label): + task_stats = controller.load_items_stats(user_id, label) + return task_stats + +def login_required(f): + @wraps(f) + def decorated_function(*args, **kwargs): + if 'user_id' not in session: + return redirect(url_for('login', next=request.url)) + return f(*args, **kwargs) + return decorated_function + +@app.route("/label/