Update directory structure, init, and config files to match fully featured example

.gitignore

@@ -162,5 +162,12 @@ cython_debug/
 # VS code
 .vscode
 
-# data heavy folders
-data/
+# data files
+*.csv
+*.xlsx
+*.RData
+*.DAT
+*.R
+
+# DS_Store files
+.DS_Store

data/Census/Census_raw/README.md

@@ -0,0 +1,3 @@
+# Census raw files README
+
+TK

data/Census/README.md

@@ -0,0 +1,3 @@
+# Census files README
+
+TK

data/NIH_child/NIH_conv/README.md

@@ -0,0 +1,3 @@
+# NIH conv files README
+
+TK

data/NIH_child/NIH_conv/notes.md

@@ -0,0 +1,18 @@
+## extract -- folder
+- pulling from .DAT
+- using r to convert to CSV
+
+## additional datas
+- public health clinics
+-- vaccination rates going down?
+-- people's proximity to access changed over time?
+- what has happened to quantity of public health clincs in each state over time?
+- census data by state
+- quantity of health insurance options by state?
+- healthdata.gov: locations of hospitals by states, perhaps?
+
+
+## dagster data
+- stream of measal cases?!
+- stream of covid cases?!
+- list of clinics??

data/NIH_child/README.md

@@ -0,0 +1,3 @@
+# NIH files README
+
+TK

data/README.md

@@ -0,0 +1,3 @@
+# Data folder README
+
+TK

requirements.txt

@@ -1,12 +1,15 @@
-dagster=1.3.6
-dagit=1.3.6
-dagster-docker
-dagster-gcp=0.19.6
-polars=0.17.5
-petl=1.7.12
-pyarrow=11.0.0
-xlsx2csv=0.8.1
-connectorx=0.3.2a5
-SQLAlchemy=2.0.9
-requests=2.31.0
-google-cloud-bigquery=2.26.0
+dagster == 1.3.7
+dagit == 1.3.7
+dagster-docker == 0.19.7
+dagster-duckdb == 0.19.7
+dagster-duckdb-pandas == 0.19.7
+dagster-gcp == 0.19.7
+openpyxl == 3.1.2
+polars == 0.17.5
+petl == 1.7.12
+pyarrow == 11.0.0
+xlsx2csv == 0.8.1
+connectorx == 0.3.2a5
+SQLAlchemy >= 1.0, < 2.0.0
+requests == 2.31.0
+google-cloud-bigquery == 3.11.0

vaccines/__init__.py

@@ -0,0 +1,32 @@
+# import os
+
+from dagster import Definitions
+
+from .assets import vaccine_assets
+# from .jobs import TK
+# from .resources import RESOURCES_LOCAL, RESOURCES_PROD, RESOURCES_STAGING
+# from .sensors import make_slack_on_failure_sensor
+
+all_assets = [*vaccine_assets]
+
+'''
+resources_by_deployment_name = {
+    "prod": RESOURCES_PROD,
+    "staging": RESOURCES_STAGING,
+    "local": RESOURCES_LOCAL,
+}
+'''
+
+# deployment_name = os.environ.get("DAGSTER_DEPLOYMENT", "local")
+
+# all_sensors = [activity_analytics_assets_sensor, recommender_assets_sensor]
+'''
+if deployment_name in ["prod", "staging"]:
+    all_sensors.append(make_slack_on_failure_sensor(base_url="my_dagit_url"))
+'''
+
+defs = Definitions(
+    assets=all_assets,
+    # resources=resources_by_deployment_name[deployment_name],
+    # sensors=all_sensors,
+)

vaccines/assets/__init__.py

@@ -0,0 +1,7 @@
+from dagster import load_assets_from_package_module
+
+from . import vaccine_assets
+
+VACCINE_ASSETS = "vaccine_assets"
+
+vaccine_assets = load_assets_from_package_module(package_module=vaccine_assets)

vaccines_test/README.md

@@ -0,0 +1,48 @@
+# vaccines
+
+This is a [Dagster](https://dagster.io/) project scaffolded with [`dagster project scaffold`](https://docs.dagster.io/getting-started/create-new-project).
+
+## Getting started
+
+First, install your Dagster code location as a Python package. By using the --editable flag, pip will install your Python package in ["editable mode"](https://pip.pypa.io/en/latest/topics/local-project-installs/#editable-installs) so that as you develop, local code changes will automatically apply.
+
+```bash
+pip install -e ".[dev]"
+```
+
+Then, start the Dagster UI web server:
+
+```bash
+dagster dev
+```
+
+Open http://localhost:3000 with your browser to see the project.
+
+You can start writing assets in `vaccines/assets.py`. The assets are automatically loaded into the Dagster code location as you define them.
+
+## Development
+
+
+### Adding new Python dependencies
+
+You can specify new Python dependencies in `setup.py`.
+
+### Unit testing
+
+Tests are in the `vaccines_tests` directory and you can run tests using `pytest`:
+
+```bash
+pytest vaccines_tests
+```
+
+### Schedules and sensors
+
+If you want to enable Dagster [Schedules](https://docs.dagster.io/concepts/partitions-schedules-sensors/schedules) or [Sensors](https://docs.dagster.io/concepts/partitions-schedules-sensors/sensors) for your jobs, the [Dagster Daemon](https://docs.dagster.io/deployment/dagster-daemon) process must be running. This is done automatically when you run `dagster dev`.
+
+Once your Dagster Daemon is running, you can start turning on schedules and sensors for your jobs.
+
+## Deploy on Dagster Cloud
+
+The easiest way to deploy your Dagster project is to use Dagster Cloud.
+
+Check out the [Dagster Cloud Documentation](https://docs.dagster.cloud) to learn more.

vaccines_test/__init__.py

@@ -0,0 +1,32 @@
+# import os
+
+from dagster import Definitions
+
+from .assets import vaccine_assets
+# from .jobs import TK
+# from .resources import RESOURCES_LOCAL, RESOURCES_PROD, RESOURCES_STAGING
+# from .sensors import make_slack_on_failure_sensor
+
+all_assets = [*vaccine_assets]
+
+'''
+resources_by_deployment_name = {
+    "prod": RESOURCES_PROD,
+    "staging": RESOURCES_STAGING,
+    "local": RESOURCES_LOCAL,
+}
+'''
+
+# deployment_name = os.environ.get("DAGSTER_DEPLOYMENT", "local")
+
+# all_sensors = [activity_analytics_assets_sensor, recommender_assets_sensor]
+'''
+if deployment_name in ["prod", "staging"]:
+    all_sensors.append(make_slack_on_failure_sensor(base_url="my_dagit_url"))
+'''
+
+defs = Definitions(
+    assets=all_assets,
+    # resources=resources_by_deployment_name[deployment_name],
+    # sensors=all_sensors,
+)

vaccines_test/assets/__init__.py

@@ -0,0 +1,7 @@
+from dagster import load_assets_from_package_module
+
+from . import vaccine_assets
+
+VACCINE_ASSETS = "vaccine_assets"
+
+vaccine_assets = load_assets_from_package_module(package_module=vaccine_assets)