From 9697443d609c8f7208674b2c8ed9a502182f8250 Mon Sep 17 00:00:00 2001 From: Sebastian Prentice Date: Thu, 7 Nov 2024 11:07:53 +0000 Subject: [PATCH 01/10] improve readme and makefirle --- Makefile | 8 ++++ README.md | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 111 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 68ba11e2..78a9c8ca 100644 --- a/Makefile +++ b/Makefile @@ -31,3 +31,11 @@ build: .PHONY: prod-push prod-push: git push heroku `git rev-parse --abbrev-ref HEAD`:master + +.PHONY: reset-db +reset-db: +.PHONY: reset-db +reset-db: + psql -h localhost -U postgres -c "DROP DATABASE IF EXISTS socket" + psql -h localhost -U postgres -c "CREATE DATABASE socket" + python tcsocket/run.py resetdb --no-input diff --git a/README.md b/README.md index f9ac5983..5b31f43c 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,109 @@ -socket-server -============= +# Socket Server ![Build Status](https://github.com/tutorcruncher/socket-server/workflows/CI/badge.svg) [![codecov](https://codecov.io/gh/tutorcruncher/socket-server/branch/master/graph/badge.svg)](https://codecov.io/gh/tutorcruncher/socket-server) Backend application for [TutorCruncher's](https://tutorcruncher.com) web integration. +## Setup and Run -# LICENSE +To set up and run this project, follow these steps: -Copyright TutorCruncher ltd. 2017 - 2022. -All rights reserved. +1. **Clone the repository:** + ```sh + git clone git@github.com:tutorcruncher/socket-server.git + cd socket-server + ``` + +2. **Install dependencies:** + ```sh + make install + ``` + +3. **Reset the database:** + ```sh + make reset-db + ``` + +3. **Run the application:** + ```sh + python tcsocket/run.py auto + ``` + +**Note:** You might have to run this with `sudo` if you are not in the `docker` group. + +## Environment Variables + +The environment variables for this project are: + +- `BIND_IP`: The IP address to bind the web server to. Default is `127.0.0.1`. +- `PORT`: The port number to bind the web server to. Default is `8000`. +- `DYNO`: Used to infer whether to run the web server or worker. If it starts with `web`, the web server will run; otherwise, the worker will run. +- `DATABASE_URL`: The URL for the database connection. +- `REDIS_URL`: The URL for the Redis connection. + +You can set these environment variables in your shell or in a `.env` file. Here is an example of how to set them in a `.env` file: + +```sh +BIND_IP=127.0.0.1 +PORT=8000 +DYNO=web.1 +DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/socket_test +REDIS_URL=redis://localhost:6379/0 +``` + +## Commands + +- **Run the application:** + ```sh + python tcsocket/run.py auto + ``` + +- **Reset the database:** + ```sh + make reset-db + ``` + +- **Open an IPython shell:** + ```sh + python tcsocket/run.py shell + ``` -## Deploying +- **Run a patch script:** + ```sh + python tcsocket/run.py patch --live + ``` +- **Format the code:** + ```sh + make format + ``` + +- **Lint the code:** + ```sh + make lint + ``` + +- **Run tests:** + ```sh + make test + ``` + +## Docker + +The project includes a `Dockerfile` for building a Docker image. To build the Docker image, run: + +```sh +make build +``` + +## Deployment + +To deploy the project to Heroku, use the following command: + +```sh +make prod-push +``` + +or To deploy socket-server, please create a new tag/release, then run the following command: @@ -20,3 +112,8 @@ git push heroku master ``` **Make sure you have checked out master and pulled all the recent changes.** + +## License + +Copyright TutorCruncher ltd. 2017 - 2022. +All rights reserved. From a39c310e27979512a8cd7e2688aa9f3a68154d7a Mon Sep 17 00:00:00 2001 From: Sebastian Prentice Date: Fri, 8 Nov 2024 09:16:00 +0000 Subject: [PATCH 02/10] add logfire and improve readme --- .gitignore | 1 + README.md | 9 ++++++-- tcsocket/app/index.html | 2 +- tcsocket/app/logs.py | 8 ++++--- tcsocket/app/main.py | 15 +++++++++++-- tcsocket/app/middleware.py | 1 + tcsocket/app/settings.py | 36 +++++++++++++----------------- tcsocket/app/validation.py | 4 ++-- tcsocket/app/views/appointments.py | 4 ++-- tcsocket/app/views/company.py | 1 + tcsocket/requirements.txt | 6 +++-- tcsocket/run.py | 22 +++++++++++++----- 12 files changed, 69 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index fc721fd9..a630a84e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ nginx/prod/keys/ activate.prod.* activate.beta.* .pytest_cache/ +/.logfire/ diff --git a/README.md b/README.md index 5b31f43c..255c86b5 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,14 @@ To set up and run this project, follow these steps: make reset-db ``` -3. **Run the application:** +4. **Run the worker:** ```sh - python tcsocket/run.py auto + python tcsocket/run.py worker + ``` + +5. **Run the web server:** + ```sh + python tcsocket/run.py web ``` **Note:** You might have to run this with `sudo` if you are not in the `docker` group. diff --git a/tcsocket/app/index.html b/tcsocket/app/index.html index ce99a48e..8f1cbee9 100644 --- a/tcsocket/app/index.html +++ b/tcsocket/app/index.html @@ -50,7 +50,7 @@
For help integrating with socket, see - help.tutorcruncher.com/tc-socket. + https://help.tutorcruncher.com/en/articles/8255881-getting-started-with-tutorcruncher-socket.
diff --git a/tcsocket/app/logs.py b/tcsocket/app/logs.py index b9e821a5..1f2315fa 100644 --- a/tcsocket/app/logs.py +++ b/tcsocket/app/logs.py @@ -25,11 +25,13 @@ def setup_logging(verbose: bool = False): 'release': os.getenv('COMMIT', None), 'name': os.getenv('SERVER_NAME', '-'), }, + 'logfire': {'class': 'logfire.integrations.logging.LogfireLoggingHandler'}, }, 'loggers': { - 'socket': {'handlers': ['socket', 'sentry'], 'level': log_level}, - 'gunicorn.error': {'handlers': ['sentry'], 'level': 'ERROR'}, - 'arq': {'handlers': ['socket', 'sentry'], 'level': log_level}, + 'socket': {'handlers': ['socket', 'sentry', 'logfire'], 'level': log_level}, + 'gunicorn.error': {'handlers': ['sentry', 'logfire'], 'level': 'ERROR'}, + 'arq': {'handlers': ['socket', 'sentry', 'logfire'], 'level': log_level}, + 'aiohttp': {'handlers': ['logfire'], 'level': log_level}, }, } logging.config.dictConfig(config) diff --git a/tcsocket/app/main.py b/tcsocket/app/main.py index b3ebd649..89aa0b65 100644 --- a/tcsocket/app/main.py +++ b/tcsocket/app/main.py @@ -1,5 +1,6 @@ import os import re +import logfire from html import escape from aiohttp import ClientSession, web @@ -32,13 +33,23 @@ async def startup(app: web.Application): redis=redis, session=ClientSession(), ) + debug(settings.logfire_token) + if bool(settings.logfire_token): + logfire.configure( + service_name='socket-server', + token=settings.logfire_token, + send_to_logfire=True, + console=False, + ) + logfire.instrument_pydantic() + logfire.instrument_requests() + logfire.instrument_aiohttp_client() async def cleanup(app: web.Application): app['pg_engine'].close() await app['pg_engine'].wait_closed() - app['redis'].close() - await app['redis'].wait_closed() + await app['redis'].close() await app['session'].close() diff --git a/tcsocket/app/middleware.py b/tcsocket/app/middleware.py index f9b06eea..0168bdb9 100644 --- a/tcsocket/app/middleware.py +++ b/tcsocket/app/middleware.py @@ -145,6 +145,7 @@ def domain_allowed(allow_domains, current_domain): async def company_middleware(request, handler): try: public_key = request.match_info.get('company') + debug(public_key) if public_key: c = sa_companies.c select_fields = c.id, c.name, c.public_key, c.private_key, c.name_display, c.options, c.domains diff --git a/tcsocket/app/settings.py b/tcsocket/app/settings.py index 801f5548..0bdab1af 100644 --- a/tcsocket/app/settings.py +++ b/tcsocket/app/settings.py @@ -1,9 +1,10 @@ from pathlib import Path -from typing import Optional +from typing import Optional, ClassVar from urllib.parse import urlparse from arq.connections import RedisSettings -from pydantic import BaseSettings, validator +from pydantic import validator, Field +from pydantic_settings import BaseSettings THIS_DIR = Path(__file__).parent BASE_DIR = THIS_DIR.parent @@ -14,20 +15,22 @@ class Settings(BaseSettings): redis_settings: RedisSettings = 'redis://localhost:6379' redis_database: int = 0 - master_key = b'this is a secret' + master_key: bytes = Field(default=b'this is a secret', env='MASTER_KEY') aws_access_key: Optional[str] = 'testing' aws_secret_key: Optional[str] = 'testing' aws_bucket_name: str = 'socket-images-beta.tutorcruncher.com' - tc_api_root = 'https://secure.tutorcruncher.com/api' - grecaptcha_secret = 'required secret for google recaptcha' - grecaptcha_url = 'https://www.google.com/recaptcha/api/siteverify' - geocoding_url = 'https://maps.googleapis.com/maps/api/geocode/json' - geocoding_key = 'required secret for google geocoding' + tc_api_root: str = 'https://secure.tutorcruncher.com/api' + grecaptcha_secret: str = 'required secret for google recaptcha' + grecaptcha_url: str = 'https://www.google.com/recaptcha/api/siteverify' + geocoding_url: str = 'https://maps.googleapis.com/maps/api/geocode/json' + geocoding_key: str = 'required secret for google geocoding' - tc_contractors_endpoint = '/public_contractors/' - tc_enquiry_endpoint = '/enquiry/' - tc_book_apt_endpoint = '/recipient_appointments/' + tc_contractors_endpoint: str = '/public_contractors/' + tc_enquiry_endpoint: str = '/enquiry/' + tc_book_apt_endpoint: str = '/recipient_appointments/' + + logfire_token: Optional[str] = '' @validator('redis_settings', always=True, pre=True) def parse_redis_settings(cls, v): @@ -68,12 +71,5 @@ def pg_port(self): return self._pg_dsn_parsed.port class Config: - fields = { - 'port': {'env': 'PORT'}, - 'database_url': {'env': 'DATABASE_URL'}, - 'redis_settings': {'env': 'REDISCLOUD_URL'}, - 'tc_api_root': {'env': 'TC_API_ROOT'}, - 'aws_access_key': {'env': 'AWS_ACCESS_KEY'}, - 'aws_secret_key': {'env': 'AWS_SECRET_KEY'}, - 'aws_bucket_name': {'env': 'AWS_BUCKET_NAME'}, - } + env_prefix = '' + env_file = '.env' diff --git a/tcsocket/app/validation.py b/tcsocket/app/validation.py index 18f95198..0dc1e4c9 100644 --- a/tcsocket/app/validation.py +++ b/tcsocket/app/validation.py @@ -3,7 +3,7 @@ from secrets import token_hex from typing import Any, List, Optional -from pydantic import BaseModel, EmailStr, NoneStr, constr, validator +from pydantic import BaseModel, EmailStr, constr, validator EXTRA_ATTR_TYPES = 'checkbox', 'text_short', 'text_extended', 'integer', 'stars', 'dropdown', 'datetime', 'date' @@ -137,7 +137,7 @@ class ContractorModel(BaseModel): town: constr(max_length=63) = None country: constr(max_length=63) = None last_updated: datetime = None - photo: NoneStr = None + photo: Optional[str] = None review_rating: float = None review_duration: int = None diff --git a/tcsocket/app/views/appointments.py b/tcsocket/app/views/appointments.py index 651eb0c7..b8b187e7 100644 --- a/tcsocket/app/views/appointments.py +++ b/tcsocket/app/views/appointments.py @@ -4,9 +4,9 @@ from datetime import datetime, timezone from operator import attrgetter from secrets import compare_digest -from typing import Dict +from typing import Dict, Protocol -from pydantic import BaseModel, Protocol, ValidationError, validator +from pydantic import BaseModel, ValidationError, validator from sqlalchemy import distinct, select from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.sql import and_, functions as sql_f diff --git a/tcsocket/app/views/company.py b/tcsocket/app/views/company.py index 7014faa9..1029a567 100644 --- a/tcsocket/app/views/company.py +++ b/tcsocket/app/views/company.py @@ -17,6 +17,7 @@ async def company_create(request): Authentication and json parsing are done by middleware. """ data = await request.json() + debug(data) update_contractors = data.pop('update_contractors', True) company: CompanyCreateModal = request['model'] existing_company = bool(company.private_key) diff --git a/tcsocket/requirements.txt b/tcsocket/requirements.txt index 631165ea..2e9af94a 100644 --- a/tcsocket/requirements.txt +++ b/tcsocket/requirements.txt @@ -3,13 +3,13 @@ aiodns==3.0.0 aiohttp==3.8.1 aiopg==1.3.4 aioredis==1.3.1 -arq==0.22 +arq==0.25.0 boto3==1.24.57 cchardet==2.1.7 gunicorn==20.1.0 python-dateutil==2.8.2 pillow==9.2.0 -pydantic[email]==1.9.1 +pydantic[email]==2.7.0 raven==6.10.0 requests==2.28.1 uvloop==0.16.0 @@ -17,3 +17,5 @@ ipython==8.4.0 pgcli==3.4.1 ipython-sql==0.4.1 yarl==1.8.1 +logfire[requests, aiohttp]~=1.3.0 +pydantic-settings~=2.6.0 diff --git a/tcsocket/run.py b/tcsocket/run.py index d5f30a4a..cbe70ca0 100755 --- a/tcsocket/run.py +++ b/tcsocket/run.py @@ -35,6 +35,7 @@ def check_app(): logger.info('app started and stopped successfully, apparently configured correctly') +@cli.command() def web(): """ Serve the application @@ -46,7 +47,9 @@ def web(): check_app() - bind = os.getenv('BIND_IP', '127.0.0.1') + f":{os.getenv('PORT', '8000')}" + bind_ip = os.getenv('BIND_IP', '127.0.0.1') + port = os.getenv('PORT', '8000') + bind = f"{bind_ip}:{port}" logger.info('Starting Web, binding to %s', bind) config = dict( @@ -67,8 +70,10 @@ def load(self): logger.info('starting gunicorn...') Application().run() + logger.info('Web server running at %s on port %s', bind_ip, port) +@cli.command() def worker(): """ Run the worker @@ -79,21 +84,26 @@ def worker(): @cli.command() -def auto(): +@click.pass_context +def auto(ctx): port_env = os.getenv('PORT') dyno_env = os.getenv('DYNO') if dyno_env: logger.info('using environment variable DYNO=%r to infer command', dyno_env) if dyno_env.lower().startswith('web'): - web() + logger.info('Running as web server') + ctx.invoke(web) else: - worker() + logger.info('Running as worker') + ctx.invoke(worker) elif port_env and port_env.isdigit(): logger.info('using environment variable PORT=%s to infer command as web', port_env) - web() + logger.info('Running as web server') + ctx.invoke(web) else: logger.info('no environment variable found to infer command, assuming worker') - worker() + logger.info('Running as worker') + ctx.invoke(worker) @cli.command() From 996e807f51323795c3d3d614e85c8f04e9bfbc7b Mon Sep 17 00:00:00 2001 From: Sebastian Prentice Date: Mon, 11 Nov 2024 15:37:55 +0000 Subject: [PATCH 03/10] wip --- tcsocket/app/main.py | 3 +-- tcsocket/app/middleware.py | 1 - tcsocket/app/views/company.py | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tcsocket/app/main.py b/tcsocket/app/main.py index 89aa0b65..8f9c004d 100644 --- a/tcsocket/app/main.py +++ b/tcsocket/app/main.py @@ -33,7 +33,6 @@ async def startup(app: web.Application): redis=redis, session=ClientSession(), ) - debug(settings.logfire_token) if bool(settings.logfire_token): logfire.configure( service_name='socket-server', @@ -41,7 +40,7 @@ async def startup(app: web.Application): send_to_logfire=True, console=False, ) - logfire.instrument_pydantic() + logfire.instrument_pydantic(app) logfire.instrument_requests() logfire.instrument_aiohttp_client() diff --git a/tcsocket/app/middleware.py b/tcsocket/app/middleware.py index 0168bdb9..f9b06eea 100644 --- a/tcsocket/app/middleware.py +++ b/tcsocket/app/middleware.py @@ -145,7 +145,6 @@ def domain_allowed(allow_domains, current_domain): async def company_middleware(request, handler): try: public_key = request.match_info.get('company') - debug(public_key) if public_key: c = sa_companies.c select_fields = c.id, c.name, c.public_key, c.private_key, c.name_display, c.options, c.domains diff --git a/tcsocket/app/views/company.py b/tcsocket/app/views/company.py index 1029a567..7014faa9 100644 --- a/tcsocket/app/views/company.py +++ b/tcsocket/app/views/company.py @@ -17,7 +17,6 @@ async def company_create(request): Authentication and json parsing are done by middleware. """ data = await request.json() - debug(data) update_contractors = data.pop('update_contractors', True) company: CompanyCreateModal = request['model'] existing_company = bool(company.private_key) From e6f0ad85a64508932f0fdde41d09877d39d7e6c5 Mon Sep 17 00:00:00 2001 From: Sebastian Prentice Date: Mon, 11 Nov 2024 16:46:28 +0000 Subject: [PATCH 04/10] update field validators --- .gitignore | 1 + tcsocket/app/settings.py | 6 +++--- tcsocket/app/validation.py | 28 ++++++++++++++-------------- tcsocket/app/views/appointments.py | 10 ++++++---- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index a630a84e..6ac98974 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ activate.prod.* activate.beta.* .pytest_cache/ /.logfire/ +.env diff --git a/tcsocket/app/settings.py b/tcsocket/app/settings.py index 0bdab1af..11661804 100644 --- a/tcsocket/app/settings.py +++ b/tcsocket/app/settings.py @@ -1,9 +1,9 @@ from pathlib import Path -from typing import Optional, ClassVar +from typing import Optional from urllib.parse import urlparse from arq.connections import RedisSettings -from pydantic import validator, Field +from pydantic import Field, field_validator from pydantic_settings import BaseSettings THIS_DIR = Path(__file__).parent @@ -32,7 +32,7 @@ class Settings(BaseSettings): logfire_token: Optional[str] = '' - @validator('redis_settings', always=True, pre=True) + @field_validator('redis_settings', mode='before') def parse_redis_settings(cls, v): conf = urlparse(v) return RedisSettings( diff --git a/tcsocket/app/validation.py b/tcsocket/app/validation.py index 0dc1e4c9..9dc52312 100644 --- a/tcsocket/app/validation.py +++ b/tcsocket/app/validation.py @@ -3,7 +3,7 @@ from secrets import token_hex from typing import Any, List, Optional -from pydantic import BaseModel, EmailStr, constr, validator +from pydantic import BaseModel, EmailStr, constr, field_validator EXTRA_ATTR_TYPES = 'checkbox', 'text_short', 'text_extended', 'integer', 'stars', 'dropdown', 'datetime', 'date' @@ -45,11 +45,11 @@ class CompanyCreateModal(BaseModel): public_key: constr(min_length=18, max_length=20) = None private_key: constr(min_length=20, max_length=50) = None - @validator('public_key', pre=True, always=True) + @field_validator('public_key', mode='before') def set_public_key(cls, v): return v or token_hex(10) - @validator('private_key', pre=True, always=True) + @field_validator('private_key', mode='before') def set_private_key(cls, v): return v or token_hex(20) @@ -132,16 +132,16 @@ class EATypeEnum(str, Enum): class ContractorModel(BaseModel): id: int deleted: bool = False - first_name: constr(max_length=255) = None - last_name: constr(max_length=255) = None - town: constr(max_length=63) = None - country: constr(max_length=63) = None + first_name: Optional[constr(max_length=255)] = None + last_name: Optional[constr(max_length=255)] = None + town: Optional[constr(max_length=63)] = None + country: Optional[constr(max_length=63)] = None last_updated: datetime = None photo: Optional[str] = None - review_rating: float = None + review_rating: Optional[float] = None review_duration: int = None - @validator('last_updated', pre=True, always=True) + @field_validator('last_updated', mode='before') def set_last_updated(cls, v): return v or datetime(2016, 1, 1) @@ -149,12 +149,12 @@ class LatitudeModel(BaseModel): latitude: Optional[float] = None longitude: Optional[float] = None - location: LatitudeModel = None + location: Optional[LatitudeModel] = None extra_attributes: List[ExtraAttributeModel] = [] class SkillModel(BaseModel): subject: str - subject_id: str + subject_id: Optional[int] = None category: str qual_level: str qual_level_id: int @@ -183,7 +183,7 @@ class EnquiryModal(BaseModel): grecaptcha_response: constr(min_length=20, max_length=1000) terms_and_conditions: bool = False - @validator('upstream_http_referrer') + @field_validator('upstream_http_referrer') def val_upstream_http_referrer(cls, v): return v[:1023] @@ -209,8 +209,8 @@ class BookingModel(BaseModel): student_id: int = None student_name: str = '' - @validator('student_name', always=True) - def check_name_or_id(cls, v, values, **kwargs): + @field_validator('student_name', mode='before') + def check_name_or_id(cls, v, values): if v == '' and values['student_id'] is None: raise ValueError('either student_id or student_name is required') return v diff --git a/tcsocket/app/views/appointments.py b/tcsocket/app/views/appointments.py index b8b187e7..8be4996c 100644 --- a/tcsocket/app/views/appointments.py +++ b/tcsocket/app/views/appointments.py @@ -6,7 +6,7 @@ from secrets import compare_digest from typing import Dict, Protocol -from pydantic import BaseModel, ValidationError, validator +from pydantic import BaseModel, ValidationError, field_validator from sqlalchemy import distinct, select from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.sql import and_, functions as sql_f @@ -270,18 +270,20 @@ class SSOData(BaseModel): expires: datetime key: str - @validator('role_type') + @field_validator('role_type') def check_role_type(cls, v): if v != 'Client': raise ValueError('must be "Client"') + return v - class Config: - fields = { + model_config = { + 'alias_generator': { 'role_type': 'rt', 'name': 'nm', 'students': 'srs', 'expires': 'exp', } + } def _get_sso_data(request, company) -> SSOData: From 7b18a940df3b6810d597c9e8a50f0b909be251b9 Mon Sep 17 00:00:00 2001 From: Sebastian Prentice Date: Tue, 12 Nov 2024 08:47:15 +0000 Subject: [PATCH 05/10] now i am getting postgres errors :bomb: --- tcsocket/app/validation.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tcsocket/app/validation.py b/tcsocket/app/validation.py index 9dc52312..d882b605 100644 --- a/tcsocket/app/validation.py +++ b/tcsocket/app/validation.py @@ -3,7 +3,8 @@ from secrets import token_hex from typing import Any, List, Optional -from pydantic import BaseModel, EmailStr, constr, field_validator +from pydantic import BaseModel, EmailStr, constr +from pydantic.v1 import validator EXTRA_ATTR_TYPES = 'checkbox', 'text_short', 'text_extended', 'integer', 'stars', 'dropdown', 'datetime', 'date' @@ -45,11 +46,11 @@ class CompanyCreateModal(BaseModel): public_key: constr(min_length=18, max_length=20) = None private_key: constr(min_length=20, max_length=50) = None - @field_validator('public_key', mode='before') + @validator('public_key', pre=True, always=True) def set_public_key(cls, v): return v or token_hex(10) - @field_validator('private_key', mode='before') + @validator('private_key', pre=True, always=True) def set_private_key(cls, v): return v or token_hex(20) @@ -136,13 +137,14 @@ class ContractorModel(BaseModel): last_name: Optional[constr(max_length=255)] = None town: Optional[constr(max_length=63)] = None country: Optional[constr(max_length=63)] = None - last_updated: datetime = None + last_updated: Optional[datetime] = None photo: Optional[str] = None review_rating: Optional[float] = None review_duration: int = None - @field_validator('last_updated', mode='before') - def set_last_updated(cls, v): + @validator('last_updated', pre=True, always=True) + def set_last_updated(cls, v: datetime) -> datetime: + print('last_updated', v) # Debugging output return v or datetime(2016, 1, 1) class LatitudeModel(BaseModel): @@ -150,7 +152,7 @@ class LatitudeModel(BaseModel): longitude: Optional[float] = None location: Optional[LatitudeModel] = None - extra_attributes: List[ExtraAttributeModel] = [] + extra_attributes: List['ExtraAttributeModel'] = [] class SkillModel(BaseModel): subject: str @@ -183,7 +185,7 @@ class EnquiryModal(BaseModel): grecaptcha_response: constr(min_length=20, max_length=1000) terms_and_conditions: bool = False - @field_validator('upstream_http_referrer') + @validator('upstream_http_referrer') def val_upstream_http_referrer(cls, v): return v[:1023] @@ -209,7 +211,7 @@ class BookingModel(BaseModel): student_id: int = None student_name: str = '' - @field_validator('student_name', mode='before') + @validator('student_name', always=True) def check_name_or_id(cls, v, values): if v == '' and values['student_id'] is None: raise ValueError('either student_id or student_name is required') From dd38be437b291a58cfb372e6deffb24ed821f8b4 Mon Sep 17 00:00:00 2001 From: Sebastian Prentice Date: Thu, 14 Nov 2024 11:39:54 +0000 Subject: [PATCH 06/10] root validator to set last_updated to release_timestamp --- tcsocket/app/processing.py | 1 - tcsocket/app/validation.py | 18 ++++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tcsocket/app/processing.py b/tcsocket/app/processing.py index 81f97d40..33b1e94f 100644 --- a/tcsocket/app/processing.py +++ b/tcsocket/app/processing.py @@ -130,7 +130,6 @@ async def contractor_set( :return: Action: created, updated or deleted """ from .worker import process_image - if contractor.deleted: if not skip_deleted: curr = await conn.execute( diff --git a/tcsocket/app/validation.py b/tcsocket/app/validation.py index d882b605..81b0ff6d 100644 --- a/tcsocket/app/validation.py +++ b/tcsocket/app/validation.py @@ -3,8 +3,9 @@ from secrets import token_hex from typing import Any, List, Optional -from pydantic import BaseModel, EmailStr, constr -from pydantic.v1 import validator +from pydantic import BaseModel, EmailStr, constr, validator, root_validator + +from tcsocket.app.views.company import logger EXTRA_ATTR_TYPES = 'checkbox', 'text_short', 'text_extended', 'integer', 'stars', 'dropdown', 'datetime', 'date' @@ -142,10 +143,15 @@ class ContractorModel(BaseModel): review_rating: Optional[float] = None review_duration: int = None - @validator('last_updated', pre=True, always=True) - def set_last_updated(cls, v: datetime) -> datetime: - print('last_updated', v) # Debugging output - return v or datetime(2016, 1, 1) + @root_validator(pre=True) + def set_last_updated(cls, values): + """ get the release_timestamp and save it to the last_updated field """ + + if 'release_timestamp' not in values: + logger.warning('release_timestamp not found in values, setting last_updated to 2016-01-01') + + values['last_updated'] = values.get('release_timestamp', datetime(2016, 1, 1)) + return values class LatitudeModel(BaseModel): latitude: Optional[float] = None From 54d1df54dce6f513c88abff92bc4aabac5f7daa2 Mon Sep 17 00:00:00 2001 From: Sebastian Prentice Date: Thu, 14 Nov 2024 11:42:00 +0000 Subject: [PATCH 07/10] change logger to stop circular import error --- tcsocket/app/validation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tcsocket/app/validation.py b/tcsocket/app/validation.py index 81b0ff6d..98dcc152 100644 --- a/tcsocket/app/validation.py +++ b/tcsocket/app/validation.py @@ -1,3 +1,4 @@ +import logging from datetime import datetime from enum import Enum, unique from secrets import token_hex @@ -5,7 +6,8 @@ from pydantic import BaseModel, EmailStr, constr, validator, root_validator -from tcsocket.app.views.company import logger +logger = logging.getLogger('socket') + EXTRA_ATTR_TYPES = 'checkbox', 'text_short', 'text_extended', 'integer', 'stars', 'dropdown', 'datetime', 'date' From 9e952150d0c0698183f4b5b7f49c237cbd7ad349 Mon Sep 17 00:00:00 2001 From: Sebastian Prentice Date: Thu, 14 Nov 2024 12:04:58 +0000 Subject: [PATCH 08/10] format --- tcsocket/app/main.py | 2 +- tcsocket/app/processing.py | 1 + tcsocket/app/validation.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tcsocket/app/main.py b/tcsocket/app/main.py index 8f9c004d..a054c767 100644 --- a/tcsocket/app/main.py +++ b/tcsocket/app/main.py @@ -1,8 +1,8 @@ import os import re -import logfire from html import escape +import logfire from aiohttp import ClientSession, web from aiopg.sa import create_engine from arq import create_pool diff --git a/tcsocket/app/processing.py b/tcsocket/app/processing.py index 33b1e94f..81f97d40 100644 --- a/tcsocket/app/processing.py +++ b/tcsocket/app/processing.py @@ -130,6 +130,7 @@ async def contractor_set( :return: Action: created, updated or deleted """ from .worker import process_image + if contractor.deleted: if not skip_deleted: curr = await conn.execute( diff --git a/tcsocket/app/validation.py b/tcsocket/app/validation.py index 98dcc152..a707b957 100644 --- a/tcsocket/app/validation.py +++ b/tcsocket/app/validation.py @@ -4,7 +4,7 @@ from secrets import token_hex from typing import Any, List, Optional -from pydantic import BaseModel, EmailStr, constr, validator, root_validator +from pydantic import BaseModel, EmailStr, constr, root_validator, validator logger = logging.getLogger('socket') @@ -147,7 +147,7 @@ class ContractorModel(BaseModel): @root_validator(pre=True) def set_last_updated(cls, values): - """ get the release_timestamp and save it to the last_updated field """ + """get the release_timestamp and save it to the last_updated field""" if 'release_timestamp' not in values: logger.warning('release_timestamp not found in values, setting last_updated to 2016-01-01') From 148e72aaec487c1e8c99eba4b14aabb6a34f0e60 Mon Sep 17 00:00:00 2001 From: Sebastian Prentice Date: Thu, 14 Nov 2024 12:09:30 +0000 Subject: [PATCH 09/10] seperate out issue fix from logfire change --- tcsocket/app/validation.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tcsocket/app/validation.py b/tcsocket/app/validation.py index a707b957..db608dd8 100644 --- a/tcsocket/app/validation.py +++ b/tcsocket/app/validation.py @@ -145,15 +145,9 @@ class ContractorModel(BaseModel): review_rating: Optional[float] = None review_duration: int = None - @root_validator(pre=True) - def set_last_updated(cls, values): - """get the release_timestamp and save it to the last_updated field""" - - if 'release_timestamp' not in values: - logger.warning('release_timestamp not found in values, setting last_updated to 2016-01-01') - - values['last_updated'] = values.get('release_timestamp', datetime(2016, 1, 1)) - return values + @validator('last_updated', pre=True, always=True) + def set_last_updated(cls, v): + return v or datetime(2016, 1, 1) class LatitudeModel(BaseModel): latitude: Optional[float] = None From 3ccabb7344a759757e39e7aeb785044b396676df Mon Sep 17 00:00:00 2001 From: Sebastian Prentice Date: Thu, 14 Nov 2024 12:56:41 +0000 Subject: [PATCH 10/10] change to use alias rather than validator --- tcsocket/app/validation.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tcsocket/app/validation.py b/tcsocket/app/validation.py index db608dd8..f7df94e7 100644 --- a/tcsocket/app/validation.py +++ b/tcsocket/app/validation.py @@ -4,7 +4,7 @@ from secrets import token_hex from typing import Any, List, Optional -from pydantic import BaseModel, EmailStr, constr, root_validator, validator +from pydantic import AliasPath, BaseModel, EmailStr, Field, constr, validator logger = logging.getLogger('socket') @@ -140,15 +140,11 @@ class ContractorModel(BaseModel): last_name: Optional[constr(max_length=255)] = None town: Optional[constr(max_length=63)] = None country: Optional[constr(max_length=63)] = None - last_updated: Optional[datetime] = None + last_updated: Optional[datetime] = Field(validation_alias=AliasPath('release_timestamp')) photo: Optional[str] = None review_rating: Optional[float] = None review_duration: int = None - @validator('last_updated', pre=True, always=True) - def set_last_updated(cls, v): - return v or datetime(2016, 1, 1) - class LatitudeModel(BaseModel): latitude: Optional[float] = None longitude: Optional[float] = None