diff --git a/.slugignore b/.slugignore new file mode 100644 index 0000000..fbd429e --- /dev/null +++ b/.slugignore @@ -0,0 +1,3 @@ +package.json +package-lock.json +yarn.lock \ No newline at end of file diff --git a/events/fetch.py b/events/fetch.py new file mode 100644 index 0000000..1fab3fb --- /dev/null +++ b/events/fetch.py @@ -0,0 +1,28 @@ +import requests +import time +import json +import sys + +if len(sys.argv) < 2: + print('usage: fetch.py ') + exit() + +mobilize_api_events_url = 'https://api.mobilize.us/v1/organizations/1316/events?timeslot_start=gte_now&is_virtual=false&per_page=100' + +next = mobilize_api_events_url + +events = [] + +while next is not None: + + r = requests.get(next) + resp = r.json() + events += resp['data'] + next = resp['next'] + print('requested {} next {}'.format(r.url, next)) + # be nice to the api + time.sleep(5) + + +with open(sys.argv[1], 'w') as f: + f.write(json.dumps(events)) \ No newline at end of file diff --git a/events/filter.py b/events/filter.py new file mode 100644 index 0000000..dd13f70 --- /dev/null +++ b/events/filter.py @@ -0,0 +1,38 @@ +import json +import sys + +events = [] + +if len(sys.argv) < 3: + print("usage: filter.py ") + exit() + +with open(sys.argv[1]) as f: + events = json.loads(f.read()) + +slim_events = [] + +for e in events: + + # if the event doesnt have a location then we can skip it. + if 'location' not in e['location'] or 'latitude' not in e['location']['location'] or e['location']['location']['latitude'] is None: + continue + + # remove attributes we arent going to display. + del e['location']['congressional_district'] + del e['location']['state_leg_district'] + del e['location']['state_senate_district'] + + slim_events.append({ + 'id': e['id'], + 'browser_url': e['browser_url'], + 'location': e['location'], + 'title': e['title'], + 'timeslots': e['timeslots'][0:3], + 'timeslot_count': len(e['timeslots']), + 'featured_image_url': e['featured_image_url'], + 'event_type': e['event_type'] + }) + +with open(sys.argv[2], 'w') as f: + f.write(json.dumps(slim_events)) \ No newline at end of file diff --git a/fetch-events.sh b/fetch-events.sh new file mode 100755 index 0000000..01b3115 --- /dev/null +++ b/fetch-events.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -ex + +python3 events/fetch.py all-events.json + +python3 events/filter.py all-events.json events.json + +gzip events.json + +aws s3 cp events.json.gz $S3_PATH/events.json --acl public-read --content-encoding gzip --content-type "application/json" + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f710b3a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests==2.22.0 +awscli==1.16.248 \ No newline at end of file diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 0000000..2af1a62 --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +python-3.7.3 \ No newline at end of file