From dcd2a9290ebc7e3f439a04673aff095ced02e728 Mon Sep 17 00:00:00 2001 From: Benjamin Staffin Date: Thu, 3 Nov 2022 20:42:31 -0400 Subject: [PATCH 01/40] Rewrite to use Flask, support Python 3 --- .envrc | 2 + .gcloudignore | 6 + .gitignore | 1 + app.yaml | 21 +- hipchatlib.py | 24 +- main.py | 772 +++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 3 + shell.nix | 15 + slacklib.py | 189 ++++++----- snippets.py | 804 ----------------------------------------------- 10 files changed, 909 insertions(+), 928 deletions(-) create mode 100644 .envrc create mode 100644 .gcloudignore create mode 100644 main.py create mode 100644 requirements.txt create mode 100644 shell.nix delete mode 100644 snippets.py diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..894b897 --- /dev/null +++ b/.envrc @@ -0,0 +1,2 @@ +use nix; +layout python3; diff --git a/.gcloudignore b/.gcloudignore new file mode 100644 index 0000000..f0f21f0 --- /dev/null +++ b/.gcloudignore @@ -0,0 +1,6 @@ +.git +.DS_Store +*.pyc +*.nix +.envrc +#!include:.gitignore diff --git a/.gitignore b/.gitignore index 0d20b64..12b17f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *.pyc +.direnv diff --git a/app.yaml b/app.yaml index c4cdfdb..f0043ad 100644 --- a/app.yaml +++ b/app.yaml @@ -1,7 +1,6 @@ -runtime: python27 -threadsafe: yes -api_version: 1 +runtime: python310 default_expiration: "365d" +app_engine_apis: true handlers: - url: /static @@ -13,23 +12,11 @@ handlers: upload: static/favicon.ico - url: /admin/.* - script: snippets.application + script: snippets.app login: admin - url: .* - script: snippets.application - -skip_files: -- .git -- .DS_Store -- .*.pyc + script: snippets.app builtins: - remote_api: on - -libraries: -- name: jinja2 - version: "2.6" -# This also brings in webapp2_extras: -- name: webapp2 - version: "2.5.1" diff --git a/hipchatlib.py b/hipchatlib.py index 9f945a3..153f909 100644 --- a/hipchatlib.py +++ b/hipchatlib.py @@ -9,18 +9,19 @@ """ import logging -import urllib -import urllib2 +import urllib.error +import urllib.parse +import urllib.request -from google.appengine.ext import webapp +import flask import models def _make_hipchat_api_call(post_dict_with_secret_token): # This is a separate function just to make it easy to mock for tests. - r = urllib2.urlopen('https://api.hipchat.com/v1/rooms/message', - urllib.urlencode(post_dict_with_secret_token)) + r = urllib.request.urlopen('https://api.hipchat.com/v1/rooms/message', + urllib.parse.urlencode(post_dict_with_secret_token)) if r.getcode() != 200: raise ValueError(r.read()) @@ -54,14 +55,13 @@ def send_to_hipchat_room(room_name, message): try: _make_hipchat_api_call(post_dict) - except Exception, why: + except Exception as why: del post_dict['auth_token'] # don't log the secret token! - logging.error('Failed sending %s to hipchat: %s' - % (post_dict, why)) + logging.error('Failed sending %s to hipchat: %s', + post_dict, why) -class TestSendToHipchat(webapp.RequestHandler): +def test_send_to_hipchat_handler(): """Send a (fixed) message to the hipchat room.""" - def get(self): - send_to_hipchat_room('HipChat Tests', 'Test of snippets-to-hipchat') - self.response.out.write('OK') + send_to_hipchat_room('HipChat Tests', 'Test of snippets-to-hipchat') + flask.response.out.write('OK') diff --git a/main.py b/main.py new file mode 100644 index 0000000..4337c6e --- /dev/null +++ b/main.py @@ -0,0 +1,772 @@ +"""Snippets server. + +The main server code for Weekly Snippets. Users can add a summary of +what they did in the last week, and browse other people's snippets. +They will also get weekly mail pointing to a webpage with everyone's +snippets in them. +""" + +__author__ = 'Craig Silverstein ' + +import datetime +import logging +import os +import re +import time +import urllib + +from google.appengine.api import mail, wrap_wsgi_app +from google.appengine.api import users +from google.appengine.ext import db +import flask +import jinja2 + +import hipchatlib +import models +import slacklib +import util + +app = flask.Flask(__name__) +app.wsgi_app = wrap_wsgi_app(app.wsgi_app) + +# This allows mocking in a different day, for testing. +_TODAY_FN = datetime.datetime.now + + +@app.template_filter("readable_date") +def _readable_date_filter(value: datetime.date): + return value.strftime('%B %d, %Y').replace(' 0', ' ') + + +@app.template_filter("iso_date") +def _iso_date_filter(value: datetime.date): + return value.strftime("%m-%d-%Y") + + +def _login_page(request): + """Redirect the user to a page where they can log in.""" + return flask.redirect(users.create_login_url(request.url)) + + +def _current_user_email(): + """Return the logged-in user's email address, converted into lowercase.""" + return users.get_current_user().email().lower() + + +def _get_or_create_user(email, put_new_user=True): + """Return the user object with the given email, creating it if needed. + + Considers the permissions scope of the currently logged in web user, + and raises an IndexError if the currently logged in user is not the same as + the queried email address (or is an admin). + + NOTE: Any access that causes _get_or_create_user() is an access that + indicates the user is active again, so they are "unhidden" in the db. + """ + user = util.get_user(email) + if user: + if user.is_hidden: + # Any access that causes _get_or_create_user() is an access + # that indicates the user is active again, so un-hide them. + # TODO(csilvers): move this get/update/put atomic into a txn + user.is_hidden = False + user.put() + elif not _logged_in_user_has_permission_for(email): + # TODO(csilvers): turn this into a 403 somewhere + raise IndexError('User "%s" not found; did you specify' + ' the full email address?' % email) + else: + # You can only create a new user under one of the app-listed domains. + try: + app_settings = models.AppSettings.get() + except ValueError: + # TODO(csilvers): do this instead: + # /admin/settings?redirect_to=user_setting + return None + + domain = email.split('@')[-1] + allowed_domains = app_settings.domains + if domain not in allowed_domains: + # TODO(csilvers): turn this into a 403 somewhere + raise RuntimeError('Permission denied: ' + 'This app is for users from %s.' + ' But you are from %s.' + % (' or '.join(allowed_domains), domain)) + + # Set the user defaults based on the global app defaults. + user = models.User(created=_TODAY_FN(), + email=email, + uses_markdown=app_settings.default_markdown, + private_snippets=app_settings.default_private, + wants_email=app_settings.default_email) + if put_new_user: + db.put(user) + db.get(user.key()) # ensure db consistency for HRD + return user + + +def _logged_in_user_has_permission_for(email): + """True if the current logged-in appengine user can edit this user.""" + return (email == _current_user_email()) or users.is_current_user_admin() + + +def _can_view_private_snippets(my_email, snippet_email): + """Return true if I have permission to view other's private snippet. + + I have permission to view if I am in the same domain as the person + who wrote the snippet (domain is everything following the @ in the + email). + + Arguments: + my_email: the email address of the currently logged in user + snippet_email: the email address of the snippet we're trying to view. + + Returns: + True if my_email has permission to view snippet_email's private + emails, or False else. + """ + my_at = my_email.rfind('@') + snippet_at = snippet_email.rfind('@') + if my_at == -1 or snippet_at == -1: + return False # be safe + return my_email[my_at:] == snippet_email[snippet_at:] + + +def _send_to_chat(msg, url_path): + """Send a message to the main room/channel for active chat integrations.""" + try: + app_settings = models.AppSettings.get() + except ValueError: + logging.warning('Not sending to chat: app settings not configured') + return + + msg = "%s %s%s" % (msg, app_settings.hostname, url_path) + + hipchat_room = app_settings.hipchat_room + if hipchat_room: + hipchatlib.send_to_hipchat_room(hipchat_room, msg) + + slack_channel = app_settings.slack_channel + if slack_channel: + slacklib.send_to_slack_channel(slack_channel, msg) + + +@app.route("/") +def user_page_handler(): + """Show all the snippets for a single user.""" + + if not users.get_current_user(): + return _login_page(flask.request) + + user_email = flask.request.args.get('u', _current_user_email()) + user = util.get_user(user_email) + + if not user: + # If there are no app settings, set those up before setting + # up the user settings. + if users.is_current_user_admin(): + try: + models.AppSettings.get() + except ValueError: + return flask.redirect( + "/admin/settings?redirect_to=user_setting" + "&msg=Welcome+to+the+snippet+server!+" + "Please+take+a+moment+to+configure+it.") + + template_values = { + 'new_user': True, + 'login_url': users.create_login_url(flask.request.uri), + 'logout_url': users.create_logout_url('/'), + 'username': user_email, + } + return flask.render_template('new_user.html', **template_values) + + snippets = util.snippets_for_user(user_email) + + if not _can_view_private_snippets(_current_user_email(), user_email): + snippets = [snippet for snippet in snippets if not snippet.private] + snippets = util.fill_in_missing_snippets(snippets, user, + user_email, _TODAY_FN()) + snippets.reverse() # get to newest snippet first + + template_values = { + 'logout_url': users.create_logout_url('/'), + 'message': flask.request.args.get('msg'), + 'username': user_email, + 'is_admin': users.is_current_user_admin(), + 'domain': user_email.split('@')[-1], + 'view_week': util.existingsnippet_monday(_TODAY_FN()), + # Snippets for the week of are due today. + 'one_week_ago': _TODAY_FN().date() - datetime.timedelta(days=7), + 'eight_days_ago': _TODAY_FN().date() - datetime.timedelta(days=8), + 'editable': (_logged_in_user_has_permission_for(user_email) and + flask.request.args.get('edit', '1') == '1'), + 'user': user, + 'snippets': snippets, + 'null_category': models.NULL_CATEGORY, + } + return flask.render_template('user_snippets.html', **template_values) + + +def _title_case(s): + """Like string.title(), but does not uppercase 'and'.""" + # Smarter would be to use 'pip install titlecase'. + SMALL = 'a|an|and|as|at|but|by|en|for|if|in|of|on|or|the|to|v\.?|via|vs\.?' + # We purposefully don't match small words at the beginning of a string. + SMALL_RE = re.compile(r' (%s)\b' % SMALL, re.I) + return SMALL_RE.sub(lambda m: ' ' + m.group(1).lower(), s.title().strip()) + + +@app.route("/weekly") +def summary_page_handler(): + """Show all the snippets for a single week.""" + + if not users.get_current_user(): + return _login_page(flask.request) + + week_string = flask.request.args.get('week') + if week_string: + week = datetime.datetime.strptime(week_string, '%m-%d-%Y').date() + else: + week = util.existingsnippet_monday(_TODAY_FN()) + + snippets_q = models.Snippet.all() + snippets_q.filter('week = ', week) + snippets = snippets_q.fetch(1000) # good for many users... + # TODO(csilvers): filter based on wants_to_view + + # Get all the user records so we can categorize snippets. + user_q = models.User.all() + results = user_q.fetch(1000) + email_to_category = {} + email_to_user = {} + for result in results: + # People aren't very good about capitalizing their + # categories consistently, so we enforce title-case, + # with exceptions for 'and'. + email_to_category[result.email] = _title_case(result.category) + email_to_user[result.email] = result + + # Collect the snippets and users by category. As we see each email, + # delete it from email_to_category. At the end of this, + # email_to_category will hold people who did not give + # snippets this week. + snippets_and_users_by_category = {} + for snippet in snippets: + # Ignore this snippet if we don't have permission to view it. + if (snippet.private and + not _can_view_private_snippets(_current_user_email(), + snippet.email)): + continue + category = email_to_category.get( + snippet.email, models.NULL_CATEGORY + ) + if snippet.email in email_to_user: + snippets_and_users_by_category.setdefault(category, []).append( + (snippet, email_to_user[snippet.email]) + ) + else: + snippets_and_users_by_category.setdefault(category, []).append( + (snippet, models.User(email=snippet.email)) + ) + + if snippet.email in email_to_category: + del email_to_category[snippet.email] + + # Add in empty snippets for the people who didn't have any -- + # unless a user is marked 'hidden'. (That's what 'hidden' + # means: pretend they don't exist until they have a non-empty + # snippet again.) + for (email, category) in email_to_category.iteritems(): + if not email_to_user[email].is_hidden: + snippet = models.Snippet(email=email, week=week) + snippets_and_users_by_category.setdefault(category, []).append( + (snippet, email_to_user[snippet.email]) + ) + + # Now get a sorted list, categories in alphabetical order and + # each snippet-author within the category in alphabetical + # order. + # The data structure is ((category, ((snippet, user), ...)), ...) + categories_and_snippets = [] + for (category, + snippets_and_users) in snippets_and_users_by_category.iteritems(): + snippets_and_users.sort(key=lambda snippet, user: snippet.email) + categories_and_snippets.append((category, snippets_and_users)) + categories_and_snippets.sort() + + template_values = { + 'logout_url': users.create_logout_url('/'), + 'message': flask.request.args.get('msg'), + # Used only to switch to 'username' mode and to modify settings. + 'username': _current_user_email(), + 'is_admin': users.is_current_user_admin(), + 'prev_week': week - datetime.timedelta(7), + 'view_week': week, + 'next_week': week + datetime.timedelta(7), + 'categories_and_snippets': categories_and_snippets, + } + return flask.render_template('weekly_snippets.html', **template_values) + + +def update_snippet(email): + week_string = flask.request.args.get('week') + week = datetime.datetime.strptime(week_string, '%m-%d-%Y').date() + assert week.weekday() == 0, 'passed-in date must be a Monday' + + text = flask.request.args.get('snippet') + + private = flask.request.args.get('private') == 'True' + is_markdown = flask.request.args.get('is_markdown') == 'True' + + # TODO(csilvers): make this get-update-put atomic. + # (maybe make the snippet id be email + week). + q = models.Snippet.all() + q.filter('email = ', email) + q.filter('week = ', week) + snippet = q.get() + + # When adding a snippet, make sure we create a user record for + # that email as well, if it doesn't already exist. + user = _get_or_create_user(email) + + # Store user's display_name in snippet so that if a user is later + # deleted, we could still show his / her display_name. + if snippet: + snippet.text = text # just update the snippet text + snippet.display_name = user.display_name + snippet.private = private + snippet.is_markdown = is_markdown + else: + # add the snippet to the db + snippet = models.Snippet(created=_TODAY_FN(), + display_name=user.display_name, + email=email, week=week, + text=text, private=private, + is_markdown=is_markdown) + db.put(snippet) + db.get(snippet.key()) # ensure db consistency for HRD + + flask.response.set_status(200) + + +@app.route("/update_snippet") +def update_snippet_handler(): + + if flask.request.method == "POST": + """handle ajax updates via POST + + in particular, return status via json rather than redirects and + hard exceptions. This isn't actually RESTy, it's just status + codes and json. + """ + # TODO(marcos): consider using PUT? + + if not users.get_current_user(): + # 403s are the catch-all 'please log in error' here + return flask.make_response({"status": 403, "message": "not logged in"}, 403) + + email = flask.request.args.get('u', _current_user_email()) + + if not _logged_in_user_has_permission_for(email): + # TODO(marcos): present these messages to the ajax client + error = ('You do not have permissions to update user' + ' snippets for %s' % email) + return flask.make_response({"status": 403, "message": error}, 403) + + update_snippet(email) + return flask.make_response({"status": 200, "message": "ok"}) + + elif flask.request.method == "GET": + if not users.get_current_user(): + return _login_page(flask.request) + + email = flask.request.args.get('u', _current_user_email()) + if not _logged_in_user_has_permission_for(email): + # TODO(csilvers): return a 403 here instead. + raise RuntimeError('You do not have permissions to update user' + ' snippets for %s' % email) + + update_snippet(email) + + email = flask.request.args.get('u', _current_user_email()) + return flask.redirect("/?msg=Snippet+saved&u=%s" % urllib.parse.quote(email)) + + +@app.route("/settings") +def settings_handler(): + """Page to display a user's settings (from class User) for modification.""" + + if not users.get_current_user(): + return _login_page(flask.request) + + user_email = flask.request.args.get('u', _current_user_email()) + if not _logged_in_user_has_permission_for(user_email): + # TODO(csilvers): return a 403 here instead. + raise RuntimeError('You do not have permissions to view user' + ' settings for %s' % user_email) + # We won't put() the new user until the settings are saved. + user = _get_or_create_user(user_email, put_new_user=False) + try: + user.key() + is_new_user = False + except db.NotSavedError: + is_new_user = True + + template_values = { + 'logout_url': users.create_logout_url('/'), + 'message': flask.request.args.get('msg'), + 'username': user.email, + 'is_admin': users.is_current_user_admin(), + 'view_week': util.existingsnippet_monday(_TODAY_FN()), + 'user': user, + 'is_new_user': is_new_user, + 'redirect_to': flask.request.args.get('redirect_to', ''), + # We could get this from user, but we want to replace + # commas with newlines for printing. + 'wants_to_view': user.wants_to_view.replace(',', '\n'), + } + return flask.render_template('settings.html', **template_values) + + +@app.route("/update_settings") +def update_settings_handler(): + """Updates the db with modifications from the Settings page.""" + + if not users.get_current_user(): + return _login_page(flask.request) + + user_email = flask.request.args.get('u', _current_user_email()) + if not _logged_in_user_has_permission_for(user_email): + # TODO(csilvers): return a 403 here instead. + raise RuntimeError('You do not have permissions to modify user' + ' settings for %s' % user_email) + # TODO(csilvers): make this get/update/put atomic (put in a txn) + user = _get_or_create_user(user_email) + + # First, check if the user clicked on 'delete' or 'hide' + # rather than 'save'. + if flask.request.args.get('hide'): + user.is_hidden = True + user.put() + time.sleep(0.1) # some time for eventual consistency + return flask.redirect('/weekly?msg=You+are+now+hidden.+Have+a+nice+day!') + elif flask.request.args.get('delete'): + db.delete(user) + return flask.redirect('/weekly?msg=Your+account+has+been+deleted.+' + '(Note+your+existing+snippets+have+NOT+been+' + 'deleted.)+Have+a+nice+day!') + + display_name = flask.request.args.get('display_name') + category = flask.request.args.get('category') + uses_markdown = flask.request.args.get('markdown') == 'yes' + private_snippets = flask.request.args.get('private') == 'yes' + wants_email = flask.request.args.get('reminder_email') == 'yes' + + # We want this list to be comma-separated, but people are + # likely to use whitespace to separate as well. Convert here. + wants_to_view = flask.request.args.get('to_view') + wants_to_view = re.sub(r'\s+', ',', wants_to_view) + wants_to_view = wants_to_view.split(',') + wants_to_view = [w for w in wants_to_view if w] # deal with ',,' + wants_to_view = ','.join(wants_to_view) # TODO(csilvers): keep as list + + # Changing their settings is the kind of activity that unhides + # someone who was hidden, unless they specifically ask to be + # hidden. + is_hidden = flask.request.args.get('is_hidden', 'no') == 'yes' + + user.is_hidden = is_hidden + user.display_name = display_name + user.category = category or models.NULL_CATEGORY + user.uses_markdown = uses_markdown + user.private_snippets = private_snippets + user.wants_email = wants_email + user.wants_to_view = wants_to_view + db.put(user) + db.get(user.key()) # ensure db consistency for HRD + + redirect_to = flask.request.args.get('redirect_to') + if redirect_to == 'snippet_entry': # true for new_user.html + return flask.redirect('/?u=%s' % urllib.parse.quote(user_email)) + else: + return flask.redirect("/settings?msg=Changes+saved&u=%s" + % urllib.parse.quote(user_email)) + + +@app.route("/admin/settings") +def admin_settings_handler(): + """Page to display settings for the whole app, for modification. + + This page should be restricted to admin users via app.yaml. + """ + my_domain = _current_user_email().split('@')[-1] + app_settings = models.AppSettings.get(create_if_missing=True, + domains=[my_domain]) + + template_values = { + 'logout_url': users.create_logout_url('/'), + 'message': flask.request.args.get('msg'), + 'username': _current_user_email(), + 'is_admin': users.is_current_user_admin(), + 'view_week': util.existingsnippet_monday(_TODAY_FN()), + 'redirect_to': flask.request.args.get('redirect_to', ''), + 'settings': app_settings, + 'slack_slash_commands': ( + slacklib.command_usage().strip()) + } + return flask.render_template('app_settings.html', **template_values) + + +@app.route("/admin/update_settings") +def admin_update_settings_handler(): + """Updates the db with modifications from the App-Settings page. + + This page should be restricted to admin users via app.yaml. + """ + _get_or_create_user(_current_user_email()) + + domains = flask.request.args.get('domains') + default_private = flask.request.args.get('private') == 'yes' + default_markdown = flask.request.args.get('markdown') == 'yes' + default_email = flask.request.args.get('reminder_email') == 'yes' + email_from = flask.request.args.get('email_from') + hipchat_room = flask.request.args.get('hipchat_room') + hipchat_token = flask.request.args.get('hipchat_token') + slack_channel = flask.request.args.get('slack_channel') + slack_token = flask.request.args.get('slack_token') + slack_slash_token = flask.request.args.get('slack_slash_token') + + # Turn domains into a list. Allow whitespace or comma to separate. + domains = re.sub(r'\s+', ',', domains) + domains = [d for d in domains.split(',') if d] + + @db.transactional + def update_settings(): + app_settings = models.AppSettings.get(create_if_missing=True, + domains=domains) + app_settings.domains = domains + app_settings.default_private = default_private + app_settings.default_markdown = default_markdown + app_settings.default_email = default_email + app_settings.email_from = email_from + app_settings.hipchat_room = hipchat_room + app_settings.hipchat_token = hipchat_token + app_settings.slack_channel = slack_channel + app_settings.slack_token = slack_token + app_settings.slack_slash_token = slack_slash_token + app_settings.put() + + update_settings() + + redirect_to = flask.request.args.get('redirect_to') + if redirect_to == 'user_setting': # true for new_user.html + return flask.redirect('/settings?redirect_to=snippet_entry' + '&msg=Now+enter+your+personal+user+settings.') + else: + return flask.redirect("/admin/settings?msg=Changes+saved") + + +@app.route("/admin/manage_users") +def admin_manage_users_handler(): + """Lets admins delete and otherwise manage users.""" + # options are 'email', 'creation_time', 'last_snippet_time' + sort_by = flask.request.args.get('sort_by', 'creation_time') + + # First, check if the user had clicked on a button. + for (name, value) in flask.request.params.iteritems(): + if name.startswith('hide '): + email_of_user_to_hide = name[len('hide '):] + # TODO(csilvers): move this get/update/put atomic into a txn + user = util.get_user_or_die(email_of_user_to_hide) + user.is_hidden = True + user.put() + time.sleep(0.1) # encourage eventual consistency + return flask.redirect('/admin/manage_users?sort_by=%s&msg=%s+hidden' + % (sort_by, email_of_user_to_hide)) + if name.startswith('unhide '): + email_of_user_to_unhide = name[len('unhide '):] + # TODO(csilvers): move this get/update/put atomic into a txn + user = util.get_user_or_die(email_of_user_to_unhide) + user.is_hidden = False + user.put() + time.sleep(0.1) # encourage eventual consistency + return flask.redirect('/admin/manage_users?sort_by=%s&msg=%s+unhidden' + % (sort_by, email_of_user_to_unhide)) + if name.startswith('delete '): + email_of_user_to_delete = name[len('delete '):] + user = util.get_user_or_die(email_of_user_to_delete) + db.delete(user) + time.sleep(0.1) # encourage eventual consistency + return flask.redirect('/admin/manage_users?sort_by=%s&msg=%s+deleted' + % (sort_by, email_of_user_to_delete)) + + user_q = models.User.all() + results = user_q.fetch(1000) + + # Tuple: (email, is-hidden, creation-time, days since last snippet) + user_data = [] + for user in results: + # Get the last snippet for that user. + last_snippet = util.most_recent_snippet_for_user(user.email) + if last_snippet: + seconds_since_snippet = ( + (_TODAY_FN().date() - last_snippet.week).total_seconds()) + weeks_since_snippet = int( + seconds_since_snippet / + datetime.timedelta(days=7).total_seconds()) + else: + weeks_since_snippet = None + user_data.append((user.email, user.is_hidden, + user.created, weeks_since_snippet)) + + # We have to use 'cmp' here since we want ascending in the + # primary key and descending in the secondary key, sometimes. + if sort_by == 'email': + user_data.sort(lambda x, y: cmp(x[0], y[0])) + elif sort_by == 'creation_time': + user_data.sort(lambda x, y: (-cmp(x[2] or datetime.datetime.min, + y[2] or datetime.datetime.min) + or cmp(x[0], y[0]))) + elif sort_by == 'last_snippet_time': + user_data.sort(lambda x, y: (-cmp(1000 if x[3] is None else x[3], + 1000 if y[3] is None else y[3]) + or cmp(x[0], y[0]))) + else: + raise ValueError('Invalid sort_by value "%s"' % sort_by) + + template_values = { + 'logout_url': users.create_logout_url('/'), + 'message': flask.request.args.get('msg'), + 'username': _current_user_email(), + 'is_admin': users.is_current_user_admin(), + 'view_week': util.existingsnippet_monday(_TODAY_FN()), + 'user_data': user_data, + 'sort_by': sort_by, + } + return flask.render_template('manage_users.html', **template_values) + + +# The following two classes are called by cron. + + +def _get_email_to_current_snippet_map(today): + """Return a map from email to True if they've written snippets this week. + + Goes through all users registered on the system, and checks if + they have a snippet in the db for the appropriate snippet-week for + 'today'. If so, they get entered into the return-map with value + True. If not, they have value False. + + Note that users whose 'wants_email' field is set to False will not + be included in either list. + + Arguments: + today: a datetime.datetime object representing the + 'current' day. We use the normal algorithm to determine what is + the most recent snippet-week for this day. + + Returns: + a map from email (user.email for each user) to True or False, + depending on if they've written snippets for this week or not. + """ + user_q = models.User.all() + users = user_q.fetch(1000) + retval = {} + for user in users: + if not user.wants_email: # ignore this user + continue + retval[user.email] = False # assume the worst, for now + + week = util.existingsnippet_monday(today) + snippets_q = models.Snippet.all() + snippets_q.filter('week = ', week) + snippets = snippets_q.fetch(1000) + for snippet in snippets: + if snippet.email in retval: # don't introduce new keys here + retval[snippet.email] = True + + return retval + + +def _maybe_send_snippets_mail(to, subject, template_path, template_values): + try: + app_settings = models.AppSettings.get() + except ValueError: + logging.error('Not sending email: app settings are not configured.') + return + if not app_settings.email_from: + return + + template_values.setdefault('hostname', app_settings.hostname) + + jinja2_instance = jinja2.get_jinja2() + mail.send_mail(sender=app_settings.email_from, + to=to, + subject=subject, + body=jinja2_instance.render_template(template_path, + **template_values)) + # Appengine has a quota of 32 emails per minute: + # https://developers.google.com/appengine/docs/quotas#Mail + # We pause 2 seconds between each email to make sure we + # don't go over that. + time.sleep(2) + + +@app.route("/admin/send_friday_reminder_chat") +def admin_send_friday_reminder_chat_handler(): + """Send a chat message to the configured chat room(s).""" + msg = 'Reminder: Weekly snippets due Monday at 5pm.' + _send_to_chat(msg, "/") + + +@app.route("/admin/send_reminder_email") +def admin_send_reminder_email_handler(): + """Send an email to everyone who doesn't have a snippet for this week.""" + + def _send_mail(email): + template_values = {} + _maybe_send_snippets_mail(email, 'Weekly snippets due today at 5pm', + 'reminder_email.txt', template_values) + + email_to_has_snippet = _get_email_to_current_snippet_map(_TODAY_FN()) + for (user_email, has_snippet) in email_to_has_snippet.iteritems(): + if not has_snippet: + _send_mail(user_email) + logging.debug('sent reminder email to %s', user_email) + else: + logging.debug('did not send reminder email to %s: ' + 'has a snippet already', user_email) + + msg = 'Reminder: Weekly snippets due today at 5pm.' + _send_to_chat(msg, "/") + + +@app.route("/admin/send_view_email") +def admin_send_view_email_handler(): + """Send an email to everyone to look at the week's snippets.""" + + def _send_mail(self, email, has_snippets): + template_values = {'has_snippets': has_snippets} + _maybe_send_snippets_mail(email, 'Weekly snippets are ready!', + 'view_email.txt', template_values) + + email_to_has_snippet = _get_email_to_current_snippet_map(_TODAY_FN()) + for (user_email, has_snippet) in email_to_has_snippet.iteritems(): + _send_mail(user_email, has_snippet) + logging.debug('sent "view" email to %s', user_email) + + msg = 'Weekly snippets are ready!' + _send_to_chat(msg, "/weekly") + + +app.add_url_rule("/slack", view_func=slacklib.slash_command_handler, + methods=["POST"]) +app.add_url_rule("/admin/test_send_to_hipchat", + view_func=hipchatlib.test_send_to_hipchat_handler) + +if __name__ == '__main__': + # This is used when running locally only. When deploying to Google App + # Engine, a webserver process such as Gunicorn will serve the app. You + # can configure startup instructions by adding `entrypoint` to app.yaml. + app.run(host='127.0.0.1', debug=True) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..96be600 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +appengine-python-standard>=1.0.0 +Flask>=2.2.2 +Jinja2>=3.1.2 diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..543e312 --- /dev/null +++ b/shell.nix @@ -0,0 +1,15 @@ +{ pkgs ? import {} }: + +with pkgs; + +mkShell { + buildInputs = [ + google-app-engine-go-sdk # dev_appserver.py is in here for some reason + (google-cloud-sdk.withExtraComponents ([ + google-cloud-sdk.components.app-engine-python + google-cloud-sdk.components.app-engine-python-extras + ])) + jre + + ]; +} diff --git a/slacklib.py b/slacklib.py index 0b0952d..89b6502 100644 --- a/slacklib.py +++ b/slacklib.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals - """Snippets server -> Slack integration. This provides HipChat integration with the snippet server, for @@ -20,13 +18,14 @@ import datetime import json import logging -import re import os +import re import textwrap -import urllib -import urllib2 -import webapp2 +import urllib.error +import urllib.parse +import urllib.request +import flask from google.appengine.ext import db from google.appengine.api import memcache @@ -59,7 +58,7 @@ def _web_api(api_method, payload): app_settings = models.AppSettings.get() payload.setdefault('token', app_settings.slack_token) uri = 'https://slack.com/api/' + api_method - r = urllib2.urlopen(uri, urllib.urlencode(payload)) + r = urllib.request.urlopen(uri, urllib.parse.urlencode(payload)) # check return code for server errors if r.getcode() != 200: @@ -116,7 +115,7 @@ def send_to_slack_channel(channel, msg): 'icon_emoji': ':pencil:', 'unfurl_links': False, # no link previews, please }) - except ValueError, why: + except ValueError as why: logging.error('Failed sending message to slack: %s', why) @@ -368,95 +367,95 @@ def command_dump(user_email): return "```{}```".format(snippet.text or 'No snippet yet for this week') -class SlashCommand(webapp2.RequestHandler): - def post(self): - """Process an incoming slash command from Slack. - - Incoming request POST looks like the following (example taken from - https://api.slack.com/slash-commands): - token=gIkuvaNzQIHg97ATvDxqgjtO - team_id=T0001 - team_domain=example - channel_id=C2147483705 - channel_name=test - user_id=U2147483697 - user_name=Steve - command=/weather - text=94070 - """ - req, res = self.request, self.response - - expected_token = models.AppSettings.get().slack_slash_token - - if not expected_token: - res.write('Slack slash commands disabled. An admin ' - 'can enable them at /admin/settings') - return +def slash_command_handler(): + """Process an incoming slash command from Slack. + + Incoming request POST looks like the following (example taken from + https://api.slack.com/slash-commands): + token=gIkuvaNzQIHg97ATvDxqgjtO + team_id=T0001 + team_domain=example + channel_id=C2147483705 + channel_name=test + user_id=U2147483697 + user_name=Steve + command=/weather + text=94070 + """ + req, res = flask.request, flask.response - # verify slash API post token for security - if _REQUIRE_SLASH_TOKEN: - token = req.get('token') - if token != expected_token: - logging.error("POST MADE WITH INVALID TOKEN") - res.write("OH NO YOU DIDNT! Security issue plz contact admin.") - return - - user_name = req.get('user_name') - user_id = req.get('user_id') - text = req.get('text') - - try: - user_email = _get_user_email_cached(user_id) - except ValueError: - logging.error("Failed getting %s email from Slack API", user_name) - res.write( - "Error getting your email address from the Slack API! " - "Please contact an admin and report the time of this error." - ) + expected_token = models.AppSettings.get().slack_slash_token + + if not expected_token: + res.write('Slack slash commands disabled. An admin ' + 'can enable them at /admin/settings') + return + + # verify slash API post token for security + if _REQUIRE_SLASH_TOKEN: + token = req.get('token') + if token != expected_token: + logging.error("POST MADE WITH INVALID TOKEN") + res.write("OH NO YOU DIDNT! Security issue plz contact admin.") return - words = text.strip().split() - if not words: - logging.info('null (list) command from user %s', user_name) + + user_name = req.get('user_name') + user_id = req.get('user_id') + text = req.get('text') + + try: + user_email = _get_user_email_cached(user_id) + except ValueError: + logging.error("Failed getting %s email from Slack API", user_name) + res.write( + "Error getting your email address from the Slack API! " + "Please contact an admin and report the time of this error." + ) + return + + words = text.strip().split() + if not words: + logging.info('null (list) command from user %s', user_name) + res.write(command_list(user_email)) + else: + cmd, args = words[0], words[1:] + if cmd == 'help': + logging.info('help command from user %s', user_name) + res.write(command_help()) + elif cmd == 'whoami': + # undocumented command to echo user email back + logging.info('whoami command from user %s', user_name) + res.write(user_email) + elif cmd == 'whoami!': + # whoami! forces a refresh of cache, for debugging + logging.info('whoami! command from user %s', user_name) + logging.info('whoami! potential cached email for %s: %s', + user_name, user_email) + refreshed = _get_user_email_cached(user_id, force_refresh=True) + logging.info('whoami! refreshed email for %s: %s', + user_name, refreshed) + res.write(refreshed) + elif cmd == 'list': + # this is the same as the null command, but support for UX + logging.info('list command from user %s', user_name) res.write(command_list(user_email)) + elif cmd == 'last': + logging.info('last command from user %s', user_name) + res.write(command_last(user_email)) + elif cmd == 'add': + logging.info('add command from user %s', user_name) + res.write(command_add(user_email, " ".join(args))) + elif cmd == 'del': + logging.info('del command from user %s', user_name) + res.write(command_del(user_email, args)) + elif cmd == 'dump': + logging.info('dump command from user %s', user_name) + res.write(command_dump(user_email)) else: - cmd, args = words[0], words[1:] - if cmd == 'help': - logging.info('help command from user %s', user_name) - res.write(command_help()) - elif cmd == 'whoami': - # undocumented command to echo user email back - logging.info('whoami command from user %s', user_name) - res.write(user_email) - elif cmd == 'whoami!': - # whoami! forces a refresh of cache, for debugging - logging.info('whoami! command from user %s', user_name) - logging.info('whoami! potential cached email for %s: %s', - user_name, user_email) - refreshed = _get_user_email_cached(user_id, force_refresh=True) - logging.info('whoami! refreshed email for %s: %s', - user_name, refreshed) - res.write(refreshed) - elif cmd == 'list': - # this is the same as the null command, but support for UX - logging.info('list command from user %s', user_name) - res.write(command_list(user_email)) - elif cmd == 'last': - logging.info('last command from user %s', user_name) - res.write(command_last(user_email)) - elif cmd == 'add': - logging.info('add command from user %s', user_name) - res.write(command_add(user_email, " ".join(args))) - elif cmd == 'del': - logging.info('del command from user %s', user_name) - res.write(command_del(user_email, args)) - elif cmd == 'dump': - logging.info('dump command from user %s', user_name) - res.write(command_dump(user_email)) - else: - logging.info('unknown command %s from user %s', cmd, user_name) - res.write( - "I don't understand what you said! " - "Perhaps you meant one of these?\n```%s```\n" - % command_usage() - ) + logging.info('unknown command %s from user %s', cmd, user_name) + res.write( + "I don't understand what you said! " + "Perhaps you meant one of these?\n```%s```\n" + % command_usage() + ) diff --git a/snippets.py b/snippets.py deleted file mode 100644 index 55afe95..0000000 --- a/snippets.py +++ /dev/null @@ -1,804 +0,0 @@ -"""Snippets server. - -The main server code for Weekly Snippets. Users can add a summary of -what they did in the last week, and browse other people's snippets. -They will also get weekly mail pointing to a webpage with everyone's -snippets in them. -""" - -__author__ = 'Craig Silverstein ' - -import datetime -import logging -import os -import re -import time -import urllib - -from google.appengine.api import mail -from google.appengine.api import users -from google.appengine.ext import db -import webapp2 -from webapp2_extras import jinja2 - -import hipchatlib -import models -import slacklib -import util - - -# This allows mocking in a different day, for testing. -_TODAY_FN = datetime.datetime.now - - -jinja2.default_config['template_path'] = os.path.join( - os.path.dirname(__file__), - "templates" -) -jinja2.default_config['filters'] = { - 'readable_date': ( - lambda value: value.strftime('%B %d, %Y').replace(' 0', ' ')), - 'iso_date': ( - lambda value: value.strftime('%m-%d-%Y')), -} - - -def _login_page(request, redirector): - """Redirect the user to a page where they can log in.""" - redirector.redirect(users.create_login_url(request.uri)) - - -def _current_user_email(): - """Return the logged-in user's email address, converted into lowercase.""" - return users.get_current_user().email().lower() - - -def _get_or_create_user(email, put_new_user=True): - """Return the user object with the given email, creating it if needed. - - Considers the permissions scope of the currently logged in web user, - and raises an IndexError if the currently logged in user is not the same as - the queried email address (or is an admin). - - NOTE: Any access that causes _get_or_create_user() is an access that - indicates the user is active again, so they are "unhidden" in the db. - """ - user = util.get_user(email) - if user: - if user.is_hidden: - # Any access that causes _get_or_create_user() is an access - # that indicates the user is active again, so un-hide them. - # TODO(csilvers): move this get/update/put atomic into a txn - user.is_hidden = False - user.put() - elif not _logged_in_user_has_permission_for(email): - # TODO(csilvers): turn this into a 403 somewhere - raise IndexError('User "%s" not found; did you specify' - ' the full email address?' % email) - else: - # You can only create a new user under one of the app-listed domains. - try: - app_settings = models.AppSettings.get() - except ValueError: - # TODO(csilvers): do this instead: - # /admin/settings?redirect_to=user_setting - return None - - domain = email.split('@')[-1] - allowed_domains = app_settings.domains - if domain not in allowed_domains: - # TODO(csilvers): turn this into a 403 somewhere - raise RuntimeError('Permission denied: ' - 'This app is for users from %s.' - ' But you are from %s.' - % (' or '.join(allowed_domains), domain)) - - # Set the user defaults based on the global app defaults. - user = models.User(created=_TODAY_FN(), - email=email, - uses_markdown=app_settings.default_markdown, - private_snippets=app_settings.default_private, - wants_email=app_settings.default_email) - if put_new_user: - db.put(user) - db.get(user.key()) # ensure db consistency for HRD - return user - - -def _logged_in_user_has_permission_for(email): - """True if the current logged-in appengine user can edit this user.""" - return (email == _current_user_email()) or users.is_current_user_admin() - - -def _can_view_private_snippets(my_email, snippet_email): - """Return true if I have permission to view other's private snippet. - - I have permission to view if I am in the same domain as the person - who wrote the snippet (domain is everything following the @ in the - email). - - Arguments: - my_email: the email address of the currently logged in user - snippet_email: the email address of the snippet we're trying to view. - - Returns: - True if my_email has permission to view snippet_email's private - emails, or False else. - """ - my_at = my_email.rfind('@') - snippet_at = snippet_email.rfind('@') - if my_at == -1 or snippet_at == -1: - return False # be safe - return my_email[my_at:] == snippet_email[snippet_at:] - - -def _send_to_chat(msg, url_path): - """Send a message to the main room/channel for active chat integrations.""" - try: - app_settings = models.AppSettings.get() - except ValueError: - logging.warning('Not sending to chat: app settings not configured') - return - - msg = "%s %s%s" % (msg, app_settings.hostname, url_path) - - hipchat_room = app_settings.hipchat_room - if hipchat_room: - hipchatlib.send_to_hipchat_room(hipchat_room, msg) - - slack_channel = app_settings.slack_channel - if slack_channel: - slacklib.send_to_slack_channel(slack_channel, msg) - - -class BaseHandler(webapp2.RequestHandler): - """Set up as per the jinja2.py docstring.""" - @webapp2.cached_property - def jinja2(self): - return jinja2.get_jinja2() - - def render_response(self, template_filename, context): - html = self.jinja2.render_template(template_filename, **context) - self.response.write(html) - - -class UserPage(BaseHandler): - """Show all the snippets for a single user.""" - - def get(self): - if not users.get_current_user(): - return _login_page(self.request, self) - - user_email = self.request.get('u', _current_user_email()) - user = util.get_user(user_email) - - if not user: - # If there are no app settings, set those up before setting - # up the user settings. - if users.is_current_user_admin(): - try: - models.AppSettings.get() - except ValueError: - self.redirect("/admin/settings?redirect_to=user_setting" - "&msg=Welcome+to+the+snippet+server!+" - "Please+take+a+moment+to+configure+it.") - return - - template_values = { - 'new_user': True, - 'login_url': users.create_login_url(self.request.uri), - 'logout_url': users.create_logout_url('/'), - 'username': user_email, - } - self.render_response('new_user.html', template_values) - return - - snippets = util.snippets_for_user(user_email) - - if not _can_view_private_snippets(_current_user_email(), user_email): - snippets = [snippet for snippet in snippets if not snippet.private] - snippets = util.fill_in_missing_snippets(snippets, user, - user_email, _TODAY_FN()) - snippets.reverse() # get to newest snippet first - - template_values = { - 'logout_url': users.create_logout_url('/'), - 'message': self.request.get('msg'), - 'username': user_email, - 'is_admin': users.is_current_user_admin(), - 'domain': user_email.split('@')[-1], - 'view_week': util.existingsnippet_monday(_TODAY_FN()), - # Snippets for the week of are due today. - 'one_week_ago': _TODAY_FN().date() - datetime.timedelta(days=7), - 'eight_days_ago': _TODAY_FN().date() - datetime.timedelta(days=8), - 'editable': (_logged_in_user_has_permission_for(user_email) and - self.request.get('edit', '1') == '1'), - 'user': user, - 'snippets': snippets, - 'null_category': models.NULL_CATEGORY, - } - self.render_response('user_snippets.html', template_values) - - -def _title_case(s): - """Like string.title(), but does not uppercase 'and'.""" - # Smarter would be to use 'pip install titlecase'. - SMALL = 'a|an|and|as|at|but|by|en|for|if|in|of|on|or|the|to|v\.?|via|vs\.?' - # We purposefully don't match small words at the beginning of a string. - SMALL_RE = re.compile(r' (%s)\b' % SMALL, re.I) - return SMALL_RE.sub(lambda m: ' ' + m.group(1).lower(), s.title().strip()) - - -class SummaryPage(BaseHandler): - """Show all the snippets for a single week.""" - - def get(self): - if not users.get_current_user(): - return _login_page(self.request, self) - - week_string = self.request.get('week') - if week_string: - week = datetime.datetime.strptime(week_string, '%m-%d-%Y').date() - else: - week = util.existingsnippet_monday(_TODAY_FN()) - - snippets_q = models.Snippet.all() - snippets_q.filter('week = ', week) - snippets = snippets_q.fetch(1000) # good for many users... - # TODO(csilvers): filter based on wants_to_view - - # Get all the user records so we can categorize snippets. - user_q = models.User.all() - results = user_q.fetch(1000) - email_to_category = {} - email_to_user = {} - for result in results: - # People aren't very good about capitalizing their - # categories consistently, so we enforce title-case, - # with exceptions for 'and'. - email_to_category[result.email] = _title_case(result.category) - email_to_user[result.email] = result - - # Collect the snippets and users by category. As we see each email, - # delete it from email_to_category. At the end of this, - # email_to_category will hold people who did not give - # snippets this week. - snippets_and_users_by_category = {} - for snippet in snippets: - # Ignore this snippet if we don't have permission to view it. - if (snippet.private and - not _can_view_private_snippets(_current_user_email(), - snippet.email)): - continue - category = email_to_category.get( - snippet.email, models.NULL_CATEGORY - ) - if snippet.email in email_to_user: - snippets_and_users_by_category.setdefault(category, []).append( - (snippet, email_to_user[snippet.email]) - ) - else: - snippets_and_users_by_category.setdefault(category, []).append( - (snippet, models.User(email=snippet.email)) - ) - - if snippet.email in email_to_category: - del email_to_category[snippet.email] - - # Add in empty snippets for the people who didn't have any -- - # unless a user is marked 'hidden'. (That's what 'hidden' - # means: pretend they don't exist until they have a non-empty - # snippet again.) - for (email, category) in email_to_category.iteritems(): - if not email_to_user[email].is_hidden: - snippet = models.Snippet(email=email, week=week) - snippets_and_users_by_category.setdefault(category, []).append( - (snippet, email_to_user[snippet.email]) - ) - - # Now get a sorted list, categories in alphabetical order and - # each snippet-author within the category in alphabetical - # order. - # The data structure is ((category, ((snippet, user), ...)), ...) - categories_and_snippets = [] - for (category, - snippets_and_users) in snippets_and_users_by_category.iteritems(): - snippets_and_users.sort(key=lambda (snippet, user): snippet.email) - categories_and_snippets.append((category, snippets_and_users)) - categories_and_snippets.sort() - - template_values = { - 'logout_url': users.create_logout_url('/'), - 'message': self.request.get('msg'), - # Used only to switch to 'username' mode and to modify settings. - 'username': _current_user_email(), - 'is_admin': users.is_current_user_admin(), - 'prev_week': week - datetime.timedelta(7), - 'view_week': week, - 'next_week': week + datetime.timedelta(7), - 'categories_and_snippets': categories_and_snippets, - } - self.render_response('weekly_snippets.html', template_values) - - -class UpdateSnippet(BaseHandler): - def update_snippet(self, email): - week_string = self.request.get('week') - week = datetime.datetime.strptime(week_string, '%m-%d-%Y').date() - assert week.weekday() == 0, 'passed-in date must be a Monday' - - text = self.request.get('snippet') - - private = self.request.get('private') == 'True' - is_markdown = self.request.get('is_markdown') == 'True' - - # TODO(csilvers): make this get-update-put atomic. - # (maybe make the snippet id be email + week). - q = models.Snippet.all() - q.filter('email = ', email) - q.filter('week = ', week) - snippet = q.get() - - # When adding a snippet, make sure we create a user record for - # that email as well, if it doesn't already exist. - user = _get_or_create_user(email) - - # Store user's display_name in snippet so that if a user is later - # deleted, we could still show his / her display_name. - if snippet: - snippet.text = text # just update the snippet text - snippet.display_name = user.display_name - snippet.private = private - snippet.is_markdown = is_markdown - else: - # add the snippet to the db - snippet = models.Snippet(created=_TODAY_FN(), - display_name=user.display_name, - email=email, week=week, - text=text, private=private, - is_markdown=is_markdown) - db.put(snippet) - db.get(snippet.key()) # ensure db consistency for HRD - - self.response.set_status(200) - - def post(self): - """handle ajax updates via POST - - in particular, return status via json rather than redirects and - hard exceptions. This isn't actually RESTy, it's just status - codes and json. - """ - # TODO(marcos): consider using PUT? - - self.response.headers['Content-Type'] = 'application/json' - - if not users.get_current_user(): - # 403s are the catch-all 'please log in error' here - self.response.set_status(403) - self.response.out.write('{"status": 403, ' - '"message": "not logged in"}') - return - - email = self.request.get('u', _current_user_email()) - - if not _logged_in_user_has_permission_for(email): - # TODO(marcos): present these messages to the ajax client - self.response.set_status(403) - error = ('You do not have permissions to update user' - ' snippets for %s' % email) - self.response.out.write('{"status": 403, ' - '"message": "%s"}' % error) - return - - self.update_snippet(email) - self.response.out.write('{"status": 200, "message": "ok"}') - - def get(self): - if not users.get_current_user(): - return _login_page(self.request, self) - - email = self.request.get('u', _current_user_email()) - if not _logged_in_user_has_permission_for(email): - # TODO(csilvers): return a 403 here instead. - raise RuntimeError('You do not have permissions to update user' - ' snippets for %s' % email) - - self.update_snippet(email) - - email = self.request.get('u', _current_user_email()) - self.redirect("/?msg=Snippet+saved&u=%s" % urllib.quote(email)) - - -class Settings(BaseHandler): - """Page to display a user's settings (from class User) for modification.""" - - def get(self): - if not users.get_current_user(): - return _login_page(self.request, self) - - user_email = self.request.get('u', _current_user_email()) - if not _logged_in_user_has_permission_for(user_email): - # TODO(csilvers): return a 403 here instead. - raise RuntimeError('You do not have permissions to view user' - ' settings for %s' % user_email) - # We won't put() the new user until the settings are saved. - user = _get_or_create_user(user_email, put_new_user=False) - try: - user.key() - is_new_user = False - except db.NotSavedError: - is_new_user = True - - template_values = { - 'logout_url': users.create_logout_url('/'), - 'message': self.request.get('msg'), - 'username': user.email, - 'is_admin': users.is_current_user_admin(), - 'view_week': util.existingsnippet_monday(_TODAY_FN()), - 'user': user, - 'is_new_user': is_new_user, - 'redirect_to': self.request.get('redirect_to', ''), - # We could get this from user, but we want to replace - # commas with newlines for printing. - 'wants_to_view': user.wants_to_view.replace(',', '\n'), - } - self.render_response('settings.html', template_values) - - -class UpdateSettings(BaseHandler): - """Updates the db with modifications from the Settings page.""" - - def get(self): - if not users.get_current_user(): - return _login_page(self.request, self) - - user_email = self.request.get('u', _current_user_email()) - if not _logged_in_user_has_permission_for(user_email): - # TODO(csilvers): return a 403 here instead. - raise RuntimeError('You do not have permissions to modify user' - ' settings for %s' % user_email) - # TODO(csilvers): make this get/update/put atomic (put in a txn) - user = _get_or_create_user(user_email) - - # First, check if the user clicked on 'delete' or 'hide' - # rather than 'save'. - if self.request.get('hide'): - user.is_hidden = True - user.put() - time.sleep(0.1) # some time for eventual consistency - self.redirect('/weekly?msg=You+are+now+hidden.+Have+a+nice+day!') - return - elif self.request.get('delete'): - db.delete(user) - self.redirect('/weekly?msg=Your+account+has+been+deleted.+' - '(Note+your+existing+snippets+have+NOT+been+' - 'deleted.)+Have+a+nice+day!') - return - - display_name = self.request.get('display_name') - category = self.request.get('category') - uses_markdown = self.request.get('markdown') == 'yes' - private_snippets = self.request.get('private') == 'yes' - wants_email = self.request.get('reminder_email') == 'yes' - - # We want this list to be comma-separated, but people are - # likely to use whitespace to separate as well. Convert here. - wants_to_view = self.request.get('to_view') - wants_to_view = re.sub(r'\s+', ',', wants_to_view) - wants_to_view = wants_to_view.split(',') - wants_to_view = [w for w in wants_to_view if w] # deal with ',,' - wants_to_view = ','.join(wants_to_view) # TODO(csilvers): keep as list - - # Changing their settings is the kind of activity that unhides - # someone who was hidden, unless they specifically ask to be - # hidden. - is_hidden = self.request.get('is_hidden', 'no') == 'yes' - - user.is_hidden = is_hidden - user.display_name = display_name - user.category = category or models.NULL_CATEGORY - user.uses_markdown = uses_markdown - user.private_snippets = private_snippets - user.wants_email = wants_email - user.wants_to_view = wants_to_view - db.put(user) - db.get(user.key()) # ensure db consistency for HRD - - redirect_to = self.request.get('redirect_to') - if redirect_to == 'snippet_entry': # true for new_user.html - self.redirect('/?u=%s' % urllib.quote(user_email)) - else: - self.redirect("/settings?msg=Changes+saved&u=%s" - % urllib.quote(user_email)) - - -class AppSettings(BaseHandler): - """Page to display settings for the whole app, for modification. - - This page should be restricted to admin users via app.yaml. - """ - - def get(self): - my_domain = _current_user_email().split('@')[-1] - app_settings = models.AppSettings.get(create_if_missing=True, - domains=[my_domain]) - - template_values = { - 'logout_url': users.create_logout_url('/'), - 'message': self.request.get('msg'), - 'username': _current_user_email(), - 'is_admin': users.is_current_user_admin(), - 'view_week': util.existingsnippet_monday(_TODAY_FN()), - 'redirect_to': self.request.get('redirect_to', ''), - 'settings': app_settings, - 'slack_slash_commands': ( - slacklib.command_usage().strip()) - } - self.render_response('app_settings.html', template_values) - - -class UpdateAppSettings(BaseHandler): - """Updates the db with modifications from the App-Settings page. - - This page should be restricted to admin users via app.yaml. - """ - - def get(self): - _get_or_create_user(_current_user_email()) - - domains = self.request.get('domains') - default_private = self.request.get('private') == 'yes' - default_markdown = self.request.get('markdown') == 'yes' - default_email = self.request.get('reminder_email') == 'yes' - email_from = self.request.get('email_from') - hipchat_room = self.request.get('hipchat_room') - hipchat_token = self.request.get('hipchat_token') - slack_channel = self.request.get('slack_channel') - slack_token = self.request.get('slack_token') - slack_slash_token = self.request.get('slack_slash_token') - - # Turn domains into a list. Allow whitespace or comma to separate. - domains = re.sub(r'\s+', ',', domains) - domains = [d for d in domains.split(',') if d] - - @db.transactional - def update_settings(): - app_settings = models.AppSettings.get(create_if_missing=True, - domains=domains) - app_settings.domains = domains - app_settings.default_private = default_private - app_settings.default_markdown = default_markdown - app_settings.default_email = default_email - app_settings.email_from = email_from - app_settings.hipchat_room = hipchat_room - app_settings.hipchat_token = hipchat_token - app_settings.slack_channel = slack_channel - app_settings.slack_token = slack_token - app_settings.slack_slash_token = slack_slash_token - app_settings.put() - - update_settings() - - redirect_to = self.request.get('redirect_to') - if redirect_to == 'user_setting': # true for new_user.html - self.redirect('/settings?redirect_to=snippet_entry' - '&msg=Now+enter+your+personal+user+settings.') - else: - self.redirect("/admin/settings?msg=Changes+saved") - - -class ManageUsers(BaseHandler): - """Lets admins delete and otherwise manage users.""" - - def get(self): - # options are 'email', 'creation_time', 'last_snippet_time' - sort_by = self.request.get('sort_by', 'creation_time') - - # First, check if the user had clicked on a button. - for (name, value) in self.request.params.iteritems(): - if name.startswith('hide '): - email_of_user_to_hide = name[len('hide '):] - # TODO(csilvers): move this get/update/put atomic into a txn - user = util.get_user_or_die(email_of_user_to_hide) - user.is_hidden = True - user.put() - time.sleep(0.1) # encourage eventual consistency - self.redirect('/admin/manage_users?sort_by=%s&msg=%s+hidden' - % (sort_by, email_of_user_to_hide)) - return - if name.startswith('unhide '): - email_of_user_to_unhide = name[len('unhide '):] - # TODO(csilvers): move this get/update/put atomic into a txn - user = util.get_user_or_die(email_of_user_to_unhide) - user.is_hidden = False - user.put() - time.sleep(0.1) # encourage eventual consistency - self.redirect('/admin/manage_users?sort_by=%s&msg=%s+unhidden' - % (sort_by, email_of_user_to_unhide)) - return - if name.startswith('delete '): - email_of_user_to_delete = name[len('delete '):] - user = util.get_user_or_die(email_of_user_to_delete) - db.delete(user) - time.sleep(0.1) # encourage eventual consistency - self.redirect('/admin/manage_users?sort_by=%s&msg=%s+deleted' - % (sort_by, email_of_user_to_delete)) - return - - user_q = models.User.all() - results = user_q.fetch(1000) - - # Tuple: (email, is-hidden, creation-time, days since last snippet) - user_data = [] - for user in results: - # Get the last snippet for that user. - last_snippet = util.most_recent_snippet_for_user(user.email) - if last_snippet: - seconds_since_snippet = ( - (_TODAY_FN().date() - last_snippet.week).total_seconds()) - weeks_since_snippet = int( - seconds_since_snippet / - datetime.timedelta(days=7).total_seconds()) - else: - weeks_since_snippet = None - user_data.append((user.email, user.is_hidden, - user.created, weeks_since_snippet)) - - # We have to use 'cmp' here since we want ascending in the - # primary key and descending in the secondary key, sometimes. - if sort_by == 'email': - user_data.sort(lambda x, y: cmp(x[0], y[0])) - elif sort_by == 'creation_time': - user_data.sort(lambda x, y: (-cmp(x[2] or datetime.datetime.min, - y[2] or datetime.datetime.min) - or cmp(x[0], y[0]))) - elif sort_by == 'last_snippet_time': - user_data.sort(lambda x, y: (-cmp(1000 if x[3] is None else x[3], - 1000 if y[3] is None else y[3]) - or cmp(x[0], y[0]))) - else: - raise ValueError('Invalid sort_by value "%s"' % sort_by) - - template_values = { - 'logout_url': users.create_logout_url('/'), - 'message': self.request.get('msg'), - 'username': _current_user_email(), - 'is_admin': users.is_current_user_admin(), - 'view_week': util.existingsnippet_monday(_TODAY_FN()), - 'user_data': user_data, - 'sort_by': sort_by, - } - self.render_response('manage_users.html', template_values) - - -# The following two classes are called by cron. - - -def _get_email_to_current_snippet_map(today): - """Return a map from email to True if they've written snippets this week. - - Goes through all users registered on the system, and checks if - they have a snippet in the db for the appropriate snippet-week for - 'today'. If so, they get entered into the return-map with value - True. If not, they have value False. - - Note that users whose 'wants_email' field is set to False will not - be included in either list. - - Arguments: - today: a datetime.datetime object representing the - 'current' day. We use the normal algorithm to determine what is - the most recent snippet-week for this day. - - Returns: - a map from email (user.email for each user) to True or False, - depending on if they've written snippets for this week or not. - """ - user_q = models.User.all() - users = user_q.fetch(1000) - retval = {} - for user in users: - if not user.wants_email: # ignore this user - continue - retval[user.email] = False # assume the worst, for now - - week = util.existingsnippet_monday(today) - snippets_q = models.Snippet.all() - snippets_q.filter('week = ', week) - snippets = snippets_q.fetch(1000) - for snippet in snippets: - if snippet.email in retval: # don't introduce new keys here - retval[snippet.email] = True - - return retval - - -def _maybe_send_snippets_mail(to, subject, template_path, template_values): - try: - app_settings = models.AppSettings.get() - except ValueError: - logging.error('Not sending email: app settings are not configured.') - return - if not app_settings.email_from: - return - - template_values.setdefault('hostname', app_settings.hostname) - - jinja2_instance = jinja2.get_jinja2() - mail.send_mail(sender=app_settings.email_from, - to=to, - subject=subject, - body=jinja2_instance.render_template(template_path, - **template_values)) - # Appengine has a quota of 32 emails per minute: - # https://developers.google.com/appengine/docs/quotas#Mail - # We pause 2 seconds between each email to make sure we - # don't go over that. - time.sleep(2) - - -class SendFridayReminderChat(BaseHandler): - """Send a chat message to the configured chat room(s).""" - - def get(self): - msg = 'Reminder: Weekly snippets due Monday at 5pm.' - _send_to_chat(msg, "/") - - -class SendReminderEmail(BaseHandler): - """Send an email to everyone who doesn't have a snippet for this week.""" - - def _send_mail(self, email): - template_values = {} - _maybe_send_snippets_mail(email, 'Weekly snippets due today at 5pm', - 'reminder_email.txt', template_values) - - def get(self): - email_to_has_snippet = _get_email_to_current_snippet_map(_TODAY_FN()) - for (user_email, has_snippet) in email_to_has_snippet.iteritems(): - if not has_snippet: - self._send_mail(user_email) - logging.debug('sent reminder email to %s' % user_email) - else: - logging.debug('did not send reminder email to %s: ' - 'has a snippet already' % user_email) - - msg = 'Reminder: Weekly snippets due today at 5pm.' - _send_to_chat(msg, "/") - - -class SendViewEmail(BaseHandler): - """Send an email to everyone to look at the week's snippets.""" - - def _send_mail(self, email, has_snippets): - template_values = {'has_snippets': has_snippets} - _maybe_send_snippets_mail(email, 'Weekly snippets are ready!', - 'view_email.txt', template_values) - - def get(self): - email_to_has_snippet = _get_email_to_current_snippet_map(_TODAY_FN()) - for (user_email, has_snippet) in email_to_has_snippet.iteritems(): - self._send_mail(user_email, has_snippet) - logging.debug('sent "view" email to %s' % user_email) - - msg = 'Weekly snippets are ready!' - _send_to_chat(msg, "/weekly") - - -application = webapp2.WSGIApplication([ - ('/', UserPage), - ('/weekly', SummaryPage), - ('/update_snippet', UpdateSnippet), - ('/settings', Settings), - ('/update_settings', UpdateSettings), - ('/admin/settings', AppSettings), - ('/admin/update_settings', UpdateAppSettings), - ('/admin/manage_users', ManageUsers), - ('/admin/send_friday_reminder_chat', SendFridayReminderChat), - ('/admin/send_reminder_email', SendReminderEmail), - ('/admin/send_view_email', SendViewEmail), - ('/admin/test_send_to_hipchat', hipchatlib.TestSendToHipchat), - ('/slack', slacklib.SlashCommand), - ], - debug=True) From 3b903ba39841c3e961945f4a55f6bef35adb77ae Mon Sep 17 00:00:00 2001 From: Benjamin Staffin Date: Fri, 4 Nov 2022 14:51:51 -0400 Subject: [PATCH 02/40] More python 3 & flask updates --- Makefile | 5 ++++- main.py | 54 ++++++++++++++++++++++++++++-------------------------- models.py | 2 +- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index fc5af25..9b5d1a2 100644 --- a/Makefile +++ b/Makefile @@ -10,4 +10,7 @@ test check: python -m unittest discover -p '*_test.py' appcfg-update deploy: - gcloud app deploy --project "${APP}" + gcloud app deploy + +create-indexes: + gcloud datastore indexes create index.yaml diff --git a/main.py b/main.py index 4337c6e..a8ead1f 100644 --- a/main.py +++ b/main.py @@ -175,7 +175,7 @@ def user_page_handler(): template_values = { 'new_user': True, - 'login_url': users.create_login_url(flask.request.uri), + 'login_url': users.create_login_url(flask.request.url), 'logout_url': users.create_logout_url('/'), 'username': user_email, } @@ -277,7 +277,7 @@ def summary_page_handler(): # unless a user is marked 'hidden'. (That's what 'hidden' # means: pretend they don't exist until they have a non-empty # snippet again.) - for (email, category) in email_to_category.iteritems(): + for (email, category) in email_to_category.items(): if not email_to_user[email].is_hidden: snippet = models.Snippet(email=email, week=week) snippets_and_users_by_category.setdefault(category, []).append( @@ -289,9 +289,9 @@ def summary_page_handler(): # order. # The data structure is ((category, ((snippet, user), ...)), ...) categories_and_snippets = [] - for (category, - snippets_and_users) in snippets_and_users_by_category.iteritems(): - snippets_and_users.sort(key=lambda snippet, user: snippet.email) + for category, snippets_and_users in snippets_and_users_by_category.items(): + # This looks stupid but python no longer allows lambda (snippet, user): snippet.email + snippets_and_users.sort(key=lambda snippet_user: snippet_user[0].email) categories_and_snippets.append((category, snippets_and_users)) categories_and_snippets.sort() @@ -309,16 +309,13 @@ def summary_page_handler(): return flask.render_template('weekly_snippets.html', **template_values) -def update_snippet(email): - week_string = flask.request.args.get('week') - week = datetime.datetime.strptime(week_string, '%m-%d-%Y').date() +def update_snippet(email: str, + week: datetime.date, + text: str, + private: bool, + is_markdown: bool): assert week.weekday() == 0, 'passed-in date must be a Monday' - text = flask.request.args.get('snippet') - - private = flask.request.args.get('private') == 'True' - is_markdown = flask.request.args.get('is_markdown') == 'True' - # TODO(csilvers): make this get-update-put atomic. # (maybe make the snippet id be email + week). q = models.Snippet.all() @@ -347,12 +344,22 @@ def update_snippet(email): db.put(snippet) db.get(snippet.key()) # ensure db consistency for HRD - flask.response.set_status(200) - -@app.route("/update_snippet") +@app.route("/update_snippet", methods=["POST", "GET"]) def update_snippet_handler(): + if flask.request.method == "POST": + data = flask.request.form + elif flask.request.method == "GET": + data = flask.request.args + + email = data.get('u', _current_user_email()) + week_string = data.get('week') + week = datetime.datetime.strptime(week_string, '%m-%d-%Y').date() + text = data.get('snippet') + private = data.get('private') == 'True' + is_markdown = data.get('is_markdown') == 'True' + if flask.request.method == "POST": """handle ajax updates via POST @@ -366,30 +373,25 @@ def update_snippet_handler(): # 403s are the catch-all 'please log in error' here return flask.make_response({"status": 403, "message": "not logged in"}, 403) - email = flask.request.args.get('u', _current_user_email()) - if not _logged_in_user_has_permission_for(email): # TODO(marcos): present these messages to the ajax client error = ('You do not have permissions to update user' ' snippets for %s' % email) return flask.make_response({"status": 403, "message": error}, 403) - update_snippet(email) + update_snippet(email, week, text, private, is_markdown) return flask.make_response({"status": 200, "message": "ok"}) elif flask.request.method == "GET": if not users.get_current_user(): return _login_page(flask.request) - email = flask.request.args.get('u', _current_user_email()) if not _logged_in_user_has_permission_for(email): # TODO(csilvers): return a 403 here instead. raise RuntimeError('You do not have permissions to update user' ' snippets for %s' % email) - update_snippet(email) - - email = flask.request.args.get('u', _current_user_email()) + update_snippet(email, week, text, private, is_markdown) return flask.redirect("/?msg=Snippet+saved&u=%s" % urllib.parse.quote(email)) @@ -574,7 +576,7 @@ def admin_manage_users_handler(): sort_by = flask.request.args.get('sort_by', 'creation_time') # First, check if the user had clicked on a button. - for (name, value) in flask.request.params.iteritems(): + for name, value in flask.request.params.items(): if name.startswith('hide '): email_of_user_to_hide = name[len('hide '):] # TODO(csilvers): move this get/update/put atomic into a txn @@ -730,7 +732,7 @@ def _send_mail(email): 'reminder_email.txt', template_values) email_to_has_snippet = _get_email_to_current_snippet_map(_TODAY_FN()) - for (user_email, has_snippet) in email_to_has_snippet.iteritems(): + for user_email, has_snippet in email_to_has_snippet.items(): if not has_snippet: _send_mail(user_email) logging.debug('sent reminder email to %s', user_email) @@ -752,7 +754,7 @@ def _send_mail(self, email, has_snippets): 'view_email.txt', template_values) email_to_has_snippet = _get_email_to_current_snippet_map(_TODAY_FN()) - for (user_email, has_snippet) in email_to_has_snippet.iteritems(): + for user_email, has_snippet in email_to_has_snippet.items(): _send_mail(user_email, has_snippet) logging.debug('sent "view" email to %s', user_email) diff --git a/models.py b/models.py index 697046b..685a301 100644 --- a/models.py +++ b/models.py @@ -50,7 +50,7 @@ class Snippet(db.Model): @property def email_md5_hash(self): m = hashlib.md5() - m.update(self.email) + m.update(self.email.encode('utf-8')) return m.hexdigest() From 6f5531f49ef2c5e8e66e1bdcdf68d918bc7350c8 Mon Sep 17 00:00:00 2001 From: Benjamin Staffin Date: Fri, 4 Nov 2022 19:50:28 -0400 Subject: [PATCH 03/40] Rewrite to use ndb datastore --- main.py | 74 ++++++++++++++++++++++++++++++------------------ models.py | 74 ++++++++++++++++++++++++------------------------ requirements.txt | 1 + slacklib.py | 9 +++--- slacklib_test.py | 6 ++-- snippets_test.py | 2 +- util.py | 27 +++++++++--------- 7 files changed, 106 insertions(+), 87 deletions(-) diff --git a/main.py b/main.py index a8ead1f..9b84683 100644 --- a/main.py +++ b/main.py @@ -17,7 +17,7 @@ from google.appengine.api import mail, wrap_wsgi_app from google.appengine.api import users -from google.appengine.ext import db +from google.cloud import ndb import flask import jinja2 @@ -26,11 +26,24 @@ import slacklib import util +# This allows mocking in a different day, for testing. +_TODAY_FN = datetime.datetime.now + app = flask.Flask(__name__) app.wsgi_app = wrap_wsgi_app(app.wsgi_app) -# This allows mocking in a different day, for testing. -_TODAY_FN = datetime.datetime.now + +class NDBMiddleware: + """WSGI middleware to wrap the app in Google Cloud NDB context""" + def __init__(self, app): + self.app = app + self.client = ndb.Client() + + def __call__(self, environ, start_response): + with self.client.context(): + return self.app(environ, start_response) + +app.wsgi_app = NDBMiddleware(app.wsgi_app) @app.template_filter("readable_date") @@ -82,7 +95,8 @@ def _get_or_create_user(email, put_new_user=True): except ValueError: # TODO(csilvers): do this instead: # /admin/settings?redirect_to=user_setting - return None + # return None + raise # TODO(benley) implement Redirect exception domain = email.split('@')[-1] allowed_domains = app_settings.domains @@ -100,8 +114,8 @@ def _get_or_create_user(email, put_new_user=True): private_snippets=app_settings.default_private, wants_email=app_settings.default_email) if put_new_user: - db.put(user) - db.get(user.key()) # ensure db consistency for HRD + user.put() + user.key.get() # ensure db consistency for HRD return user @@ -230,13 +244,14 @@ def summary_page_handler(): else: week = util.existingsnippet_monday(_TODAY_FN()) - snippets_q = models.Snippet.all() - snippets_q.filter('week = ', week) + snippets_q = models.Snippet.query( + models.Snippet.week == week + ) snippets = snippets_q.fetch(1000) # good for many users... # TODO(csilvers): filter based on wants_to_view # Get all the user records so we can categorize snippets. - user_q = models.User.all() + user_q = models.User.query() results = user_q.fetch(1000) email_to_category = {} email_to_user = {} @@ -318,9 +333,10 @@ def update_snippet(email: str, # TODO(csilvers): make this get-update-put atomic. # (maybe make the snippet id be email + week). - q = models.Snippet.all() - q.filter('email = ', email) - q.filter('week = ', week) + q = models.Snippet.query( + models.Snippet.email == email, + models.Snippet.week == week + ) snippet = q.get() # When adding a snippet, make sure we create a user record for @@ -341,8 +357,8 @@ def update_snippet(email: str, email=email, week=week, text=text, private=private, is_markdown=is_markdown) - db.put(snippet) - db.get(snippet.key()) # ensure db consistency for HRD + snippet.put() + snippet.key.get() # ensure db consistency for HRD @app.route("/update_snippet", methods=["POST", "GET"]) @@ -409,10 +425,13 @@ def settings_handler(): ' settings for %s' % user_email) # We won't put() the new user until the settings are saved. user = _get_or_create_user(user_email, put_new_user=False) - try: - user.key() + + # NOTE: this will break if you explicitly set the key when creating the + # user entity! + # See https://groups.google.com/g/google-appengine/c/Tm8NDWIvc70 + if user.key and user.key.id(): is_new_user = False - except db.NotSavedError: + else: is_new_user = True template_values = { @@ -454,7 +473,7 @@ def update_settings_handler(): time.sleep(0.1) # some time for eventual consistency return flask.redirect('/weekly?msg=You+are+now+hidden.+Have+a+nice+day!') elif flask.request.args.get('delete'): - db.delete(user) + user.delete() return flask.redirect('/weekly?msg=Your+account+has+been+deleted.+' '(Note+your+existing+snippets+have+NOT+been+' 'deleted.)+Have+a+nice+day!') @@ -485,8 +504,8 @@ def update_settings_handler(): user.private_snippets = private_snippets user.wants_email = wants_email user.wants_to_view = wants_to_view - db.put(user) - db.get(user.key()) # ensure db consistency for HRD + user.put() + user.key.get() # ensure db consistency for HRD redirect_to = flask.request.args.get('redirect_to') if redirect_to == 'snippet_entry': # true for new_user.html @@ -543,7 +562,7 @@ def admin_update_settings_handler(): domains = re.sub(r'\s+', ',', domains) domains = [d for d in domains.split(',') if d] - @db.transactional + @ndb.transactional def update_settings(): app_settings = models.AppSettings.get(create_if_missing=True, domains=domains) @@ -576,7 +595,7 @@ def admin_manage_users_handler(): sort_by = flask.request.args.get('sort_by', 'creation_time') # First, check if the user had clicked on a button. - for name, value in flask.request.params.items(): + for name, value in flask.request.form.items(): if name.startswith('hide '): email_of_user_to_hide = name[len('hide '):] # TODO(csilvers): move this get/update/put atomic into a txn @@ -598,12 +617,12 @@ def admin_manage_users_handler(): if name.startswith('delete '): email_of_user_to_delete = name[len('delete '):] user = util.get_user_or_die(email_of_user_to_delete) - db.delete(user) + user.delete() time.sleep(0.1) # encourage eventual consistency return flask.redirect('/admin/manage_users?sort_by=%s&msg=%s+deleted' % (sort_by, email_of_user_to_delete)) - user_q = models.User.all() + user_q = models.User.query() results = user_q.fetch(1000) # Tuple: (email, is-hidden, creation-time, days since last snippet) @@ -672,7 +691,7 @@ def _get_email_to_current_snippet_map(today): a map from email (user.email for each user) to True or False, depending on if they've written snippets for this week or not. """ - user_q = models.User.all() + user_q = models.User.query() users = user_q.fetch(1000) retval = {} for user in users: @@ -681,8 +700,9 @@ def _get_email_to_current_snippet_map(today): retval[user.email] = False # assume the worst, for now week = util.existingsnippet_monday(today) - snippets_q = models.Snippet.all() - snippets_q.filter('week = ', week) + snippets_q = models.Snippet.query( + models.Snippet.week == week + ) snippets = snippets_q.fetch(1000) for snippet in snippets: if snippet.email in retval: # don't introduce new keys here diff --git a/models.py b/models.py index 685a301..93e92c8 100644 --- a/models.py +++ b/models.py @@ -2,7 +2,7 @@ import hashlib import os -from google.appengine.ext import db +from google.cloud import ndb from google.appengine.api import users @@ -21,31 +21,31 @@ # support that later. -class User(db.Model): +class User(ndb.Model): """User preferences.""" - created = db.DateTimeProperty() - last_modified = db.DateTimeProperty(auto_now=True) - email = db.StringProperty(required=True) # The key to this record - is_hidden = db.BooleanProperty(default=False) # hide 'empty' snippets - category = db.StringProperty(default=NULL_CATEGORY) # groups snippets - uses_markdown = db.BooleanProperty(default=True) # interpret snippet text - private_snippets = db.BooleanProperty(default=False) # private by default? - wants_email = db.BooleanProperty(default=True) # get nag emails? + created = ndb.DateTimeProperty() + last_modified = ndb.DateTimeProperty(auto_now=True) + email = ndb.StringProperty(required=True) # The key to this record + is_hidden = ndb.BooleanProperty(default=False) # hide 'empty' snippets + category = ndb.StringProperty(default=NULL_CATEGORY) # groups snippets + uses_markdown = ndb.BooleanProperty(default=True) # interpret snippet text + private_snippets = ndb.BooleanProperty(default=False) # private by default? + wants_email = ndb.BooleanProperty(default=True) # get nag emails? # TODO(csilvers): make a ListProperty instead. - wants_to_view = db.TextProperty(default='all') # comma-separated list - display_name = db.TextProperty(default='') # display name of the user + wants_to_view = ndb.TextProperty(default='all') # comma-separated list + display_name = ndb.TextProperty(default='') # display name of the user -class Snippet(db.Model): +class Snippet(ndb.Model): """Every snippet is identified by the monday of the week it goes with.""" - created = db.DateTimeProperty() - last_modified = db.DateTimeProperty(auto_now=True) - display_name = db.StringProperty() # display name of the user - email = db.StringProperty(required=True) # week+email: key to this record - week = db.DateProperty(required=True) # the monday of the week - text = db.TextProperty() - private = db.BooleanProperty(default=False) # snippet is private? - is_markdown = db.BooleanProperty(default=False) # text is markdown? + created = ndb.DateTimeProperty() + last_modified = ndb.DateTimeProperty(auto_now=True) + display_name = ndb.StringProperty() # display name of the user + email = ndb.StringProperty(required=True) # week+email: key to this record + week = ndb.DateProperty(required=True) # the monday of the week + text = ndb.TextProperty() + private = ndb.BooleanProperty(default=False) # snippet is private? + is_markdown = ndb.BooleanProperty(default=False) # text is markdown? @property def email_md5_hash(self): @@ -54,23 +54,23 @@ def email_md5_hash(self): return m.hexdigest() -class AppSettings(db.Model): +class AppSettings(ndb.Model): """Application-wide preferences.""" - created = db.DateTimeProperty() - last_modified = db.DateTimeProperty(auto_now=True) + created = ndb.DateTimeProperty() + last_modified = ndb.DateTimeProperty(auto_now=True) # Application settings - domains = db.StringListProperty(required=True) - hostname = db.StringProperty(required=True) # used for emails - default_private = db.BooleanProperty(default=False) # new-user default - default_markdown = db.BooleanProperty(default=True) # new-user default - default_email = db.BooleanProperty(default=True) # new-user default + domains = ndb.StringProperty(repeated=True) + hostname = ndb.StringProperty(required=True) # used for emails + default_private = ndb.BooleanProperty(default=False) # new-user default + default_markdown = ndb.BooleanProperty(default=True) # new-user default + default_email = ndb.BooleanProperty(default=True) # new-user default # Chat and email settings - email_from = db.StringProperty(default='') - hipchat_room = db.StringProperty(default='') - hipchat_token = db.StringProperty(default='') - slack_channel = db.StringProperty(default='') - slack_token = db.StringProperty(default='') - slack_slash_token = db.StringProperty(default='') + email_from = ndb.StringProperty(default='') + hipchat_room = ndb.StringProperty(default='') + hipchat_token = ndb.StringProperty(default='') + slack_channel = ndb.StringProperty(default='') + slack_token = ndb.StringProperty(default='') + slack_slash_token = ndb.StringProperty(default='') @staticmethod def get(create_if_missing=False, domains=None): @@ -81,12 +81,12 @@ def get(create_if_missing=False, domains=None): are initialized with the given value for 'domains'. The new entity is *not* put to the datastore. """ - retval = AppSettings.get_by_key_name('global_settings') + retval = AppSettings.get_by_id('global_settings') if retval: return retval elif create_if_missing: # We default to sending email, and having it look like it's - # comint from the current user. We add a '+snippets' in there + # coming from the current user. We add a '+snippets' in there # to allow for filtering email_address = users.get_current_user().email() email_address = email_address.replace('@', '+snippets@') diff --git a/requirements.txt b/requirements.txt index 96be600..0dac537 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ appengine-python-standard>=1.0.0 Flask>=2.2.2 Jinja2>=3.1.2 +google-cloud-ndb diff --git a/slacklib.py b/slacklib.py index 89b6502..b650f42 100644 --- a/slacklib.py +++ b/slacklib.py @@ -26,7 +26,6 @@ import urllib.request import flask -from google.appengine.ext import db from google.appengine.api import memcache import models @@ -304,8 +303,8 @@ def command_add(user_email, new_item): snippet.is_markdown = True # TODO(mroth): we should abstract out DB writes to a library wrapper - db.put(snippet) - db.get(snippet.key()) # ensure db consistency for HRD + snippet.put() + snippet.key.get() # ensure db consistency for HRD return "Added *{}* to your weekly snippets.".format(new_item) @@ -353,8 +352,8 @@ def command_del(user_email, args): snippet.text = _markdown_list(items) snippet.is_markdown = True - db.put(snippet) - db.get(snippet.key()) # ensure db consistency for HRD + snippet.put() + snippet.key.get() # ensure db consistency for HRD return "Removed *{}* from your weekly snippets.".format(removed_item) diff --git a/slacklib_test.py b/slacklib_test.py index 11cc2c7..b06fb61 100644 --- a/slacklib_test.py +++ b/slacklib_test.py @@ -92,9 +92,9 @@ def _mock_data(self): )) def _most_recent_snippet(self, user_email): - snippets_q = models.Snippet.all() - snippets_q.filter('email = ', user_email) - snippets_q.order('-week') # newest snippet first + snippets_q = models.Snippet.query( + models.Snippet.email == user_email + ).order('-week') # newest snippet first return snippets_q.fetch(1)[0] def setUp(self): diff --git a/snippets_test.py b/snippets_test.py index a70adad..56c49e4 100755 --- a/snippets_test.py +++ b/snippets_test.py @@ -804,7 +804,7 @@ def testViewSnippetAfterAUserIsDeleted(self): self.request_fetcher.get(url) # Now delete user 2 - u = models.User.all().filter('email =', '2@example.com').get() + u = models.User.query(models.User.email == '2@example.com').get() u.delete() response = self.request_fetcher.get('/weekly?week=02-20-2012') diff --git a/util.py b/util.py index 8315e96..5c157d4 100644 --- a/util.py +++ b/util.py @@ -3,16 +3,13 @@ from models import Snippet from models import User - # Functions for retrieving a user -def get_user(email): +def get_user(email: str): """Return the user object with the given email, or None if not found.""" - q = User.all() - q.filter('email = ', email) - return q.get() + return User.query(User.email == email).get() -def get_user_or_die(email): +def get_user_or_die(email: str): user = get_user(email) if not user: raise ValueError('User "%s" not found' % email) @@ -21,18 +18,20 @@ def get_user_or_die(email): def snippets_for_user(user_email): """Return all snippets for a given user, oldest snippet first.""" - snippets_q = Snippet.all() - snippets_q.filter('email = ', user_email) - snippets_q.order('week') # this puts oldest snippet first - return snippets_q.fetch(1000) # good for many years... + return ( + Snippet.query(User.email == user_email) + .order('week') # this puts oldest snippet first + .fetch(1000) # good for many years... + ) def most_recent_snippet_for_user(user_email): """Return the most recent snippet for a given user, or None.""" - snippets_q = Snippet.all() - snippets_q.filter('email = ', user_email) - snippets_q.order('-week') # this puts newest snippet first - return snippets_q.get() + return ( + Snippet.query(User.email == user_email) + .order('-week') # this puts newest snippet first + .get() + ) # Functions around filling in snippets From 5ce9c0135044a8ce6f2cb1967a1378e3180b7d5b Mon Sep 17 00:00:00 2001 From: Benjamin Staffin Date: Fri, 4 Nov 2022 20:29:20 -0400 Subject: [PATCH 04/40] Fix python2->3 cmp sorting thing --- main.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/main.py b/main.py index 9b84683..3e6095a 100644 --- a/main.py +++ b/main.py @@ -9,6 +9,7 @@ __author__ = 'Craig Silverstein ' import datetime +from functools import cmp_to_key import logging import os import re @@ -588,6 +589,11 @@ def update_settings(): return flask.redirect("/admin/settings?msg=Changes+saved") +def cmp(a, b): + """Python 2's cmp function, used in /admin/manage_users handler""" + return (a > b) - (a < b) + + @app.route("/admin/manage_users") def admin_manage_users_handler(): """Lets admins delete and otherwise manage users.""" @@ -644,15 +650,17 @@ def admin_manage_users_handler(): # We have to use 'cmp' here since we want ascending in the # primary key and descending in the secondary key, sometimes. if sort_by == 'email': - user_data.sort(lambda x, y: cmp(x[0], y[0])) + user_data.sort(key=cmp_to_key(lambda x, y: cmp(x[0], y[0]))) elif sort_by == 'creation_time': - user_data.sort(lambda x, y: (-cmp(x[2] or datetime.datetime.min, - y[2] or datetime.datetime.min) - or cmp(x[0], y[0]))) + user_data.sort(key=cmp_to_key( + lambda x, y: (-cmp(x[2] or datetime.datetime.min, + y[2] or datetime.datetime.min) + or cmp(x[0], y[0])))) elif sort_by == 'last_snippet_time': - user_data.sort(lambda x, y: (-cmp(1000 if x[3] is None else x[3], - 1000 if y[3] is None else y[3]) - or cmp(x[0], y[0]))) + user_data.sort(key=cmp_to_key( + lambda x, y: (-cmp(1000 if x[3] is None else x[3], + 1000 if y[3] is None else y[3]) + or cmp(x[0], y[0])))) else: raise ValueError('Invalid sort_by value "%s"' % sort_by) From 1b9a66cc02512b27b035a1d105ea142b68878a65 Mon Sep 17 00:00:00 2001 From: Benjamin Staffin Date: Mon, 7 Nov 2022 14:59:19 -0500 Subject: [PATCH 05/40] cleanup app.yaml --- app.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app.yaml b/app.yaml index f0043ad..0f40b05 100644 --- a/app.yaml +++ b/app.yaml @@ -12,11 +12,8 @@ handlers: upload: static/favicon.ico - url: /admin/.* - script: snippets.app + script: auto login: admin -- url: .* - script: snippets.app - builtins: - remote_api: on From d979f0b9989c4cff1df03347d8e7eb23dfc5dd8d Mon Sep 17 00:00:00 2001 From: Benjamin Staffin Date: Mon, 7 Nov 2022 14:59:52 -0500 Subject: [PATCH 06/40] Use google cloud logging library --- main.py | 5 +++++ requirements.txt | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 3e6095a..16c8d6a 100644 --- a/main.py +++ b/main.py @@ -19,6 +19,7 @@ from google.appengine.api import mail, wrap_wsgi_app from google.appengine.api import users from google.cloud import ndb +import google.cloud.logging import flask import jinja2 @@ -27,6 +28,10 @@ import slacklib import util +# Set up cloud logging +logging_client = google.cloud.logging.Client() +logging_client.setup_logging(log_level=logging.INFO) + # This allows mocking in a different day, for testing. _TODAY_FN = datetime.datetime.now diff --git a/requirements.txt b/requirements.txt index 0dac537..a7f2e5b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ appengine-python-standard>=1.0.0 Flask>=2.2.2 Jinja2>=3.1.2 -google-cloud-ndb +google-cloud-ndb>=1.11.1 +google-cloud-logging>=3.2.5 From b5f59fade1185d0a841f615fd75289ec89610be2 Mon Sep 17 00:00:00 2001 From: Benjamin Staffin Date: Wed, 9 Nov 2022 12:51:17 -0500 Subject: [PATCH 07/40] Always use SSL --- app.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app.yaml b/app.yaml index 0f40b05..bcd38ba 100644 --- a/app.yaml +++ b/app.yaml @@ -14,6 +14,11 @@ handlers: - url: /admin/.* script: auto login: admin + secure: always + +- url: /.* + script: auto + secure: always builtins: - remote_api: on From 8488bde963665efaa15b529703dc166b8a92a8d9 Mon Sep 17 00:00:00 2001 From: Benjamin Staffin Date: Wed, 9 Nov 2022 12:51:32 -0500 Subject: [PATCH 08/40] Fix /admin/update_settings, fix hostname setting --- main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 16c8d6a..22d7313 100644 --- a/main.py +++ b/main.py @@ -560,6 +560,7 @@ def admin_update_settings_handler(): email_from = flask.request.args.get('email_from') hipchat_room = flask.request.args.get('hipchat_room') hipchat_token = flask.request.args.get('hipchat_token') + hostname = flask.request.args.get('hostname') slack_channel = flask.request.args.get('slack_channel') slack_token = flask.request.args.get('slack_token') slack_slash_token = flask.request.args.get('slack_slash_token') @@ -568,10 +569,10 @@ def admin_update_settings_handler(): domains = re.sub(r'\s+', ',', domains) domains = [d for d in domains.split(',') if d] - @ndb.transactional + @ndb.transactional() def update_settings(): app_settings = models.AppSettings.get(create_if_missing=True, - domains=domains) + domains=domains) app_settings.domains = domains app_settings.default_private = default_private app_settings.default_markdown = default_markdown @@ -579,6 +580,7 @@ def update_settings(): app_settings.email_from = email_from app_settings.hipchat_room = hipchat_room app_settings.hipchat_token = hipchat_token + app_settings.hostname = hostname app_settings.slack_channel = slack_channel app_settings.slack_token = slack_token app_settings.slack_slash_token = slack_slash_token @@ -650,7 +652,7 @@ def admin_manage_users_handler(): else: weeks_since_snippet = None user_data.append((user.email, user.is_hidden, - user.created, weeks_since_snippet)) + user.created, weeks_since_snippet)) # We have to use 'cmp' here since we want ascending in the # primary key and descending in the secondary key, sometimes. From 6b234051f85101f6bcb68e192ca474e0ba82dc61 Mon Sep 17 00:00:00 2001 From: Benjamin Staffin Date: Wed, 9 Nov 2022 14:02:07 -0500 Subject: [PATCH 09/40] use POST for /admin/update_settings --- main.py | 29 ++++++++++++++--------------- templates/app_settings.html | 2 +- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/main.py b/main.py index 22d7313..72be4c0 100644 --- a/main.py +++ b/main.py @@ -539,13 +539,12 @@ def admin_settings_handler(): 'view_week': util.existingsnippet_monday(_TODAY_FN()), 'redirect_to': flask.request.args.get('redirect_to', ''), 'settings': app_settings, - 'slack_slash_commands': ( - slacklib.command_usage().strip()) + 'slack_slash_commands': slacklib.command_usage().strip() } return flask.render_template('app_settings.html', **template_values) -@app.route("/admin/update_settings") +@app.route("/admin/update_settings", methods=["POST"]) def admin_update_settings_handler(): """Updates the db with modifications from the App-Settings page. @@ -553,17 +552,17 @@ def admin_update_settings_handler(): """ _get_or_create_user(_current_user_email()) - domains = flask.request.args.get('domains') - default_private = flask.request.args.get('private') == 'yes' - default_markdown = flask.request.args.get('markdown') == 'yes' - default_email = flask.request.args.get('reminder_email') == 'yes' - email_from = flask.request.args.get('email_from') - hipchat_room = flask.request.args.get('hipchat_room') - hipchat_token = flask.request.args.get('hipchat_token') - hostname = flask.request.args.get('hostname') - slack_channel = flask.request.args.get('slack_channel') - slack_token = flask.request.args.get('slack_token') - slack_slash_token = flask.request.args.get('slack_slash_token') + domains = flask.request.form.get('domains') + default_private = flask.request.form.get('private') == 'yes' + default_markdown = flask.request.form.get('markdown') == 'yes' + default_email = flask.request.form.get('reminder_email') == 'yes' + email_from = flask.request.form.get('email_from') + hipchat_room = flask.request.form.get('hipchat_room') + hipchat_token = flask.request.form.get('hipchat_token') + hostname = flask.request.form.get('hostname') + slack_channel = flask.request.form.get('slack_channel') + slack_token = flask.request.form.get('slack_token') + slack_slash_token = flask.request.form.get('slack_slash_token') # Turn domains into a list. Allow whitespace or comma to separate. domains = re.sub(r'\s+', ',', domains) @@ -588,7 +587,7 @@ def update_settings(): update_settings() - redirect_to = flask.request.args.get('redirect_to') + redirect_to = flask.request.form.get('redirect_to') if redirect_to == 'user_setting': # true for new_user.html return flask.redirect('/settings?redirect_to=snippet_entry' '&msg=Now+enter+your+personal+user+settings.') diff --git a/templates/app_settings.html b/templates/app_settings.html index 71fb7b5..364e2ad 100644 --- a/templates/app_settings.html +++ b/templates/app_settings.html @@ -9,7 +9,7 @@

Manage Users

hide users

-