deeper status checks

CHANGELOG.markdown

@@ -2,9 +2,13 @@
 
 * add: instance hidelist
   <https://github.com/codl/forget/pull/590>
+* changed: more comprehensive checks in `/api/status_check`
+  <https://github.com/codl/forget/pull/418>
 * removed: migration path for known instances list from cookie to localstorage
   <https://github.com/codl/forget/pull/545>
 
+This release involves a database migration. Run `flask db upgrade` after updating.
+
 ## v2.1.0
 
 Released 2022-03-04

migrations/versions/2a46ac75281c_add_celery_checkin.py

@@ -0,0 +1,29 @@
+"""add celery checkin
+
+Revision ID: 2a46ac75281c
+Revises: 7b0e9b8e0887
+Create Date: 2022-03-04 20:43:58.455920
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2a46ac75281c'
+down_revision = '7b0e9b8e0887'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('worker_checkins',
+    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_worker_checkins'))
+    )
+
+
+def downgrade():
+    op.drop_table('worker_checkins')

model.py

@@ -408,3 +408,7 @@ class MisskeyInstance(db.Model):
 
     def bump(self, value=1):
         self.popularity = (self.popularity or 10) + value
+
+class WorkerCheckin(db.Model, TimestampMixin):
+    __tablename__ = 'worker_checkins'
+    id = db.Column(db.Integer, primary_key=True)

routes/api.py

@@ -1,10 +1,11 @@
 from app import app, db, imgproxy
 from libforget.auth import require_auth_api, get_viewer
 from flask import jsonify, redirect, make_response, request, Response
-from model import Account
+from model import Account, WorkerCheckin
 import libforget.settings
 import libforget.json
 import random
+from datetime import datetime, timedelta
 
 @app.route('/api/health_check') # deprecated 2021-03-12
 @app.route('/api/status_check')
@@ -19,6 +20,24 @@ def api_status_check():
     except Exception:
         return ('Redis bad', 500)
 
+    CHECKIN_EVENTS = 5
+    CHECKIN_PERIOD = timedelta(minutes=10)
+    # sorry about the obtuse variable names, this trips if the frequency is
+    # lower than events/period
+    checkin_count = db.session.query(WorkerCheckin)\
+            .filter(WorkerCheckin.created_at > db.func.now() - CHECKIN_PERIOD)\
+            .count()
+    if checkin_count < events:
+        return ('Celery slow, {} check-ins in {}'.format(
+            checkin_count, CHECKIN_PERIOD
+        ), 500)
+
+    CHECKIN_LATENESS_THRESHOLD = timedelta(minutes=5)
+    checkin = db.session.query(WorkerCheckin.created_at)\
+            .order_by(db.desc(WorkerCheckin.created_at)).first()
+    if checkin + CHECKIN_LATENESS_THRESHOLD < datetime.utcnow():
+        return ('Celery late, last check-in was {}'.format(checkin), 500)
+
     return 'OK'
 
 

tasks.py

@@ -2,7 +2,7 @@
 from app import app as flaskapp
 from app import db
 from model import Session, Account, TwitterArchive, Post, OAuthToken,\
-                  MastodonInstance, MisskeyInstance
+                  MastodonInstance, MisskeyInstance, WorkerCheckin
 import libforget.twitter
 import libforget.mastodon
 import libforget.misskey
@@ -398,6 +398,11 @@ def periodic_cleanup():
         you have restored access and you can now re-enable Forget if you wish.
         """.format(service=account.service.capitalize())
 
+    # delete worker check-ins after 48 hours
+    (WorkerCheckin.query.filter(
+        WorkerCheckin.updated_at < (db.func.now() - timedelta(hours=48)))
+     .delete(synchronize_session=False))
+
     db.session.commit()
 
 
@@ -513,6 +518,10 @@ def update_misskey_instances_popularity():
     })
     db.session.commit()
 
+@app.task
+def report_in():
+    db.session.add(WorkerCheckin())
+    db.session.commit()
 
 app.add_periodic_task(40, queue_fetch_for_most_stale_accounts)
 app.add_periodic_task(9, queue_deletes)
@@ -521,6 +530,7 @@ def update_misskey_instances_popularity():
 app.add_periodic_task(300, periodic_cleanup)
 app.add_periodic_task(300, update_mastodon_instances_popularity)
 app.add_periodic_task(300, update_misskey_instances_popularity)
+app.add_periodic_task(60, report_in)
 
 if __name__ == '__main__':
     app.worker_main()