feat(phase-5): on-prem sync poll hardening + stale job reclaim

Harden the Hub-side polling protocol for on-prem TapTrack agents:
- sync/poll: accept job_failed_ids (retry/fail on-prem delivery failures)
- sync/poll: deduct credits + write ledger when on-prem reports sent jobs
- sync/poll: return feature_flags (tier-based) + suspended flag in config
- sync/poll: skip job dispatch for suspended/expired schools
- sms_jobs: add delivered_via (pull|push) + processing_started_at columns
- tasks/sms: new sms.reclaim_stale_jobs task resets processing→pending if on-prem
  goes offline (jobs stuck >5 min), enabling Celery push fallback
- tasks/sms: tag Celery-sent jobs as delivered_via='push'
- worker: schedule reclaim_stale_jobs every 5 minutes
- migration: 001_phase5 adds delivered_via + processing_started_at to sms_jobs

backend/app/tasks/sms.py

@@ -1,7 +1,7 @@
-"""Celery task: process pending SMS jobs via Semaphore."""
+"""Celery task: process pending SMS jobs via Semaphore (push path)."""
 import logging
 import os
-from datetime import datetime, timezone
+from datetime import datetime, timezone, timedelta
 
 import httpx
 from sqlalchemy import create_engine, select, update, and_
@@ -19,9 +19,17 @@ def _make_sync_engine():
 _engine = _make_sync_engine()
 _Session = sessionmaker(bind=_engine)
 
+
 @celery_app.task(name="sms.process_queue")
 def process_sms_queue():
-    """Process up to 20 pending SMS jobs per run via Semaphore API."""
+    """
+    Push path: process up to 20 pending SMS jobs per run via Semaphore API.
+
+    Only picks up `pending` jobs. Jobs in `processing` are being handled
+    by the on-prem pull agent via /api/sync/poll — do not touch them here.
+    The reclaim_stale_jobs task will reset stale `processing` jobs back to
+    `pending` if the on-prem agent stops polling.
+    """
     from app.models.sms import SmsJob, SmsJobStatus
     from app.models.school import School
 
@@ -44,6 +52,7 @@ def process_sms_queue():
                 job.status = SmsJobStatus.sent
                 job.sent_at = datetime.now(timezone.utc)
                 job.semaphore_message_id = result.get("message_id")
+                job.delivered_via = "push"
                 # Deduct credit
                 from app.models.sms import SmsCreditLedger, SmsCreditTx
                 school.sms_credits = float(school.sms_credits) - 1.0
@@ -52,7 +61,7 @@ def process_sms_queue():
                     tx_type=SmsCreditTx.deduct,
                     amount=-1.0,
                     balance_after=float(school.sms_credits),
-                    description=f"SMS sent to {job.recipient_phone}",
+                    description=f"SMS sent via Semaphore to {job.recipient_phone}",
                     reference_id=job.id,
                 ))
                 # Low credit alert
@@ -67,6 +76,46 @@ def process_sms_queue():
     finally:
         db.close()
 
+
+@celery_app.task(name="sms.reclaim_stale_jobs")
+def reclaim_stale_jobs():
+    """
+    Graceful degradation: reset `processing` jobs that have been stuck for
+    more than 5 minutes back to `pending`.
+
+    This handles the case where an on-prem TapTrack instance went offline
+    after receiving jobs from /api/sync/poll but before reporting them back.
+    Once reclaimed, the jobs are eligible to be picked up by sms.process_queue
+    (push path via Semaphore) or by the next on-prem poll.
+    """
+    from app.models.sms import SmsJob, SmsJobStatus
+
+    stale_cutoff = datetime.now(timezone.utc) - timedelta(minutes=5)
+    db = _Session()
+    try:
+        result = db.execute(
+            update(SmsJob)
+            .where(
+                and_(
+                    SmsJob.status == SmsJobStatus.processing,
+                    SmsJob.processing_started_at != None,  # noqa: E711
+                    SmsJob.processing_started_at < stale_cutoff,
+                )
+            )
+            .values(
+                status=SmsJobStatus.pending,
+                processing_started_at=None,
+                error_message="Reclaimed: on-prem agent did not report back within 5 minutes",
+            )
+        )
+        reclaimed = result.rowcount
+        db.commit()
+        if reclaimed:
+            logger.info("sms.reclaim_stale_jobs: reclaimed %d stale processing jobs → pending", reclaimed)
+    finally:
+        db.close()
+
+
 def _send_semaphore(phone: str, message: str, sender: str) -> dict:
     try:
         with httpx.Client(timeout=15) as client:
@@ -84,6 +133,7 @@ def _send_semaphore(phone: str, message: str, sender: str) -> dict:
     except Exception as e:
         return {"success": False, "error": str(e)}
 
+
 @celery_app.task(name="sms.send_low_credit_alert")
 def send_low_credit_alert(school_id: str):
     """Send low credit warning email to school billing contact."""