Files
TapTrack-Hub/backend/app/tasks/sms.py
kevin-asprec 132290957c feat(phase-5): on-prem sync poll hardening + stale job reclaim
Harden the Hub-side polling protocol for on-prem TapTrack agents:
- sync/poll: accept job_failed_ids (retry/fail on-prem delivery failures)
- sync/poll: deduct credits + write ledger when on-prem reports sent jobs
- sync/poll: return feature_flags (tier-based) + suspended flag in config
- sync/poll: skip job dispatch for suspended/expired schools
- sms_jobs: add delivered_via (pull|push) + processing_started_at columns
- tasks/sms: new sms.reclaim_stale_jobs task resets processing→pending if on-prem
  goes offline (jobs stuck >5 min), enabling Celery push fallback
- tasks/sms: tag Celery-sent jobs as delivered_via='push'
- worker: schedule reclaim_stale_jobs every 5 minutes
- migration: 001_phase5 adds delivered_via + processing_started_at to sms_jobs
2026-03-16 12:30:37 +08:00

153 lines
5.7 KiB
Python

"""Celery task: process pending SMS jobs via Semaphore (push path)."""
import logging
import os
from datetime import datetime, timezone, timedelta
import httpx
from sqlalchemy import create_engine, select, update, and_
from sqlalchemy.orm import sessionmaker
from app.worker import celery_app
from app.config import settings
logger = logging.getLogger(__name__)
def _make_sync_engine():
db_url = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@db:5432/taptrack_hub")
return create_engine(db_url.replace("postgresql+asyncpg://", "postgresql://"), pool_pre_ping=True, pool_size=2)
_engine = _make_sync_engine()
_Session = sessionmaker(bind=_engine)
@celery_app.task(name="sms.process_queue")
def process_sms_queue():
"""
Push path: process up to 20 pending SMS jobs per run via Semaphore API.
Only picks up `pending` jobs. Jobs in `processing` are being handled
by the on-prem pull agent via /api/sync/poll — do not touch them here.
The reclaim_stale_jobs task will reset stale `processing` jobs back to
`pending` if the on-prem agent stops polling.
"""
from app.models.sms import SmsJob, SmsJobStatus
from app.models.school import School
db = _Session()
try:
jobs = db.execute(
select(SmsJob).where(SmsJob.status == SmsJobStatus.pending).limit(20)
).scalars().all()
for job in jobs:
school = db.get(School, job.school_id)
if not school or float(school.sms_credits) <= 0:
job.status = SmsJobStatus.cancelled
job.error_message = "Insufficient credits"
db.commit()
continue
result = _send_semaphore(job.recipient_phone, job.message, job.sender_name)
if result["success"]:
job.status = SmsJobStatus.sent
job.sent_at = datetime.now(timezone.utc)
job.semaphore_message_id = result.get("message_id")
job.delivered_via = "push"
# Deduct credit
from app.models.sms import SmsCreditLedger, SmsCreditTx
school.sms_credits = float(school.sms_credits) - 1.0
db.add(SmsCreditLedger(
school_id=school.id,
tx_type=SmsCreditTx.deduct,
amount=-1.0,
balance_after=float(school.sms_credits),
description=f"SMS sent via Semaphore to {job.recipient_phone}",
reference_id=job.id,
))
# Low credit alert
if float(school.sms_credits) <= school.sms_credit_low_threshold:
send_low_credit_alert.delay(school.id)
else:
job.retry_count += 1
if job.retry_count >= 5:
job.status = SmsJobStatus.failed
job.error_message = result.get("error")
db.commit()
finally:
db.close()
@celery_app.task(name="sms.reclaim_stale_jobs")
def reclaim_stale_jobs():
"""
Graceful degradation: reset `processing` jobs that have been stuck for
more than 5 minutes back to `pending`.
This handles the case where an on-prem TapTrack instance went offline
after receiving jobs from /api/sync/poll but before reporting them back.
Once reclaimed, the jobs are eligible to be picked up by sms.process_queue
(push path via Semaphore) or by the next on-prem poll.
"""
from app.models.sms import SmsJob, SmsJobStatus
stale_cutoff = datetime.now(timezone.utc) - timedelta(minutes=5)
db = _Session()
try:
result = db.execute(
update(SmsJob)
.where(
and_(
SmsJob.status == SmsJobStatus.processing,
SmsJob.processing_started_at != None, # noqa: E711
SmsJob.processing_started_at < stale_cutoff,
)
)
.values(
status=SmsJobStatus.pending,
processing_started_at=None,
error_message="Reclaimed: on-prem agent did not report back within 5 minutes",
)
)
reclaimed = result.rowcount
db.commit()
if reclaimed:
logger.info("sms.reclaim_stale_jobs: reclaimed %d stale processing jobs → pending", reclaimed)
finally:
db.close()
def _send_semaphore(phone: str, message: str, sender: str) -> dict:
try:
with httpx.Client(timeout=15) as client:
resp = client.post(settings.SEMAPHORE_URL, data={
"apikey": settings.SEMAPHORE_API_KEY,
"number": phone,
"message": message,
"sendername": sender,
})
if resp.status_code == 200:
data = resp.json()
msg_id = str(data[0].get("message_id", "")) if isinstance(data, list) and data else None
return {"success": True, "message_id": msg_id}
return {"success": False, "error": f"HTTP {resp.status_code}"}
except Exception as e:
return {"success": False, "error": str(e)}
@celery_app.task(name="sms.send_low_credit_alert")
def send_low_credit_alert(school_id: str):
"""Send low credit warning email to school billing contact."""
from app.services.email import send_email
from app.models.school import School
db = _Session()
try:
school = db.get(School, school_id)
if school and school.billing_email:
send_email(
to=school.billing_email,
subject=f"[TapTrack Hub] Low SMS Credits — {school.name}",
body=f"Your SMS credit balance for {school.name} is low ({float(school.sms_credits):.0f} remaining). Please top up to continue sending SMS notifications.",
)
finally:
db.close()