Files
TapTrack-Hub/backend/app/routers/sync.py
kevin-asprec 132290957c feat(phase-5): on-prem sync poll hardening + stale job reclaim
Harden the Hub-side polling protocol for on-prem TapTrack agents:
- sync/poll: accept job_failed_ids (retry/fail on-prem delivery failures)
- sync/poll: deduct credits + write ledger when on-prem reports sent jobs
- sync/poll: return feature_flags (tier-based) + suspended flag in config
- sync/poll: skip job dispatch for suspended/expired schools
- sms_jobs: add delivered_via (pull|push) + processing_started_at columns
- tasks/sms: new sms.reclaim_stale_jobs task resets processing→pending if on-prem
  goes offline (jobs stuck >5 min), enabling Celery push fallback
- tasks/sms: tag Celery-sent jobs as delivered_via='push'
- worker: schedule reclaim_stale_jobs every 5 minutes
- migration: 001_phase5 adds delivered_via + processing_started_at to sms_jobs
2026-03-16 12:30:37 +08:00

173 lines
7.0 KiB
Python

"""On-prem sync endpoint — polled by TapTrack every 30s to get SMS jobs and config."""
import uuid
from datetime import datetime, timezone
from fastapi import APIRouter, HTTPException, Request
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, and_, update
from fastapi import Depends
from app.database import get_db
from app.models.license import License, LicenseStatus
from app.models.school import School, SchoolStatus
from app.models.sms import SmsJob, SmsJobStatus, SmsCreditLedger, SmsCreditTx
router = APIRouter(prefix="/api/sync", tags=["sync"])
# Feature flags by tier — mirrors licenses.py
def _tier_features(tier: str) -> dict:
base = {"sms": True, "reports": True, "websocket": True, "multi_terminal": True}
if tier == "premium":
base.update({"api_keys": True, "webhooks": True, "bulk_enrollment": True})
elif tier == "basic":
base.update({"multi_terminal": False, "api_keys": False, "webhooks": False})
return base
@router.post("/poll")
async def sync_poll(
request: Request,
db: AsyncSession = Depends(get_db),
):
"""
Called by on-prem TapTrack every 30s.
Returns pending SMS jobs and current config (sender_name, credits, feature flags).
Body:
{
"license_key": "TTUB-XXXXX-XXXXX-XXXXX",
"report_sent_ids": ["uuid1", "uuid2"], // confirmed-sent job IDs → deduct credits
"job_failed_ids": ["uuid3"] // jobs on-prem could NOT deliver → retry/fail
}
"""
body = await request.json()
key = body.get("license_key", "")
sent_ids: list[str] = body.get("report_sent_ids", [])
failed_ids: list[str] = body.get("job_failed_ids", [])
# Validate license
lic = (await db.execute(select(License).where(License.key == key))).scalar_one_or_none()
if not lic or lic.status == LicenseStatus.revoked:
raise HTTPException(403, detail={"reason": "revoked", "message": "Invalid or revoked license"})
if lic.status == LicenseStatus.expired:
raise HTTPException(403, detail={"reason": "expired", "message": "License has expired"})
school = (await db.execute(select(School).where(School.id == lic.school_id))).scalar_one_or_none()
if not school:
raise HTTPException(404, detail="School not found")
now = datetime.now(timezone.utc)
# ── Mark completed jobs (on-prem confirmed delivery) ──────────────────────
if sent_ids:
# Fetch jobs to deduct credits (only for this school, only processing/pending)
jobs_sent = (await db.execute(
select(SmsJob).where(
and_(
SmsJob.id.in_(sent_ids),
SmsJob.school_id == school.id,
SmsJob.status.in_([SmsJobStatus.processing, SmsJobStatus.pending]),
)
)
)).scalars().all()
if jobs_sent:
credit_deductions = len(jobs_sent)
new_balance = float(school.sms_credits) - credit_deductions
new_balance = max(new_balance, 0.0) # floor at 0
# Deduct credits from school
school.sms_credits = new_balance
# Write a single consolidated ledger entry for this batch
db.add(SmsCreditLedger(
id=str(uuid.uuid4()),
school_id=school.id,
tx_type=SmsCreditTx.deduct,
amount=-float(credit_deductions),
balance_after=new_balance,
description=f"SMS delivered via on-prem agent ({credit_deductions} messages)",
reference_id=sent_ids[0] if len(sent_ids) == 1 else None,
))
# Mark jobs as sent, tag delivered_via=pull
await db.execute(
update(SmsJob)
.where(and_(SmsJob.id.in_([j.id for j in jobs_sent]), SmsJob.school_id == school.id))
.values(status=SmsJobStatus.sent, sent_at=now, delivered_via="pull")
)
# Low credit alert — fire if below threshold
if new_balance <= float(school.sms_credit_low_threshold):
try:
from app.tasks.sms import send_low_credit_alert
send_low_credit_alert.delay(school.id)
except Exception:
pass # don't fail the poll if alert fails to queue
# ── Handle jobs on-prem failed to send ────────────────────────────────────
if failed_ids:
failed_jobs = (await db.execute(
select(SmsJob).where(
and_(
SmsJob.id.in_(failed_ids),
SmsJob.school_id == school.id,
)
)
)).scalars().all()
for job in failed_jobs:
job.retry_count = (job.retry_count or 0) + 1
if job.retry_count >= 5:
job.status = SmsJobStatus.failed
job.error_message = "Max retries reached — on-prem delivery failed"
else:
# Return to pending so it can be re-dispatched (pull or push)
job.status = SmsJobStatus.pending
job.error_message = f"On-prem delivery failed (attempt {job.retry_count})"
job.processing_started_at = None
# ── Fetch pending jobs for this school ────────────────────────────────────
# Suspended schools get an empty job list — on-prem should show banner
pending_jobs: list[SmsJob] = []
if school.status == SchoolStatus.active:
pending_jobs = (await db.execute(
select(SmsJob)
.where(and_(SmsJob.school_id == school.id, SmsJob.status == SmsJobStatus.pending))
.limit(50)
)).scalars().all()
# Mark dispatched jobs as processing
if pending_jobs:
job_ids = [j.id for j in pending_jobs]
await db.execute(
update(SmsJob)
.where(SmsJob.id.in_(job_ids))
.values(status=SmsJobStatus.processing, processing_started_at=now)
)
# ── Update license heartbeat ───────────────────────────────────────────────
lic.last_validated_at = now
lic.last_seen_ip = request.client.host if request.client else None
await db.commit()
suspended = school.status in (SchoolStatus.suspended, SchoolStatus.expired)
return {
"sms_jobs": [
{
"id": j.id,
"recipient_phone": j.recipient_phone,
"message": j.message,
"sender_name": j.sender_name,
}
for j in pending_jobs
],
"config": {
"sms_sender_name": school.sms_sender_name,
"sms_credits": float(school.sms_credits),
"school_status": school.status.value,
"suspended": suspended,
"feature_flags": _tier_features(lic.tier),
},
}