feat(phase-5): on-prem sync poll hardening + stale job reclaim
Harden the Hub-side polling protocol for on-prem TapTrack agents: - sync/poll: accept job_failed_ids (retry/fail on-prem delivery failures) - sync/poll: deduct credits + write ledger when on-prem reports sent jobs - sync/poll: return feature_flags (tier-based) + suspended flag in config - sync/poll: skip job dispatch for suspended/expired schools - sms_jobs: add delivered_via (pull|push) + processing_started_at columns - tasks/sms: new sms.reclaim_stale_jobs task resets processing→pending if on-prem goes offline (jobs stuck >5 min), enabling Celery push fallback - tasks/sms: tag Celery-sent jobs as delivered_via='push' - worker: schedule reclaim_stale_jobs every 5 minutes - migration: 001_phase5 adds delivered_via + processing_started_at to sms_jobs
This commit is contained in:
kevin-asprec
@@ -1,4 +1,5 @@
|
||||
"""On-prem sync endpoint — polled by TapTrack every 30s to get SMS jobs and config."""
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
@@ -7,11 +8,20 @@ from fastapi import Depends
|
||||
|
||||
from app.database import get_db
|
||||
from app.models.license import License, LicenseStatus
|
||||
from app.models.school import School
|
||||
from app.models.sms import SmsJob, SmsJobStatus
|
||||
from app.models.school import School, SchoolStatus
|
||||
from app.models.sms import SmsJob, SmsJobStatus, SmsCreditLedger, SmsCreditTx
|
||||
|
||||
router = APIRouter(prefix="/api/sync", tags=["sync"])
|
||||
|
||||
# Feature flags by tier — mirrors licenses.py
|
||||
def _tier_features(tier: str) -> dict:
|
||||
base = {"sms": True, "reports": True, "websocket": True, "multi_terminal": True}
|
||||
if tier == "premium":
|
||||
base.update({"api_keys": True, "webhooks": True, "bulk_enrollment": True})
|
||||
elif tier == "basic":
|
||||
base.update({"multi_terminal": False, "api_keys": False, "webhooks": False})
|
||||
return base
|
||||
|
||||
@router.post("/poll")
|
||||
async def sync_poll(
|
||||
request: Request,
|
||||
@@ -20,58 +30,143 @@ async def sync_poll(
|
||||
"""
|
||||
Called by on-prem TapTrack every 30s.
|
||||
Returns pending SMS jobs and current config (sender_name, credits, feature flags).
|
||||
Body: { license_key: str, report_sent_ids: [str] } (completed job IDs to mark as sent)
|
||||
|
||||
Body:
|
||||
{
|
||||
"license_key": "TTUB-XXXXX-XXXXX-XXXXX",
|
||||
"report_sent_ids": ["uuid1", "uuid2"], // confirmed-sent job IDs → deduct credits
|
||||
"job_failed_ids": ["uuid3"] // jobs on-prem could NOT deliver → retry/fail
|
||||
}
|
||||
"""
|
||||
body = await request.json()
|
||||
key = body.get("license_key", "")
|
||||
sent_ids = body.get("report_sent_ids", [])
|
||||
sent_ids: list[str] = body.get("report_sent_ids", [])
|
||||
failed_ids: list[str] = body.get("job_failed_ids", [])
|
||||
|
||||
# Validate license
|
||||
lic = (await db.execute(select(License).where(License.key == key))).scalar_one_or_none()
|
||||
if not lic or lic.status == LicenseStatus.revoked:
|
||||
raise HTTPException(403, "Invalid or revoked license")
|
||||
raise HTTPException(403, detail={"reason": "revoked", "message": "Invalid or revoked license"})
|
||||
if lic.status == LicenseStatus.expired:
|
||||
raise HTTPException(403, detail={"reason": "expired", "message": "License has expired"})
|
||||
|
||||
school = (await db.execute(select(School).where(School.id == lic.school_id))).scalar_one_or_none()
|
||||
if not school:
|
||||
raise HTTPException(404)
|
||||
raise HTTPException(404, detail="School not found")
|
||||
|
||||
# Mark completed jobs
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
# ── Mark completed jobs (on-prem confirmed delivery) ──────────────────────
|
||||
if sent_ids:
|
||||
await db.execute(
|
||||
update(SmsJob)
|
||||
.where(and_(SmsJob.id.in_(sent_ids), SmsJob.school_id == school.id))
|
||||
.values(status=SmsJobStatus.sent, sent_at=datetime.now(timezone.utc))
|
||||
)
|
||||
# Fetch jobs to deduct credits (only for this school, only processing/pending)
|
||||
jobs_sent = (await db.execute(
|
||||
select(SmsJob).where(
|
||||
and_(
|
||||
SmsJob.id.in_(sent_ids),
|
||||
SmsJob.school_id == school.id,
|
||||
SmsJob.status.in_([SmsJobStatus.processing, SmsJobStatus.pending]),
|
||||
)
|
||||
)
|
||||
)).scalars().all()
|
||||
|
||||
# Get pending jobs (max 50 per poll)
|
||||
pending_jobs = (await db.execute(
|
||||
select(SmsJob)
|
||||
.where(and_(SmsJob.school_id == school.id, SmsJob.status == SmsJobStatus.pending))
|
||||
.limit(50)
|
||||
)).scalars().all()
|
||||
if jobs_sent:
|
||||
credit_deductions = len(jobs_sent)
|
||||
new_balance = float(school.sms_credits) - credit_deductions
|
||||
new_balance = max(new_balance, 0.0) # floor at 0
|
||||
|
||||
# Mark as processing
|
||||
job_ids = [j.id for j in pending_jobs]
|
||||
if job_ids:
|
||||
await db.execute(
|
||||
update(SmsJob)
|
||||
.where(SmsJob.id.in_(job_ids))
|
||||
.values(status=SmsJobStatus.processing)
|
||||
)
|
||||
# Deduct credits from school
|
||||
school.sms_credits = new_balance
|
||||
|
||||
# Update last seen
|
||||
lic.last_validated_at = datetime.now(timezone.utc)
|
||||
# Write a single consolidated ledger entry for this batch
|
||||
db.add(SmsCreditLedger(
|
||||
id=str(uuid.uuid4()),
|
||||
school_id=school.id,
|
||||
tx_type=SmsCreditTx.deduct,
|
||||
amount=-float(credit_deductions),
|
||||
balance_after=new_balance,
|
||||
description=f"SMS delivered via on-prem agent ({credit_deductions} messages)",
|
||||
reference_id=sent_ids[0] if len(sent_ids) == 1 else None,
|
||||
))
|
||||
|
||||
# Mark jobs as sent, tag delivered_via=pull
|
||||
await db.execute(
|
||||
update(SmsJob)
|
||||
.where(and_(SmsJob.id.in_([j.id for j in jobs_sent]), SmsJob.school_id == school.id))
|
||||
.values(status=SmsJobStatus.sent, sent_at=now, delivered_via="pull")
|
||||
)
|
||||
|
||||
# Low credit alert — fire if below threshold
|
||||
if new_balance <= float(school.sms_credit_low_threshold):
|
||||
try:
|
||||
from app.tasks.sms import send_low_credit_alert
|
||||
send_low_credit_alert.delay(school.id)
|
||||
except Exception:
|
||||
pass # don't fail the poll if alert fails to queue
|
||||
|
||||
# ── Handle jobs on-prem failed to send ────────────────────────────────────
|
||||
if failed_ids:
|
||||
failed_jobs = (await db.execute(
|
||||
select(SmsJob).where(
|
||||
and_(
|
||||
SmsJob.id.in_(failed_ids),
|
||||
SmsJob.school_id == school.id,
|
||||
)
|
||||
)
|
||||
)).scalars().all()
|
||||
|
||||
for job in failed_jobs:
|
||||
job.retry_count = (job.retry_count or 0) + 1
|
||||
if job.retry_count >= 5:
|
||||
job.status = SmsJobStatus.failed
|
||||
job.error_message = "Max retries reached — on-prem delivery failed"
|
||||
else:
|
||||
# Return to pending so it can be re-dispatched (pull or push)
|
||||
job.status = SmsJobStatus.pending
|
||||
job.error_message = f"On-prem delivery failed (attempt {job.retry_count})"
|
||||
job.processing_started_at = None
|
||||
|
||||
# ── Fetch pending jobs for this school ────────────────────────────────────
|
||||
# Suspended schools get an empty job list — on-prem should show banner
|
||||
pending_jobs: list[SmsJob] = []
|
||||
if school.status == SchoolStatus.active:
|
||||
pending_jobs = (await db.execute(
|
||||
select(SmsJob)
|
||||
.where(and_(SmsJob.school_id == school.id, SmsJob.status == SmsJobStatus.pending))
|
||||
.limit(50)
|
||||
)).scalars().all()
|
||||
|
||||
# Mark dispatched jobs as processing
|
||||
if pending_jobs:
|
||||
job_ids = [j.id for j in pending_jobs]
|
||||
await db.execute(
|
||||
update(SmsJob)
|
||||
.where(SmsJob.id.in_(job_ids))
|
||||
.values(status=SmsJobStatus.processing, processing_started_at=now)
|
||||
)
|
||||
|
||||
# ── Update license heartbeat ───────────────────────────────────────────────
|
||||
lic.last_validated_at = now
|
||||
lic.last_seen_ip = request.client.host if request.client else None
|
||||
|
||||
await db.commit()
|
||||
|
||||
suspended = school.status in (SchoolStatus.suspended, SchoolStatus.expired)
|
||||
|
||||
return {
|
||||
"sms_jobs": [
|
||||
{"id": j.id, "recipient_phone": j.recipient_phone,
|
||||
"message": j.message, "sender_name": j.sender_name}
|
||||
{
|
||||
"id": j.id,
|
||||
"recipient_phone": j.recipient_phone,
|
||||
"message": j.message,
|
||||
"sender_name": j.sender_name,
|
||||
}
|
||||
for j in pending_jobs
|
||||
],
|
||||
"config": {
|
||||
"sms_sender_name": school.sms_sender_name,
|
||||
"sms_credits": float(school.sms_credits),
|
||||
"school_status": school.status.value,
|
||||
"suspended": suspended,
|
||||
"feature_flags": _tier_features(lic.tier),
|
||||
},
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user