feat: initial BantayCam scaffold
- FastAPI backend with async SQLAlchemy - Camera RTSP management (add, start, stop) - Vehicle detection (YOLO + fast-alpr) - Type: car, motorcycle, truck, jeepney - Color detection (HSV) - License plate OCR - Motorcycle person count - Face detection + InsightFace ArcFace embedding - pgvector identity grouping (auto-cluster same face) - Vehicle + person movement trail APIs - Docker Compose with pgvector/pg16 - Models: Camera, VehicleIdentity, VehicleEvent, PersonIdentity, FaceEvent
This commit is contained in:
Nemo
0
backend/app/services/__init__.py
Normal file
0
backend/app/services/__init__.py
Normal file
88
backend/app/services/face_service.py
Normal file
88
backend/app/services/face_service.py
Normal file
@@ -0,0 +1,88 @@
|
||||
"""
|
||||
FaceService — InsightFace-based face detection, embedding, and identity matching.
|
||||
Uses pgvector cosine similarity for fast 1:N identity search.
|
||||
"""
|
||||
import cv2
|
||||
import numpy as np
|
||||
import logging
|
||||
from typing import Optional
|
||||
from dataclasses import dataclass, field
|
||||
from uuid import UUID
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FaceDetection:
|
||||
bbox: dict = field(default_factory=dict) # {"x","y","w","h"}
|
||||
embedding: Optional[np.ndarray] = None # 512-dim vector
|
||||
detection_confidence: float = 0.0
|
||||
face_crop: Optional[np.ndarray] = None
|
||||
matched_identity_id: Optional[UUID] = None
|
||||
similarity_score: float = 0.0
|
||||
|
||||
|
||||
class FaceService:
|
||||
def __init__(self):
|
||||
self._app = None
|
||||
self._ready = False
|
||||
|
||||
def load(self):
|
||||
"""Load InsightFace model — call once at startup."""
|
||||
try:
|
||||
from insightface.app import FaceAnalysis
|
||||
self._app = FaceAnalysis(
|
||||
name=settings.FACE_MODEL,
|
||||
allowed_modules=["detection", "recognition"],
|
||||
)
|
||||
self._app.prepare(
|
||||
ctx_id=settings.GPU_DEVICE,
|
||||
det_size=(640, 640),
|
||||
)
|
||||
self._ready = True
|
||||
logger.info(f"InsightFace loaded (model={settings.FACE_MODEL})")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load InsightFace: {e}")
|
||||
|
||||
def detect(self, frame: np.ndarray) -> list[FaceDetection]:
|
||||
"""
|
||||
Detect all faces in a frame and extract embeddings.
|
||||
Returns list of FaceDetection objects.
|
||||
"""
|
||||
if not self._ready or self._app is None:
|
||||
return []
|
||||
|
||||
try:
|
||||
faces = self._app.get(frame)
|
||||
except Exception as e:
|
||||
logger.debug(f"Face detection error: {e}")
|
||||
return []
|
||||
|
||||
detections = []
|
||||
for face in faces:
|
||||
bbox = face.bbox.astype(int)
|
||||
x1, y1, x2, y2 = bbox[0], bbox[1], bbox[2], bbox[3]
|
||||
|
||||
det = FaceDetection(
|
||||
bbox={"x": x1, "y": y1, "w": x2 - x1, "h": y2 - y1},
|
||||
embedding=face.normed_embedding, # Already L2-normalized
|
||||
detection_confidence=float(face.det_score),
|
||||
face_crop=frame[max(0, y1):y2, max(0, x1):x2],
|
||||
)
|
||||
detections.append(det)
|
||||
|
||||
return detections
|
||||
|
||||
def cosine_similarity(self, emb1: np.ndarray, emb2: np.ndarray) -> float:
|
||||
"""Cosine similarity between two normalized embeddings."""
|
||||
return float(np.dot(emb1, emb2))
|
||||
|
||||
def embeddings_to_list(self, embedding: np.ndarray) -> list[float]:
|
||||
"""Convert numpy embedding to list for pgvector storage."""
|
||||
return embedding.tolist()
|
||||
|
||||
|
||||
# Singleton
|
||||
face_service = FaceService()
|
||||
153
backend/app/services/stream_manager.py
Normal file
153
backend/app/services/stream_manager.py
Normal file
@@ -0,0 +1,153 @@
|
||||
"""
|
||||
StreamManager — Manages RTSP stream connections and dispatches frames to workers.
|
||||
Each camera runs in its own asyncio task with a frame queue.
|
||||
"""
|
||||
import asyncio
|
||||
import cv2
|
||||
import logging
|
||||
from typing import Dict, Optional
|
||||
from uuid import UUID
|
||||
from datetime import datetime
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StreamWorker:
|
||||
"""
|
||||
Handles one RTSP camera stream.
|
||||
Reads frames and puts them into a queue for the AI pipeline to consume.
|
||||
"""
|
||||
|
||||
def __init__(self, camera_id: UUID, camera_name: str, rtsp_url: str):
|
||||
self.camera_id = camera_id
|
||||
self.camera_name = camera_name
|
||||
self.rtsp_url = rtsp_url
|
||||
self.frame_queue: asyncio.Queue = asyncio.Queue(maxsize=10)
|
||||
self.is_running = False
|
||||
self._task: Optional[asyncio.Task] = None
|
||||
self.frame_count = 0
|
||||
self.last_frame_at: Optional[datetime] = None
|
||||
self.error: Optional[str] = None
|
||||
|
||||
async def start(self):
|
||||
self.is_running = True
|
||||
self._task = asyncio.create_task(self._read_loop())
|
||||
logger.info(f"[{self.camera_name}] Stream started")
|
||||
|
||||
async def stop(self):
|
||||
self.is_running = False
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
logger.info(f"[{self.camera_name}] Stream stopped")
|
||||
|
||||
async def _read_loop(self):
|
||||
"""Read frames from RTSP in a thread pool (OpenCV is blocking)."""
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
while self.is_running:
|
||||
try:
|
||||
cap = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: cv2.VideoCapture(self.rtsp_url)
|
||||
)
|
||||
|
||||
if not cap.isOpened():
|
||||
self.error = "Cannot open RTSP stream"
|
||||
logger.error(f"[{self.camera_name}] {self.error}")
|
||||
await asyncio.sleep(5)
|
||||
continue
|
||||
|
||||
self.error = None
|
||||
frame_idx = 0
|
||||
|
||||
while self.is_running:
|
||||
ret, frame = await loop.run_in_executor(None, cap.read)
|
||||
|
||||
if not ret:
|
||||
logger.warning(f"[{self.camera_name}] Frame read failed, reconnecting...")
|
||||
break
|
||||
|
||||
frame_idx += 1
|
||||
self.frame_count += 1
|
||||
self.last_frame_at = datetime.utcnow()
|
||||
|
||||
# Skip frames for performance
|
||||
if frame_idx % settings.PROCESS_EVERY_N_FRAMES != 0:
|
||||
continue
|
||||
|
||||
# Non-blocking put — drop frame if queue is full
|
||||
try:
|
||||
self.frame_queue.put_nowait({
|
||||
"frame": frame,
|
||||
"frame_idx": frame_idx,
|
||||
"captured_at": self.last_frame_at,
|
||||
})
|
||||
except asyncio.QueueFull:
|
||||
pass # AI pipeline is slow — drop frame
|
||||
|
||||
cap.release()
|
||||
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.error = str(e)
|
||||
logger.exception(f"[{self.camera_name}] Stream error: {e}")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
|
||||
class StreamManager:
|
||||
"""
|
||||
Singleton that manages all active camera streams.
|
||||
"""
|
||||
_instance = None
|
||||
|
||||
def __init__(self):
|
||||
self._workers: Dict[str, StreamWorker] = {}
|
||||
|
||||
@classmethod
|
||||
def get(cls) -> "StreamManager":
|
||||
if cls._instance is None:
|
||||
cls._instance = StreamManager()
|
||||
return cls._instance
|
||||
|
||||
async def start_stream(self, camera_id: UUID, camera_name: str, rtsp_url: str):
|
||||
key = str(camera_id)
|
||||
if key in self._workers:
|
||||
await self.stop_stream(camera_id)
|
||||
|
||||
worker = StreamWorker(camera_id, camera_name, rtsp_url)
|
||||
self._workers[key] = worker
|
||||
await worker.start()
|
||||
return worker
|
||||
|
||||
async def stop_stream(self, camera_id: UUID):
|
||||
key = str(camera_id)
|
||||
if key in self._workers:
|
||||
await self._workers[key].stop()
|
||||
del self._workers[key]
|
||||
|
||||
def get_worker(self, camera_id: UUID) -> Optional[StreamWorker]:
|
||||
return self._workers.get(str(camera_id))
|
||||
|
||||
def get_all_status(self) -> list:
|
||||
return [
|
||||
{
|
||||
"camera_id": str(k),
|
||||
"is_running": w.is_running,
|
||||
"frame_count": w.frame_count,
|
||||
"last_frame_at": w.last_frame_at,
|
||||
"error": w.error,
|
||||
}
|
||||
for k, w in self._workers.items()
|
||||
]
|
||||
|
||||
async def stop_all(self):
|
||||
for worker in list(self._workers.values()):
|
||||
await worker.stop()
|
||||
self._workers.clear()
|
||||
227
backend/app/services/vehicle_detector.py
Normal file
227
backend/app/services/vehicle_detector.py
Normal file
@@ -0,0 +1,227 @@
|
||||
"""
|
||||
VehicleDetector — YOLOv8-based vehicle detection, classification, and person counting.
|
||||
Also handles license plate OCR via fast-alpr.
|
||||
"""
|
||||
import cv2
|
||||
import numpy as np
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VEHICLE_CLASSES = {
|
||||
2: "car",
|
||||
3: "motorcycle",
|
||||
5: "bus",
|
||||
7: "truck",
|
||||
}
|
||||
|
||||
# COCO person class
|
||||
PERSON_CLASS = 0
|
||||
|
||||
# Color detection ranges (HSV)
|
||||
COLOR_RANGES = {
|
||||
"red": [(0, 70, 50), (10, 255, 255)],
|
||||
"red2": [(170, 70, 50), (180, 255, 255)],
|
||||
"blue": [(100, 70, 50), (130, 255, 255)],
|
||||
"white": [(0, 0, 180), (180, 30, 255)],
|
||||
"black": [(0, 0, 0), (180, 255, 50)],
|
||||
"silver": [(0, 0, 100), (180, 30, 180)],
|
||||
"yellow": [(20, 70, 50), (35, 255, 255)],
|
||||
"green": [(35, 70, 50), (85, 255, 255)],
|
||||
"orange": [(10, 70, 50), (20, 255, 255)],
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class VehicleDetection:
|
||||
vehicle_type: str = "unknown"
|
||||
color: Optional[str] = None
|
||||
plate_number: Optional[str] = None
|
||||
plate_confidence: float = 0.0
|
||||
person_count: int = 1
|
||||
detection_confidence: float = 0.0
|
||||
vehicle_bbox: dict = field(default_factory=dict)
|
||||
plate_bbox: dict = field(default_factory=dict)
|
||||
vehicle_crop: Optional[np.ndarray] = None
|
||||
plate_crop: Optional[np.ndarray] = None
|
||||
|
||||
|
||||
class VehicleDetector:
|
||||
def __init__(self):
|
||||
self._yolo = None
|
||||
self._alpr = None
|
||||
self._ready = False
|
||||
|
||||
def load(self):
|
||||
"""Load models — call once at startup."""
|
||||
try:
|
||||
from ultralytics import YOLO
|
||||
self._yolo = YOLO(settings.YOLO_MODEL)
|
||||
logger.info("YOLO model loaded")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load YOLO: {e}")
|
||||
return
|
||||
|
||||
try:
|
||||
from fast_alpr import ALPR
|
||||
self._alpr = ALPR(
|
||||
detector_model="plate-detection-v1-large",
|
||||
ocr_model="global-plates-mobile-vit-v2-model",
|
||||
device="cuda" if settings.GPU_DEVICE >= 0 else "cpu",
|
||||
)
|
||||
logger.info("ALPR model loaded")
|
||||
except Exception as e:
|
||||
logger.warning(f"ALPR not loaded (will skip plate reading): {e}")
|
||||
|
||||
self._ready = True
|
||||
|
||||
def detect(self, frame: np.ndarray) -> list[VehicleDetection]:
|
||||
"""
|
||||
Run vehicle detection on a frame.
|
||||
Returns list of VehicleDetection objects.
|
||||
"""
|
||||
if not self._ready or self._yolo is None:
|
||||
return []
|
||||
|
||||
results = self._yolo(
|
||||
frame,
|
||||
conf=settings.DETECTION_CONFIDENCE_THRESHOLD,
|
||||
verbose=False,
|
||||
)
|
||||
|
||||
detections = []
|
||||
h, w = frame.shape[:2]
|
||||
|
||||
for result in results:
|
||||
boxes = result.boxes
|
||||
if boxes is None:
|
||||
continue
|
||||
|
||||
# Group: find all vehicles and all persons
|
||||
vehicles = []
|
||||
persons = []
|
||||
|
||||
for box in boxes:
|
||||
cls_id = int(box.cls[0])
|
||||
conf = float(box.conf[0])
|
||||
x1, y1, x2, y2 = map(int, box.xyxy[0])
|
||||
|
||||
if cls_id in VEHICLE_CLASSES:
|
||||
vehicles.append({
|
||||
"type": VEHICLE_CLASSES[cls_id],
|
||||
"conf": conf,
|
||||
"bbox": {"x": x1, "y": y1, "w": x2 - x1, "h": y2 - y1},
|
||||
"crop": frame[y1:y2, x1:x2],
|
||||
})
|
||||
elif cls_id == PERSON_CLASS:
|
||||
persons.append({"bbox": {"x": x1, "y": y1, "w": x2 - x1, "h": y2 - y1}})
|
||||
|
||||
for vehicle in vehicles:
|
||||
det = VehicleDetection(
|
||||
vehicle_type=vehicle["type"],
|
||||
detection_confidence=vehicle["conf"],
|
||||
vehicle_bbox=vehicle["bbox"],
|
||||
vehicle_crop=vehicle["crop"],
|
||||
)
|
||||
|
||||
# Detect color
|
||||
det.color = self._detect_color(vehicle["crop"])
|
||||
|
||||
# Count persons on/near motorcycle
|
||||
if vehicle["type"] == "motorcycle":
|
||||
det.person_count = self._count_persons_on_motorcycle(
|
||||
vehicle["bbox"], persons
|
||||
)
|
||||
|
||||
# Read plate
|
||||
if self._alpr is not None:
|
||||
plate_result = self._read_plate(frame, vehicle["bbox"])
|
||||
if plate_result:
|
||||
det.plate_number = plate_result["text"]
|
||||
det.plate_confidence = plate_result["confidence"]
|
||||
det.plate_bbox = plate_result["bbox"]
|
||||
bx = plate_result["bbox"]
|
||||
det.plate_crop = frame[
|
||||
bx["y"]:bx["y"] + bx["h"],
|
||||
bx["x"]:bx["x"] + bx["w"]
|
||||
]
|
||||
|
||||
detections.append(det)
|
||||
|
||||
return detections
|
||||
|
||||
def _read_plate(self, frame: np.ndarray, vehicle_bbox: dict) -> Optional[dict]:
|
||||
"""Run ALPR on vehicle region."""
|
||||
if self._alpr is None:
|
||||
return None
|
||||
try:
|
||||
# Expand vehicle crop slightly for plate detection
|
||||
x, y, w, h = vehicle_bbox["x"], vehicle_bbox["y"], vehicle_bbox["w"], vehicle_bbox["h"]
|
||||
crop = frame[max(0, y):y + h, max(0, x):x + w]
|
||||
results = self._alpr.run(crop)
|
||||
if results:
|
||||
best = results[0]
|
||||
ocr = best.ocr
|
||||
if ocr and ocr.text:
|
||||
# Offset bbox back to full frame coordinates
|
||||
pb = best.detection.bounding_box
|
||||
return {
|
||||
"text": ocr.text.upper().replace(" ", ""),
|
||||
"confidence": float(ocr.confidence),
|
||||
"bbox": {
|
||||
"x": x + int(pb.x1),
|
||||
"y": y + int(pb.y1),
|
||||
"w": int(pb.x2 - pb.x1),
|
||||
"h": int(pb.y2 - pb.y1),
|
||||
}
|
||||
}
|
||||
except Exception as e:
|
||||
logger.debug(f"ALPR error: {e}")
|
||||
return None
|
||||
|
||||
def _detect_color(self, crop: np.ndarray) -> str:
|
||||
"""Detect dominant vehicle color using HSV histogram."""
|
||||
if crop is None or crop.size == 0:
|
||||
return "unknown"
|
||||
try:
|
||||
hsv = cv2.cvtColor(crop, cv2.COLOR_BGR2HSV)
|
||||
max_pixels = 0
|
||||
detected_color = "unknown"
|
||||
|
||||
for color_name, (lower, upper) in COLOR_RANGES.items():
|
||||
mask = cv2.inRange(hsv, np.array(lower), np.array(upper))
|
||||
pixel_count = cv2.countNonZero(mask)
|
||||
if pixel_count > max_pixels:
|
||||
max_pixels = pixel_count
|
||||
detected_color = color_name.replace("2", "") # red2 → red
|
||||
|
||||
return detected_color
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
def _count_persons_on_motorcycle(self, moto_bbox: dict, persons: list) -> int:
|
||||
"""Count persons whose center falls within or near the motorcycle bbox."""
|
||||
count = 0
|
||||
mx, my, mw, mh = moto_bbox["x"], moto_bbox["y"], moto_bbox["w"], moto_bbox["h"]
|
||||
|
||||
for person in persons:
|
||||
px, py, pw, ph = person["bbox"]["x"], person["bbox"]["y"], person["bbox"]["w"], person["bbox"]["h"]
|
||||
# Person center
|
||||
cx = px + pw // 2
|
||||
cy = py + ph // 2
|
||||
# Check if center is within motorcycle bounding box (with some margin)
|
||||
margin = 30
|
||||
if (mx - margin <= cx <= mx + mw + margin and
|
||||
my - margin <= cy <= my + mh + margin):
|
||||
count += 1
|
||||
|
||||
return max(count, 1) # At least 1 rider assumed
|
||||
|
||||
|
||||
# Singleton
|
||||
vehicle_detector = VehicleDetector()
|
||||
Reference in New Issue
Block a user