Rate Limiter with Redis Backend

Response:
 ```python
"""
Redis-based Rate Limiter for SaaS Voice Widget Platform.

Implements sliding window rate limiting with support for:
- Per-IP request limiting (requests/minute)
- Per-user tiered API limiting (configurable per tier)
- Per-widget concurrent call limiting
- Atomic operations via Lua scripts

File: Sunaiva/talking-widget/backend/rate_limiter_redis.py
"""

from __future__ import annotations

import logging
import time
import uuid
from contextlib import asynccontextmanager
from dataclasses import dataclass
from enum import Enum, auto
from typing import Any, AsyncGenerator, Callable, Dict, List, Optional, Protocol

import redis.asyncio as redis
from fastapi import FastAPI, HTTPException, Request, Response, status
from fastapi.responses import JSONResponse
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
from starlette.types import ASGIApp

__all__ = [
    "RateLimitTier",
    "RateLimitRule",
    "RateLimitExceeded",
    "ConcurrentLimitExceeded",
    "RedisRateLimiter",
    "RateLimitMiddleware",
    "WidgetRateLimitDependency",
    "setup_rate_limiting",
]

logger = logging.getLogger(__name__)

class RateLimitError(Exception):
    """Base exception for rate limiting errors."""

    def __init__(self, message: str, key: Optional[str] = None):
        super().__init__(message)
        self.key = key

class RateLimitExceeded(RateLimitError):
    """Raised when sliding window rate limit is exceeded."""

    def __init__(
        self,
        limit: int,
        window: int,
        retry_after: int,
        key: Optional[str] = None
    ):
        super().__init__(
            f"Rate limit exceeded: {limit} requests per {window}s. Retry after {retry_after}s.",
            key=key
        )
        self.limit = limit
        self.window = window
        self.retry_after = retry_after

class ConcurrentLimitExceeded(RateLimitError):
    """Raised when concurrent call limit is exceeded."""

    def __init__(
        self,
        current: int,
        limit: int,
        key: Optional[str] = None
    ):
        super().__init__(
            f"Concurrent limit exceeded: {current}/{limit}",
            key=key
        )
        self.current = current
        self.limit = limit

class RateLimitTier(Enum):
    """Subscription tiers with different rate limits."""
    FREE = "free"
    PRO = "pro"
    ENTERPRISE = "enterprise"
    CUSTOM = "custom"

@dataclass(frozen=True)
class RateLimitRule:
    """
    Configuration for rate limiting rules per tier.

    Attributes:
        requests_per_minute: Maximum requests allowed per minute (sliding window)
        concurrent_calls: Maximum concurrent WebSocket/call connections allowed
        burst_size: Optional token bucket burst capacity (for future implementation)
    """
    requests_per_minute: int
    concurrent_calls: int
    burst_size: int = 0

# Default tier configurations - modify as needed or load from database
DEFAULT_TIER_CONFIG: Dict[RateLimitTier, RateLimitRule] = {
    RateLimitTier.FREE: RateLimitRule(
        requests_per_minute=60,
        concurrent_calls=2,
        burst_size=5
    ),
    RateLimitTier.PRO: RateLimitRule(
        requests_per_minute=300,
        concurrent_calls=10,
        burst_size=15
    ),
    RateLimitTier.ENTERPRISE: RateLimitRule(
        requests_per_minute=1000,
        concurrent_calls=50,
        burst_size=50
    ),
    RateLimitTier.CUSTOM: RateLimitRule(
        requests_per_minute=5000,
        concurrent_calls=200,
        burst_size=100
    ),
}

class RedisClientProtocol(Protocol):
    """Protocol for Redis client to allow dependency injection."""

    async def eval(self, script: str, num_keys: int, *keys_and_args: Any) -> Any:
        ...

    async def zcard(self, key: str) -> int:
        ...

    async def ttl(self, key: str) -> int:
        ...

    async def close(self) -> None:
        ...

class RedisRateLimiter:
    """
    Production-ready async rate limiter using Redis.

    Implements sliding window algorithm using Redis Sorted Sets for O(log N)
    cleanup and counting. Uses Lua scripts for atomic operations to prevent
    race conditions in high-concurrency scenarios.
    """

    def __init__(
        self,
        redis_client: redis.Redis,
        tier_config: Optional[Dict[RateLimitTier, RateLimitRule]] = None,
        default_tier: RateLimitTier = RateLimitTier.FREE,
        key_prefix: str = "sunaiva:rl",
        fail_open: bool = False
    ):
        """
        Initialize the rate limiter.

        Args:
            redis_client: Configured async Redis client
            tier_config: Mapping of tiers to rate limit rules
            default_tier: Default tier for unauthenticated requests
            key_prefix: Redis key prefix for namespacing
            fail_open: If True, allow requests when Redis is unavailable
        """
        self.redis = redis_client
        self.tier_config = tier_config or DEFAULT_TIER_CONFIG
        self.default_tier = default_tier
        self.key_prefix = key_prefix
        self.fail_open = fail_open

        # Register Lua scripts for atomic operations
        self._sliding_window_lua = self.redis.register_script("""
            local key = KEYS[1]
            local window = tonumber(ARGV[1])
            local limit = tonumber(ARGV[2])
            local now = tonumber(ARGV[3])
            local member = ARGV[4]

            -- Remove entries outside the sliding window
            redis.call('ZREMRANGEBYSCORE', key, '-inf', now - window)

            -- Count current entries within window
            local current = redis.call('ZCARD', key)

            if current < limit then
                -- Add current request timestamp
                redis.call('ZADD', key, now, member)
                -- Set expiry to window size for auto-cleanup
                redis.call('EXPIRE', key, window)
                return {1, current + 1, limit, 0}
            else
                -- Calculate retry-after based on oldest entry
                local oldest = redis.call('ZRANGE', key, 0, 0, 'WITHSCORES')
                local retry_after = 0
                if #oldest >= 2 then
                    retry_after = math.ceil((tonumber(oldest[2]) + window) - now)
                end
                return {0, current, limit, retry_after}
            end
        """)

        self._concurrent_acquire_lua = self.redis.register_script("""
            local key = KEYS[1]
            local limit = tonumber(ARGV[1])
            local ttl = tonumber(ARGV[2])

            local current = redis.call('GET', key)
            if not current then
                current = 0
            else
                current = tonumber(current)
            end

            if current < limit then
                local new_val = redis.call('INCR', key)
                redis.call('EXPIRE', key, ttl)
                return {1, new_val, limit}
            else
                return {0, current, limit}
            end
        """)

        self._concurrent_release_lua = self.redis.register_script("""
            local key = KEYS[1]
            local current = redis.call('GET', key)
            if current and tonumber(current) > 0 then
                local new_val = redis.call('DECR', key)
                return new_val
            end
            return 0
        """)

    def _make_key(self, *parts: str) -> str:
        """Construct Redis key with prefix."""
        return f"{self.key_prefix}:{':'.join(parts)}"

    async def check_sliding_window(
        self,
        key: str,
        limit: int,
        window_seconds: int,
        request_id: Optional[str] = None
    ) -> tuple[bool, int, int]:
        """
        Check rate limit using sliding window algorithm.

        Args:
            key: Unique identifier for this limit (e.g., "ip:192.168.1.1")
            limit: Maximum number of requests allowed
            window_seconds: Time window in seconds
            request_id: Unique identifier for this specific request (for deduplication)

        Returns:
            Tuple of (allowed: bool, current_count: int, retry_after: int)
        """
        now = time.time()
        member = request_id or f"{now}:{uuid.uuid4().hex[:8]}"
        full_key = self._make_key(key)

        try:
            result = await self._sliding_window_lua(
                keys=[full_key],
                args=[window_seconds, limit, now, member]
            )

            allowed = bool(result[0])
            current = result[1]
            retry_after = result[3] if len(result) > 3 else 0

            if not allowed:
                logger.warning(
                    "Rate limit exceeded",
                    extra={
                        "key": full_key,
                        "current": current,
                        "limit": limit,
                        "window": window_seconds,
                        "retry_after": retry_after
                    }
                )

            return allowed, current, retry_after

        except redis.RedisError as e:
            logger.error(f"Redis error in sliding window check: {e}", exc_info=True)
            if self.fail_open:
                return True, 0, 0
            raise

    async def check_ip_limit(
        self,
        ip: str,
        requests_per_minute: int = 60
    ) -> None:
        """
        Check per-IP rate limit.

        Raises:
            RateLimitExceeded: If limit is exceeded
        """
        allowed, current, retry_after = await self.check_sliding_window(
            key=f"ip:{ip}",
            limit=requests_per_minute,
            window_seconds=60
        )

        if not allowed:
            raise RateLimitExceeded(
                limit=requests_per_minute,
                window=60,
                retry_after=retry_after,
                key=f"ip:{ip}"
            )

    async def check_user_limit(
        self,
        user_id: str,
        tier: Optional[RateLimitTier] = None
    ) -> None:
        """
        Check per-user API rate limit based on subscription tier.

        Args:
            user_id: Unique user identifier
            tier: User's subscription tier (defaults to default_tier)

        Raises:
            RateLimitExceeded: If limit is exceeded
        """
        tier = tier or self.default_tier
        rule = self.tier_config.get(tier, self.tier_config[self.default_tier])

        allowed, current, retry_after = await self.check_sliding_window(
            key=f"user:{user_id}:{tier.value}",
            limit=rule.requests_per_minute,
            window_seconds=60
        )

        if not allowed:
            raise RateLimitExceeded(
                limit=rule.requests_per_minute,
                window=60,
                retry_after=retry_after,
                key=f"user:{user_id}"
            )

    @asynccontextmanager
    async def acquire_widget_slot(
        self,
        widget_id: str,
        max_concurrent: Optional[int] = None,
        ttl_seconds: int = 3600
    ) -> AsyncGenerator[Dict[str, Any], None]:
        """
        Context manager for per-widget concurrent call limiting.

        Uses atomic INCR/DECR with expiry to handle crashed connections.

        Args:
            widget_id: Unique widget identifier
            max_concurrent: Max concurrent calls (defaults to tier config or 10)
            ttl_seconds: Time-to-live for the counter (cleanup safety)

        Yields:
            Dict with current usage info: {"current": int, "limit": int}

        Raises:
            ConcurrentLimitExceeded: If no slots available
        """
        if max_concurrent is None:
            max_concurrent = self.tier_config[self.default_tier].concurrent_calls

        key = self._make_key("concurrent", "widget", widget_id)

        try:
            # Try to acquire slot
            result = await self._concurrent_acquire_lua(
                keys=[key],
                args=[max_concurrent, ttl_seconds]
            )

            allowed = bool(result[0])
            current = result[1]
            limit = result[2]

            if not allowed:
                raise ConcurrentLimitExceeded(
                    current=current,
                    limit=limit,
                    key=f"widget:{widget_id}"
                )

            try:
                yield {"current": current, "limit": limit, "widget_id": widget_id}
            finally:
                # Release slot
                try:
                    await self._concurrent_release_lua(keys=[key], args=[])
                except redis.RedisError as e:
                    logger.error(f"Failed to release widget slot: {e}", exc_info=True)

        except redis.RedisError as e:
            logger.error(f"Redis error in widget slot acquisition: {e}", exc_info=True)
            if self.fail_open:
                yield {"current": -1, "limit": max_concurrent, "widget_id": widget_id}
            else:
                raise

    async def get_limit_status(
        self,
        key: str,
        window_seconds: int = 60
    ) -> Dict[str, Any]:
        """
        Get current rate limit status for monitoring/debugging.

        Args:
            key: The rate limit key (e.g., "ip:192.168.1.1")
            window_seconds: Window size to calculate remaining time

        Returns:
            Dict with current count, TTL, and limit info
        """
        full_key = self._make_key(key)
        try:
            pipe = self.redis.pipeline()
            pipe.zcard(full_key)
            pipe.ttl(full_key)
            results = await pipe.execute()

            current = results[0]
            ttl = results[1]

            return {
                "key": key,
                "current_requests": current,
                "ttl_seconds": ttl,
                "window_seconds": window_seconds,
                "reset_in_seconds": max(0, ttl) if ttl > 0 else window_seconds
            }
        except redis.RedisError as e:
            logger.error(f"Error getting limit status: {e}")
            return {"key": key, "error": str(e), "current_requests": -1}

class RateLimitMiddleware(BaseHTTPMiddleware):
    """
    FastAPI middleware for global rate limiting.

    Applies per-IP limits to all requests and per-user limits when
    authentication info is available.
    """

    def __init__(
        self,
        app: ASGIApp,
        rate_limiter: RedisRateLimiter,
        exempt_paths: Optional[List[str]] = None,
        ip_header: str = "X-Forwarded-For",
        user_id_func: Optional[Callable[[Request], Optional[str]]] = None,
        tier_func: Optional[Callable[[Request], Optional[RateLimitTier]]] = None,
        ip_limit_per_minute: int = 60,
        skip_on_error: bool =


Call Recording Storage Manager

Response:
```python
"""
Recording Manager Module for SaaS Voice Widget Platform.

Handles S3-compatible storage operations, signed URL generation,
retention policies, and storage analytics for call recordings.
"""

import asyncio
import hashlib
import logging
from contextlib import asynccontextmanager
from datetime import datetime, timedelta
from enum import Enum
from typing import Optional, List, BinaryIO, AsyncGenerator, Dict, Any
from uuid import UUID, uuid4

import aioboto3
from botocore.config import Config
from botocore.exceptions import ClientError, BotoCoreError
from pydantic import BaseModel, Field, validator
from sqlalchemy import (
    Column, String, DateTime, Integer, BigInteger, ForeignKey,
    Enum as SQLEnum, Index, select, func, and_, delete, update
)
from sqlalchemy.dialects.postgresql import UUID as PGUUID, JSONB
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import declarative_base, relationship, selectinload
from sqlalchemy.exc import SQLAlchemyError

# Configure module logger
logger = logging.getLogger(__name__)

# -----------------------------------------------------------------------------
# Configuration
# -----------------------------------------------------------------------------

class StorageSettings(BaseSettings):
    """S3/MinIO storage configuration."""

    S3_ENDPOINT: str = "https://s3.elest.io"
    S3_ACCESS_KEY: str
    S3_SECRET_KEY: str
    S3_BUCKET_NAME: str = "voice-recordings"
    S3_REGION: str = "us-east-1"
    S3_USE_SSL: bool = True

    # Recording settings
    RECORDING_RETENTION_DAYS: int = 90
    SIGNED_URL_EXPIRY_HOURS: int = 24
    MAX_UPLOAD_SIZE_MB: int = 500

    class Config:
        env_prefix = "APP_"
        case_sensitive = True

settings = StorageSettings()

# -----------------------------------------------------------------------------
# Database Models
# -----------------------------------------------------------------------------

Base = declarative_base()

class TranscriptStatus(str, Enum):
    """Transcription processing status."""
    PENDING = "pending"
    PROCESSING = "processing"
    COMPLETED = "completed"
    FAILED = "failed"
    NOT_REQUIRED = "not_required"

class Recording(Base):
    """Call recording metadata and storage reference."""

    __tablename__ = "recordings"

    id = Column(PGUUID(as_uuid=True), primary_key=True, default=uuid4)
    widget_id = Column(PGUUID(as_uuid=True), ForeignKey("widgets.id", ondelete="CASCADE"), nullable=False, index=True)

    # Storage reference
    s3_key = Column(String(512), nullable=False, unique=True, index=True)
    s3_bucket = Column(String(128), nullable=False, default=settings.S3_BUCKET_NAME)
    filename = Column(String(256), nullable=False)
    content_type = Column(String(128), default="audio/wav")
    checksum_sha256 = Column(String(64), nullable=True)  # Integrity verification

    # File metadata
    size_bytes = Column(BigInteger, nullable=False, default=0)
    duration_seconds = Column(Integer, nullable=True)

    # Caller information
    caller_number = Column(String(32), nullable=True, index=True)
    caller_name = Column(String(128), nullable=True)
    caller_metadata = Column(JSONB, default=dict)  # Extra caller context

    # Transcription tracking
    transcript_status = Column(SQLEnum(TranscriptStatus), default=TranscriptStatus.PENDING, index=True)
    transcript_text = Column(String, nullable=True)
    transcript_confidence = Column(String(10), nullable=True)

    # Lifecycle
    created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.utcnow, index=True)
    expires_at = Column(DateTime(timezone=True), nullable=False, index=True)
    deleted_at = Column(DateTime(timezone=True), nullable=True)
    deletion_reason = Column(String(64), nullable=True)

    # Relationships
    widget = relationship("Widget", back_populates="recordings")

    __table_args__ = (
        Index('ix_recordings_widget_created', 'widget_id', 'created_at'),
        Index('ix_recordings_expires_cleanup', 'expires_at', 'deleted_at'),
    )

class WidgetStorageStats(Base):
    """Aggregated storage statistics per widget for fast querying."""

    __tablename__ = "widget_storage_stats"

    widget_id = Column(PGUUID(as_uuid=True), ForeignKey("widgets.id", ondelete="CASCADE"), primary_key=True)
    total_recordings = Column(Integer, default=0)
    total_bytes = Column(BigInteger, default=0)
    last_updated = Column(DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow)

    # Optional: daily breakdown stored as JSONB for analytics
    daily_stats = Column(JSONB, default=dict)

# Assuming Widget model exists elsewhere, referenced here for relationships
class Widget(Base):
    """Placeholder for Widget model reference."""
    __tablename__ = "widgets"

    id = Column(PGUUID(as_uuid=True), primary_key=True, default=uuid4)
    recordings = relationship("Recording", back_populates="widget", lazy="dynamic")

# -----------------------------------------------------------------------------
# Pydantic Schemas
# -----------------------------------------------------------------------------

class RecordingMetadata(BaseModel):
    """Input metadata for recording creation."""
    widget_id: UUID
    filename: str
    content_type: str = "audio/wav"
    duration_seconds: Optional[int] = None
    caller_number: Optional[str] = None
    caller_name: Optional[str] = None
    caller_metadata: Optional[Dict[str, Any]] = None

    @validator('filename')
    def validate_filename(cls, v):
        if not v or len(v) > 256:
            raise ValueError("Filename must be between 1 and 256 characters")
        # Sanitize filename
        return v.replace("..", "").replace("/", "_")

class RecordingResponse(BaseModel):
    """Recording data response model."""
    id: UUID
    widget_id: UUID
    filename: str
    size_bytes: int
    duration_seconds: Optional[int]
    content_type: str
    transcript_status: TranscriptStatus
    created_at: datetime
    expires_at: datetime
    playback_url: Optional[str] = None  # Only populated when explicitly requested

    class Config:
        from_attributes = True

class StorageStats(BaseModel):
    """Storage usage statistics per widget."""
    widget_id: UUID
    total_recordings: int
    total_bytes: int
    total_gb: float = Field(..., description="Total storage in gigabytes")
    average_file_size_mb: float
    last_updated: datetime

class SignedUrlResponse(BaseModel):
    """Presigned URL response."""
    recording_id: UUID
    url: str
    expires_at: datetime
    method: str = "GET"

# -----------------------------------------------------------------------------
# Exceptions
# -----------------------------------------------------------------------------

class RecordingManagerError(Exception):
    """Base exception for recording manager."""
    pass

class StorageError(RecordingManagerError):
    """S3/MinIO storage operation failed."""
    pass

class RecordingNotFoundError(RecordingManagerError):
    """Recording not found in database."""
    pass

class ValidationError(RecordingManagerError):
    """Input validation failed."""
    pass

class QuotaExceededError(RecordingManagerError):
    """Storage quota exceeded for widget."""
    pass

# -----------------------------------------------------------------------------
# Recording Manager
# -----------------------------------------------------------------------------

class RecordingManager:
    """
    Manages call recording lifecycle including upload, retrieval,
    retention policies, and storage analytics.
    """

    def __init__(self, db_session: AsyncSession):
        self.db = db_session
        self.s3_config = Config(
            signature_version="s3v4",
            retries={"max_attempts": 3, "mode": "adaptive"}
        )
        self._s3_session = aioboto3.Session()

    @asynccontextmanager
    async def _get_s3_client(self):
        """
        Async context manager for S3 client.

        Yields:
            S3 client instance configured for MinIO/Elestio.
        """
        client = self._s3_session.client(
            "s3",
            endpoint_url=settings.S3_ENDPOINT,
            aws_access_key_id=settings.S3_ACCESS_KEY,
            aws_secret_access_key=settings.S3_SECRET_KEY,
            region_name=settings.S3_REGION,
            config=self.s3_config,
            use_ssl=settings.S3_USE_SSL
        )
        try:
            yield client
        finally:
            await client.close()

    def _generate_s3_key(self, widget_id: UUID, filename: str) -> str:
        """
        Generate hierarchical S3 key for organization.

        Format: widgets/{widget_id}/{year}/{month}/{uuid}_{filename}
        """
        now = datetime.utcnow()
        unique_id = str(uuid4())[:8]
        safe_filename = filename.replace(" ", "_")
        return (
            f"widgets/{widget_id}/{now.year}/{now.month:02d}/"
            f"{unique_id}_{safe_filename}"
        )

    async def upload_recording(
        self,
        file_data: BinaryIO,
        metadata: RecordingMetadata,
        calculate_checksum: bool = True
    ) -> RecordingResponse:
        """
        Upload a recording to S3 and create database record.

        Args:
            file_data: Binary stream of the audio file
            metadata: Recording metadata
            calculate_checksum: Whether to calculate SHA256 checksum

        Returns:
            RecordingResponse with created record details

        Raises:
            StorageError: If S3 upload fails
            ValidationError: If file size or metadata invalid
        """
        try:
            # Calculate file size and checksum
            file_data.seek(0, 2)  # Seek to end
            file_size = file_data.tell()
            file_data.seek(0)  # Reset to beginning

            if file_size > settings.MAX_UPLOAD_SIZE_MB * 1024 * 1024:
                raise ValidationError(
                    f"File size {file_size} exceeds maximum "
                    f"{settings.MAX_UPLOAD_SIZE_MB}MB"
                )

            # Calculate checksum if requested
            checksum = None
            if calculate_checksum:
                sha256_hash = hashlib.sha256()
                while chunk := file_data.read(8192):
                    sha256_hash.update(chunk)
                checksum = sha256_hash.hexdigest()
                file_data.seek(0)

            # Generate S3 key and expiry
            s3_key = self._generate_s3_key(metadata.widget_id, metadata.filename)
            expires_at = datetime.utcnow() + timedelta(days=settings.RECORDING_RETENTION_DAYS)

            # Upload to S3
            async with self._get_s3_client() as s3:
                try:
                    await s3.upload_fileobj(
                        file_data,
                        settings.S3_BUCKET_NAME,
                        s3_key,
                        ExtraArgs={
                            "ContentType": metadata.content_type,
                            "Metadata": {
                                "widget-id": str(metadata.widget_id),
                                "checksum-sha256": checksum or "",
                                "original-filename": metadata.filename
                            }
                        }
                    )
                    logger.info(
                        f"Uploaded recording to S3: {s3_key}, "
                        f"size: {file_size} bytes"
                    )
                except (ClientError, BotoCoreError) as e:
                    logger.error(f"S3 upload failed: {str(e)}")
                    raise StorageError(f"Failed to upload file: {str(e)}")

            # Create database record
            recording = Recording(
                widget_id=metadata.widget_id,
                s3_key=s3_key,
                s3_bucket=settings.S3_BUCKET_NAME,
                filename=metadata.filename,
                content_type=metadata.content_type,
                size_bytes=file_size,
                duration_seconds=metadata.duration_seconds,
                caller_number=metadata.caller_number,
                caller_name=metadata.caller_name,
                caller_metadata=metadata.caller_metadata or {},
                checksum_sha256=checksum,
                expires_at=expires_at
            )

            self.db.add(recording)

            # Update storage stats
            await self._increment_storage_stats(
                metadata.widget_id,
                file_size
            )

            await self.db.commit()
            await self.db.refresh(recording)

            logger.info(
                f"Created recording record {recording.id} for widget "
                f"{metadata.widget_id}"
            )

            return RecordingResponse.from_orm(recording)

        except SQLAlchemyError as e:
            await self.db.rollback()
            logger.error(f"Database error during upload: {str(e)}")
            raise StorageError(f"Database operation failed: {str(e)}")
        except Exception:
            await self.db.rollback()
            raise

    async def get_playback_url(
        self,
        recording_id: UUID,
        widget_id: Optional[UUID] = None,
        expiry_hours: Optional[int] = None
    ) -> SignedUrlResponse:
        """
        Generate a presigned URL for recording playback.

        Args:
            recording_id: UUID of the recording
            widget_id: Optional widget ID for additional validation
            expiry_hours: Custom expiry time (default 24h)

        Returns:
            SignedUrlResponse with presigned URL

        Raises:
            RecordingNotFoundError: If recording not found or expired
            StorageError: If URL generation fails
        """
        expiry = expiry_hours or settings.SIGNED_URL_EXPIRY_HOURS

        try:
            # Fetch recording
            query = select(Recording).where(
                and_(
                    Recording.id == recording_id,
                    Recording.deleted_at.is_(None),
                    Recording.expires_at > datetime.utcnow()
                )
            )

            if widget_id:
                query = query.where(Recording.widget_id == widget_id)

            result = await self.db.execute(query)
            recording = result.scalar_one_or_none()

            if not recording:
                raise RecordingNotFoundError(
                    f"Recording {recording_id} not found or expired"
                )

            # Generate presigned URL
            async with self._get_s3_client() as s3:
                try:
                    url = await s3.generate_presigned_url(
                        "get_object",
                        Params={
                            "Bucket": recording.s3_bucket,
                            "Key": recording.s3_key,
                            "ResponseContentDisposition":
                                f'inline; filename="{recording.filename}"',
                            "ResponseContentType": recording.content_type
                        },
                        ExpiresIn=expiry * 3600
                    )

                    expires_at = datetime.utcnow() + timedelta(hours=expiry)

                    logger.info(
                        f"Generated playback URL for recording {recording_id}, "
                        f"expires at {expires_at}"
                    )

                    return SignedUrlResponse(
                        recording_id=recording_id,
                        url=url,
                        expires_at=expires_at,
                        method="GET"
                    )

                except (ClientError, BotoCoreError) as e:
                    logger.error(f"Failed to generate presigned URL: {str(e)}")
                    raise StorageError(f"URL generation failed: {str(e)}")

        except SQLAlchemyError as e:
            logger.error(f"Database error fetching recording: {str(e)}")
            raise StorageError(f"Database operation failed: {str(e)}")

    async def delete_recording(
        self,
        recording_id: UUID,
        reason: str = "user_requested",
        soft_delete: bool = False
    ) -> bool:
        """
        Delete a recording from S3 and database.

        Args:
            recording_id: UUID of recording to delete
            reason: Reason for deletion (for audit logs)
            soft_delete: If True, mark as deleted without removing from S3

        Returns:
            True if successful

        Raises:
            RecordingNotFoundError: If recording not found
            StorageError: If deletion fails
        """
        try:
            result = await self.db.execute(
                select(Recording).where(
                    and_(
                        Recording.id == recording_id,
                        Recording.deleted_at.is_(None)
                    )
                )
            )
            recording = result.scalar_one_or_none()

            if not recording:
                raise RecordingNotFoundError(
                    f"Recording {recording_id} not found"
                )

            if not soft_delete:
                # Delete from S3
                async with self._get_s3_client() as s3:
                    try:
                        await s3.delete_object(
                            Bucket=recording.s3_bucket,
                            Key=recording.s3_key
                        )
                        logger.info(
                            f"Deleted recording {recording_id} from S3: "
                            f"{recording.s3_key}"
                        )
                    except (ClientError, BotoCoreError) as e:
                        logger.error(f"S3 deletion failed: {str(e)}")
                        # Continue to mark as deleted in DB even if S3 fails
                        # (idempotency - S3 delete might have succeeded previously)

            # Update database record
            recording.deleted_at = datetime.utcnow()
            recording.deletion_reason = reason

            # Update storage stats
            await self._decrement_storage_stats(
                recording.widget_id,
                recording.size_bytes
            )

            await self.db.commit()

            logger.info(
                f"Marked recording {recording_id} as deleted. "
                f"Reason: {reason}"
            )

            return True

        except SQLAlchemyError as e:
            await self.db.rollback()
            logger.error(f"Database error during deletion: {str(e)}")
            raise StorageError(f"Deletion failed: {str(e)}")

    async def enforce_retention_policy(self, batch_size: int = 100) -> Dict[str, int]:
        """
        Background task to enforce 90-day retention policy.
        Deletes expired recordings in batches.

        Args:
            batch_size: Number of records to process per batch

        Returns:
            Dict with deletion statistics
        """
        stats = {"processed": 0, "deleted": 0, "errors": 0}
        cutoff_date = datetime.utcnow()

        try:
            while True:
                # Fetch batch of expired recordings
                result = await self.db.execute(
                    select(Recording)
                    .where(
                        and_(
                            Recording.expires_at < cutoff_date,
                            Recording.deleted_at.is_(None)
                        )
                    )
                    .limit(batch_size)
                )
                recordings = result.scalars().all()

                if not recordings:
                    break

                for recording in recordings:
                    try:
                        await self.delete_recording(
                            recording.id,
                            reason="retention_policy",
                            soft_delete=False
                        )
                        stats["deleted"] += 1
                    except Exception as e:
                        logger.error(
                            f"Failed to delete expired recording "
                            f"{recording.id}: {str(e)}"
                        )
                        stats["errors"] += 1

                    stats["processed"] += 1

                # Commit batch
                await self.db.commit()

                logger.info(
                    f"Retention policy batch processed: "
                    f"{stats['processed']} total, {stats['deleted']} deleted"
                )

                if len(recordings) < batch_size:
                    break

            return stats

        except Exception as e:
            await self.db.rollback()
            logger.error(f"Retention policy enforcement failed: {str(e)}")
            raise

    async def get_storage_stats(self, widget_id: UUID) -> StorageStats:
        """
        Get storage usage statistics for a widget.

        Args:
            widget_id: Widget UUID

        Returns:
            StorageStats with usage metrics
        """
        try:
            # Try to get cached stats first
            result = await self.db.execute(
                select(WidgetStorageStats).where(
                    WidgetStorageStats.widget_id == widget_id
                )
            )
            cached = result.scalar_one_or_none()

            if cached and (datetime.utcnow() - cached.last_updated).seconds < 300:
                # Use cached stats if less than 5 minutes old
                return StorageStats(
                    widget_id=widget_id,
                    total_recordings=cached.total_recordings,
                    total_bytes=cached.total_bytes,
                    total_gb=round(cached.total_bytes / (1024**3), 4),
                    average_file_size_mb=(
                        (cached.total_bytes / cached.total_recordings / (1024**2))
                        if cached.total_recordings > 0 else 0.0
                    ),
                    last_updated=cached.last_updated
                )

            # Calculate fresh stats
            result = await self.db.execute(
                select(
                    func.count(Recording.id).label("count"),
                    func.coalesce(func.sum(Recording.size_bytes), 0).label("total_bytes")
                )
                .where(
                    and_(
                        Recording.widget_id == widget_id,
                        Recording.deleted_at.is_(None)
                    )
                )
            )
            row = result.one()

            stats = StorageStats(
                widget_id=widget_id,
                total_recordings=row.count,
                total_bytes=row.total_bytes,
                total_gb=round(row.total_bytes / (1024**3), 4),
                average_file_size_mb=(
                    (row.total_bytes / row.count / (1024**2))
                    if row.count > 0 else 0.0
                ),
                last_updated=datetime.utcnow()
            )

            # Update cache
            await self.db.execute(
                update(WidgetStorageStats)
                .where(WidgetStorageStats.widget_id == widget_id)
                .values(
                    total_recordings=stats.total_recordings,
                    total_bytes=stats.total_bytes,
                    last_updated=stats.last_updated
                )
            )
            await self.db.commit()

            return stats

        except SQLAlchemyError as e:
            logger.error(f"Failed to get storage stats: {str(e)}")
            raise StorageError(f"Stats calculation failed: {str(e)}")

    async def update_transcript_status(
        self,
        recording_id: UUID,
        status: TranscriptStatus,
        transcript_text: Optional[str] = None,
        confidence: Optional[str] = None
    ) -> RecordingResponse:
        """
        Update transcription status and optionally store transcript.

        Args:
            recording_id: Recording UUID
            status: New transcript status
            transcript_text: Optional transcript content
            confidence: Optional confidence score

        Returns:
            Updated RecordingResponse
        """
        try:
            result = await self.db.execute(
                select(Recording).where(Recording.id == recording_id)
            )
            recording = result.scalar_one_or_none()

            if not recording:
                raise RecordingNotFoundError(
                    f"Recording {recording_id} not found"
                )

            recording.transcript_status = status
            if transcript_text is not None:
                recording.transcript_text = transcript_text
            if confidence is not None:
                recording.transcript_confidence = confidence

            await self.db.commit()
            await self.db.refresh(recording)

            logger.info(
                f"Updated transcript status for {recording_id} to {status.value}"
            )

            return RecordingResponse.from_orm(recording)

        except SQLAlchemyError as e:
            await self.db.rollback()
            raise StorageError(f"Status update failed: {str(e)}")

    async def list_recordings(
        self,
        widget_id: UUID,
        limit: int = 50,
        offset: int = 0,
        transcript_status: Optional[TranscriptStatus] = None,
        start_date: Optional[datetime] = None,
        end_date: Optional[datetime] = None
    ) -> List[RecordingResponse]:
        """
        List recordings for a widget with optional filtering.

        Args:
            widget_id: Widget UUID
            limit: Pagination limit
            offset: Pagination offset
            transcript_status: Filter by transcript status
            start_date: Filter by creation date (inclusive)
            end_date: Filter by creation date (inclusive)

        Returns:
            List of RecordingResponse
        """
        try:
            query = select(Recording).where(
                and_(
                    Recording.widget_id == widget_id,
                    Recording.deleted_at.is_(None)
                )
            )

            if transcript_status:
                query = query.where(
                    Recording.transcript_status == transcript_status
                )

            if start_date:
                query = query.where(Recording.created_at >= start_date)

            if end_date:
                query = query.where(Recording.created_at <= end_date)

            query = query.order_by(Recording.created_at.desc())
            query = query.limit(limit).offset(offset)

            result = await self.db.execute(query)
            recordings = result.scalars().all()

            return [RecordingResponse.from_orm(r) for r in recordings]

        except SQLAlchemyError as e:
            logger.error(f"Failed to list recordings: {str(e)}")
            raise StorageError(f"List operation failed: {str(e)}")

    # -------------------------------------------------------------------------
    # Private Helper Methods
    # -------------------------------------------------------------------------

    async def _increment_storage_stats(
        self,
        widget_id: UUID,
        size_bytes: int
    ) -> None:
        """Increment storage stats for widget."""
        await self.db.execute(
            update(WidgetStorageStats)
            .where(WidgetStorageStats.widget_id == widget_id)
            .values(
                total_recordings=WidgetStorageStats.total_recordings + 1,
                total_bytes=WidgetStorageStats.total_bytes + size_bytes,
                last_updated=datetime.utcnow()
            )
        )

    async def _decrement_storage_stats(
        self,
        widget_id: UUID,
        size_bytes: int
    ) -> None:
        """Decrement storage stats for widget."""
        await self.db.execute(
            update(WidgetStorageStats)
            .where(WidgetStorageStats.widget_id == widget_id)
            .values(
                total_recordings=func.greatest(
                    WidgetStorageStats.total_recordings - 1,
                    0
                ),
                total_bytes=func.greatest(
                    WidgetStorageStats.total_bytes - size_bytes,
                    0
                ),
                last_updated=datetime.utcnow()
            )
        )

# -----------------------------------------------------------------------------
# Background Task Scheduler Integration
# -----------------------------------------------------------------------------

async def run_retention_cleanup(db_session_factory):
    """
    Background task to run retention policy enforcement.
    Intended to be run by APScheduler or Celery beat.

    Usage:
        from apscheduler.schedulers.asyncio import AsyncIOScheduler

        scheduler = AsyncIOScheduler()
        scheduler.add_job(
            run_retention_cleanup,
            'cron',
            hour=2,
            args=[session_factory]
        )
    """
    async with db_session_factory() as session:
        manager = RecordingManager(session)
        stats = await manager.enforce_retention_policy()
        logger.info(f"Retention cleanup completed: {stats}")

# -----------------------------------------------------------------------------
# FastAPI Dependency Injection
# -----------------------------------------------------------------------------

async def get_recording_manager(
    db: AsyncSession
) -> RecordingManager:
    """
    FastAPI dependency for RecordingManager.

    Usage:
        @router.post("/upload")
        async def upload(
            file: UploadFile,
            manager: RecordingManager = Depends(get_recording_manager)
        ):
            ...
    """
    return RecordingManager(db)
```


Widget Deployment Health Monitor


Agency Partner Commission Calculator

Response:
```python
"""
Commission Tracking Module for Agency Partners.

This module provides comprehensive commission calculation, tracking, and reporting
for agency partner referrals. Supports recurring commissions, prorated calculations
for subscription changes, and custom commission rates per partner.

File: Sunaiva/talking-widget/backend/commission_calculator.py
"""

import logging
from datetime import datetime, timedelta
from decimal import Decimal, ROUND_HALF_UP
from enum import Enum
from typing import List, Optional, Dict, Any, Tuple
from uuid import UUID, uuid4

import asyncpg
from pydantic import BaseModel, Field, validator, ConfigDict
from sqlalchemy import (
    Column, String, Numeric, DateTime, ForeignKey, Enum as SQLEnum,
    Index, select, and_, func, update
)
from sqlalchemy.dialects.postgresql import UUID as SQLUUID
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.orm import declarative_base, relationship, selectinload
from sqlalchemy.exc import IntegrityError, SQLAlchemyError

# Configure structured logging
logger = logging.getLogger(__name__)

# =============================================================================
# Exceptions
# =============================================================================

class CommissionError(Exception):
    """Base exception for commission-related errors."""
    pass

class PartnerNotFoundError(CommissionError):
    """Raised when a partner is not found."""
    pass

class ReferralNotFoundError(CommissionError):
    """Raised when a referral is not found."""
    pass

class CalculationError(CommissionError):
    """Raised when commission calculation fails."""
    pass

class DuplicateEventError(CommissionError):
    """Raised when attempting to process a duplicate subscription event."""
    pass

# =============================================================================
# Enums
# =============================================================================

class CommissionStatus(str, Enum):
    """Status of a commission record."""
    PENDING = "pending"
    APPROVED = "approved"
    PAID = "paid"
    CANCELLED = "cancelled"
    DISPUTED = "disputed"

class PartnerStatus(str, Enum):
    """Status of a partner account."""
    ACTIVE = "active"
    SUSPENDED = "suspended"
    INACTIVE = "inactive"

class SubscriptionEventType(str, Enum):
    """Types of subscription lifecycle events."""
    CREATED = "created"
    RENEWAL = "renewal"
    UPGRADE = "upgrade"
    DOWNGRADE = "downgrade"
    CANCELLATION = "cancellation"

# =============================================================================
# SQLAlchemy Models (Database Layer)
# =============================================================================

Base = declarative_base()

class Partner(Base):
    """Database model for agency partners."""
    __tablename__ = "partners"

    id = Column(SQLUUID(as_uuid=True), primary_key=True, default=uuid4)
    name = Column(String(255), nullable=False)
    email = Column(String(255), unique=True, nullable=False, index=True)
    default_commission_rate = Column(Numeric(5, 4), nullable=False, default=Decimal("0.30"))
    custom_commission_rate = Column(Numeric(5, 4), nullable=True)  # Override default
    status = Column(SQLEnum(PartnerStatus), default=PartnerStatus.ACTIVE, nullable=False)
    payout_method = Column(String(50), nullable=True)  # e.g., 'bank_transfer', 'paypal'
    payout_details = Column(String(500), nullable=True)  # JSON or encrypted details
    created_at = Column(DateTime(timezone=True), default=datetime.utcnow, nullable=False)
    updated_at = Column(DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow)

    # Relationships
    referrals = relationship("Referral", back_populates="partner", lazy="selectin")

    def get_effective_rate(self) -> Decimal:
        """Return the effective commission rate for this partner."""
        return self.custom_commission_rate or self.default_commission_rate

class Referral(Base):
    """Database model for customer referrals."""
    __tablename__ = "referrals"

    id = Column(SQLUUID(as_uuid=True), primary_key=True, default=uuid4)
    partner_id = Column(SQLUUID(as_uuid=True), ForeignKey("partners.id"), nullable=False, index=True)
    customer_email = Column(String(255), nullable=False, index=True)
    widget_subscription_id = Column(String(100), unique=True, nullable=False, index=True)
    subscription_amount = Column(Numeric(10, 2), nullable=False)  # Current monthly amount
    currency = Column(String(3), default="USD", nullable=False)
    status = Column(String(20), default="active", nullable=False)  # active, cancelled, churned
    created_at = Column(DateTime(timezone=True), default=datetime.utcnow, nullable=False)
    cancelled_at = Column(DateTime(timezone=True), nullable=True)

    # Relationships
    partner = relationship("Partner", back_populates="referrals")
    commissions = relationship("Commission", back_populates="referral", lazy="selectin")
    events = relationship("SubscriptionEvent", back_populates="referral", lazy="selectin")

    __table_args__ = (
        Index('idx_referral_partner_status', 'partner_id', 'status'),
    )

class Commission(Base):
    """Database model for individual commission records."""
    __tablename__ = "commissions"

    id = Column(SQLUUID(as_uuid=True), primary_key=True, default=uuid4)
    referral_id = Column(SQLUUID(as_uuid=True), ForeignKey("referrals.id"), nullable=False, index=True)
    amount = Column(Numeric(10, 2), nullable=False)
    rate_used = Column(Numeric(5, 4), nullable=False)  # Snapshot of rate at calculation time
    subscription_amount = Column(Numeric(10, 2), nullable=False)  # Base amount commission was calculated on
    period_start = Column(DateTime(timezone=True), nullable=False)
    period_end = Column(DateTime(timezone=True), nullable=False)
    status = Column(SQLEnum(CommissionStatus), default=CommissionStatus.PENDING, nullable=False)
    event_type = Column(SQLEnum(SubscriptionEventType), nullable=False)
    calculated_at = Column(DateTime(timezone=True), default=datetime.utcnow, nullable=False)
    paid_at = Column(DateTime(timezone=True), nullable=True)
    notes = Column(String(500), nullable=True)

    # Relationships
    referral = relationship("Referral", back_populates="commissions")

    __table_args__ = (
        Index('idx_commission_status_period', 'status', 'period_start', 'period_end'),
        Index('idx_commission_referral_status', 'referral_id', 'status'),
    )

class SubscriptionEvent(Base):
    """Audit log for subscription lifecycle events."""
    __tablename__ = "subscription_events"

    id = Column(SQLUUID(as_uuid=True), primary_key=True, default=uuid4)
    referral_id = Column(SQLUUID(as_uuid=True), ForeignKey("referrals.id"), nullable=False)
    event_type = Column(SQLEnum(SubscriptionEventType), nullable=False)
    old_amount = Column(Numeric(10, 2), nullable=True)
    new_amount = Column(Numeric(10, 2), nullable=False)
    effective_date = Column(DateTime(timezone=True), nullable=False)
    processed = Column(String(100), unique=True, nullable=False, index=True)  # Idempotency key
    created_at = Column(DateTime(timezone=True), default=datetime.utcnow, nullable=False)

    # Relationships
    referral = relationship("Referral", back_populates="events")

# =============================================================================
# Pydantic Schemas (API Layer)
# =============================================================================

class CommissionCalculationRequest(BaseModel):
    """Request model for commission calculation."""
    referral_id: UUID
    subscription_amount: Decimal = Field(..., gt=0, decimal_places=2)
    event_type: SubscriptionEventType
    period_start: datetime
    period_end: datetime
    proration_factor: Optional[Decimal] = Field(default=Decimal("1.0"), ge=0, le=1)

    @validator('period_end')
    def validate_period(cls, v, values):
        if 'period_start' in values and v <= values['period_start']:
            raise ValueError('period_end must be after period_start')
        return v

class CommissionResponse(BaseModel):
    """Response model for commission data."""
    model_config = ConfigDict(from_attributes=True)

    id: UUID
    referral_id: UUID
    partner_id: UUID
    partner_name: str
    amount: Decimal
    rate_used: Decimal
    subscription_amount: Decimal
    period_start: datetime
    period_end: datetime
    status: CommissionStatus
    event_type: SubscriptionEventType
    calculated_at: datetime

class PayoutReportItem(BaseModel):
    """Individual item in a payout report."""
    partner_id: UUID
    partner_name: str
    partner_email: str
    total_commissions: Decimal
    commission_count: int
    period_start: datetime
    period_end: datetime
    payout_method: Optional[str]
    details: List[CommissionResponse]

class SubscriptionChangeEvent(BaseModel):
    """Event model for subscription upgrades/downgrades."""
    referral_id: UUID
    event_type: SubscriptionEventType
    old_amount: Optional[Decimal]
    new_amount: Decimal
    effective_date: datetime
    idempotency_key: str

# =============================================================================
# Repository Layer
# =============================================================================

class PartnerRepository:
    """Repository for partner data access."""

    def __init__(self, session: AsyncSession):
        self.session = session

    async def get_by_id(self, partner_id: UUID) -> Optional[Partner]:
        """Fetch partner by ID with referrals."""
        result = await self.session.execute(
            select(Partner)
            .options(selectinload(Partner.referrals))
            .where(Partner.id == partner_id)
        )
        return result.scalar_one_or_none()

    async def get_by_email(self, email: str) -> Optional[Partner]:
        """Fetch partner by email."""
        result = await self.session.execute(
            select(Partner).where(Partner.email == email)
        )
        return result.scalar_one_or_none()

    async def update_custom_rate(
        self,
        partner_id: UUID,
        new_rate: Decimal
    ) -> Partner:
        """Update custom commission rate for a partner."""
        partner = await self.get_by_id(partner_id)
        if not partner:
            raise PartnerNotFoundError(f"Partner {partner_id} not found")

        partner.custom_commission_rate = new_rate
        partner.updated_at = datetime.utcnow()
        await self.session.commit()
        return partner

class ReferralRepository:
    """Repository for referral data access."""

    def __init__(self, session: AsyncSession):
        self.session = session

    async def get_by_id(self, referral_id: UUID) -> Optional[Referral]:
        """Fetch referral with commissions and partner."""
        result = await self.session.execute(
            select(Referral)
            .options(selectinload(Referral.commissions), selectinload(Referral.partner))
            .where(Referral.id == referral_id)
        )
        return result.scalar_one_or_none()

    async def get_by_subscription_id(self, subscription_id: str) -> Optional[Referral]:
        """Fetch referral by widget subscription ID."""
        result = await self.session.execute(
            select(Referral)
            .options(selectinload(Referral.partner))
            .where(Referral.widget_subscription_id == subscription_id)
        )
        return result.scalar_one_or_none()

    async def update_subscription_amount(
        self,
        referral_id: UUID,
        new_amount: Decimal
    ) -> Referral:
        """Update the current subscription amount for a referral."""
        referral = await self.get_by_id(referral_id)
        if not referral:
            raise ReferralNotFoundError(f"Referral {referral_id} not found")

        referral.subscription_amount = new_amount
        await self.session.commit()
        return referral

class CommissionRepository:
    """Repository for commission data access."""

    def __init__(self, session: AsyncSession):
        self.session = session

    async def create(self, commission: Commission) -> Commission:
        """Create a new commission record."""
        self.session.add(commission)
        try:
            await self.session.commit()
            await self.session.refresh(commission)
            return commission
        except IntegrityError as e:
            await self.session.rollback()
            logger.error(f"Integrity error creating commission: {e}")
            raise

    async def get_pending_for_partner(
        self,
        partner_id: UUID,
        start_date: Optional[datetime] = None,
        end_date: Optional[datetime] = None
    ) -> List[Commission]:
        """Get pending commissions for a partner."""
        query = (
            select(Commission)
            .join(Referral)
            .where(
                and_(
                    Referral.partner_id == partner_id,
                    Commission.status == CommissionStatus.PENDING
                )
            )
        )

        if start_date:
            query = query.where(Commission.period_start >= start_date)
        if end_date:
            query = query.where(Commission.period_end <= end_date)

        result = await self.session.execute(query)
        return result.scalars().all()

    async def approve_for_payout(
        self,
        commission_ids: List[UUID]
    ) -> int:
        """Approve commissions for payout."""
        result = await self.session.execute(
            update(Commission)
            .where(Commission.id.in_(commission_ids))
            .where(Commission.status == CommissionStatus.PENDING)
            .values(status=CommissionStatus.APPROVED)
        )
        await self.session.commit()
        return result.rowcount

    async def mark_as_paid(
        self,
        commission_ids: List[UUID]
    ) -> int:
        """Mark commissions as paid."""
        result = await self.session.execute(
            update(Commission)
            .where(Commission.id.in_(commission_ids))
            .where(Commission.status == CommissionStatus.APPROVED)
            .values(
                status=CommissionStatus.PAID,
                paid_at=datetime.utcnow()
            )
        )
        await self.session.commit()
        return result.rowcount

    async def get_monthly_report(
        self,
        year: int,
        month: int,
        status: Optional[CommissionStatus] = None
    ) -> List[Dict[str, Any]]:
        """Generate aggregated monthly report by partner."""
        start_date = datetime(year, month, 1)
        if month == 12:
            end_date = datetime(year + 1, 1, 1)
        else:
            end_date = datetime(year, month + 1, 1)

        query = (
            select(
                Partner.id.label("partner_id"),
                Partner.name.label("partner_name"),
                Partner.email.label("partner_email"),
                Partner.payout_method,
                func.count(Commission.id).label("commission_count"),
                func.sum(Commission.amount).label("total_amount")
            )
            .join(Referral, Commission.referral_id == Referral.id)
            .join(Partner, Referral.partner_id == Partner.id)
            .where(Commission.period_start >= start_date)
            .where(Commission.period_end < end_date)
            .group_by(Partner.id, Partner.name, Partner.email, Partner.payout_method)
        )

        if status:
            query = query.where(Commission.status == status)

        result = await self.session.execute(query)
        return result.mappings().all()

class SubscriptionEventRepository:
    """Repository for subscription event tracking."""

    def __init__(self, session: AsyncSession):
        self.session = session

    async def create(self, event: SubscriptionEvent) -> SubscriptionEvent:
        """Create a subscription event with idempotency check."""
        self.session.add(event)
        try:
            await self.session.commit()
            return event
        except IntegrityError:
            await self.session.rollback()
            raise DuplicateEventError(
                f"Event with processed key {event.processed} already exists"
            )

    async def exists(self, idempotency_key: str) -> bool:
        """Check if an event has already been processed."""
        result = await self.session.execute(
            select(SubscriptionEvent).where(SubscriptionEvent.processed == idempotency_key)
        )
        return result.scalar_one_or_none() is not None

# =============================================================================
# Core Business Logic
# =============================================================================

class CommissionCalculator:
    """
    Pure business logic for commission calculations.

    Handles recurring commissions, prorations, and rate applications.
    All calculations use Decimal for precision.
    """

    DEFAULT_RATE = Decimal("0.30")  # 30% default
    MINIMUM_COMMISSION = Decimal("0.01")

    @staticmethod
    def calculate_proration_factor(
        effective_date: datetime,
        period_start: datetime,
        period_end: datetime
    ) -> Decimal:
        """
        Calculate the proration factor for mid-cycle changes.

        Args:
            effective_date: When the change takes effect
            period_start: Start of billing period
            period_end: End of billing period

        Returns:
            Decimal between 0 and 1 representing remaining period
        """
        if effective_date <= period_start:
            return Decimal("1.0")
        if effective_date >= period_end:
            return Decimal("0.0")

        total_days = (period_end - period_start).days
        remaining_days = (period_end - effective_date).days

        if total_days <= 0:
            return Decimal("1.0")

        factor = Decimal(remaining_days) / Decimal(total_days)
        return factor.quantize(Decimal("0.0001"), rounding=ROUND_HALF_UP)

    @classmethod
    def calculate_commission(
        cls,
        subscription_amount: Decimal,
        commission_rate: Decimal,
        proration_factor: Decimal = Decimal("1.0"),
        event_type: SubscriptionEventType = SubscriptionEventType.RENEWAL
    ) -> Decimal:
        """
        Calculate commission amount with precision handling.

        Args:
            subscription_amount: Base subscription amount
            commission_rate: Rate as decimal (e.g., 0.30 for 30%)
            proration_factor: 0.0 to 1.0 for partial periods
            event_type: Type of subscription event

        Returns:
            Calculated commission amount

        Raises:
            CalculationError: If calculation inputs are invalid
        """
        try:
            # Validate inputs
            if subscription_amount < 0:
                raise ValueError("Subscription amount cannot be negative")
            if commission_rate < 0 or commission_rate > 1:
                raise ValueError("Commission rate must be between 0 and 1")
            if proration_factor < 0 or proration_factor > 1:
                raise ValueError("Proration factor must be between 0 and 1")

            # Base calculation
            base_commission = subscription_amount * commission_rate

            # Apply proration
            final_commission = base_commission * proration_factor

            # Round to 2 decimal places (currency)
            final_commission = final_commission.quantize(
                Decimal("0.01"),
                rounding=ROUND_HALF_UP
            )

            # Handle minimum commission threshold
            if final_commission > 0 and final_commission < cls.MINIMUM_COMMISSION:
                final_commission = cls.MINIMUM_COMMISSION

            return final_commission

        except Exception as e:
            logger.error(
                "Commission calculation failed",
                extra={
                    "subscription_amount": str(subscription_amount),
                    "commission_rate": str(commission_rate),
                    "error": str(e)
                }
            )
            raise CalculationError(f"Failed to calculate commission: {e}")

    @classmethod
    def calculate_upgrade_adjustment(
        cls,
        old_amount: Decimal,
        new_amount: Decimal,
        commission_rate: Decimal,
        effective_date: datetime,
        period_start: datetime,
        period_end: datetime
    ) -> Tuple[Decimal, Decimal]:
        """
        Calculate commission for subscription upgrades.

        Handles the difference between old and new amounts with proration.

        Returns:
            Tuple of (original_commission, adjustment_commission)
        """
        # Calculate original full-period commission
        original_commission = cls.calculate_commission(
            old_amount, commission_rate, Decimal("1.0")
        )

        # Calculate proration factor
        proration = cls.calculate_proration_factor(
            effective_date, period_start, period_end
        )

        # Calculate new full period commission
        new_full_commission = cls.calculate_commission(
            new_amount, commission_rate, Decimal("1.0")
        )

        # Calculate adjustment (difference prorated for remaining period)
        difference = new_amount - old_amount
        if difference <= 0:
            return original_commission, Decimal("0.00")

        adjustment = cls.calculate_commission(
            difference, commission_rate, proration
        )

        return original_commission, adjustment

    @classmethod
    def calculate_downgrade_adjustment(
        cls,
        old_amount: Decimal,
        new_amount: Decimal,
        commission_rate: Decimal,
        effective_date: datetime,
        period_start: datetime,
        period_end: datetime
    ) -> Tuple[Decimal, Decimal]:
        """
        Calculate commission adjustment for downgrades.

        Returns:
            Tuple of (original_commission, negative_adjustment)
        """
        proration = cls.calculate_proration_factor(
            effective_date, period_start, period_end
        )

        difference = old_amount - new_amount
        if difference <= 0:
            return Decimal("0.00"), Decimal("0.00")

        # Calculate commission to claw back for remaining period
        adjustment = cls.calculate_commission(
            difference, commission_rate, proration
        )

        return Decimal("0.00"), adjustment * Decimal("-1")

# =============================================================================
# Service Layer
# =============================================================================

class CommissionService:
    """
    High-level service for commission operations.

    Coordinates between repositories and calculator to handle
    business workflows like processing renewals, changes, and payouts.
    """

    def __init__(self, session: AsyncSession):
        self.session = session
        self.partner_repo = PartnerRepository(session)
        self.referral_repo = ReferralRepository(session)
        self.commission_repo = CommissionRepository(session)
        self.event_repo = SubscriptionEventRepository(session)
        self.calculator = CommissionCalculator()

    async def process_renewal(
        self,
        referral_id: UUID,
        subscription_amount: Decimal,
        period_start: datetime,
        period_end: datetime,
        idempotency_key: str
    ) -> Commission:
        """
        Process a monthly subscription renewal and create commission.

        Args:
            referral_id: UUID of the referral
            subscription_amount: Amount charged for this period
            period_start: Start of billing period
            period_end: End of billing period
            idempotency_key: Unique key to prevent duplicate processing

        Returns:
            Created Commission record
        """
        # Check for duplicate
        if await self.event_repo.exists(idempotency_key):
            raise DuplicateEventError(f"Renewal {idempotency_key} already processed")

        # Fetch referral with partner
        referral = await self.referral_repo.get_by_id(referral_id)
        if not referral:
            raise ReferralNotFoundError(f"Referral {referral_id} not found")

        if referral.partner.status != PartnerStatus.ACTIVE:
            raise CommissionError(f"Partner {referral.partner_id} is not active")

        # Calculate commission
        rate = referral.partner.get_effective_rate()
        amount = self.calculator.calculate_commission(
            subscription_amount=subscription_amount,
            commission_rate=rate,
            proration_factor=Decimal("1.0"),
            event_type=SubscriptionEventType.RENEWAL
        )

        # Create commission record
        commission = Commission(
            id=uuid4(),
            referral_id=referral_id,
            amount=amount,
            rate_used=rate,
            subscription_amount=subscription_amount,
            period_start=period_start,
            period_end=period_end,
            status=CommissionStatus.PENDING,
            event_type=SubscriptionEventType.RENEWAL
        )

        # Log event for idempotency
        event = SubscriptionEvent(
            id=uuid4(),
            referral_id=referral_id,
            event_type=SubscriptionEventType.RENEWAL,
            old_amount=referral.subscription_amount,
            new_amount=subscription_amount,
            effective_date=period_start,
            processed=idempotency_key
        )

        await self.event_repo.create(event)
        created = await self.commission_repo.create(commission)

        logger.info(
            "Processed renewal commission",
            extra={
                "referral_id": str(referral_id),
                "amount": str(amount),
                "rate": str(rate),
                "idempotency_key": idempotency_key
            }
        )

        return created

    async def process_subscription_change(
        self,
        event: SubscriptionChangeEvent
    ) -> List[Commission]:
        """
        Handle subscription upgrades or downgrades with prorated commissions.

        Args:
            event: Subscription change event details

        Returns:
            List of created/modified commission records
        """
        # Check for duplicate
        if await self.event_repo.exists(event.idempotency_key):
            raise DuplicateEventError(f"Event {event.idempotency_key} already processed")

        referral = await self.referral_repo.get_by_id(event.referral_id)
        if not referral:
            raise ReferralNotFoundError(f"Referral {event.referral_id} not found")

        rate = referral.partner.get_effective_rate()
        commissions = []

        # Log the event first
        event_record = SubscriptionEvent(
            id=uuid4(),
            referral_id=event.referral_id,
            event_type=event.event_type,
            old_amount=event.old_amount,
            new_amount=event.new_amount,
            effective_date=event.effective_date,
            processed=event.idempotency_key
        )
        await self.event_repo.create(event_record)

        if event.event_type == SubscriptionEventType.UPGRADE:
            # Calculate upgrade commission
            original, adjustment = self.calculator.calculate_upgrade_adjustment(
                old_amount=event.old_amount or Decimal("0"),
                new_amount=event.new_amount,
                commission_rate=rate,
                effective_date=event.effective_date,
                period_start=event.effective_date.replace(day=1),  # Simplified: assume month boundary
                period_end=(event.effective_date.replace(day=1) + timedelta(days=32)).replace(day=1) - timedelta(days=1)
            )

            # Create adjustment commission
            if adjustment > 0:
                adj_commission = Commission(
                    id=uuid4(),
                    referral_id=event.referral_id,
                    amount=adjustment,
                    rate_used=rate,
                    subscription_amount=event.new_amount - (event.old_amount or Decimal("0")),
                    period_start=event.effective_date,
                    period_end=event_record.effective_date.replace(day=1) + timedelta(days=32),
                    status=CommissionStatus.PENDING,
                    event_type=SubscriptionEventType.UPGRADE,
                    notes=f"Upgrade adjustment from {event.old_amount} to {event.new_amount}"
                )
                created = await self.commission_repo.create(adj_commission)
                commissions.append(created)

        elif event.event_type == SubscriptionEventType.DOWNGRADE:
            # For downgrades, we might create a negative commission or adjust future payouts
            original, adjustment = self.calculator.calculate_downgrade_adjustment(
                old_amount=event.old_amount or Decimal("0"),
                new_amount=event.new_amount,
                commission_rate=rate,
                effective_date=event.effective_date,
                period_start=event.effective_date.replace(day=1),
                period_end=(event.effective_date.replace(day=1) + timedelta(days=32)).replace(day=1) - timedelta(days=1)
            )

            if adjustment < 0:
                adj_commission = Commission(
                    id=uuid4(),
                    referral_id=event.referral_id,
                    amount=adjustment,  # Negative amount
                    rate_used=rate,
                    subscription_amount=event.old_amount - event.new_amount,
                    period_start=event.effective_date,
                    period_end=event.effective_date.replace(day=1) + timedelta(days=32),
                    status=CommissionStatus.PENDING,
                    event_type=SubscriptionEventType.DOWNGRADE,
                    notes=f"Downgrade adjustment from {event.old_amount} to {event.new_amount}"
                )
                created = await self.commission_repo.create(adj_commission)
                commissions.append(created)

        # Update referral's current amount
        await self.referral_repo.update_subscription_amount(
            event.referral_id,
            event.new_amount
        )

        logger.info(
            "Processed subscription change",
            extra={
                "referral_id": str(event.referral_id),
                "event_type": event.event_type.value,
                "old_amount": str(event.old_amount),
                "new_amount": str(event.new_amount),
                "commissions_created": len(commissions)
            }
        )

        return commissions

    async def generate_monthly_payout_report(
        self,
        year: int,
        month: int,
        status: CommissionStatus = CommissionStatus.PENDING
    ) -> List[PayoutReportItem]:
        """
        Generate monthly payout report aggregated by partner.

        Args:
            year: Year for report
            month: Month for report (1-12)
            status: Commission status to include

        Returns:
            List of payout report items
        """
        raw_data = await self.commission_repo.get_monthly_report(year, month, status)

        reports = []
        for row in raw_data:
            # Fetch detailed commissions for this partner
            partner_commissions = await self.commission_repo.get_pending_for_partner(
                row["partner_id"],
                datetime(year, month, 1),
                datetime(year, month + 1, 1) if month < 12 else datetime(year + 1, 1, 1)
            )

            # Convert to response models
            commission_responses = []
            for comm in partner_commissions:
                commission_responses.append(
                    CommissionResponse(
                        id=comm.id,
                        referral_id=comm.referral_id,
                        partner_id=row["partner_id"],
                        partner_name=row["partner_name"],
                        amount=comm.amount,
                        rate_used=comm.rate_used,
                        subscription_amount=comm.subscription_amount,
                        period_start=comm.period_start,
                        period_end=comm.period_end,
                        status=comm.status,
                        event_type=comm.event_type,
                        calculated_at=comm.calculated_at
                    )
                )

            reports.append(
                PayoutReportItem(
                    partner_id=row["partner_id"],
                    partner_name=row["partner_name"],
                    partner_email=row["partner_email"],
                    total_commissions=row["total_amount"] or Decimal("0"),
                    commission_count=row["commission_count"],
                    period_start=datetime(year, month, 1),
                    period_end=datetime(year, month + 1, 1) if month < 12 else datetime(year + 1, 1, 1),
                    payout_method=row["payout_method"],
                    details=commission_responses
                )
            )

        logger.info(
            "Generated payout report",
            extra={"year": year, "month": month, "partner_count": len(reports)}
        )

        return reports

    async def approve_commissions_for_payout(
        self,
        commission_ids: List[UUID]
    ) -> int:
        """
        Approve commissions for payout.

        Args:
            commission_ids: List of commission UUIDs to approve

        Returns:
            Number of commissions approved
        """
        count = await self.commission_repo.approve_for_payout(commission_ids)
        logger.info(f"Approved {count} commissions for payout")
        return count

    async def process_payout_completion(
        self,
        commission_ids: List[UUID]
    ) -> int:
        """
        Mark commissions as paid after payout completion.

        Args:
            commission_ids: List of commission UUIDs that were paid

        Returns:
            Number of commissions marked as paid
        """
        count = await self.commission_repo.mark_as_paid(commission_ids)
        logger.info(f"Marked {count} commissions as paid")
        return count

    async def set_partner_custom_rate(
        self,
        partner_id: UUID,
        new_rate: Decimal
    ) -> Partner:
        """
        Set a custom commission rate for a specific partner.

        Args:
            partner_id: Partner UUID
            new_rate: New rate as decimal (e.g., 0.40 for 40%)

        Returns:
            Updated Partner record
        """
        if new_rate < 0 or new_rate > 1:
            raise ValueError("Commission rate must be between 0 and 1")

        partner = await self.partner_repo.update_custom_rate(partner_id, new_rate)

        logger.info(
            "Updated partner custom rate",
            extra={
                "partner_id": str(partner_id),
                "new_rate": str(new_rate)
            }
        )

        return partner

# =============================================================================
# FastAPI Integration (Optional Usage Example)
# =============================================================================

"""
Example FastAPI router implementation:

from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.ext.asyncio import AsyncSession

router = APIRouter(prefix="/commissions", tags=["commissions"])

async def get_db() -> AsyncSession:
    # Your database session dependency
    pass

@router.post("/calculate", response_model=CommissionResponse)
async def calculate_commission_endpoint(
    request: CommissionCalculationRequest,
    db: AsyncSession = Depends(get_db)
):
    service = CommissionService(db)
    try:
        commission = await service.process_renewal(
            referral_id=request.referral_id,
            subscription_amount=request.subscription_amount,
            period_start=request.period_start,
            period_end=request.period_end,
            idempotency_key=f"{request.referral_id}-{request.period_start.isoformat()}"
        )
        return commission
    except (ReferralNotFoundError, DuplicateEventError) as e:
        raise HTTPException(status_code=400, detail=str(e))
    except Exception as e:
        logger.error(f"Commission calculation failed: {e}")
        raise HTTPException(status_code=500, detail="Internal server error")

@router.get("/reports/monthly", response_model=List[PayoutReportItem])
async def get_monthly_report(
    year: int,
    month: int,
    db: AsyncSession = Depends(get_db)
):
    service = CommissionService(db)
    return await service.generate_monthly_payout_report(year, month)
"""
```

This production-ready commission tracking module provides:

**Core Features:**
- **30% Default Commission**: Configurable per-partner override via `custom_commission_rate`
- **Recurring Tracking**: Each renewal creates a `Commission` record linked to the referral
- **Proration Logic**: Handles mid-cycle upgrades/downgrades with `calculate_proration_factor()`
- **Monthly Reports**: Aggregated payout reports with `generate_monthly_payout_report()`
- **Idempotency**: Prevents duplicate processing via `SubscriptionEvent.processed` keys

**Technical Implementation:**
- **Async-First**: All database operations use SQLAlchemy async sessions
- **Type Safety**: Full Pydantic models for API boundaries, SQLAlchemy for ORM
- **Precision**: Decimal types for all monetary calculations to avoid floating-point errors
- **Error Handling**: Custom exceptions (`CommissionError`, `DuplicateEventError`) with proper logging
- **Database Design**: PostgreSQL-optimized with proper indexes on `partner_id`, `status`, and date ranges

**Key Classes:**
1. `CommissionCalculator`: Pure business logic for math operations
2. `CommissionService`: High-level orchestration of workflows
3. Repository pattern for data access (Partner, Referral, Commission repositories)

**Usage Flow:**
1. Partner refers customer → Create `Referral` record
2. Monthly renewal → Call `process_renewal()` creates `Commission` (status=PENDING)
3. Upgrade/downgrade → Call `process_subscription_change()` with proration
4. End of month → Call `generate_monthly_payout_report()` to get aggregation
5. After payment → Call `process_payout_completion()` to mark PAID


Widget A/B Testing Framework