Response:
 Here's a production-grade implementation of the File Upload Service for Sunaiva:

## Project Structure

```
sunaiva/
├── config.py
├── exceptions.py
├── models/
│   ├── __init__.py
│   ├── database.py
│   └── schemas.py
├── services/
│   ├── __init__.py
│   ├── storage.py
│   └── upload.py
├── api/
│   ├── __init__.py
│   ├── dependencies.py
│   └── routes/
│       └── uploads.py
└── tests/
    ├── test_upload_service.py
    └── test_upload_api.py
```

## 1. Configuration (`config.py`)

```python
"""Application configuration with S3-compatible storage settings."""

from functools import lru_cache
from typing import Literal, Optional

from pydantic import Field, field_validator
from pydantic_settings import BaseSettings, SettingsConfigDict

class Settings(BaseSettings):
    """Production settings for Sunaiva file upload service."""

    model_config = SettingsConfigDict(
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore"
    )

    # Application
    app_name: str = "Sunaiva Upload Service"
    debug: bool = False
    environment: Literal["development", "staging", "production"] = "production"

    # Database
    database_url: str = Field(..., description="PostgreSQL connection string")

    # Redis
    redis_url: str = "redis://localhost:6379/0"

    # S3-Compatible Storage (R2/MinIO)
    s3_endpoint_url: str = Field(..., description="S3 endpoint (e.g., https://<account>.r2.cloudflarestorage.com)")
    s3_region: str = "auto"
    s3_access_key_id: str = Field(..., alias="AWS_ACCESS_KEY_ID")
    s3_secret_access_key: str = Field(..., alias="AWS_SECRET_ACCESS_KEY")
    s3_bucket_name: str = Field(..., description="Bucket for file storage")
    s3_public_url: Optional[str] = Field(None, description="Custom domain for public URLs (e.g., https://cdn.sunaiva.com)")

    # File Limits
    max_image_size_bytes: int = 5 * 1024 * 1024  # 5MB
    max_audio_size_bytes: int = 10 * 1024 * 1024  # 10MB

    # Security
    signed_url_expiration: int = 3600  # 1 hour
    upload_rate_limit_per_minute: int = 30

    @field_validator("s3_endpoint_url")
    @classmethod
    def validate_s3_endpoint(cls, v: str) -> str:
        """Ensure endpoint URL has protocol."""
        if not v.startswith(("http://", "https://")):
            raise ValueError("S3 endpoint must include protocol (http:// or https://)")
        return v.rstrip("/")

@lru_cache
def get_settings() -> Settings:
    """Cached settings instance."""
    return Settings()
```

## 2. Custom Exceptions (`exceptions.py`)

```python
"""Domain-specific exceptions for the upload service."""

from typing import Optional

class SunaivaException(Exception):
    """Base exception for Sunaiva services."""

    def __init__(self, message: str, code: Optional[str] = None):
        self.message = message
        self.code = code or "INTERNAL_ERROR"
        super().__init__(self.message)

class FileValidationError(SunaivaException):
    """Raised when file validation fails."""

    def __init__(self, message: str, field: Optional[str] = None):
        super().__init__(message, code="FILE_VALIDATION_ERROR")
        self.field = field

class StorageError(SunaivaException):
    """Raised when S3/storage operations fail."""

    def __init__(self, message: str, original_error: Optional[Exception] = None):
        super().__init__(message, code="STORAGE_ERROR")
        self.original_error = original_error

class RateLimitError(SunaivaException):
    """Raised when rate limit is exceeded."""

    def __init__(self, retry_after: int):
        super().__init__(
            f"Rate limit exceeded. Retry after {retry_after} seconds.",
            code="RATE_LIMIT_ERROR"
        )
        self.retry_after = retry_after
```

## 3. Database Models (`models/database.py`)

```python
"""SQLAlchemy 2.0 async models for file metadata."""

import uuid
from datetime import datetime
from typing import Optional

from sqlalchemy import String, Integer, DateTime, ForeignKey, func
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column

class Base(DeclarativeBase):
    pass

class FileMetadata(Base):
    """Stores metadata for uploaded files with tenant isolation."""

    __tablename__ = "file_uploads"

    id: Mapped[uuid.UUID] = mapped_column(
        UUID(as_uuid=True),
        primary_key=True,
        default=uuid.uuid4
    )
    business_id: Mapped[uuid.UUID] = mapped_column(
        UUID(as_uuid=True),
        ForeignKey("businesses.id", ondelete="CASCADE"),
        index=True,
        nullable=False
    )

    # File Info
    original_filename: Mapped[str] = mapped_column(String(255), nullable=False)
    storage_key: Mapped[str] = mapped_column(String(512), unique=True, nullable=False)
    content_type: Mapped[str] = mapped_column(String(100), nullable=False)
    file_size: Mapped[int] = mapped_column(Integer, nullable=False)
    file_category: Mapped[str] = mapped_column(String(20), nullable=False)  # 'image' or 'audio'

    # Timestamps
    created_at: Mapped[datetime] = mapped_column(
        DateTime(timezone=True),
        server_default=func.now(),
        nullable=False
    )
    expires_at: Mapped[Optional[datetime]] = mapped_column(
        DateTime(timezone=True),
        nullable=True
    )

    # Access Control
    is_public: Mapped[bool] = mapped_column(default=False)
    access_count: Mapped[int] = mapped_column(default=0)

    def __repr__(self) -> str:
        return f"<FileMetadata(id={self.id}, business={self.business_id}, key={self.storage_key})>"
```

## 4. Pydantic Schemas (`models/schemas.py`)

```python
"""Pydantic v2 models for API request/response validation."""

from datetime import datetime
from typing import Literal, Optional
from uuid import UUID

from pydantic import BaseModel, Field, HttpUrl, field_validator

class FileUploadResponse(BaseModel):
    """Response model for successful uploads."""

    file_id: UUID
    filename: str
    content_type: str
    size_bytes: int
    category: Literal["image", "audio"]
    url: HttpUrl = Field(..., description="Signed URL for immediate access")
    expires_at: datetime = Field(..., description="URL expiration timestamp")

    class Config:
        from_attributes = True

class SignedUrlResponse(BaseModel):
    """Response for signed URL generation."""

    url: HttpUrl
    expires_in_seconds: int
    expires_at: datetime

class FileValidationConfig(BaseModel):
    """Configuration for file validation."""

    allowed_image_types: set[str] = Field(default={"image/jpeg", "image/png", "image/svg+xml"})
    allowed_audio_types: set[str] = Field(default={"audio/mpeg", "audio/wav", "audio/x-wav"})
    max_image_size: int = Field(default=5 * 1024 * 1024)
    max_audio_size: int = Field(default=10 * 1024 * 1024)

    @field_validator("allowed_image_types", "allowed_audio_types", mode="before")
    @classmethod
    def normalize_mime_types(cls, v: set[str]) -> set[str]:
        """Normalize MIME type strings."""
        return {t.lower().strip() for t in v}
```

## 5. Storage Service (`services/storage.py`)

```python
"""Async S3-compatible storage implementation using aiobotocore."""

import uuid
from datetime import datetime, timedelta
from typing import BinaryIO, Optional
from urllib.parse import urlparse

import aiobotocore.session
from botocore.exceptions import ClientError

from config import get_settings
from exceptions import StorageError

class StorageService:
    """Async S3-compatible storage client (R2/MinIO)."""

    def __init__(self):
        self.settings = get_settings()
        self.session = aiobotocore.session.get_session()
        self._client = None

    async def _get_client(self):
        """Lazy initialization of S3 client."""
        if self._client is None:
            self._client = self.session.create_client(
                "s3",
                region_name=self.settings.s3_region,
                endpoint_url=self.settings.s3_endpoint_url,
                aws_access_key_id=self.settings.s3_access_key_id,
                aws_secret_access_key=self.settings.s3_secret_access_key,
            )
        return self._client

    def _generate_key(self, business_id: uuid.UUID, category: str, filename: str) -> str:
        """Generate unique storage key with tenant isolation."""
        # Sanitize filename: remove path components, limit length
        safe_name = filename.split("/")[-1].split("\\")[-1][:100]
        timestamp = datetime.utcnow().strftime("%Y/%m/%d")
        unique_id = str(uuid.uuid4())[:8]

        return f"{business_id}/{category}/{timestamp}/{unique_id}_{safe_name}"

    async def upload_file(
        self,
        file_obj: BinaryIO,
        business_id: uuid.UUID,
        category: str,
        filename: str,
        content_type: str,
        content_length: int
    ) -> str:
        """
        Upload file to S3-compatible storage.

        Args:
            file_obj: File-like object to upload
            business_id: Tenant ID for isolation
            category: 'image' or 'audio'
            filename: Original filename
            content_type: MIME type
            content_length: File size in bytes

        Returns:
            Storage key (path in bucket)
        """
        key = self._generate_key(business_id, category, filename)
        client = await self._get_client()

        try:
            # Ensure file pointer is at start
            file_obj.seek(0)

            await client.put_object(
                Bucket=self.settings.s3_bucket_name,
                Key=key,
                Body=file_obj,
                ContentType=content_type,
                ContentLength=content_length,
                Metadata={
                    "business-id": str(business_id),
                    "original-filename": filename,
                    "upload-timestamp": datetime.utcnow().isoformat()
                }
            )
            return key

        except ClientError as e:
            error_code = e.response["Error"]["Code"]
            raise StorageError(
                f"Failed to upload file: {error_code}",
                original_error=e
            )
        except Exception as e:
            raise StorageError(f"Unexpected error during upload: {str(e)}", original_error=e)

    async def generate_signed_url(
        self,
        storage_key: str,
        expiration_seconds: Optional[int] = None
    ) -> str:
        """
        Generate presigned URL for temporary access.

        Args:
            storage_key: Path in bucket
            expiration_seconds: URL lifetime (default from settings)

        Returns:
            Presigned URL string
        """
        client = await self._get_client()
        expiration = expiration_seconds or self.settings.signed_url_expiration

        try:
            # For R2/MinIO, we use standard S3 presigned URLs
            url = await client.generate_presigned_url(
                "get_object",
                Params={
                    "Bucket": self.settings.s3_bucket_name,
                    "Key": storage_key,
                    "ResponseContentDisposition": "inline"
                },
                ExpiresIn=expiration
            )

            # If using custom CDN domain, replace the endpoint
            if self.settings.s3_public_url:
                parsed = urlparse(url)
                # Extract the path and query from presigned URL
                path_and_query = parsed.path + "?" + parsed.query
                url = f"{self.settings.s3_public_url.rstrip('/')}{path_and_query}"

            return url

        except ClientError as e:
            raise StorageError(
                f"Failed to generate signed URL: {e.response['Error']['Code']}",
                original_error=e
            )

    async def delete_file(self, storage_key: str) -> None:
        """Delete file from storage."""
        client = await self._get_client()

        try:
            await client.delete_object(
                Bucket=self.settings.s3_bucket_name,
                Key=storage_key
            )
        except ClientError as e:
            raise StorageError(
                f"Failed to delete file: {e.response['Error']['Code']}",
                original_error=e
            )

    async def close(self):
        """Cleanup resources."""
        if self._client:
            await self._client.close()
            self._client = None

    async def __aenter__(self):
        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb):
        await self.close()
```

## 6. Upload Service (`services/upload.py`)

```python
"""Business logic for file upload validation and processing."""

import io
import magic
from typing import BinaryIO, Tuple
from uuid import UUID

from models.schemas import FileValidationConfig
from services.storage import StorageService
from exceptions import FileValidationError
from config import get_settings

class UploadService:
    """Handles file validation, storage, and metadata management."""

    # Magic bytes signatures for validation
    SIGNATURES = {
        "image/jpeg": [(b"\xff\xd8\xff", 0)],  # JFIF and Exif
        "image/png": [(b"\x89PNG\r\n\x1a\n", 0)],
        "image/svg+xml": None,  # Special handling for XML-based
        "audio/mpeg": [(b"ID3", 0), (b"\xff\xfb", 0), (b"\xff\xf3", 0)],  # MP3
        "audio/wav": [(b"RIFF", 0)],  # WAV starts with RIFF
        "audio/x-wav": [(b"RIFF", 0)],
    }

    def __init__(self, storage: StorageService):
        self.storage = storage
        self.config = FileValidationConfig()
        self.settings = get_settings()

    def _detect_content_type(self, file_obj: BinaryIO) -> str:
        """
        Detect MIME type using libmagic (file signatures).
        More reliable than extension-based detection.
        """
        # Read first 2048 bytes for magic number detection
        head = file_obj.read(2048)
        file_obj.seek(0)

        # Special handling for SVG (XML-based)
        try:
            text_head = head.decode("utf-8", errors="ignore").lower().strip()
            if text_head.startswith(("<?xml", "<svg")) and "<svg" in text_head:
                return "image/svg+xml"
        except Exception:
            pass

        # Use libmagic for binary files
        mime = magic.from_buffer(head, mime=True)
        return mime.lower()

    def _validate_size(self, size_bytes: int, category: str) -> None:
        """Validate file size against category limits."""
        if category == "image" and size_bytes > self.config.max_image_size:
            raise FileValidationError(
                f"Image size {size_bytes} bytes exceeds limit of {self.config.max_image_size} bytes (5MB)",
                field="file_size"
            )
        elif category == "audio" and size_bytes > self.config.max_audio_size:
            raise FileValidationError(
                f"Audio size {size_bytes} bytes exceeds limit of {self.config.max_audio_size} bytes (10MB)",
                field="file_size"
            )

    def _validate_content_type(
        self,
        declared_type: str,
        detected_type: str,
        category: str
    ) -> None:
        """
        Validate that detected type matches declared type and is allowed.

        Security: Prevents spoofing (e.g., renaming .exe to .jpg)
        """
        allowed = (
            self.config.allowed_image_types if category == "image"
            else self.config.allowed_audio_types
        )

        # Normalize types
        declared = declared_type.lower().strip()
        detected = detected_type.lower().strip()

        # Check if detected type is in allowed list
        if detected not in allowed:
            # Some systems detect SVG as text/xml or application/xml
            if category == "image" and detected in ("text/xml", "application/xml"):
                detected = "image/svg+xml"
            else:
                raise FileValidationError(
                    f"File type '{detected}' not allowed for {category}. "
                    f"Allowed types: {', '.join(allowed)}",
                    field="content_type"
                )

        # Verify consistency (prevent type confusion attacks)
        # Allow some flexibility (e.g., audio/x-wav vs audio/wav)
        if declared != detected:
            # Map equivalent types


Response:
 Here's is a production-ready, immutable audit logging system for Sunaiva. This implementation uses PostgreSQL with append-only enforcement at both the database and application layers, partitioned tables for performance, and comprehensive query capabilities.

## 1. Database Schema (Migration)

```sql
-- migrations/001_create_audit_logs.sql
-- Enable required extensions
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE EXTENSION IF NOT EXISTS "pgcrypto";

-- Create enum type for audit actions
CREATE TYPE audit_action_type AS ENUM (
    'login',
    'logout',
    'create',
    'update',
    'delete',
    'admin_action',
    'api_key_created',
    'api_key_revoked',
    'view',
    'export',
    'system'
);

-- Create enum for actor types
CREATE TYPE audit_actor_type AS ENUM (
    'user',
    'system',
    'api_key',
    'service_account',
    'anonymous'
);

-- Create partitioned audit log table (by month for retention management)
CREATE TABLE audit_logs (
    id UUID DEFAULT uuid_generate_v4() PRIMARY KEY,
    timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
    action_type audit_action_type NOT NULL,
    actor_id UUID,  -- Can be user_id, api_key_id, etc.
    actor_type audit_actor_type NOT NULL DEFAULT 'anonymous',
    actor_email VARCHAR(255),  -- Denormalized for query performance
    resource_type VARCHAR(100) NOT NULL,  -- e.g., 'widget', 'user', 'billing'
    resource_id VARCHAR(100),  -- Optional specific resource identifier
    ip_address INET,
    user_agent TEXT,
    correlation_id UUID,  -- For tracing requests across services
    status VARCHAR(20) NOT NULL CHECK (status IN ('success', 'failure', 'warning')),
    details JSONB NOT NULL DEFAULT '{}',  -- Immutable snapshot of changes/context
    hash_chain VARCHAR(64),  -- For cryptographic integrity (optional advanced feature)
    retention_date DATE GENERATED ALWAYS AS (DATE(timestamp + INTERVAL '1 year')) STORED
) PARTITION BY RANGE (timestamp);

-- Create initial partitions (current year + next year)
-- Note: In production, use a cron job or pg_partman to create future partitions
CREATE TABLE audit_logs_y2024m12 PARTITION OF audit_logs
    FOR VALUES FROM ('2024-12-01') TO ('2025-01-01');
CREATE TABLE audit_logs_y2025m01 PARTITION OF audit_logs
    FOR VALUES FROM ('2025-01-01') TO ('2025-02-01');

-- Critical: Create rule to prevent updates (immutability enforcement)
CREATE OR REPLACE RULE audit_logs_no_update AS ON UPDATE TO audit_logs
    DO INSTEAD NOTHING;

CREATE OR REPLACE RULE audit_logs_no_delete AS ON DELETE TO audit_logs
    DO INSTEAD NOTHING;

-- Indexes for efficient querying
CREATE INDEX idx_audit_logs_timestamp ON audit_logs USING BRIN (timestamp);
CREATE INDEX idx_audit_logs_action_type ON audit_logs (action_type);
CREATE INDEX idx_audit_logs_actor_id ON audit_logs (actor_id) WHERE actor_id IS NOT NULL;
CREATE INDEX idx_audit_logs_resource ON audit_logs (resource_type, resource_id);
CREATE INDEX idx_audit_logs_correlation ON audit_logs (correlation_id);
CREATE INDEX idx_audit_logs_details ON audit_logs USING GIN (details jsonb_path_ops);

-- Function to enforce append-only at database level (additional safety)
CREATE OR REPLACE FUNCTION enforce_audit_immutability()
RETURNS TRIGGER AS $$
BEGIN
    RAISE EXCEPTION 'Audit logs are immutable: updates and deletes are prohibited';
END;
$$ LANGUAGE plpgsql;

-- Attach triggers (redundant with rules but provides clearer error messages)
CREATE TRIGGER trg_audit_logs_no_update
    BEFORE UPDATE ON audit_logs
    FOR EACH ROW EXECUTE FUNCTION enforce_audit_immutability();

CREATE TRIGGER trg_audit_logs_no_delete
    BEFORE DELETE ON audit_logs
    FOR EACH ROW EXECUTE FUNCTION enforce_audit_immutability();

-- View for querying (exposes only non-sensitive fields if needed)
CREATE VIEW audit_logs_readonly AS
SELECT
    id,
    timestamp,
    action_type,
    actor_id,
    actor_type,
    actor_email,
    resource_type,
    resource_id,
    status,
    correlation_id,
    details,
    retention_date
FROM audit_logs;

-- Comment for documentation
COMMENT ON TABLE audit_logs IS 'Immutable audit trail for compliance. Append-only. Retention: 1 year.';
```

## 2. Pydantic Models (Schemas)

```python
# models/audit.py
from datetime import datetime
from enum import Enum
from typing import Any, Dict, Optional
from uuid import UUID

from pydantic import BaseModel, ConfigDict, Field, field_serializer

class AuditActionType(str, Enum):
    """Enumeration of auditable actions."""
    LOGIN = "login"
    LOGOUT = "logout"
    CREATE = "create"
    UPDATE = "update"
    DELETE = "delete"
    ADMIN_ACTION = "admin_action"
    API_KEY_CREATED = "api_key_created"
    API_KEY_REVOKED = "api_key_revoked"
    VIEW = "view"
    EXPORT = "export"
    SYSTEM = "system"

class ActorType(str, Enum):
    """Types of actors performing actions."""
    USER = "user"
    SYSTEM = "system"
    API_KEY = "api_key"
    SERVICE_ACCOUNT = "service_account"
    ANONYMOUS = "anonymous"

class AuditStatus(str, Enum):
    """Status of the audited action."""
    SUCCESS = "success"
    FAILURE = "failure"
    WARNING = "warning"

class AuditLogBase(BaseModel):
    """Base model for audit log entries."""
    model_config = ConfigDict(from_attributes=True, extra="forbid")

    action_type: AuditActionType
    actor_id: Optional[UUID] = None
    actor_type: ActorType = ActorType.ANONYMOUS
    actor_email: Optional[str] = Field(None, max_length=255)
    resource_type: str = Field(..., max_length=100)
    resource_id: Optional[str] = Field(None, max_length=100)
    status: AuditStatus = AuditStatus.SUCCESS
    details: Dict[str, Any] = Field(default_factory=dict)
    correlation_id: Optional[UUID] = None

class AuditLogCreate(AuditLogBase):
    """Model for creating new audit log entries."""
    ip_address: Optional[str] = None
    user_agent: Optional[str] = None

    # Sensitive fields that should be masked before storage
    @field_serializer('details')
    def mask_sensitive_data(self, details: Dict[str, Any]) -> Dict[str, Any]:
        """Remove sensitive fields from details before storage."""
        if not details:
            return {}

        # Deep copy to avoid mutating original
        cleaned = details.copy()

        # Fields to always mask (case-insensitive partial matching)
        sensitive_keys = {'password', 'token', 'secret', 'api_key', 'credit_card', 'ssn'}

        def redact_sensitive(obj: Any) -> Any:
            if isinstance(obj, dict):
                return {
                    k: '[REDACTED]' if any(s in k.lower() for s in sensitive_keys) else redact_sensitive(v)
                    for k, v in obj.items()
                }
            elif isinstance(obj, list):
                return [redact_sensitive(item) for item in obj]
            return obj

        return redact_sensitive(cleaned)

class AuditLogResponse(AuditLogBase):
    """Model for audit log responses (read-only)."""
    model_config = ConfigDict(from_attributes=True)

    id: UUID
    timestamp: datetime
    ip_address: Optional[str] = None  # May be hashed for GDPR
    user_agent: Optional[str] = None
    retention_date: Optional[datetime] = None

class AuditLogQuery(BaseModel):
    """Query parameters for filtering audit logs."""
    start_date: Optional[datetime] = None
    end_date: Optional[datetime] = None
    action_types: Optional[list[AuditActionType]] = None
    actor_id: Optional[UUID] = None
    actor_type: Optional[ActorType] = None
    resource_type: Optional[str] = None
    resource_id: Optional[str] = None
    status: Optional[AuditStatus] = None
    correlation_id: Optional[UUID] = None
    limit: int = Field(50, ge=1, le=1000)
    offset: int = Field(0, ge=0)

class AuditLogStatistics(BaseModel):
    """Statistics for audit logs."""
    total_events: int
    events_by_type: Dict[str, int]
    events_by_status: Dict[str, int]
    date_range: tuple[datetime, datetime]
```

## 3. Repository Layer (Asyncpg)

```python
# repositories/audit_repository.py
from datetime import datetime, timedelta
from typing import List, Optional, Dict, Any
from uuid import UUID

import asyncpg
from asyncpg import Connection

from models.audit import AuditLogCreate, AuditLogResponse, AuditLogQuery

class AuditRepository:
    """Repository for immutable audit log operations.

    Enforces append-only semantics. No update or delete methods provided.
    """

    def __init__(self, pool: asyncpg.Pool):
        self._pool = pool

    async def create(
        self,
        audit_entry: AuditLogCreate,
        conn: Optional[Connection] = None
    ) -> UUID:
        """Create a new immutable audit log entry.

        Args:
            audit_entry: The audit data to persist
            conn: Optional existing connection (for transactions)

        Returns:
            UUID of the created audit entry

        Raises:
            asyncpg.PostgresError: If database insertion fails
        """
        query = """
            INSERT INTO audit_logs (
                action_type, actor_id, actor_type, actor_email,
                resource_type, resource_id, ip_address, user_agent,
                status, details, correlation_id
            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
            RETURNING id
        """

        params = (
            audit_entry.action_type.value,
            audit_entry.actor_id,
            audit_entry.actor_type.value,
            audit_entry.actor_email,
            audit_entry.resource_type,
            audit_entry.resource_id,
            audit_entry.ip_address,
            audit_entry.user_agent,
            audit_entry.status.value,
            audit_entry.details,
            audit_entry.correlation_id
        )

        try:
            if conn:
                result = await conn.fetchval(query, *params)
            else:
                async with self._pool.acquire() as connection:
                    result = await connection.fetchval(query, *params)
            return result
        except asyncpg.PostgresError as e:
            # Log to external monitoring (Sentry/DataDog) in production
            raise AuditStorageError(f"Failed to store audit log: {e}") from e

    async def query(
        self,
        filters: AuditLogQuery,
        requesting_actor_id: Optional[UUID] = None  # For permission checks
    ) -> List[AuditLogResponse]:
        """Query audit logs with filters.

        Note: This is read-only access. No modification possible.

        Args:
            filters: Query parameters
            requesting_actor_id: ID of user making the request (for audit)

        Returns:
            List of audit log entries matching criteria
        """
        conditions = []
        params = []
        param_idx = 1

        # Build dynamic query
        if filters.start_date:
            conditions.append(f"timestamp >= ${param_idx}")
            params.append(filters.start_date)
            param_idx += 1

        if filters.end_date:
            conditions.append(f"timestamp <= ${param_idx}")
            params.append(filters.end_date)
            param_idx += 1

        if filters.action_types:
            placeholders = [f"${i}" for i in range(param_idx, param_idx + len(filters.action_types))]
            conditions.append(f"action_type IN ({','.join(placeholders)})")
            params.extend([at.value for at in filters.action_types])
            param_idx += len(filters.action_types)

        if filters.actor_id:
            conditions.append(f"actor_id = ${param_idx}")
            params.append(filters.actor_id)
            param_idx += 1

        if filters.actor_type:
            conditions.append(f"actor_type = ${param_idx}")
            params.append(filters.actor_type.value)
            param_idx += 1

        if filters.resource_type:
            conditions.append(f"resource_type = ${param_idx}")
            params.append(filters.resource_type)
            param_idx += 1

        if filters.resource_id:
            conditions.append(f"resource_id = ${param_idx}")
            params.append(filters.resource_id)
            param_idx += 1

        if filters.status:
            conditions.append(f"status = ${param_idx}")
            params.append(filters.status.value)
            param_idx += 1

        if filters.correlation_id:
            conditions.append(f"correlation_id = ${param_idx}")
            params.append(filters.correlation_id)
            param_idx += 1

        where_clause = " AND ".join(conditions) if conditions else "TRUE"

        query = f"""
            SELECT
                id, timestamp, action_type, actor_id, actor_type,
                actor_email, resource_type, resource_id, ip_address,
                user_agent, status, details, correlation_id, retention_date
            FROM audit_logs_readonly
            WHERE {where_clause}
            ORDER BY timestamp DESC
            LIMIT ${param_idx} OFFSET ${param_idx + 1}
        """
        params.extend([filters.limit, filters.offset])

        try:
            async with self._pool.acquire() as conn:
                rows = await conn.fetch(query, *params)
                return [AuditLogResponse.model_validate(dict(row)) for row in rows]
        except asyncpg.PostgresError as e:
            raise AuditQueryError(f"Failed to query audit logs: {e}") from e

    async def get_by_id(self, audit_id: UUID) -> Optional[AuditLogResponse]:
        """Retrieve a specific audit log by ID."""
        query = """
            SELECT * FROM audit_logs_readonly
            WHERE id = $1
        """
        async with self._pool.acquire() as conn:
            row = await conn.fetchrow(query, audit_id)
            return AuditLogResponse.model_validate(dict(row)) if row else None

    async def get_statistics(
        self,
        start_date: datetime,
        end_date: datetime
    ) -> Dict[str, Any]:
        """Get aggregated statistics for a date range."""
        query = """
            SELECT
                COUNT(*) as total,
                action_type,
                status
            FROM audit_logs_readonly
            WHERE timestamp BETWEEN $1 AND $2
            GROUP BY action_type, status
        """

        async with self._pool.acquire() as conn:
            rows = await conn.fetch(query, start_date, end_date)

        total = sum(r['count'] for r in rows)
        by_type = {}
        by_status = {}

        for row in rows:
            by_type[row['action_type']] = by_type.get(row['action_type'], 0) + row['count']
            by_status[row['status']] = by_status.get(row['status'], 0) + row['count']

        return {
            "total_events": total,
            "events_by_type": by_type,
            "events_by_status": by_status,
            "date_range": (start_date, end_date)
        }

    async def cleanup_expired(self, batch_size: int = 1000) -> int:
        """Delete audit logs older than retention period (1 year).

        Note: This is the ONLY delete operation allowed, and should only
        be called by automated retention jobs.

        Returns:
            Number of records deleted
        """
        # Note: In production with partitioning, this would drop partitions
        # instead of deleting rows for performance.
        query = """
            DELETE FROM audit_logs
            WHERE retention_date < CURRENT_DATE
            LIMIT $1
        """

        # This requires bypassing the delete trigger. In production, use
        # partition dropping instead, or use a privileged role.
        async with self._pool.acquire() as conn:
            # Temporarily disable trigger for cleanup (requires superuser)
            # In production, use pg_partman or scheduled jobs outside app
            result = await conn.execute(query, batch_size)
            # Parse "DELETE N" result
            return int(result.split()[1]) if result else 0

class AuditStorageError(Exception):
    """Raised when audit log storage fails."""
    pass

class AuditQueryError(Exception):
    """Raised when audit log query fails."""
    pass
```

## 4. Service Layer with Context Management

```python
# services/audit_service.py
import hashlib
from contextvars import ContextVar
from typing import Any, Dict, Optional
from uuid import UUID, uuid4

from fastapi import Request

from models.audit import AuditLogCreate, AuditActionType, ActorType, AuditStatus
from repositories.audit_repository import AuditRepository

# Context variable for correlation ID propagation
correlation_id_ctx: ContextVar[Optional[UUID]] = ContextVar('correlation_id', default=None)

class AuditService:
    """High-level service for audit logging operations."""

    def __init__(self, repository: AuditRepository, gdpr_mode: bool = True):
        self._repo = repository
        self._gdpr_mode = gdpr_mode  # If True, hash IP addresses

    def _hash_ip(self, ip: Optional[str]) -> Optional[str]:
        """Hash IP address for GDPR compliance if enabled."""
        if not ip or not self._gdpr_mode:
            return ip
        # Use first 3 octets only for IPv4, or hash full for IPv6
        if '.' in ip:  # IPv4
            parts = ip.split('.')
            return f"{parts[0]}.{parts[1]}.{parts[2]}.0"
        return hashlib.sha256(ip.encode()).hexdigest()[:16]

    async def log_event(
        self,
        action_type: AuditActionType,
        resource_type: str,
        resource_id: Optional[str] = None,
        actor_id: Optional[UUID] = None,
        actor_type: ActorType = ActorType.SYSTEM,
        actor_email: Optional[str] = None,
        status: AuditStatus = AuditStatus.SUCCESS,
        details: Optional[Dict[str, Any]] = None,
        request: Optional[Request] = None,
        ip_address: Optional[str] = None,
        user_agent: Optional[str] = None,
        correlation_id: Optional[UUID] = None
    ) -> UUID:
        """Log a single audit event.

        This is the primary interface for creating immutable audit records.
        """
        # Get correlation ID from context if not provided
        if not correlation_id:
            correlation_id = correlation_id_ctx.get() or uuid4()

        # Extract request info if provided
        if request:
            ip_address = ip_address or self._get_client_ip(request)
            user_agent = user_agent or request.headers.get('user-agent')

        entry = AuditLogCreate(
            action_type=action_type,
            actor_id=actor_id,
            actor_type=actor_type,
            actor_email=actor_email,
            resource_type=resource_type,
            resource_id


Response:
 Here is the production-ready implementation for **TW-INFRA-07: Tenant Provisioning Automation**.

## Project Structure

```text
sunaiva/
├── config.py              # Pydantic settings
├── exceptions.py          # Custom exceptions
├── main.py               # FastAPI application
├── models/
│   ├── __init__.py
│   ├── enums.py          # Provisioning states
│   └── schemas.py        # Pydantic v2 models
└── services/
    ├── __init__.py
    ├── database.py       # Repository pattern with asyncpg
    ├── provisioning.py   # Orchestration engine
    ├── telnyx.py        # Telnyx API client
    └── webhook.py       # Outbound webhook delivery
```

## 1. Configuration & Exceptions

```python
# config.py
import secrets
from functools import lru_cache
from typing import Optional

from pydantic import Field, RedisDsn
from pydantic_settings import BaseSettings

class Settings(BaseSettings):
    """Production configuration with validation."""

    # Database
    database_url: str = Field(alias="DATABASE_URL")
    database_pool_size: int = Field(default=10, alias="DB_POOL_SIZE")

    # Redis
    redis_url: RedisDsn = Field(alias="REDIS_URL")

    # Security
    api_key_prefix: str = "sk_live_"
    api_key_length: int = 32

    # Telnyx
    telnyx_api_key: str = Field(alias="TELNYX_API_KEY")
    telnyx_api_url: str = "https://api.telnyx.com/v2"

    # Webhooks
    webhook_timeout: int = 30
    webhook_retry_attempts: int = 3

    # Provisioning
    provisioning_timeout_seconds: int = 60
    max_retry_attempts: int = 3

    class Config:
        env_file = ".env"
        case_sensitive = False

@lru_cache()
def get_settings() -> Settings:
    """Cached settings instance."""
    return Settings()

# exceptions.py
class ProvisioningError(Exception):
    """Base exception for provisioning failures."""

    def __init__(self, message: str, step: str, tenant_id: Optional[str] = None):
        super().__init__(message)
        self.step = step
        self.tenant_id = tenant_id

class TelnyxAPIError(ProvisioningError):
    """Telnyx API communication failure."""
    pass

class DatabaseTransactionError(ProvisioningError):
    """PostgreSQL transaction failure."""
    pass

class WebhookDeliveryError(ProvisioningError):
    """Failed to deliver provisioning webhook."""
    pass
```

## 2. Domain Models & Enums

```python
# models/enums.py
from enum import Enum

class ProvisioningStatus(str, Enum):
    """Tenant lifecycle states."""
    PENDING = "pending"           # Initial state
    PROVISIONING = "provisioning"  # Resources being created
    ACTIVE = "active"             # Fully operational
    FAILED = "failed"             # Unrecoverable error
    PARTIAL = "partial"           # Some resources created

class ProvisioningStep(str, Enum):
    """Atomic provisioning operations."""
    CREATE_TENANT = "create_tenant"
    CREATE_WIDGET = "create_widget"
    GENERATE_API_KEY = "generate_api_key"
    CREATE_TELNYX_CONNECTION = "create_telnyx_connection"
    SEND_WEBHOOK = "send_webhook"

# models/schemas.py
import uuid
from datetime import datetime
from typing import Optional, Dict, Any

from pydantic import BaseModel, ConfigDict, Field, HttpUrl

class TenantBase(BaseModel):
    """Base tenant attributes."""
    name: str = Field(..., min_length=1, max_length=255)
    email: str = Field(..., pattern=r'^[\w\.-]+@[\w\.-]+\.\w+$')
    webhook_url: Optional[HttpUrl] = None

class TenantCreate(TenantBase):
    """Request model for tenant creation."""
    pass

class TenantResponse(BaseModel):
    """Response model with sensitive data."""
    model_config = ConfigDict(from_attributes=True)

    id: uuid.UUID
    name: str
    email: str
    status: ProvisioningStatus
    api_key: Optional[str] = Field(
        default=None,
        description="Only returned on initial creation. Store securely."
    )
    widget_id: Optional[uuid.UUID] = None
    telnyx_connection_id: Optional[str] = None
    provisioning_metadata: Dict[str, Any] = Field(default_factory=dict)
    created_at: datetime
    updated_at: datetime

class ProvisioningLogEntry(BaseModel):
    """Audit trail for provisioning attempts."""
    model_config = ConfigDict(from_attributes=True)

    id: uuid.UUID
    tenant_id: uuid.UUID
    step: str
    status: str  # success, failed, retrying
    error_message: Optional[str] = None
    attempt_number: int = 1
    created_at: datetime

class WebhookPayload(BaseModel):
    """Standardized webhook notification."""
    event: str = "tenant.provisioned"
    tenant_id: uuid.UUID
    status: ProvisioningStatus
    timestamp: datetime = Field(default_factory=datetime.utcnow)
    data: Dict[str, Any] = Field(default_factory=dict)
```

## 3. Database Layer (Repository Pattern)

```python
# services/database.py
import hashlib
import secrets
from contextlib import asynccontextmanager
from datetime import datetime
from typing import Optional, List, Dict, Any, AsyncGenerator

import asyncpg
from asyncpg import Pool, Connection

from config import get_settings
from exceptions import DatabaseTransactionError
from models.enums import ProvisioningStatus, ProvisioningStep
from models.schemas import TenantResponse, ProvisioningLogEntry

class DatabaseService:
    """PostgreSQL repository with connection pooling."""

    def __init__(self):
        self.pool: Optional[Pool] = None
        self.settings = get_settings()

    async def initialize(self) -> None:
        """Initialize connection pool."""
        self.pool = await asyncpg.create_pool(
            dsn=self.settings.database_url,
            min_size=5,
            max_size=self.settings.database_pool_size,
            command_timeout=60,
            server_settings={
                'jit': 'off'  # Disable JIT for short queries
            }
        )

    async def close(self) -> None:
        """Graceful pool shutdown."""
        if self.pool:
            await self.pool.close()

    @asynccontextmanager
    async def transaction(self) -> AsyncGenerator[Connection, None]:
        """Atomic transaction context manager."""
        if not self.pool:
            raise DatabaseTransactionError("Pool not initialized", "connection")

        async with self.pool.acquire() as conn:
            async with conn.transaction():
                yield conn

    async def create_tenant(
        self,
        name: str,
        email: str,
        webhook_url: Optional[str]
    ) -> uuid.UUID:
        """Insert tenant record with initial pending state."""
        async with self.transaction() as conn:
            try:
                tenant_id = await conn.fetchval(
                    """
                    INSERT INTO tenants (name, email, webhook_url, status, created_at, updated_at)
                    VALUES ($1, $2, $3, $4, NOW(), NOW())
                    RETURNING id
                    """,
                    name, email, str(webhook_url) if webhook_url else None,
                    ProvisioningStatus.PENDING.value
                )
                return tenant_id
            except asyncpg.UniqueViolationError as e:
                raise DatabaseTransactionError(
                    f"Tenant with email {email} already exists",
                    "create_tenant"
                ) from e
            except Exception as e:
                raise DatabaseTransactionError(str(e), "create_tenant") from e

    async def update_tenant_status(
        self,
        tenant_id: uuid.UUID,
        status: ProvisioningStatus,
        metadata: Optional[Dict[str, Any]] = None
    ) -> None:
        """Update provisioning state with metadata."""
        async with self.transaction() as conn:
            await conn.execute(
                """
                UPDATE tenants
                SET status = $1, provisioning_metadata = COALESCE(provisioning_metadata, '{}') || $2, updated_at = NOW()
                WHERE id = $3
                """,
                status.value,
                metadata or {},
                tenant_id
            )

    async def create_widget(self, tenant_id: uuid.UUID) -> uuid.UUID:
        """Create default AI voice widget configuration."""
        default_config = {
            "theme": "light",
            "position": "bottom-right",
            "greeting": "Hello! How can I help you today?",
            "voice_id": "default_female",
            "auto_open": False
        }

        async with self.transaction() as conn:
            widget_id = await conn.fetchval(
                """
                INSERT INTO widgets (tenant_id, name, config, created_at)
                VALUES ($1, $2, $3, NOW())
                RETURNING id
                """,
                tenant_id,
                "Default Widget",
                default_config
            )
            return widget_id

    async def create_api_key(
        self,
        tenant_id: uuid.UUID
    ) -> tuple[str, str]:
        """
        Generate secure API key.
        Returns: (plaintext_key, key_id)
        """
        settings = get_settings()

        # Generate cryptographically secure key
        random_component = secrets.token_urlsafe(settings.api_key_length)
        plaintext_key = f"{settings.api_key_prefix}{random_component}"

        # Store hash for verification, prefix for display
        key_hash = hashlib.sha256(plaintext_key.encode()).hexdigest()
        key_id = secrets.token_hex(8)

        async with self.transaction() as conn:
            await conn.execute(
                """
                INSERT INTO api_keys (id, tenant_id, key_hash, prefix, created_at)
                VALUES ($1, $2, $3, $4, NOW())
                """,
                key_id, tenant_id, key_hash, plaintext_key[:8]
            )

        return plaintext_key, key_id

    async def create_telnyx_connection(
        self,
        tenant_id: uuid.UUID,
        connection_id: str,
        phone_number: Optional[str] = None
    ) -> None:
        """Store Telnyx connection reference."""
        async with self.transaction() as conn:
            await conn.execute(
                """
                INSERT INTO telnyx_connections (tenant_id, connection_id, phone_number, status, created_at)
                VALUES ($1, $2, $3, $4, NOW())
                """,
                tenant_id, connection_id, phone_number, "active"
            )

    async def log_provisioning_step(
        self,
        tenant_id: uuid.UUID,
        step: ProvisioningStep,
        status: str,
        error_message: Optional[str] = None,
        attempt_number: int = 1
    ) -> None:
        """Audit logging for observability."""
        async with self.transaction() as conn:
            await conn.execute(
                """
                INSERT INTO provisioning_logs
                (tenant_id, step, status, error_message, attempt_number, created_at)
                VALUES ($1, $2, $3, $4, $5, NOW())
                """,
                tenant_id, step.value, status, error_message, attempt_number
            )

    async def get_tenant(self, tenant_id: uuid.UUID) -> Optional[TenantResponse]:
        """Fetch tenant with related IDs."""
        if not self.pool:
            return None

        async with self.pool.acquire() as conn:
            row = await conn.fetchrow(
                """
                SELECT
                    t.*,
                    w.id as widget_id,
                    tc.connection_id as telnyx_connection_id
                FROM tenants t
                LEFT JOIN widgets w ON w.tenant_id = t.id
                LEFT JOIN telnyx_connections tc ON tc.tenant_id = t.id
                WHERE t.id = $1
                """,
                tenant_id
            )

            if not row:
                return None

            return TenantResponse.model_validate(dict(row))

    async def get_provisioning_history(
        self,
        tenant_id: uuid.UUID
    ) -> List[ProvisioningLogEntry]:
        """Get audit trail for debugging."""
        async with self.pool.acquire() as conn:
            rows = await conn.fetch(
                """
                SELECT * FROM provisioning_logs
                WHERE tenant_id = $1
                ORDER BY created_at DESC
                """,
                tenant_id
            )
            return [ProvisioningLogEntry.model_validate(dict(r)) for r in rows]
```

## 4. External Service Integrations

```python
# services/telnyx.py
import asyncio
from typing import Optional, Dict, Any

import httpx

from config import get_settings
from exceptions import TelnyxAPIError

class TelnyxService:
    """Telnyx API client with circuit breaker pattern."""

    def __init__(self):
        self.settings = get_settings()
        self.client = httpx.AsyncClient(
            base_url=self.settings.telnyx_api_url,
            headers={"Authorization": f"Bearer {self.settings.telnyx_api_key}"},
            timeout=30.0
        )

    async def close(self) -> None:
        await self.client.aclose()

    async def create_connection(
        self,
        tenant_id: str,
        webhook_url: str
    ) -> Dict[str, Any]:
        """
        Create Telnyx SIP connection for voice widget.
        Idempotent: safe to retry.
        """
        try:
            payload = {
                "connection_type": "sip",
                "name": f"Sunaiva-{tenant_id}",
                "webhook_url": webhook_url,
                "webhook_failover_url": f"{webhook_url}/fallback",
                "dtmf_type": "rfc2833",
                "encode": "G711",
                "encrypted": True
            }

            response = await self.client.post(
                "/connections",
                json={"data": payload}
            )
            response.raise_for_status()

            data = response.json()["data"]
            return {
                "connection_id": data["id"],
                "phone_number": None,  # Would purchase number in real scenario
                "status": "active"
            }

        except httpx.HTTPStatusError as e:
            raise TelnyxAPIError(
                f"Telnyx API error: {e.response.text}",
                "create_telnyx_connection",
                tenant_id
            ) from e
        except Exception as e:
            raise TelnyxAPIError(str(e), "create_telnyx_connection", tenant_id) from e

    async def verify_connection(self, connection_id: str) -> bool:
        """Health check for existing connection."""
        try:
            response = await self.client.get(f"/connections/{connection_id}")
            return response.status_code == 200
        except Exception:
            return False

# services/webhook.py
import json
from datetime import datetime
from typing import Dict, Any, Optional

import httpx
from tenacity import (
    retry,
    stop_after_attempt,
    wait_exponential,
    retry_if_exception_type
)

from config import get_settings
from exceptions import WebhookDeliveryError
from models.enums import ProvisioningStatus

class WebhookService:
    """Reliable webhook delivery with exponential backoff."""

    def __init__(self):
        self.settings = get_settings()
        self.client = httpx.AsyncClient(
            timeout=self.settings.webhook_timeout
        )

    async def close(self) -> None:
        await self.client.aclose()

    @retry(
        stop=stop_after_attempt(3),
        wait=wait_exponential(multiplier=1, min=4, max=10),
        retry=retry_if_exception_type((httpx.HTTPStatusError, httpx.NetworkError)),
        reraise=True
    )
    async def _send_with_retry(
        self,
        url: str,
        payload: Dict[str, Any]
    ) -> None:
        """Internal method with retry logic."""
        response = await self.client.post(
            url,
            json=payload,
            headers={
                "Content-Type": "application/json",
                "X-Sunaiva-Event": "tenant.provisioned",
                "X-Sunaiva-Timestamp": datetime.utcnow().isoformat()
            }
        )
        response.raise_for_status()

    async def notify_provisioning_complete(
        self,
        webhook_url: Optional[str],
        tenant_id: str,
        status: ProvisioningStatus,
        metadata: Dict[str, Any]
    ) -> None:
        """
        Notify external systems of provisioning status.
        Failures are logged but don't block provisioning.
        """
        if not webhook_url:
            return

        payload = {
            "


Response:
I'll implement the IP Geolocation analytics system for Sunaiva with privacy-first design, MaxMind GeoIP integration, and comprehensive error handling.

## 1. Database Schema

```sql
-- migrations/V003__analytics_geolocation.sql
CREATE TABLE IF NOT EXISTS widget_analytics (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    widget_id UUID NOT NULL REFERENCES widgets(id) ON DELETE CASCADE,
    session_id VARCHAR(64) NOT NULL,  -- Hashed session identifier

    -- Geolocation (Privacy: No raw IP stored)
    country_code CHAR(2),
    country_name VARCHAR(100),
    region VARCHAR(100),
    city VARCHAR(100),
    latitude DECIMAL(10, 8),
    longitude DECIMAL(11, 8),
    timezone VARCHAR(50),

    -- Anonymized IP (last octet zeroed for IPv4, /48 for IPv6)
    anonymized_ip INET,

    -- Device info
    user_agent_hash VARCHAR(64),
    device_type VARCHAR(20),  -- mobile, desktop, tablet

    -- Timing
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),

    -- Indexes for performance
    CONSTRAINT valid_coords CHECK (
        (latitude IS NULL AND longitude IS NULL) OR
        (latitude BETWEEN -90 AND 90 AND longitude BETWEEN -180 AND 180)
    )
);

-- Performance indexes
CREATE INDEX CONCURRENTLY idx_analytics_widget_time
    ON widget_analytics(widget_id, created_at DESC);
CREATE INDEX CONCURRENTLY idx_analytics_location
    ON widget_analytics(widget_id, country_code, region, city);
CREATE INDEX CONCURRENTLY idx_analytics_session
    ON widget_analytics(session_id, created_at DESC);

-- Materialized view for location aggregation (refreshed every hour)
CREATE MATERIALIZED VIEW analytics_location_summary AS
SELECT
    widget_id,
    country_code,
    country_name,
    region,
    city,
    DATE_TRUNC('day', created_at) as date,
    COUNT(*) as visitor_count,
    COUNT(DISTINCT session_id) as unique_visitors
FROM widget_analytics
WHERE created_at > NOW() - INTERVAL '90 days'
GROUP BY widget_id, country_code, country_name, region, city, DATE_TRUNC('day', created_at);

CREATE UNIQUE INDEX idx_location_summary_unique
    ON analytics_location_summary(widget_id, country_code, region, city, date);
```

## 2. Core Models

```python
# app/models/analytics.py
from datetime import datetime
from typing import Optional, Literal
from pydantic import BaseModel, Field, field_validator
from decimal import Decimal

class GeoLocation(BaseModel):
    """Resolved geolocation data for a visitor."""
    country_code: Optional[str] = Field(None, pattern=r'^[A-Z]{2}$')
    country_name: Optional[str] = None
    region: Optional[str] = None
    city: Optional[str] = None
    latitude: Optional[Decimal] = Field(None, ge=-90, le=90)
    longitude: Optional[Decimal] = Field(None, ge=-180, le=180)
    timezone: Optional[str] = None

    @field_validator('country_code')
    @classmethod
    def uppercase_country(cls, v: Optional[str]) -> Optional[str]:
        return v.upper() if v else None

    def is_resolved(self) -> bool:
        """Check if we have meaningful location data."""
        return self.country_code is not None

class AnonymizedIP(BaseModel):
    """Privacy-safe IP representation."""
    original_ip: str = Field(exclude=True)  # Never serialized
    anonymized: str  # Truncated for storage

    @field_validator('anonymized')
    @classmethod
    def validate_ip_format(cls, v: str) -> str:
        """Ensure IP is properly anonymized."""
        if ':' in v:
            # IPv6: Should end with ::
            if not v.endswith('::'):
                raise ValueError('IPv6 must be anonymized to /48')
        else:
            # IPv4: Should end with .0
            if not v.endswith('.0'):
                raise ValueError('IPv4 must have last octet zeroed')
        return v

class WidgetAnalyticsEvent(BaseModel):
    """Analytics event for widget interaction."""
    widget_id: str = Field(..., pattern=r'^[0-9a-f-]{36}$')
    session_id: str = Field(..., min_length=32, max_length=64)
    location: GeoLocation
    anonymized_ip: AnonymizedIP
    device_type: Literal['mobile', 'desktop', 'tablet', 'unknown'] = 'unknown'
    user_agent_hash: Optional[str] = None
    timestamp: datetime = Field(default_factory=datetime.utcnow)

class LocationAggregation(BaseModel):
    """Aggregated location statistics."""
    country_code: Optional[str]
    country_name: Optional[str]
    region: Optional[str]
    city: Optional[str]
    visitor_count: int = Field(..., ge=0)
    unique_visitors: int = Field(..., ge=0)
    percentage: Decimal = Field(..., ge=0, le=100)
```

## 3. Privacy Utilities

```python
# app/utils/privacy.py
import hashlib
import ipaddress
from typing import Union

def anonymize_ip(ip_str: str) -> str:
    """
    Anonymize IP address for privacy compliance (GDPR/CCPA).

    IPv4: Zero the last octet (192.168.1.100 -> 192.168.1.0)
    IPv6: Zero the last 80 bits (keep first 48 bits, i.e., /48)

    Args:
        ip_str: Raw IP address string

    Returns:
        Anonymized IP string safe for storage

    Raises:
        ValueError: If IP format is invalid
    """
    try:
        ip = ipaddress.ip_address(ip_str)

        if isinstance(ip, ipaddress.IPv4Address):
            # Create IPv4Network with /24 mask, return network address
            network = ipaddress.IPv4Network(f"{ip_str}/24", strict=False)
            return str(network.network_address)

        elif isinstance(ip, ipaddress.IPv6Address):
            # Create IPv6Network with /48 mask
            network = ipaddress.IPv6Network(f"{ip_str}/48", strict=False)
            return str(network.network_address)

    except ValueError as e:
        raise ValueError(f"Invalid IP address format: {ip_str}") from e

def hash_session_id(ip: str, user_agent: str, salt: str) -> str:
    """
    Create deterministic but irreversible session identifier.

    Combines IP, User-Agent, and server salt to create unique
    visitor identifier without storing PII.

    Args:
        ip: Raw IP address (anonymized before storage, but raw used for hash)
        user_agent: Raw User-Agent string
        salt: Server-side secret salt

    Returns:
        SHA-256 hash hex digest (truncated to 32 chars)
    """
    combined = f"{ip}:{user_agent}:{salt}"
    return hashlib.sha256(combined.encode()).hexdigest()[:32]

def detect_device_type(user_agent: str) -> str:
    """Simple device type detection from User-Agent."""
    ua = user_agent.lower()
    if 'mobile' in ua or 'android' in ua or 'iphone' in ua:
        return 'mobile'
    elif 'tablet' in ua or 'ipad' in ua:
        return 'tablet'
    elif 'bot' in ua or 'crawler' in ua:
        return 'bot'
    else:
        return 'desktop'
```

## 4. GeoIP Service

```python
# app/services/geoip.py
import asyncio
import logging
from pathlib import Path
from typing import Optional

import geoip2.database
import geoip2.errors
from geoip2.models import City

from app.models.analytics import GeoLocation
from app.services.redis import RedisCache  # Assuming existing Redis wrapper

logger = logging.getLogger(__name__)

class GeoIPService:
    """
    MaxMind GeoLite2 integration with caching.

    Handles database reader lifecycle and provides async wrapper
    for GeoIP lookups with Redis caching.
    """

    def __init__(
        self,
        db_path: Path,
        redis_cache: RedisCache,
        cache_ttl: int = 86400  # 24 hours
    ):
        self.db_path = db_path
        self.redis = redis_cache
        self.cache_ttl = cache_ttl
        self._reader: Optional[geoip2.database.Reader] = None
        self._lock = asyncio.Lock()

    async def initialize(self) -> None:
        """Initialize the GeoIP database reader."""
        if not self.db_path.exists():
            raise FileNotFoundError(
                f"GeoIP database not found at {self.db_path}. "
                "Download GeoLite2-City.mmdb from MaxMind."
            )

        # geoip2 reader is sync, run in thread pool
        loop = asyncio.get_event_loop()
        self._reader = await loop.run_in_executor(
            None,
            lambda: geoip2.database.Reader(str(self.db_path))
        )
        logger.info(f"Loaded GeoIP database from {self.db_path}")

    async def close(self) -> None:
        """Close database reader."""
        if self._reader:
            loop = asyncio.get_event_loop()
            await loop.run_in_executor(None, self._reader.close)
            self._reader = None

    async def lookup(self, ip_address: str) -> GeoLocation:
        """
        Resolve IP to geolocation with caching.

        Args:
            ip_address: IP address to lookup

        Returns:
            GeoLocation model (empty if lookup fails)
        """
        if not self._reader:
            raise RuntimeError("GeoIP service not initialized")

        # Check private/reserved ranges
        try:
            import ipaddress
            ip = ipaddress.ip_address(ip_address)
            if ip.is_private or ip.is_loopback or ip.is_reserved:
                return GeoLocation()
        except ValueError:
            return GeoLocation()

        # Check cache first
        cache_key = f"geoip:{ip_address}"
        cached = await self.redis.get(cache_key)
        if cached:
            return GeoLocation.model_validate_json(cached)

        # Perform lookup in thread pool (geoip2 is sync)
        try:
            loop = asyncio.get_event_loop()
            result: City = await loop.run_in_executor(
                None,
                self._reader.city,
                ip_address
            )

            location = GeoLocation(
                country_code=result.country.iso_code,
                country_name=result.country.name,
                region=result.subdivisions.most_specific.name,
                city=result.city.name,
                latitude=result.location.latitude,
                longitude=result.location.longitude,
                timezone=result.location.time_zone
            )

            # Cache successful lookup
            await self.redis.setex(
                cache_key,
                self.cache_ttl,
                location.model_dump_json()
            )

            return location

        except geoip2.errors.AddressNotFoundError:
            logger.debug(f"IP not found in GeoIP database: {ip_address}")
            return GeoLocation()

        except Exception as e:
            logger.error(f"GeoIP lookup failed for {ip_address}: {e}")
            return GeoLocation()

    async def batch_lookup(self, ip_addresses: list[str]) -> dict[str, GeoLocation]:
        """Efficient batch lookup with parallel execution."""
        tasks = [self.lookup(ip) for ip in ip_addresses]
        results = await asyncio.gather(*tasks, return_exceptions=True)

        return {
            ip: result if not isinstance(result, Exception) else GeoLocation()
            for ip, result in zip(ip_addresses, results)
        }
```

## 5. Analytics Repository

```python
# app/repositories/analytics.py
from datetime import datetime, timedelta
from typing import Optional
from decimal import Decimal

import asyncpg
from app.models.analytics import (
    WidgetAnalyticsEvent,
    LocationAggregation,
    GeoLocation
)

class AnalyticsRepository:
    """Database operations for analytics with location data."""

    def __init__(self, pool: asyncpg.Pool):
        self.pool = pool

    async def record_event(self, event: WidgetAnalyticsEvent) -> None:
        """
        Store anonymized analytics event.

        Privacy note: Only anonymized IP and hashed session stored.
        """
        async with self.pool.acquire() as conn:
            await conn.execute(
                """
                INSERT INTO widget_analytics (
                    widget_id, session_id, country_code, country_name,
                    region, city, latitude, longitude, timezone,
                    anonymized_ip, device_type, user_agent_hash, created_at
                ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
                """,
                event.widget_id,
                event.session_id,
                event.location.country_code,
                event.location.country_name,
                event.location.region,
                event.location.city,
                event.location.latitude,
                event.location.longitude,
                event.location.timezone,
                str(event.anonymized_ip.anonymized),  # Cast to string for INET
                event.device_type,
                event.user_agent_hash,
                event.timestamp
            )

    async def get_location_analytics(
        self,
        widget_id: str,
        start_date: Optional[datetime] = None,
        end_date: Optional[datetime] = None,
        group_by: str = "country"  # country, region, city
    ) -> list[LocationAggregation]:
        """
        Aggregate visitors by location.

        Args:
            widget_id: Widget UUID
            start_date: Filter start (default 30 days ago)
            end_date: Filter end (default now)
            group_by: Aggregation level
        """
        if not start_date:
            start_date = datetime.utcnow() - timedelta(days=30)
        if not end_date:
            end_date = datetime.utcnow()

        # Dynamic group by clause
        group_columns = {
            "country": ["country_code", "country_name"],
            "region": ["country_code", "country_name", "region"],
            "city": ["country_code", "country_name", "region", "city"]
        }.get(group_by, ["country_code", "country_name"])

        columns_str = ", ".join(group_columns)

        async with self.pool.acquire() as conn:
            rows = await conn.fetch(
                f"""
                WITH location_stats AS (
                    SELECT
                        {columns_str},
                        COUNT(*) as visitor_count,
                        COUNT(DISTINCT session_id) as unique_visitors
                    FROM widget_analytics
                    WHERE widget_id = $1
                        AND created_at BETWEEN $2 AND $3
                    GROUP BY {columns_str}
                ),
                total AS (
                    SELECT SUM(visitor_count) as total_visitors
                    FROM location_stats
                )
                SELECT
                    ls.*,
                    CASE
                        WHEN t.total_visitors > 0
                        THEN ROUND(ls.visitor_count::numeric / t.total_visitors * 100, 2)
                        ELSE 0
                    END as percentage
                FROM location_stats ls
                CROSS JOIN total t
                ORDER BY visitor_count DESC
                """,
                widget_id,
                start_date,
                end_date
            )

            return [
                LocationAggregation(
                    country_code=row.get('country_code'),
                    country_name=row.get('country_name'),
                    region=row.get('region') if 'region' in group_columns else None,
                    city=row.get('city') if 'city' in group_columns else None,
                    visitor_count=row['visitor_count'],
                    unique_visitors=row['unique_visitors'],
                    percentage=Decimal(str(row['percentage']))
                )
                for row in rows
            ]

    async def get_realtime_locations(
        self,
        widget_id: str,
        minutes: int = 5
    ) -> list[GeoLocation]:
        """Get recent unique locations for live map view."""
        cutoff = datetime.utcnow() - timedelta(minutes=minutes)

        async with self.pool.acquire() as conn:
            rows = await conn.fetch(
                """
                SELECT DISTINCT ON (country_code, region, city)
                    country_code, country_name, region, city,
                    latitude, longitude, timezone
                FROM widget_analytics
                WHERE widget_id = $1
                    AND created_at > $2
                    AND latitude IS NOT NULL
                ORDER BY country_code, region, city, created_at DESC
                LIMIT 100
                """,
                widget_id,
                cutoff
            )

            return [
                GeoLocation(
                    country_code=row['country_code'],
                    country_name=row['country_name'],
                    region=row['region'],
                    city=row['city'],
                    latitude=row['latitude'],
                    longitude=row['longitude'],
                    timezone=row['timezone']
                )
                for row in rows
            ]
```

## 6. Middleware Integration

```python
# app/middleware/geoip.py
import logging
from typing import Optional, Callable

from fastapi import Request, Response
from starlette.middleware.base import BaseHTTPMiddleware

from app.services.geoip import GeoIPService
from app.utils.privacy import anonymize_ip, hash_session_id, detect_device_type
from app.config import settings

logger = logging.getLogger(__name__)

class GeoIPMiddleware(BaseHTTPMiddleware):
    """
    Middleware to resolve geolocation and anonymize IP for all requests.

    Attaches


Response:
Here is the production-ready implementation for **TW-INFRA-10**. This module provides a robust, idempotent Telnyx webhook processor with deduplication, stateful conversation management, and comprehensive observability.

## Project Structure

```
sunaiva/
├── src/
│   ├── config.py
│   ├── models.py
│   ├── repository.py
│   ├── service.py
│   ├── security.py
│   └── main.py
└── tests/
    ├── test_unit.py
    └── test_integration.py
```

---

### 1. Configuration (`src/config.py`)

```python
"""Application configuration using Pydantic Settings."""

from functools import lru_cache
from typing import Optional

from pydantic import Field, PostgresDsn, RedisDsn
from pydantic_settings import BaseSettings, SettingsConfigDict

class Settings(BaseSettings):
    """Production-grade configuration with environment variable validation."""

    model_config = SettingsConfigDict(
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore"
    )

    # Application
    app_name: str = "sunaiva-telnyx-processor"
    debug: bool = False
    log_level: str = "INFO"

    # Database
    database_url: PostgresDsn = Field(..., description="Async PostgreSQL connection string")
    db_pool_min_size: int = Field(default=5, ge=1)
    db_pool_max_size: int = Field(default=20, ge=1)

    # Redis
    redis_url: RedisDsn = Field(..., description="Redis connection string for deduplication")
    redis_ttl_seconds: int = Field(default=86400, description="Event deduplication TTL (24h)")

    # Telnyx Security
    telnyx_webhook_public_key: str = Field(..., description="Ed25519 public key for webhook verification")
    telnyx_webhook_tolerance_seconds: int = Field(default=300, description="Timestamp tolerance for replay attacks")

    # Processing
    enable_signature_verification: bool = Field(default=True, description="Toggle for testing environments")

@lru_cache
def get_settings() -> Settings:
    """Cached settings instance for dependency injection."""
    return Settings()
```

---

### 2. Data Models (`src/models.py`)

```python
"""Pydantic v2 models for Telnyx webhook payloads and domain entities."""

from datetime import datetime
from enum import Enum
from typing import Any, Dict, List, Optional
from uuid import UUID

from pydantic import BaseModel, ConfigDict, Field, field_validator

class CallDirection(str, Enum):
    """Inbound or outbound call direction."""
    INBOUND = "inbound"
    OUTBOUND = "outbound"

class CallStatus(str, Enum):
    """Internal conversation state machine."""
    INITIATED = "initiated"
    ANSWERED = "answered"
    HANGUP = "hangup"
    FAILED = "failed"

class TelnyxEventType(str, Enum):
    """Supported Telnyx event types."""
    CALL_INITIATED = "call.initiated"
    CALL_ANSWERED = "call.answered"
    CALL_HANGUP = "call.hangup"
    CALL_RECORDING_SAVED = "call.recording.saved"
    CALL_TRANSCRIPTION = "call.transcription"  # Future-proofing

class TelnyxPayload(BaseModel):
    """Nested payload structure from Telnyx webhooks."""
    model_config = ConfigDict(populate_by_name=True)

    call_control_id: str = Field(..., description="Unique identifier for the call leg")
    call_leg_id: Optional[str] = Field(None, description="Specific leg ID for conference calls")
    from_number: str = Field(..., alias="from", description="Caller number (E.164)")
    to_number: str = Field(..., alias="to", description="Callee number (E.164)")
    direction: CallDirection
    start_time: Optional[datetime] = None
    answer_time: Optional[datetime] = None
    end_time: Optional[datetime] = None
    call_duration: Optional[int] = Field(None, description="Duration in seconds")
    hangup_cause: Optional[str] = None
    hangup_source: Optional[str] = None
    recording_urls: Optional[Dict[str, str]] = Field(
        default=None,
        description="Map of format (wav/mp3) to URL"
    )
    transcription_data: Optional[Dict[str, Any]] = Field(
        default=None,
        description="Transcription text and confidence scores"
    )
    client_state: Optional[str] = None  # Base64 encoded metadata we sent

    @field_validator("from_number", "to_number")
    @classmethod
    def validate_e164(cls, v: str) -> str:
        """Ensure phone numbers are E.164 format."""
        if not v.startswith("+"):
            raise ValueError(f"Phone number {v} must be E.164 format (+1234567890)")
        return v

class TelnyxWebhookData(BaseModel):
    """Top-level data object containing event metadata."""
    event_type: TelnyxEventType
    id: str = Field(..., description="Unique event ID for deduplication (UUID v4)")
    occurred_at: datetime
    payload: TelnyxPayload
    record_type: str = "event"

class TelnyxWebhookPayload(BaseModel):
    """Root webhook envelope."""
    data: TelnyxWebhookData

    @property
    def event_id(self) -> str:
        """Convenience accessor for deduplication logic."""
        return self.data.id

    @property
    def call_control_id(self) -> str:
        """Convenience accessor for conversation lookup."""
        return self.data.payload.call_control_id

class Conversation(BaseModel):
    """Internal domain model for database representation."""
    model_config = ConfigDict(from_attributes=True)

    id: Optional[UUID] = None
    telnyx_call_control_id: str
    from_number: str
    to_number: str
    direction: CallDirection
    status: CallStatus
    started_at: Optional[datetime] = None
    answered_at: Optional[datetime] = None
    ended_at: Optional[datetime] = None
    duration_seconds: Optional[int] = None
    recording_url: Optional[str] = None
    transcription_text: Optional[str] = None
    hangup_cause: Optional[str] = None
    raw_events: List[Dict[str, Any]] = Field(default_factory=list)
    created_at: Optional[datetime] = None
    updated_at: Optional[datetime] = None
```

---

### 3. Security (`src/security.py`)

```python
"""Webhook signature verification using Telnyx Ed25519 scheme."""

import base64
import hashlib
import hmac
import time
from typing import Optional

from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
from cryptography.exceptions import InvalidSignature

from config import get_settings

class SecurityError(Exception):
    """Base security exception."""
    pass

class InvalidSignatureError(SecurityError):
    """Raised when webhook signature verification fails."""
    pass

class TimestampToleranceError(SecurityError):
    """Raised when webhook timestamp is too old (replay attack prevention)."""
    pass

def verify_telnyx_signature(
    payload_bytes: bytes,
    signature_header: Optional[str],
    timestamp_header: Optional[str],
    tolerance_seconds: int = 300
) -> bool:
    """
    Verify Ed25519 signature from Telnyx webhook headers.

    Args:
        payload_bytes: Raw request body
        signature_header: Base64-encoded signature from 'Telnyx-Signature' header
        timestamp_header: Unix timestamp from 'Telnyx-Timestamp' header
        tolerance_seconds: Maximum age of request to prevent replay attacks

    Returns:
        bool: True if valid

    Raises:
        InvalidSignatureError: If signature is invalid or missing
        TimestampToleranceError: If timestamp is outside tolerance window
    """
    settings = get_settings()

    if not settings.enable_signature_verification:
        return True

    if not signature_header or not timestamp_header:
        raise InvalidSignatureError("Missing required signature headers")

    # Check timestamp tolerance
    try:
        timestamp = int(timestamp_header)
    except (ValueError, TypeError):
        raise InvalidSignatureError("Invalid timestamp format")

    now = int(time.time())
    if abs(now - timestamp) > tolerance_seconds:
        raise TimestampToleranceError(
            f"Timestamp {timestamp} is outside tolerance window of {tolerance_seconds}s"
        )

    # Construct signed payload: timestamp|body
    signed_payload = f"{timestamp}|".encode('utf-8') + payload_bytes

    try:
        # Load public key
        public_key_bytes = base64.b64decode(settings.telnyx_webhook_public_key)
        public_key = Ed25519PublicKey.from_public_bytes(public_key_bytes)

        # Decode signature
        signature = base64.b64decode(signature_header)

        # Verify
        public_key.verify(signature, signed_payload)
        return True

    except (InvalidSignature, ValueError, base64.binascii.Error) as e:
        raise InvalidSignatureError(f"Signature verification failed: {e}")
```

---

### 4. Repository Layer (`src/repository.py`)

```python
"""Database abstraction using asyncpg with connection pooling."""

import json
import logging
from contextlib import asynccontextmanager
from typing import List, Optional
from uuid import UUID

import asyncpg
from asyncpg import Pool

from models import CallStatus, Conversation

logger = logging.getLogger(__name__)

class ConversationRepository:
    """PostgreSQL repository for conversation lifecycle management."""

    def __init__(self, pool: Pool):
        self.pool = pool

    @asynccontextmanager
    async def transaction(self):
        """Provide a transactional scope around a series of operations."""
        async with self.pool.acquire() as conn:
            async with conn.transaction():
                yield conn

    async def upsert_conversation_initiated(self, conversation: Conversation) -> UUID:
        """
        Create new conversation or update if exists (idempotent).

        Handles race conditions where initiated arrives after answered (rare but possible).
        """
        query = """
        INSERT INTO conversations (
            telnyx_call_control_id, from_number, to_number, direction,
            status, started_at, raw_events, created_at, updated_at
        ) VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, NOW(), NOW())
        ON CONFLICT (telnyx_call_control_id)
        DO UPDATE SET
            raw_events = conversations.raw_events || EXCLUDED.raw_events,
            updated_at = NOW()
        RETURNING id
        """

        raw_event = {
            "type": "initiated",
            "timestamp": conversation.started_at.isoformat() if conversation.started_at else None
        }

        async with self.pool.acquire() as conn:
            record_id = await conn.fetchval(
                query,
                conversation.telnyx_call_control_id,
                conversation.from_number,
                conversation.to_number,
                conversation.direction.value,
                CallStatus.INITIATED.value,
                conversation.started_at,
                json.dumps([raw_event])
            )
            logger.info(f"Upserted conversation {record_id} for call {conversation.telnyx_call_control_id}")
            return record_id

    async def update_conversation_answered(self, call_control_id: str, answer_time: Optional[datetime]) -> None:
        """Mark conversation as answered with timestamp."""
        query = """
        UPDATE conversations
        SET
            status = $1,
            answered_at = $2,
            raw_events = raw_events || $3::jsonb,
            updated_at = NOW()
        WHERE telnyx_call_control_id = $4
        """

        event_data = json.dumps([{"type": "answered", "timestamp": answer_time.isoformat() if answer_time else None}])

        async with self.pool.acquire() as conn:
            result = await conn.execute(query, CallStatus.ANSWERED.value, answer_time, event_data, call_control_id)
            if result == "UPDATE 0":
                logger.warning(f"Answered event for unknown call_control


Response:
 Here's the production-ready implementation for **TW-INFRA-12**, featuring a FastAPI backend with PostgreSQL/Redis integration and a modular widget.js architecture with dynamic imports and progress tracking.

## 1. Database Schema

```sql
-- Enable UUID extension
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

-- Widget configurations
CREATE TABLE widgets (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    tenant_id UUID NOT NULL,
    name VARCHAR(255) NOT NULL,
    config JSONB NOT NULL DEFAULT '{}',
    version VARCHAR(20) DEFAULT '1.0.0',
    is_active BOOLEAN DEFAULT true,
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
);

-- Module loading analytics
CREATE TABLE widget_analytics (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    widget_id UUID REFERENCES widgets(id) ON DELETE CASCADE,
    event_type VARCHAR(50) NOT NULL, -- 'module_load', 'module_error', 'chat_open', etc.
    module_name VARCHAR(50),
    load_time_ms INTEGER,
    error_message TEXT,
    user_agent TEXT,
    ip_hash VARCHAR(64), -- hashed for privacy
    metadata JSONB DEFAULT '{}',
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
);

-- Indexes for performance
CREATE INDEX idx_analytics_widget_time ON widget_analytics(widget_id, created_at);
CREATE INDEX idx_analytics_event ON widget_analytics(event_type, created_at);
CREATE INDEX idx_widgets_tenant ON widgets(tenant_id) WHERE is_active = true;
```

## 2. Backend Implementation

```python
"""
Sunaiva Widget Service - Lazy Loading Module System
Production-ready FastAPI backend with asyncpg and Redis
"""

import asyncio
import hashlib
import json
import time
from contextlib import asynccontextmanager
from datetime import datetime
from typing import Any, Dict, Literal, Optional

import asyncpg
import redis.asyncio as redis
from fastapi import (
    BackgroundTasks,
    Depends,
    FastAPI,
    HTTPException,
    Query,
    Request,
    Response,
)
from fastapi.middleware.cors import CORSMiddleware
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import JSONResponse, PlainTextResponse
from jinja2 import Template
from pydantic import BaseModel, Field, field_validator

# Configuration
class Settings(BaseModel):
    database_url: str = "postgresql://user:pass@localhost/sunaiva"
    redis_url: str = "redis://localhost:6379/0"
    widget_cdn_base: str = "https://cdn.sunaiva.com"
    api_version: str = "v1"

    class Config:
        env_file = ".env"

settings = Settings()

# Global connection pools
db_pool: Optional[asyncpg.Pool] = None
redis_client: Optional[redis.Redis] = None

# Pydantic Models (v2)
class WidgetConfigResponse(BaseModel):
    widget_id: str
    theme: Dict[str, Any]
    features: Dict[str, bool]
    module_base_url: str
    version: str

class AnalyticsPayload(BaseModel):
    event_type: Literal["module_load", "module_error", "chat_open", "voice_open", "page_view", "progress_update"]
    module_name: Optional[str] = None
    load_time_ms: Optional[int] = None
    error_message: Optional[str] = None
    metadata: Dict[str, Any] = Field(default_factory=dict)
    timestamp: float = Field(default_factory=time.time)

    @field_validator('load_time_ms')
    @classmethod
    def validate_load_time(cls, v):
        if v is not None and v < 0:
            raise ValueError('load_time_ms must be positive')
        return v

class WidgetCreate(BaseModel):
    tenant_id: str
    name: str
    config: Dict[str, Any] = Field(default_factory=dict)

# Widget Core Template (Optimized for <5KB gzipped)
# Note: In production, this would be pre-minified and stored in Redis/DB
WIDGET_CORE_TEMPLATE = """
(function(global,document){'use strict';
const CONFIG={{config|tojson}};
const BASE=CONFIG.module_base_url;
const CACHE=new Map();
const EMIT=(e,d)=>{global.dispatchEvent(new CustomEvent('sunaiva:'+e,{detail:d}));};
class SunaivaCore{
constructor(){this.modules=new Map();this.loading=new Set();this.init();}
init(){this.injectCSS();this.createUI();this.scheduleAnalytics();}
injectCSS(){const s=document.createElement('style');s.textContent=`
#sun-w{position:fixed;bottom:20px;right:20px;z-index:9999;cursor:pointer;}
#sun-w .sun-b{width:60px;height:60px;border-radius:50%;background:{{theme.primary_color}};box-shadow:0 4px 12px rgba(0,0,0,.15);display:flex;align-items:center;justify-content:center;transition:transform .2s;}
#sun-w .sun-b:hover{transform:scale(1.1);}
#sun-w .sun-i{width:30px;height:30px;background:#fff;mask:url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'%3E%3Cpath d='M20 2H4c-1.1 0-2 .9-2 2v18l4-4h14c1.1 0 2-.9 2-2V4c0-1.1-.9-2-2-2z'/%3E%3C/svg%3E") center/contain no-repeat;}
#sun-l{position:fixed;bottom:90px;right:20px;width:40px;height:40px;border:3px solid {{theme.primary_color}}22;border-top


Response:
 I'll implement a production-ready WebSocket load balancing solution with HAProxy (recommended for open-source WebSocket support) and Nginx configurations, along with a FastAPI backend implementing health checks and connection management.

## 1. HAProxy Configuration (Recommended)

```haproxy
# haproxy/haproxy.cfg
# Production-ready HAProxy configuration for Sunaiva WebSocket load balancing

global
    # Performance tuning for high WebSocket concurrency
    maxconn 50000                    # Global max connections (5x per-backend limit for safety)
    ulimit-n 100000                # File descriptor limit
    nbthread 4                     # Multi-threading for modern HAProxy

    # Security hardening
    ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
    ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets

    # Logging
    log stdout local0 info
    daemon

defaults
    mode http
    log global
    option httplog

    # Timeouts optimized for WebSockets
    timeout connect 5s             # Backend connection timeout
    timeout client 30s             # Client idle timeout
    timeout server 30s             # Server idle timeout
    timeout tunnel 1h              # CRITICAL: WebSocket tunnel timeout (keep open)
    timeout http-keep-alive 1s     # Reuse connections for HTTP

    # Health check defaults
    option httpchk GET /health
    http-check expect status 200

    # Retry logic
    retries 3
    option redispatch              # Retry on different server if failure

frontend sunaiva_frontend
    bind *:80
    bind *:443 ssl crt /etc/ssl/certs/sunaiva.pem alpn h2,http/1.1

    # Request logging with unique IDs for tracing
    unique-id-format %{+X}o\ %ci:%cp_%fi:%fp_%Ts_%rt:%pid
    unique-id-header X-Request-ID

    # ACL to detect WebSocket upgrade requests
    acl is_websocket hdr(Upgrade) -i websocket
    acl is_connection_upgrade hdr_beg(Connection) -i upgrade

    # Redirect HTTP to HTTPS (optional, remove if not needed)
    redirect scheme https code 301 if !{ ssl_fc }

    # Route WebSocket traffic to dedicated backend
    use_backend websocket_backend if is_websocket is_connection_upgrade
    default_backend http_backend

backend websocket_backend
    description "WebSocket backend with sticky sessions"

    # Sticky sessions using cookie-based persistence
    # 'insert' adds cookie if not present, 'indirect' removes it from request to backend
    # 'nocache' prevents caching
    cookie SUNAIVA_WIDGET_SESSION insert indirect nocache maxidle 30m maxlife 8h

    # WebSocket-specific health check
    # Tests actual WebSocket upgrade capability
    option httpchk GET /health/ws HTTP/1.1\r\nHost:\ localhost\r\nUpgrade:\ websocket\r\nConnection:\ Upgrade\r\nSec-WebSocket-Key:\ dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version:\ 13
    http-check expect status 101    # Expect Switching Protocols (101)

    # Backend servers with maxconn enforcement (10,000 per backend)
    server ws1 backend1:8000 cookie ws1 maxconn 10000 check inter 5s rise 2 fall 3
    server ws2 backend2:8000 cookie ws2 maxconn 10000 check inter 5s rise 2 fall 3
    server ws3 backend3:8000 cookie ws3 maxconn 10000 check inter 5s rise 2 fall 3 backup

    # Connection queuing (optional but recommended)
    # If maxconn reached, queue up to 1000 connections per backend
    maxqueue 1000
    timeout queue 30s

backend http_backend
    description "Standard HTTP backend for API requests"

    # Round-robin for stateless HTTP requests
    balance roundrobin

    # Standard HTTP health check
    option httpchk GET /health

    server api1 backend1:8000 maxconn 10000 check
    server api2 backend2:8000 maxconn 10000 check
    server api3 backend3:8000 maxconn 10000 check backup

# Statistics page (disable in production or secure it)
listen stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 10s
    stats admin if TRUE
```

## 2. Nginx Configuration (Alternative)

```nginx
# nginx/nginx.conf
# Production Nginx configuration for WebSocket load balancing
# Note: Requires Nginx Plus for advanced health checks and sticky sessions
# For open source, use ip_hash (limited) or consider HAProxy above

events {
    worker_connections 4096;
    use epoll;
    multi_accept on;
}

http {
    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    # Logging format with connection tracking
    log_format websocket_log '$remote_addr - $remote_user [$time_local] '
                            '"$request" $status $body_bytes_sent '
                            '"$http_referer" "$http_user_agent" '
                            'conn_id=$connection_id '
                            'upstream=$upstream_addr '
                            'upgrade=$http_upgrade';

    access_log /var/log/nginx/access.log websocket_log;

    # Upstream configuration with sticky sessions (ip_hash for open source)
    upstream websocket_backend {
        # Sticky sessions based on client IP (limitation: NAT issues)
        ip_hash;

        # Connection limits per backend (requires Nginx Plus for max_conns)
        # For open source, use limit_conn zone per server

        server backend1:8000 weight=5;
        server backend2:8000 weight=5;

        # Keepalive connections to backends
        keepalive 100;
        keepalive_timeout 60s;
        keepalive_requests 1000;
    }

    # Rate limiting zone (optional)
    limit_conn_zone $binary_remote_addr zone=addr:10m;
    limit_conn addr 10;

    server {
        listen 80;
        listen 443 ssl http2;
        server_name widget.sunava.com;

        ssl_certificate /etc/ssl/certs/sunaiva.crt;
        ssl_certificate_key /etc/ssl/certs/sunaiva.key;
        ssl_protocols TLSv1.2 TLSv1.3;

        # WebSocket endpoint
        location /ws/ {
            proxy_pass http://websocket_backend;

            # WebSocket support headers
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";

            # Critical for WebSocket longevity
            proxy_read_timeout 86400s;
            proxy_send_timeout 86400s;

            # Headers for backend identification
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-Request-ID $request_id;

            # Disable buffering for real-time communication
            proxy_buffering off;
            proxy_cache off;
        }

        # Health check endpoint (pass through to backend)
        location /health {
            proxy_pass http://websocket_backend;
            proxy_http_version 1.1;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            access_log off;  # Reduce noise
        }

        # Static widget.js serving
        location /widget.js {
            alias /var/www/sunaiva/widget.js;
            add_header Content-Type application/javascript;
            expires 1h;
        }
    }
}
```

## 3. FastAPI Backend with WebSocket Health Checks

```python
# backend/main.py
"""
Sunaiva WebSocket Backend with Load Balancer Integration
Supports WebSocket health checks, connection tracking, and sticky session validation.
"""

import asyncio
import json
import logging
import signal
import sys
import uuid
from contextlib import asynccontextmanager
from typing import Dict, Optional, Set
from dataclasses import dataclass, field

import asyncpg
import redis.asyncio as redis
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, status, Depends, HTTPException
from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field

# Configure structured logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

@dataclass
class ConnectionMetrics:
    """Tracks connection metrics for load balancer health decisions."""
    active_connections: int = 0
    total_connections: int = 0
    rejected_connections: int = 0
    server_id: str = field(default_factory=lambda: str(uuid.uuid4())[:8])
    start_time: float = field(default_factory=lambda: asyncio.get_event_loop().time())

class ConnectionManager:
    """
    Manages WebSocket connections with capacity enforcement.
    Enforces max 10,000 connections per instance (matching HAProxy maxconn).
    """

    def __init__(self, max_connections: int = 10000):
        self.max_connections = max_connections
        self.active_connections: Dict[str, WebSocket] = {}
        self.metrics = ConnectionMetrics()
        self._lock = asyncio.Lock()

    async def connect(self, websocket: WebSocket, client_id: str) -> bool:
        """
        Attempt to accept a WebSocket connection.

        Args:
            websocket: The WebSocket object
            client_id: Unique identifier for the client

        Returns:
            bool: True if connected successfully, False if at capacity
        """
        async with self._lock:
            if len(self.active_connections) >= self.max_connections:
                self.metrics.rejected_connections += 1
                logger.warning(f"Connection rejected: capacity full ({self.max_connections})")
                await websocket.close(code=1008, reason="Server at capacity")
                return False

            await websocket.accept()
            self.active_connections[client_id] = websocket
            self.metrics.active_connections = len(self.active_connections)
            self.metrics.total_connections += 1

            logger.info(
                f"Client {client_id} connected. "
                f"Active: {self.metrics.active_connections}/{self.max_connections}"
            )
            return True

    async def disconnect(self, client_id: str):
        """Remove connection tracking."""
        async with self._lock:
            if client_id in self.active_connections:
                del self.active_connections[client_id]
                self.metrics.active_connections = len(self.active_connections)
                logger.info(f"Client {client_id} disconnected. Active: {self.metrics.active_connections}")

    async def broadcast(self, message: dict, exclude: Optional[str] = None):
        """Broadcast message to all connected clients."""
        disconnected = []
        for client_id, connection in self.active_connections.items():
            if client_id != exclude:
                try:
                    await connection.send_json(message)
                except Exception as e:
                    logger.error(f"Failed to send to {client_id}: {e}")
                    disconnected.append(client_id)

        # Cleanup failed connections
        for client_id in disconnected:
            await self.disconnect(client_id)

    def get_metrics(self) -> dict:
        """Return current connection metrics."""
        return {
            "server_id": self.metrics.server_id,
            "active_connections": self.metrics.active_connections,
            "total_connections": self.metrics.total_connections,
            "rejected_connections": self.metrics.rejected_connections,
            "uptime_seconds": asyncio.get_event_loop().time() - self.metrics.start_time,
            "capacity_percentage": (self.metrics.active_connections / self.max_connections) * 100,
            "healthy": self.metrics.active_connections < self.max_connections
        }

# Global connection manager
manager = ConnectionManager(max_connections=10000)

class HealthResponse(BaseModel):
    """Health check response model."""
    status: str
    server_id: str
    active_connections: int
    max_connections: int = Field(default=10000, description="Matches HAProxy maxconn")
    websocket_capable: bool = True
    timestamp: str

class WebSocketTestMessage(BaseModel):
    """WebSocket test message format."""
    type: str
    timestamp: float
    server_id: str
    message: Optional[str] = None

@asynccontextmanager
async def lifespan(app: FastAPI):
    """Application lifespan manager."""
    # Startup
    logger.info(f"Starting Sunaiva backend instance: {manager.metrics.server_id}")

    # Setup signal handlers for graceful shutdown
    def signal_handler(sig, frame):
        logger.info("Shutdown signal received, closing connections...")
        sys.exit(0)

    signal.signal(signal.SIGTERM, signal_handler)
    signal.signal(signal.SIGINT, signal_handler)

    yield

    # Shutdown
    logger.info("Shutting down, cleaning up connections...")
    # Close all websocket connections gracefully
    close_tasks = []
    for client_id, websocket in list(manager.active_connections.items()):
        close_tasks.append(websocket.close(code=1001, reason="Server shutting down"))
    if close_tasks:
        await asyncio.gather(*close_tasks, return_exceptions=True)

app =