Response:
I'll implement the widget performance optimization story with a complete FastAPI backend serving an optimized widget.js file, along with the TypeScript widget implementation featuring lazy loading, Shadow DOM, and performance optimizations.

## 1. Widget Service Layer

```python
# src/services/widget_service.py
"""
Widget generation and optimization service for Sunaiva.
Handles dynamic widget.js generation, minification, and caching.
"""

import hashlib
import json
import logging
from typing import Optional, Dict, Any, AsyncIterator
from datetime import datetime, timedelta

import aiohttp
from fastapi import HTTPException, status
from redis.asyncio import Redis

from src.models.widget import WidgetConfig, WidgetCacheEntry, PerformanceMetrics
from src.core.config import settings

logger = logging.getLogger(__name__)

class WidgetService:
    """
    Service for generating optimized widget JavaScript bundles.
    Implements code splitting, dynamic imports, and aggressive caching.
    """

    def __init__(self, redis_client: Redis, db_pool: Optional[Any] = None):
        self.redis = redis_client
        self.db_pool = db_pool
        self.widget_template = self._load_widget_template()
        self.chunk_manifest: Dict[str, str] = {}

    def _load_widget_template(self) -> str:
        """Load the base widget template with performance optimizations."""
        # In production, this loads from compiled TypeScript output
        return """
        /* Sunaiva AI Widget - Performance Optimized Build */
        (function() {
            'use strict';

            // Preconnect hints for critical resources
            const preconnect = () => {
                const hosts = {{preconnect_hosts}};
                hosts.forEach(host => {
                    const link = document.createElement('link');
                    link.rel = 'preconnect';
                    link.href = host;
                    link.crossOrigin = 'anonymous';
                    document.head.appendChild(link);
                });
            };

            // Shadow DOM initialization for style isolation
            const initShadowDOM = (config) => {
                const container = document.getElementById(config.containerId) || document.body;
                const shadow = container.attachShadow({ mode: 'open', delegatesFocus: true });

                // Inject critical CSS inline to prevent FOUC
                const style = document.createElement('style');
                style.textContent = {{critical_css}};
                shadow.appendChild(style);

                return shadow;
            };

            // Lazy loader for audio/video components
            const loadMediaComponents = async () => {
                const [{ AudioEngine }, { VideoComponent }] = await Promise.all([
                    import(/* webpackChunkName: "audio" */ '{{audio_chunk_url}}'),
                    import(/* webpackChunkName: "video" */ '{{video_chunk_url}}')
                ]);
                return { AudioEngine, VideoComponent };
            };

            // Initialize widget with performance budget checks
            const init = async (config) => {
                const startTime = performance.now();

                try {
                    preconnect();
                    const shadow = initShadowDOM(config);

                    // Lazy load heavy components only when needed
                    if (config.enableVoice) {
                        const mediaComponents = await loadMediaComponents();
                        window.__sunaivaMedia = mediaComponents;
                    }

                    // Mark performance metrics
                    const initTime = performance.now() - startTime;
                    if (initTime > 50) {
                        console.warn('[Sunaiva] Widget initialization exceeded budget:', initTime);
                    }

                    // Initialize React/Vue components inside shadow DOM
                    const root = document.createElement('div');
                    root.id = 'sunaiva-root';
                    shadow.appendChild(root);

                    // Dynamic import of main app component
                    const { default: App } = await import('{{main_chunk_url}}');
                    App.mount(root, config);

                } catch (error) {
                    console.error('[Sunaiva] Widget initialization failed:', error);
                    throw error;
                }
            };

            // Expose global API
            window.Sunaiva = { init };
        })();
        """

    async def generate_widget_script(
        self,
        business_id: str,
        config: Optional[WidgetConfig] = None
    ) -> str:
        """
        Generate optimized widget.js for a specific business.

        Args:
            business_id: Unique identifier for the business
            config: Optional custom configuration overrides

        Returns:
            Minified JavaScript string ready for delivery

        Raises:
            HTTPException: If generation fails or business not found
        """
        try:
            # Check cache first
            cache_key = f"widget:js:{business_id}"
            cached = await self.redis.get(cache_key)

            if cached:
                logger.debug(f"Cache hit for widget {business_id}")
                return cached.decode('utf-8')

            # Fetch business configuration from DB
            db_config = await self._fetch_business_config(business_id)
            if not db_config:
                raise HTTPException(
                    status_code=status.HTTP_404_NOT_FOUND,
                    detail=f"Business {business_id} not found"
                )

            # Merge configurations
            final_config = {**db_config, **(config.dict() if config else {})}

            # Generate script with substitutions
            script = await self._render_widget_script(final_config)

            # Minify (in production, use terser or esbuild)
            minified = self._minify_js(script)

            # Cache with TTL based on config stability
            ttl = 3600 if not config else 300  # 1hr for standard, 5min for custom
            await self.redis.setex(cache_key, ttl, minified)

            # Store metrics
            await self._record_generation_metrics(business_id, len(minified))

            return minified

        except HTTPException:
            raise
        except Exception as e:
            logger.error(f"Widget generation failed for {business_id}: {e}")
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail="Widget generation failed"
            )

    async def _fetch_business_config(self, business_id: str) -> Optional[Dict[str, Any]]:
        """Fetch widget configuration from database."""
        if not self.db_pool:
            # Return default config if no DB available
            return {
                "containerId": "sunaiva-widget",
                "enableVoice": True,
                "enableVideo": True,
                "theme": "light",
                "telnyxToken": None,
                "webrtcServers": ["wss://rtc.telnyx.com"]
            }

        async with self.db_pool.acquire() as conn:
            row = await conn.fetchrow(
                "SELECT config FROM widgets WHERE business_id = $1 AND active = true",
                business_id
            )
            return json.loads(row['config']) if row else None

    async def _render_widget_script(self, config: Dict[str, Any]) -> str:
        """Render widget template with configuration substitutions."""
        # Critical CSS for preventing FOUC (Flash of Unstyled Content)
        critical_css = """
            :host { all: initial; font-family: system-ui, -apple-system, sans-serif; }
            #sunaiva-root { position: fixed; bottom: 20px; right: 20px; z-index: 9999; }
            .sunaiva-button { width: 60px; height: 60px; border-radius: 50%; /* ... */ }
        """

        # Preconnect to Telnyx and CDN
        preconnect_hosts = json.dumps([
            "https://rtc.telnyx.com",
            "https://cdn.sunaiva.com",
            *config.get("webrtcServers", [])
        ])

        # Chunk URLs with cache-busting hashes
        audio_chunk = f"/static/js/audio.{self.chunk_manifest.get('audio', 'latest')}.js"
        video_chunk = f"/static/js/video.{self.chunk_manifest.get('video', 'latest')}.js"
        main_chunk = f"/static/js/main.{self.chunk_manifest.get('main', 'latest')}.js"

        script = self.widget_template
        script = script.replace("{{preconnect_hosts}}", preconnect_hosts)
        script = script.replace("{{critical_css}}", json.dumps(critical_css))
        script = script.replace("{{audio_chunk_url}}", audio_chunk)
        script = script.replace("{{video_chunk_url}}", video_chunk)
        script = script.replace("{{main_chunk_url}}", main_chunk)

        # Inject runtime configuration
        runtime_config = json.dumps({
            **config,
            "timestamp": datetime.utcnow().isoformat(),
            "version": settings.WIDGET_VERSION
        })

        return f"{script}\nwindow.__SUNAIVA_CONFIG__ = {runtime_config};"

    def _minify_js(self, script: str) -> str:
        """
        Basic JavaScript minification.
        In production, use esbuild or terser via subprocess or Node API.
        """
        # Remove comments
        import re
        # Remove single-line comments
        script = re.sub(r'//.*$', '', script, flags=re.MULTILINE)
        # Remove multi-line comments
        script = re.sub(r'/\*[\s\S]*?\*/', '', script)
        # Remove extra whitespace
        script = re.sub(r'\s+', ' ', script)
        return script.strip()

    async def _record_generation_metrics(self, business_id: str, size_bytes: int):
        """Record widget generation metrics for monitoring."""
        metrics_key = f"metrics:widget:{business_id}"
        timestamp = datetime.utcnow().isoformat()

        pipeline = self.redis.pipeline()
        pipeline.hincrby(metrics_key, "generations", 1)
        pipeline.hset(metrics_key, "last_size", size_bytes)
        pipeline.hset(metrics_key, "last_generated", timestamp)
        pipeline.expire(metrics_key, 86400)  # 24 hour retention
        await pipeline.execute()

    async def invalidate_cache(self, business_id: str):
        """Invalidate widget cache for a business (on config update)."""
        await self.redis.delete(f"widget:js:{business_id}")
        logger.info(f"Cache invalidated for business {business_id}")

    async def get_performance_metrics(self, business_id: str) -> PerformanceMetrics:
        """Retrieve performance metrics for widget instances."""
        metrics_key = f"metrics:widget:{business_id}"
        data = await self.redis.hgetall(metrics_key)

        if not data:
            return PerformanceMetrics(business_id=business_id)

        return PerformanceMetrics(
            business_id=business_id,
            generations=int(data.get(b"generations", 0)),
            last_size=int(data.get(b"last_size", 0)),
            avg_load_time_ms=float(data.get(b"avg_load_time", 0)),
            cache_hit_rate=float(data.get(b"cache_hit_rate", 0))
        )
```

## 2. Widget API Routes

```python
# src/api/routes/widget.py
"""
Widget delivery API endpoints with performance optimizations.
Includes compression, HTTP/2 Server Push hints, and aggressive caching.
"""

import gzip
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Request, Response, Header
from fastapi.responses import PlainTextResponse, StreamingResponse
from fastapi.middleware.cors import CORSMiddleware
from redis.asyncio import Redis
import asyncpg
import logging

from src.services.widget_service import WidgetService
from src.models.widget import WidgetConfig, WidgetResponse, PerformanceMetrics
from src.core.dependencies import get_redis, get_db_pool
from src.core.config import settings

logger = logging.getLogger(__name__)
router = APIRouter(prefix="/widget", tags=["widget"])

async def get_widget_service(
    redis: Redis = Depends(get_redis),
    db_pool: asyncpg.Pool = Depends(get_db_pool)
) -> WidgetService:
    """Dependency injection for widget service."""
    return WidgetService(redis, db_pool)

@router.get(
    "/{business_id}.js",
    response_class=PlainTextResponse,
    responses={
        200: {"description": "Widget JavaScript bundle"},
        404: {"description": "Business not found"},
        500: {"description": "Generation error"}
    }
)
async def get_widget_script(
    business_id: str,
    request: Request,
    widget_service: WidgetService = Depends(get_widget_service),
    accept_encoding: Optional[str] = Header(None),
    if_none_match: Optional[str] = Header(None)
) -> Response:
    """
    Serve optimized widget.js for a business.

    Features:
    - ETag-based caching (304 Not Modified support)
    - Brotli/Gzip compression based on client Accept-Encoding
    - Preconnect headers for WebRTC servers
    - Immutable cache headers for long-term caching
    """
    try:
        # Generate ETag based on business_id and version
        etag = f'W/"{hash(business_id + settings.WIDGET_VERSION)}"'

        # Check if client has fresh copy
        if if_none_match == etag:
            return Response(status_code=304)

        # Generate or retrieve cached script
        script = await widget_service.generate_widget_script(business_id)

        # Compression handling
        content = script.encode('utf-8')
        headers = {
            "Content-Type": "application/javascript; charset=utf-8",
            "ETag": etag,
            "Cache-Control": "public, max-age=3600, immutable",
            "Vary": "Accept-Encoding",
            # Performance headers
            "X-DNS-Prefetch-Control": "on",
            "Link": "<https://rtc.telnyx.com>; rel=preconnect, "
                   "<https://cdn.sunaiva.com>; rel=preconnect",
            # Security headers
            "X-Content-Type-Options": "nosniff",
            "Referrer-Policy": "strict-origin-when-cross-origin"
        }

        # Apply compression if supported and beneficial (>1KB)
        if accept_encoding and len(content) > 1024:
            if "br" in accept_encoding and settings.ENABLE_BROTLI:
                # In production, use brotli compression
                # For now, skip if brotli not available
                pass
            elif "gzip" in accept_encoding:
                content = gzip.compress(content, compresslevel=6)
                headers["Content-Encoding"] = "gzip"

        return Response(
            content=content,
            headers=headers,
            media_type="application/javascript"
        )

    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error serving widget for {business_id}: {e}")
        raise HTTPException(status_code=500, detail="Failed to serve widget")

@router.post(
    "/{business_id}/config",
    response_model=WidgetResponse
)
async def update_widget_config(
    business_id: str,
    config: WidgetConfig,
    widget_service: WidgetService = Depends(get_widget_service)
):
    """
    Update widget configuration and invalidate cache.
    Triggers regeneration on next request.
    """
    try:
        await widget_service.invalidate_cache(business_id)

        # Store new config in DB (simplified)
        # await widget_service.update_config(business_id, config)

        return WidgetResponse(
            business_id=business_id,
            status="updated",
            cache_invalidated=True
        )
    except Exception as e:
        logger.error(f"Config update failed for {business_id}: {e}")
        raise HTTPException(status_code=500, detail="Config update failed")

@router.get(
    "/{business_id}/metrics",
    response_model=PerformanceMetrics
)
async def get_widget_metrics(
    business_id: str,
    widget_service: WidgetService = Depends(get_widget_service)
):
    """Get performance metrics for widget delivery."""
    return await widget_service.get_performance_metrics(business_id)

@router.head("/{business_id}.js")
async def check_widget(
    business_id: str,
    widget_service: WidgetService = Depends(get_widget_service)
):
    """
    HEAD endpoint for checking widget availability without downloading body.
    Useful for preflight checks.
    """
    exists = await widget_service.redis.exists(f"widget:js:{business_id}")
    if not exists:
        # Check DB without generating
        config = await widget_service._fetch_business_config(business_id)
        if not config:
            raise HTTPException(status_code=404)

    return Response(
        headers={
            "Cache-Control": "public, max-age=3600",
            "X-Widget-Version": settings.WIDGET_VERSION
        }
    )
```

## 3. TypeScript Widget Implementation

```typescript
// src/widget-core/index.ts
/**
 * Sunaiva AI Widget Core - Performance Optimized
 *
 * Targets:
 * - Total Blocking Time: < 50ms
 * - First Meaningful Paint: < 500ms
 * - Lighthouse Performance: > 90
 */

// Feature detection for progressive enhancement
const supportsESModules = () => {
  const script = document.createElement('script');
  return 'noModule' in script;
};

const supportsShadowDOM = () => {
  return Boolean(document.body.attachShadow);
};

const supportsIntersectionObserver = () => {
  return 'IntersectionObserver' in window;
};

// Performance budget tracking
interface PerformanceBudget {
  startTime: number;
  checkpoints: Map<string, number>;
}

class PerformanceMonitor {
  private budget: PerformanceBudget;
  private readonly TBT_THRESHOLD = 50; // 50ms blocking time

  constructor() {
    this.budget = {
      startTime: performance.now(),
      checkpoints: new Map()
    };
  }

  mark(checkpoint: string): void {
    this.budget.checkpoints.set(checkpoint, performance.now());
  }

  measure(taskName: string): number {
    const now = performance.now();
    const duration = now - this.budget.startTime;

    if (duration > this.TBT_THRESHOLD) {
      console.warn(`[Sunaiva] Task "${taskName}" exceeded budget: ${duration.toFixed(2)}ms`);
      // Send to monitoring service
      this.reportViolation(taskName, duration);
    }

    return duration;
  }

  private reportViolation(taskName: string, duration: number): void {
    if ('sendBeacon' in navigator) {
      navigator.sendBeacon('/api/metrics/performance', JSON.stringify({
        task: taskName,
        duration,
        timestamp: Date.now(),
        url: window.location.href
      }));
    }
  }
}

// Shadow DOM Manager for style isolation
class ShadowDOMManager {
  private shadowRoot: ShadowRoot | null = null;
  private container: HTMLElement;

  constructor(containerId: string) {
    const el = document.getElementById(containerId);
    this.container = el || this.createDefaultContainer();
  }

  private createDefaultContainer(): HTMLElement {
    const div = document.createElement('div');
    div.id = 'sunaiva-widget-container';
    Object.assign(div.style, {
      position: 'fixed',
      bottom: '20px',
      right: '20px',
      zIndex: '2147483647', // Max safe z-index
      pointerEvents: 'none' // Allow clicks through until loaded
    });
    document.body.appendChild(div);
    return div;
  }

  attach(): ShadowRoot {
    if (!supportsShadowDOM()) {
      // Fallback to iframe for older browsers
      return this.createIframeFallback();
    }

    this.shadowRoot = this.container.attachShadow({
      mode: 'open',
      delegatesFocus: true // Accessibility improvement
    });

    this.injectCriticalCSS();
    return this.shadowRoot;
  }

  private injectCriticalCSS(): void {
    const style = document.createElement('style');
    style.textContent = `
      :host {
        all: initial;
        font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
        --sunaiva-primary: #007bff;
        --sunaiva-bg: #ffffff;
      }

      #sunaiva-root {
        pointer-events: auto;
        contain: layout style paint;
        content-visibility: auto;
      }

      .sunaiva-button {
        width: 60px;
        height: 60px;
        border-radius: 50%;
        background: var(--sunaiva-primary);
        border: none;
        box-shadow: 0 4px 12px rgba(0,0,0,0.15);
        cursor: pointer;
        transition: transform 0.2s, box-shadow 0.2s;
        will-change: transform;
      }

      .sunaiva-button:hover {
        transform: scale(1.05);
        box-shadow: 0 6px 16px rgba(0,0,0,0.2);
      }

      /* Prevent layout shift */
      .sunaiva-skeleton {
        width: 60px;
        height: 60px;
        background: linear-gradient(90deg, #f0f0f0 25%, #e0e0e0 50%, #f0f0f0 75%);
        background-size: 200% 100%;
        animation: loading 1.5s infinite;
        border-radius: 50%;
      }

      @keyframes loading {
        0% { background-position: 200% 0; }
        100% { background-position: -200% 0; }
      }
    `;
    this.shadowRoot!.appendChild(style);
  }

  private createIframeFallback(): ShadowRoot {
    // Legacy browser support using iframe isolation
    const iframe = document.createElement('iframe');
    iframe.style.cssText = 'border:none;width:100%;height:100%;position:absolute;';
    this.container.appendChild(iframe);

    // Return iframe document as pseudo-shadow root
    return iframe.contentDocument as any;
  }

  getRoot(): HTMLElement {
    if (!this.shadowRoot)


Response:
 ```python
# app/monitoring/metrics.py
"""
Core metrics definitions for Sunaiva platform monitoring.
Implements Prometheus metrics collection with custom collectors
for FastAPI application performance monitoring.
"""

import time
from contextlib import asynccontextmanager
from typing import Callable, Optional

from prometheus_client import (
    Counter,
    Histogram,
    Gauge,
    Info,
    generate_latest,
    CONTENT_TYPE_LATEST,
    CollectorRegistry,
)
from fastapi import FastAPI, Request, Response
from fastapi.responses import PlainTextResponse
import structlog

logger = structlog.get_logger(__name__)

# Create a dedicated registry for application metrics
APP_REGISTRY = CollectorRegistry()

# Application metadata
APP_INFO = Info(
    "app_info",
    "Application version and build information",
    registry=APP_REGISTRY,
)

# Request metrics
REQUEST_COUNT = Counter(
    "http_request_count_total",
    "Total number of HTTP requests",
    ["method", "endpoint", "status_code"],
    registry=APP_REGISTRY,
)

REQUEST_DURATION = Histogram(
    "http_request_duration_seconds",
    "HTTP request duration in seconds",
    ["method", "endpoint"],
    buckets=[0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, float("inf")],
    registry=APP_REGISTRY,
)

# Connection metrics
ACTIVE_CONNECTIONS = Gauge(
    "http_active_connections",
    "Number of active HTTP connections being processed",
    registry=APP_REGISTRY,
)

# Error tracking
ERROR_COUNT = Counter(
    "http_error_count_total",
    "Total number of HTTP errors by type",
    ["method", "endpoint", "error_type"],
    registry=APP_REGISTRY,
)

# Database metrics
DB_CONNECTION_POOL_SIZE = Gauge(
    "db_connection_pool_size",
    "Current database connection pool size",
    ["pool_name"],
    registry=APP_REGISTRY,
)

DB_CONNECTION_POOL_AVAILABLE = Gauge(
    "db_connection_pool_available",
    "Available database connections in pool",
    ["pool_name"],
    registry=APP_REGISTRY,
)

# Custom business metrics (Sunaiva specific)
WIDGET_LOAD_COUNT = Counter(
    "widget_load_count_total",
    "Total number of widget.js loads",
    ["customer_id", "domain"],
    registry=APP_REGISTRY,
)

VOICE_SESSION_DURATION = Histogram(
    "voice_session_duration_seconds",
    "Duration of voice widget sessions",
    ["customer_id"],
    buckets=[5.0, 15.0, 30.0, 60.0, 120.0, 300.0, 600.0],
    registry=APP_REGISTRY,
)

class MetricsMiddleware:
    """
    FastAPI middleware for automatic Prometheus metrics collection.
    Tracks request count, duration, error rates, and active connections.
    """

    def __init__(
        self,
        app: FastAPI,
        exclude_paths: Optional[list[str]] = None,
        exclude_methods: Optional[list[str]] = None,
    ):
        self.app = app
        self.exclude_paths = set(exclude_paths or ["/metrics", "/health", "/ready"])
        self.exclude_methods = set(exclude_methods or ["OPTIONS"])

    async def __call__(self, scope, receive, send):
        if scope["type"] != "http":
            await self.app(scope, receive, send)
            return

        request = Request(scope, receive)
        method = request.method
        path = request.url.path

        # Skip excluded paths and methods
        if path in self.exclude_paths or method in self.exclude_methods:
            await self.app(scope, receive, send)
            return

        # Track active connections
        ACTIVE_CONNECTIONS.inc()
        start_time = time.perf_counter()

        # Wrap send to capture status code
        status_code = 200
        async def wrapped_send(message):
            nonlocal status_code
            if message["type"] == "http.response.start":
                status_code = message["status"]
            await send(message)

        try:
            await self.app(scope, receive, wrapped_send)
        except Exception as exc:
            status_code = 500
            ERROR_COUNT.labels(
                method=method,
                endpoint=self._get_endpoint_name(path),
                error_type=type(exc).__name__,
            ).inc()
            raise
        finally:
            duration = time.perf_counter() - start_time
            ACTIVE_CONNECTIONS.dec()

            # Record metrics
            endpoint = self._get_endpoint_name(path)
            REQUEST_COUNT.labels(
                method=method,
                endpoint=endpoint,
                status_code=str(status_code),
            ).inc()

            REQUEST_DURATION.labels(
                method=method,
                endpoint=endpoint,
            ).observe(duration)

    def _get_endpoint_name(self, path: str) -> str:
        """
        Normalize path for metric labeling to prevent cardinality explosion.
        Replaces dynamic segments with placeholders.
        """
        # Simple normalization - in production, use route patterns
        parts = path.strip("/").split("/")
        normalized = []

        for part in parts:
            # Replace UUIDs, IDs, and hashes with placeholders
            if len(part) == 32 and all(c in "0123456789abcdef" for c in part.lower()):
                normalized.append("<hash>")
            elif part.isdigit():
                normalized.append("<id>")
            else:
                normalized.append(part)

        return "/" + "/".join(normalized) if normalized else "/root"

def setup_metrics(app: FastAPI, app_version: str = "unknown") -> None:
    """
    Configure metrics collection for the FastAPI application.

    Args:
        app: FastAPI application instance
        app_version: Current application version string
    """
    # Set application info
    APP_INFO.info({"version": app_version, "name": "sunaiva-api"})

    # Add metrics endpoint
    @app.get("/metrics", response_class=PlainTextResponse)
    async def metrics_endpoint():
        """
        Prometheus metrics endpoint exposing all collected metrics
        in OpenMetrics/Prometheus exposition format.
        """
        return PlainTextResponse(
            content=generate_latest(APP_REGISTRY),
            media_type=CONTENT_TYPE_LATEST,
        )

    # Add middleware
    app.add_middleware(MetricsMiddleware)

    logger.info("metrics_initialized", version=app_version)

# app/monitoring/health.py
"""
Health check endpoints with dependency validation for PostgreSQL and Redis.
Includes metrics exposure for health status monitoring.
"""

from typing import Optional
import asyncio
import time

from fastapi import APIRouter, HTTPException, Depends
from pydantic import BaseModel, Field
import asyncpg
import redis.asyncio as redis

from app.config import settings
from app.monitoring.metrics import APP_REGISTRY
from prometheus_client import Gauge

router = APIRouter(tags=["health"])

# Health status metric
HEALTH_STATUS = Gauge(
    "health_check_status",
    "Health check status (1 = healthy, 0 = unhealthy)",
    ["check_type"],
    registry=APP_REGISTRY,
)

HEALTH_CHECK_DURATION = Gauge(
    "health_check_duration_seconds",
    "Time taken for health check to complete",
    ["check_type"],
    registry=APP_REGISTRY,
)

class HealthCheck(BaseModel):
    """Health check response model."""
    status: str = Field(..., description="Overall health status")
    version: str = Field(..., description="Application version")
    checks: dict = Field(default_factory=dict, description="Individual check results")
    timestamp: float = Field(default_factory=time.time, description="Check timestamp")

class DependencyCheck:
    """Base class for dependency health checks."""

    def __init__(self, name: str, timeout: float = 5.0):
        self.name = name
        self.timeout = timeout
        self.last_check: Optional[float] = None
        self.last_result: Optional[bool] = None

    async def check(self) -> tuple[bool, Optional[str]]:
        """
        Perform health check.
        Returns: (is_healthy, error_message)
        """
        raise NotImplementedError

    async def execute(self) -> dict:
        """Execute check with timing and metric recording."""
        start = time.perf_counter()
        try:
            healthy, error = await asyncio.wait_for(
                self.check(),
                timeout=self.timeout
            )
            duration = time.perf_counter() - start

            # Record metrics
            HEALTH_STATUS.labels(check_type=self.name).set(1 if healthy else 0)
            HEALTH_CHECK_DURATION.labels(check_type=self.name).set(duration)

            self.last_check = time.time()
            self.last_result = healthy

            return {
                "status": "healthy" if healthy else "unhealthy",
                "response_time_ms": round(duration * 1000, 2),
                "error": error,
            }
        except asyncio.TimeoutError:
            HEALTH_STATUS.labels(check_type=self.name).set(0)
            return {
                "status": "unhealthy",
                "error": f"Timeout after {self.timeout}s",
                "response_time_ms": self.timeout * 1000,
            }
        except Exception as e:
            HEALTH_STATUS.labels(check_type=self.name).set(0)
            return {
                "status": "unhealthy",
                "error": str(e),
                "response_time_ms": round((time.perf_counter() - start) * 1000, 2),
            }

class DatabaseCheck(DependencyCheck):
    """PostgreSQL health check."""

    def __init__(self, dsn: str):
        super().__init__("database", timeout=3.0)
        self.dsn = dsn

    async def check(self) -> tuple[bool, Optional[str]]:
        try:
            conn = await asyncpg.connect(dsn=self.dsn, command_timeout=2.0)
            try:
                result = await conn.fetchval("SELECT 1")
                if result == 1:
                    # Check pool stats if available
                    pool = await asyncpg.create_pool(
                        dsn=self.dsn,
                        min_size=1,
                        max_size=10,
                        command_timeout=2.0
                    )
                    # Note: In real implementation, inject actual pool reference
                    return True, None
                return False, "Unexpected query result"
            finally:
                await conn.close()
        except asyncpg.PostgresError as e:
            return False, f"PostgreSQL error: {str(e)}"
        except Exception as e:
            return False, f"Connection failed: {str(e)}"

class RedisCheck(DependencyCheck):
    """Redis health check."""

    def __init__(self, redis_url: str):
        super().__init__("redis", timeout=2.0)
        self.redis_url = redis_url

    async def check(self) -> tuple[bool, Optional[str]]:
        try:
            client = redis.from_url(self.redis_url, socket_connect_timeout=2.0)
            try:
                pong = await client.ping()
                if pong:
                    # Check memory usage
                    info = await client.info("memory")
                    used_memory = info.get("used_memory", 0)
                    return True, None
                return False, "PING failed"
            finally:
                await client.close()
        except redis.ConnectionError as e:
            return False, f"Redis connection error: {str(e)}"
        except Exception as e:
            return False, f"Redis check failed: {str(e)}"

# Dependency instances (singleton pattern)
_db_check: Optional[DatabaseCheck] = None
_redis_check: Optional[RedisCheck] = None

def get_health_checks() -> tuple[DatabaseCheck, RedisCheck]:
    """Get or initialize health check instances."""
    global _db_check, _redis_check

    if _db_check is None:
        _db_check = DatabaseCheck(settings.DATABASE_URL)
    if _redis_check is None:
        _redis_check = RedisCheck(settings.REDIS_URL)

    return _db_check, _redis_check

@router.get("/health", response_model=HealthCheck)
async def health_check():
    """
    Comprehensive health check endpoint verifying all critical dependencies.
    Returns 503 if any dependency is unhealthy.
    """
    db_check, redis_check = get_health_checks()

    # Run checks concurrently
    db_result, redis_result = await asyncio.gather(
        db_check.execute(),
        redis_check.execute(),
        return_exceptions=True,
    )

    checks = {
        "database": db_result if not isinstance(db_result, Exception) else {
            "status": "error",
            "error": str(db_result),
        },
        "redis": redis_result if not isinstance(redis_result, Exception) else {
            "status": "error",
            "error": str(redis_result),
        },
    }

    # Determine overall health
    all_healthy = all(
        c.get("status") == "healthy"
        for c in checks.values()
        if isinstance(c, dict)
    )

    if not all_healthy:
        raise HTTPException(
            status_code=503,
            detail=HealthCheck(
                status="unhealthy",
                version=settings.APP_VERSION,
                checks=checks,
            ).model_dump(),
        )

    return HealthCheck(
        status="healthy",
        version=settings.APP_VERSION,
        checks=checks,
    )

@router.get("/ready")
async def readiness_check():
    """
    Kubernetes-style readiness probe.
    Returns 200 when ready to serve traffic, 503 otherwise.
    """
    # Quick check without detailed diagnostics
    db_check, redis_check = get_health_checks()

    try:
        db_healthy, _ = await asyncio.wait_for(db_check.check(), timeout=1.0)
        if not db_healthy:
            raise HTTPException(status_code=503, detail="Database not ready")
    except asyncio.TimeoutError:
        raise HTTPException(status_code=503, detail="Database check timeout")

    return {"status": "ready"}

@router.get("/live")
async def liveness_check():
    """
    Kubernetes-style liveness probe.
    Basic check that the process is running.
    """
    return {"status": "alive"}

# prometheus/alerts.yml
"""
Prometheus alerting rules for Sunaiva platform.
Apply these rules to your Prometheus server configuration.
"""

groups:
  - name: sunaiva_api_alerts
    interval: 15s
    rules:
      # High Error Rate Alert
      - alert: HighErrorRate
        expr: |
          (
            sum(rate(http_request_count_total{status_code=~"5.."}[5m]))
            /
            sum(rate(http_request_count_total[5m]))
          ) > 0.05
        for: 2m
        labels:
          severity: critical
          team: devops
        annotations:
          summary: "High error rate detected on {{ $labels.instance }}"
          description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes"
          runbook_url: "https://wiki.sunaiva.com/runbooks/high-error-rate"

      # P99 Latency Alert
      - alert: HighLatency
        expr: |
          histogram_quantile(
            0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
          ) > 2
        for: 3m
        labels:
          severity: warning
          team: backend
        annotations:
          summary: "High P99 latency on {{ $labels.instance }}"
          description: "P99 latency is {{ $value }}s, exceeding 2s threshold"
          dashboard_url: "https://grafana.sunaiva.com/d/api-latency"

      # Health Check Failure
      - alert: HealthCheckFailure
        expr: health_check_status == 0
        for: 1m
        labels:
          severity: critical
          team: devops
        annotations:
          summary: "Health check failing for {{ $labels.check_type }}"
          description: "{{ $labels.check_type }} has been unhealthy for more than 1 minute"

      # Database Connection Pool Exhaustion
      - alert: DatabasePoolExhausted
        expr: |
          (
            db_connection_pool_available
            /
            db_connection_pool_size
          ) < 0.1
        for: 5m
        labels:
          severity: warning
          team: database
        annotations:
          summary: "Database connection pool nearly exhausted"
          description: "Only {{ $value | humanizePercentage }} connections available"

      # Redis Connection Loss
      - alert: RedisUnavailable
        expr: health_check_status{check_type="redis"} == 0
        for: 30s
        labels:
          severity: critical
          team: cache
        annotations:
          summary: "Redis cache unavailable"
          description: "Redis health check has failed"

      # Instance Down (Blackbox exporter style)
      - alert: InstanceDown
        expr: up{job="sunaiva-api"} == 0
        for: 1m
        labels:
          severity: critical
          team: devops
        annotations:
          summary: "Instance {{ $labels.instance }} down"
          description: "Prometheus target scrape has failed for 1 minute"

# tests/test_monitoring.py
"""
Unit and integration tests for monitoring components.
"""

import pytest
import time
from unittest.mock import AsyncMock, MagicMock, patch
from fastapi import FastAPI, Request
from fastapi.testclient import TestClient

from app.monitoring.metrics import (
    setup_metrics,
    REQUEST_COUNT,
    REQUEST_DURATION,
    ACTIVE_CONNECTIONS,
    MetricsMiddleware,
    APP_REGISTRY,
)
from app.monitoring.health import (
    DatabaseCheck,
    RedisCheck,
    HealthCheck,
    get_health_checks,
)

class TestMetricsMiddleware:
    """Test suite for metrics collection middleware."""

    @pytest.fixture
    def app(self):
        """Create test FastAPI app with metrics."""
        app = FastAPI()
        setup_metrics(app, app_version="test-1.0.0")

        @app.get("/test/{item_id}")
        async def test_endpoint(item_id: int):
            return {"id": item_id}

        @app.get("/error")
        async def error_endpoint():
            raise ValueError("Test error")

        return app

    @pytest.fixture
    def client(self, app):
        return TestClient(app)

    def test_metrics_endpoint_exists(self, client):
        """Verify /metrics endpoint returns Prometheus format."""
        response = client.get("/metrics")
        assert response.status_code == 200
        assert "text/plain" in response.headers["content-type"]
        # Check for our custom metrics
        assert "http_request_count_total" in response.text
        assert "app_info" in response.text

    def test_request_count_increment(self, client):
        """Verify request counter increments on HTTP requests."""
        # Get initial count (may be 0 or from previous tests)
        initial_response = client.get("/metrics")

        # Make requests
        for _ in range(5):
            client.get("/test/123")

        # Check metrics
        metrics_response = client.get("/metrics")
        assert metrics_response.status_code == 200

        # Parse prometheus format to find our specific counter
        content = metrics_response.text
        # Should contain increment for our test endpoint
        assert 'method="GET"' in content
        assert 'endpoint="/test/<id>"' in content or 'endpoint="/test/123"' in content

    def test_error_rate_tracking(self, client):
        """Verify error status codes are tracked separately."""
        # Make error request
        response = client.get("/error")
        assert response.status_code == 500

        metrics = client.get("/metrics").text
        # Should have status_code="500" label
        assert 'status_code="500"' in metrics

    def test_active_connections_gauge(self, client):
        """Verify active connections gauge tracks concurrent requests."""
        # This is tricky to test without actual concurrency, but we can verify
        # the gauge exists and has a value
        metrics = client.get("/metrics").text
        assert "http_active_connections" in metrics

    def test_request_duration_histogram(self, client):
        """Verify request duration is recorded in histogram."""
        # Make request with artificial delay would be ideal, but we'll verify
        # the histogram exists
        client.get("/test/456")
        metrics = client.get("/metrics").text
        assert "http_request_duration_seconds_bucket" in metrics
        assert "http_request_duration_seconds_sum" in metrics
        assert "http_request_duration_seconds_count" in metrics

class TestHealthChecks:
    """Test suite for health check endpoints."""

    @pytest.mark.asyncio
    async def test_database_check_success(self):
        """Test successful database health check."""
        with patch("asyncpg.connect") as mock_connect:
            mock_conn = AsyncMock()
            mock_conn.fetchval.return_value = 1
            mock_connect.return_value = mock_conn

            check = DatabaseCheck("postgresql://test")
            result = await check.execute()

            assert result["status"] == "healthy"
            assert "response_time_ms" in result
            assert result.get("error") is None

    @pytest.mark.asyncio
    async def test_database_check_failure(self):
        """Test database health check with connection failure."""
        with patch("asyncpg.connect") as mock_connect:
            mock_connect.side_effect = Exception("Connection refused")

            check = DatabaseCheck("postgresql://test")
            result = await check.execute()

            assert result["status"] == "unhealthy"
            assert "Connection refused" in result["error"]

    @pytest.mark.asyncio
    async def test_redis_check_success(self):
        """Test successful Redis health check."""
        with patch("redis.asyncio.from_url") as mock_from_url:
            mock_client = AsyncMock()
            mock_client.ping.return_value = True
            mock_client.info.return_value = {"used_memory": 1024}
            mock_from_url.return_value = mock_client

            check = RedisCheck("redis://localhost")
            result = await check.execute()


Response:
Here is the production-grade implementation for **TW-API-26**, featuring a layered architecture with strict type safety, comprehensive error handling, and enterprise-grade patterns.

## 1. Database Schema (PostgreSQL)

```sql
-- migrations/003_prompt_templates.sql
CREATE TYPE industry_type AS ENUM (
    'plumber', 'dentist', 'real_estate', 'restaurant', 'legal', 'hvac', 'custom'
);

CREATE TABLE prompt_templates (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    industry industry_type NOT NULL,
    version VARCHAR(20) NOT NULL DEFAULT '1.0.0',
    name VARCHAR(255) NOT NULL,
    content TEXT NOT NULL,
    is_active BOOLEAN NOT NULL DEFAULT true,
    is_default BOOLEAN NOT NULL DEFAULT false,
    traffic_weight INTEGER DEFAULT 100 CHECK (traffic_weight BETWEEN 0 AND 100),
    variables_required JSONB NOT NULL DEFAULT '[]',
    metadata JSONB DEFAULT '{}', -- For A/B test tags, author, etc.
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    UNIQUE(industry, version)
);

-- Index for fast A/B test selection
CREATE INDEX idx_templates_industry_active ON prompt_templates(industry, is_active)
    WHERE is_active = true;

-- Widget override support
ALTER TABLE widgets ADD COLUMN custom_template_id UUID REFERENCES prompt_templates(id) ON DELETE SET NULL;
ALTER TABLE widgets ADD COLUMN ab_test_group VARCHAR(50) DEFAULT NULL;

-- Trigger to ensure only one default per industry
CREATE OR REPLACE FUNCTION enforce_single_default_template()
RETURNS TRIGGER AS $$
BEGIN
    IF NEW.is_default THEN
        UPDATE prompt_templates
        SET is_default = false
        WHERE industry = NEW.industry
        AND id != NEW.id;
    END IF;
    RETURN NEW;
END;
$$ LANGUAGE plpgsql;

CREATE TRIGGER trigger_single_default_template
    BEFORE INSERT OR UPDATE ON prompt_templates
    FOR EACH ROW EXECUTE FUNCTION enforce_single_default_template();
```

## 2. Core Domain Layer

```python
# src/core/exceptions.py
from typing import Optional

class SunaivaException(Exception):
    """Base exception for Sunaiva platform."""
    def __init__(self, message: str, code: Optional[str] = None):
        super().__init__(message)
        self.message = message
        self.code = code or "INTERNAL_ERROR"

class TemplateNotFoundError(SunaivaException):
    """Raised when template is not found."""
    def __init__(self, identifier: str):
        super().__init__(f"Template not found: {identifier}", "TEMPLATE_NOT_FOUND")
        self.identifier = identifier

class TemplateRenderError(SunaivaException):
    """Raised when template rendering fails."""
    def __init__(self, message: str, missing_vars: Optional[list[str]] = None):
        super().__init__(message, "TEMPLATE_RENDER_ERROR")
        self.missing_vars = missing_vars or []

class InvalidIndustryError(SunaivaException):
    """Raised when invalid industry is specified."""
    def __init__(self, industry: str):
        super().__init__(f"Invalid industry: {industry}", "INVALID_INDUSTRY")
        self.industry = industry
```

```python
# src/models/template.py
from __future__ import annotations

import enum
from dataclasses import dataclass
from datetime import datetime
from typing import Any
from uuid import UUID

class Industry(str, enum.Enum):
    """Supported industry verticals."""
    PLUMBER = "plumber"
    DENTIST = "dentist"
    REAL_ESTATE = "real_estate"
    RESTAURANT = "restaurant"
    LEGAL = "legal"
    HVAC = "hvac"
    CUSTOM = "custom"

@dataclass(frozen=True)
class TemplateVariables:
    """Type-safe container for template substitution variables."""
    business_name: str
    industry: str
    services: str
    hours: str
    location: str

    def to_dict(self) -> dict[str, str]:
        return {
            "business_name": self.business_name,
            "industry": self.industry,
            "services": self.services,
            "hours": self.hours,
            "location": self.location,
        }

@dataclass
class PromptTemplate:
    """Domain model for AI conversation templates."""
    id: UUID
    industry: Industry
    version: str
    name: str
    content: str
    is_active: bool
    is_default: bool
    traffic_weight: int
    variables_required: list[str]
    metadata: dict[str, Any]
    created_at: datetime
    updated_at: datetime

    def validate_variables(self, variables: TemplateVariables) -> list[str]:
        """Check if all required variables are present."""
        provided = set(variables.to_dict().keys())
        required = set(self.variables_required)
        return list(required - provided)
```

## 3. Schemas (Pydantic v2)

```python
# src/schemas/template.py
from __future__ import annotations

from datetime import datetime
from typing import Any, Literal
from uuid import UUID

from pydantic import BaseModel, Field, field_validator

from models.template import Industry

class TemplateBase(BaseModel):
    """Base schema for templates."""
    name: str = Field(..., min_length=1, max_length=255)
    content: str = Field(..., min_length=10, description="Jinja2 template content")
    industry: Industry
    traffic_weight: int = Field(default=100, ge=0, le=100)
    variables_required: list[Literal["business_name", "industry", "services", "hours", "location"]] = Field(
        default=["business_name", "industry", "services", "hours", "location"]
    )
    metadata: dict[str, Any] = Field(default_factory=dict)

class TemplateCreate(TemplateBase):
    """Schema for creating new templates."""
    version: str = Field(default="1.0.0", pattern=r"^\d+\.\d+\.\d+$")
    is_default: bool = False

    @field_validator("content")
    @classmethod
    def validate_jinja_syntax(cls, v: str) -> str:
        """Basic validation that content is valid Jinja2."""
        try:
            from jinja2 import Environment, meta
            env = Environment()
            ast = env.parse(v)
            # Ensure only allowed variables are used
            undeclared = meta.find_undeclared_variables(ast)
            allowed = {"business_name", "industry", "services", "hours", "location"}
            invalid = undeclared - allowed
            if invalid:
                raise ValueError(f"Invalid template variables: {invalid}")
            return v
        except Exception as e:
            raise ValueError(f"Invalid template syntax: {e}")

class TemplateUpdate(BaseModel):
    """Schema for updating templates (immutable versions)."""
    is_active: bool | None = None
    traffic_weight: int | None = Field(None, ge=0, le=100)
    metadata: dict[str, Any] | None = None

class TemplateResponse(TemplateBase):
    """Schema for template responses."""
    id: UUID
    version: str
    is_active: bool
    is_default: bool
    created_at: datetime
    updated_at: datetime

    class Config:
        from_attributes = True

class TemplateRenderRequest(BaseModel):
    """Schema for testing template rendering."""
    template_id: UUID | None = None
    industry: Industry | None = None
    version: str | None = None
    variables: dict[str, str] = Field(..., description="Template variables")

class TemplateRenderResponse(BaseModel):
    """Schema for rendered template output."""
    rendered_content: str
    template_id: UUID
    version: str
    variables_used: dict[str, str]

class WidgetTemplateConfig(BaseModel):
    """Schema for configuring widget template overrides."""
    custom_template_id: UUID | None = None
    ab_test_group: str | None = Field(None, max_length=50)
```

## 4. Repository Layer (asyncpg)

```python
# src/repositories/template_repo.py
from __future__ import annotations

import json
from typing import Any
from uuid import UUID

import asyncpg
from asyncpg import Pool

from core.exceptions import TemplateNotFoundError
from models.template import Industry, PromptTemplate

class TemplateRepository:
    """PostgreSQL repository for prompt templates."""

    def __init__(self, pool: Pool):
        self._pool = pool

    async def get_by_id(self, template_id: UUID) -> PromptTemplate:
        """Fetch template by UUID."""
        async with self._pool.acquire() as conn:
            row = await conn.fetchrow(
                """
                SELECT id, industry, version, name, content, is_active,
                       is_default, traffic_weight, variables_required,
                       metadata, created_at, updated_at
                FROM prompt_templates
                WHERE id = $1
                """,
                template_id
            )
            if not row:
                raise TemplateNotFoundError(str(template_id))
            return self._row_to_model(row)

    async def get_by_industry_and_version(
        self,
        industry: Industry,
        version: str
    ) -> PromptTemplate:
        """Fetch specific version of industry template."""
        async with self._pool.acquire() as conn:
            row = await conn.fetchrow(
                """
                SELECT id, industry, version, name, content, is_active,
                       is_default, traffic_weight, variables_required,
                       metadata, created_at, updated_at
                FROM prompt_templates
                WHERE industry = $1 AND version = $2
                """,
                industry.value,
                version
            )
            if not row:
                raise TemplateNotFoundError(f"{industry.value}:{version}")
            return self._row_to_model(row)

    async def get_active_by_industry(self, industry: Industry) -> list[PromptTemplate]:
        """Get all active templates for industry (for A/B testing)."""
        async with self._pool.acquire() as conn:
            rows = await conn.fetch(
                """
                SELECT id, industry, version, name, content, is_active,
                       is_default, traffic_weight, variables_required,
                       metadata, created_at, updated_at
                FROM prompt_templates
                WHERE industry = $1 AND is_active = true
                ORDER BY traffic_weight DESC
                """,
                industry.value
            )
            return [self._row_to_model(row) for row in rows]

    async def get_default_for_industry(self, industry: Industry) -> PromptTemplate:
        """Get default template for industry."""
        async with self._pool.acquire() as conn:
            row = await conn.fetchrow(
                """
                SELECT id, industry, version, name, content, is_active,
                       is_default, traffic_weight, variables_required,
                       metadata, created_at, updated_at
                FROM prompt_templates
                WHERE industry = $1 AND is_default = true AND is_active = true
                LIMIT 1
                """,
                industry.value
            )
            if not row:
                raise TemplateNotFoundError(f"default:{industry.value}")
            return self._row_to_model(row)

    async def create(self, template: PromptTemplate) -> PromptTemplate:
        """Create new template version."""
        async with self._pool.acquire() as conn:
            row = await conn.fetchrow(
                """
                INSERT INTO prompt_templates
                (industry, version, name, content, is_active, is_default,
                 traffic_weight, variables_required, metadata)
                VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
                RETURNING id, created_at, updated_at
                """,
                template.industry.value,
                template.version,
                template.name,
                template.content,
                template.is_active,
                template.is_default,
                template.traffic_weight,
                json.dumps(template.variables_required),
                json.dumps(template.metadata)
            )
            template.id = row["id"]
            template.created_at = row["created_at"]
            template.updated_at = row["updated_at"]
            return template

    async def update(self, template_id: UUID, **kwargs: Any) -> PromptTemplate:
        """Update template metadata (not content - immutable versions)."""
        allowed_fields = {"is_active", "traffic_weight", "metadata"}
        updates = {k: v for k, v in kwargs.items() if k in allowed_fields}

        if not updates:
            return await self.get_by_id(template_id)

        set_clause = ", ".join(f"{k} = ${i+2}" for i, k in enumerate(updates.keys()))
        values = list(updates.values())

        async with self._pool.acquire() as conn:
            row = await conn.fetchrow(
                f"""
                UPDATE prompt_templates
                SET {set_clause}, updated_at = NOW()
                WHERE id = $1
                RETURNING id, industry, version, name, content, is_active,
                          is_default, traffic_weight, variables_required,
                          metadata, created_at, updated_at
                """,
                template_id,
                *values
            )
            if not row:
                raise TemplateNotFoundError(str(template_id))
            return self._row_to_model(row)

    def _row_to_model(self, row: asyncpg.Record) -> PromptTemplate:
        """Convert DB row to domain model."""
        return PromptTemplate(
            id=row["id"],
            industry=Industry(row["industry"]),
            version=row["version"],
            name=row["name"],
            content=row["content"],
            is_active=row["is_active"],
            is_default=row["is_default"],
            traffic_weight=row["traffic_weight"],
            variables_required=json.loads(row["variables_required"]),
            metadata=json.loads(row["metadata"]) if row["metadata"] else {},
            created_at=row["created_at"],
            updated_at=row["updated_at"]
        )
```

## 5. Service Layer with Caching

```python
# src/services/template_service.py
from __future__ import annotations

import random
import re
from typing import Any
from uuid import UUID

import structlog
from jinja2 import Environment, StrictUndefined, TemplateError, meta
from jinja2.sandbox import SandboxedEnvironment
from redis.asyncio import Redis

from core.exceptions import TemplateNotFoundError, TemplateRenderError
from models.template import Industry, PromptTemplate, TemplateVariables
from repositories.template_repo import TemplateRepository

logger = structlog.get_logger()

class TemplateService:
    """Business logic for prompt template management."""

    # Industry presets for seeding
    PRESETS = {
        Industry.PLUMBER: """You are a helpful assistant for {{ business_name }}, a professional plumbing service.
Services offered: {{ services }}
Business Hours: {{ hours }}
Location: {{ location }}

Guidelines:
- Emphasize 24/7 emergency availability if mentioned in services
- Ask about the type of plumbing issue (leak, clog, installation)
- Mention licensed and insured status
- Schedule appointments within business hours""",

        Industry.DENTIST: """You are a dental office assistant for {{ business_name }}.
Services: {{ services }}
Hours: {{ hours }}
Location: {{ location }}

Guidelines:
- Be empathetic about dental anxiety
- Ask about insurance provider
- Mention new patient specials if applicable
- Schedule cleanings, exams, or emergency visits""",

        Industry.REAL_ESTATE: """You are a real estate assistant for {{ business_name }}.
Specialties: {{ services }}
Office Hours: {{ hours }}
Areas Served: {{ location }}

Guidelines:
- Qualify leads (buying, selling, renting, investing)
- Ask about timeline and budget
- Mention recent sales in the area
- Schedule property tours or consultations""",

        Industry.RESTAURANT: """You are a reservation assistant for {{ business_name }}.
Cuisine/Services: {{ services }}
Hours: {{ hours }}
Address: {{ location }}

Guidelines:
- Ask about party size and preferred time
- Mention dietary accommodations
- Ask about special occasions
- Confirm reservation details""",

        Industry.LEGAL: """You are a legal intake assistant for {{ business_name }}.
Practice Areas: {{ services }}
Office Hours: {{ hours }}
Location: {{ location }}

Guidelines:
- Provide general info only, no legal advice
- Ask about case type and urgency
- Mention consultation fees if applicable
- Schedule confidential consultations""",

        Industry.HVAC: """You are a HVAC assistant for {{ business_name }}.
Services: {{ services }}
Hours: {{ hours }}
Service Area: {{ location }}

Guidelines:
- Ask about heating or cooling issues
- Mention seasonal maintenance plans
- Emphasize emergency repair availability
- Schedule service calls or estimates"""
    }

    def __init__(self, repo: TemplateRepository, redis: Redis):
        self.repo = repo
        self.redis = redis
        self.env = SandboxedEnvironment(undefined=StrictUndefined)
        self.cache_ttl = 3600  # 1 hour

    async def seed_industry_presets(self) -> None:
        """Initialize default templates for all industries."""
        for industry, content in self.PRESETS.items():
            try:
                # Check if default exists
                await self.repo.get_default_for_industry(industry)
                logger.info("Preset already exists", industry=industry.value)
            except TemplateNotFoundError:
                template = PromptTemplate(
                    id=UUID(int=0),  # Will be replaced by DB
                    industry=industry,
                    version="1.0.0",
                    name=f"{industry.value.title()} Default Template",
                    content=content,
                    is_active=True,
                    is_default=True,
                    traffic_weight=100,
                    variables_required=["business_name", "industry", "services", "hours", "location"],
                    metadata={"source": "system_preset"},
                    created_at=None,  # type: ignore
                    updated_at=None,  # type: ignore
                )
                await self.repo.create(template)
                logger.info("Created preset template", industry=industry.value)

    async def get_template_for_widget(
        self,
        widget_id: UUID,
        industry: Industry,
        custom_template_id: UUID | None = None,
        ab_test_group: str | None = None
    ) -> PromptTemplate:
        """
        Resolve which template to use for a widget conversation.

        Priority:
        1. Custom template override (if valid and active)
        2. A/B test selection (if group specified)
        3. Default industry template
        """
        cache_key = f"template:widget:{widget_id}:{industry.value}:{ab_test_group}"

        # Check cache
        cached = await self.redis.get(cache_key)
        if cached:
            # Deserialize from cache (simplified - use proper serialization in prod)
            pass

        template = None

        # 1. Check custom override
        if custom_template_id:
            try:
                template = await self.repo.get_by_id(custom_template_id)
                if template.is_active:
                    logger.info("Using custom template", widget_id=str(widget_id), template_id=str(custom_template_id))
                    return template
            except TemplateNotFoundError:
                logger.warning("Custom template not found, falling back", widget_id=str(widget_id))

        # 2. A/B Test selection
        if ab_test_group:
            template = await self._select_ab_test_template(industry, ab_test_group)
            if template:
                await self.redis.setex(cache_key, self.cache_ttl, str(template.id))
                return template

        # 3. Default industry template
        template = await self.repo.get_default_for_industry(industry)
        await self.redis.setex(cache_key, self.cache_ttl, str(template.id))
        return template

    async def _select_ab_test_template(
        self,
        industry: Industry,
        group: str
    ) -> PromptTemplate | None:
        """Select template based on A/B test weights."""
        candidates = await self.repo.get_active_by_industry(industry)
        if not candidates:
            return None

        # Simple weighted random selection
        total_weight = sum(t.traffic_weight for t in candidates)
        if total_weight == 0:
            return candidates[0] if candidates else None

        # Group-based deterministic selection (consistent for same group)
        # Use hash of group name to pick bucket
        group_hash = sum(ord(c) for c in group) % total_weight
        current = 0

        for template in candidates:
            current += template.traffic_weight
            if group_hash < current:
                logger.info("A/B test selection", industry=industry.value, group=group, version=template.version)
                return template

        return candidates[-1]

    async def render_template(
        self,
        template_id: UUID,
        variables: TemplateVariables
    ) -> tuple[str, PromptTemplate]:
        """
        Render template with variable substitution.

        Returns:
            Tuple of (rendered_content, template_used)
        """
        template = await self.repo.get_by_id(template_id)

        # Validate variables
        missing = template.validate_variables(variables)
        if missing:
            raise TemplateRenderError(
                f"Missing required variables: {missing}",
                missing_vars=missing
            )

        try:
            jinja_template = self.env.from_string(template.content)
            rendered = jinja_template.render(**variables.to_dict())

            # Post-processing: clean up extra whitespace
            rendered = re.sub(r'\n{3,}', '\n\n', rendered).strip()

            logger.info(
                "Template rendered successfully",
                template_id=str(template_id),
                industry=template.industry.value
            )

            return rendered, template

        except TemplateError as e:
            logger.error(
                "Template rendering failed",
                template_id=str(template_id),
                error=str(e)
            )
            raise TemplateRenderError(f"Jinja2 rendering error: {e}")

    async def validate_template_content(self, content: str) -> tuple[bool, list[str]]:
        """Validate template syntax and variable usage."""
        try:
            ast = self.env.parse(content)
            undeclared = meta.find_undeclared_variables(ast)
            allowed = {"business_name", "industry", "services", "hours", "location"}
            invalid = undeclared - allowed

            if invalid:
                return False, [f"Invalid variables: {invalid}"]

            # Try to render with dummy data
            dummy = TemplateVariables(
                business_name="Test Biz",
                industry="test",
                services="test services",
                hours="9-5",
                location="Test City"
            )
            template = self.env.from_string(content)
            template.render(**dummy.to_dict())

            return True, []

        except Exception as e:
            return False, [str(e)]
```

## 6. API Layer (FastAPI)

```python
# src/api/routes/templates.py
from __future__ import annotations

from typing import Annotated
from uuid import UUID

import structlog
from fastapi import APIRouter, Depends, HTTPException, Query, status
from redis.asyncio import Redis

from core.exceptions import SunaivaException, TemplateNotFoundError, TemplateRenderError
from models.template import Industry, TemplateVariables
from repositories.template_repo import TemplateRepository
from schemas.template import (
    TemplateCreate,
    TemplateRenderRequest,
    TemplateRenderResponse,
    TemplateResponse,
    TemplateUpdate,
    WidgetTemplateConfig,
)
from services.template_service import TemplateService

logger = structlog.get_logger()
router = APIRouter(prefix="/templates", tags=["templates"])

async def get_template_service(
    repo: Annotated[TemplateRepository, Depends()],
    redis: Annotated[Redis, Depends()]
) -> TemplateService:
    """Dependency injection for template service."""
    return TemplateService(repo, redis)

@router.post("", response_model=TemplateResponse, status_code=status.HTTP_201_CREATED)
async def create_template(
    data: TemplateCreate,
    service: Annotated[TemplateService, Depends(get_template_service)]
) -> TemplateResponse:
    """
    Create a new template version.

    If is_default=True, automatically unsets previous default for this industry.
    """
    try:
        from models.template import PromptTemplate

        template = PromptTemplate(
            id=UUID(int=0),  # Will be set by DB
            industry=data.industry,
            version=data.version,
            name=data.name,
            content=data.content,
            is_active=True,
            is_default=data.is_default,
            traffic_weight=data.traffic_weight,
            variables_required=data.variables_required,
            metadata=data.metadata,
            created_at=None,  # type: ignore
            updated_at=None,  # type: ignore
        )

        created = await service.repo.create(template)
        return TemplateResponse.model_validate(created)

    except Exception as e:
        logger.error("Failed to create template", error=str(e))
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail="Failed to create template"
        )

@router.get("/industry/{industry}", response_model=list[TemplateResponse])
async def list_templates_by_industry(
    industry: Industry,
    active_only: bool = Query(True, description="Only return active templates"),
    service: Annotated[TemplateService, Depends(get_template_service)]
) -> list[TemplateResponse]:
    """List all templates for an industry (for A/B test management)."""
    try:
        if active_only:
            templates = await service.repo.get_active_by_industry(industry)
        else:
            # Would need a new repo method for all versions
            raise NotImplementedError("active_only=False requires additional implementation")
        return [TemplateResponse.model_validate(t) for t in templates]
    except SunaivaException as e:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=e.message)

@router.post("/render", response_model=TemplateRenderResponse)
async def render_template(
    request: TemplateRenderRequest,
    service: Annotated[TemplateService, Depends


Visitor Identification and Session Tracking


Response:
 I'll implement a production-grade outbound call scheduling system with Telnyx integration, timezone awareness, and distributed job scheduling.

## Project Structure

```
sunaiva/
├── app/
│   ├── api/
│   │   ├── deps.py
│   │   └── v1/
│   │       └── calls.py
│   ├── core/
│   │   ├── config.py
│   │   ├── exceptions.py
│   │   └── security.py
│   ├── db/
│   │   ├── session.py
│   │   └── repositories/
│   │       └── call.py
│   ├── models/
│   │   └── call.py
│   ├── schemas/
│   │   └── call.py
│   ├── services/
│   │   ├── scheduler.py
│   │   ├── telnyx_client.py
│   │   └── call_service.py
│   └── main.py
└── tests/
    └── test_calls.py
```

## 1. Core Configuration & Exceptions

```python
# app/core/config.py
from functools import lru_cache
from typing import Any

from pydantic_settings import BaseSettings, SettingsConfigDict

class Settings(BaseSettings):
    """Application configuration with validation."""

    model_config = SettingsConfigDict(env_file=".env", extra="ignore")

    # Database
    database_url: str = "postgresql://user:pass@localhost/sunaiva"
    redis_url: str = "redis://localhost:6379/0"

    # Telnyx
    telnyx_api_key: str
    telnyx_public_key: str | None = None  # For webhook verification
    telnyx_app_id: str  # Telnyx connection/app ID for outbound calls

    # Application
    app_name: str = "Sunaiva API"
    debug: bool = False
    base_url: str = "https://api.sunaiva.com"

    # Security
    secret_key: str
    api_key_header: str = "X-API-Key"

    @property
    def telnyx_webhook_url(self) -> str:
        """Public webhook endpoint for Telnyx callbacks."""
        return f"{self.base_url}/api/v1/webhooks/telnyx"

@lru_cache
def get_settings() -> Settings:
    """Cached settings instance."""
    return Settings()
```

```python
# app/core/exceptions.py
from typing import Any

from fastapi import HTTPException, status

class SunaivaException(Exception):
    """Base application exception."""

    def __init__(
        self,
        message: str,
        details: dict[str, Any] | None = None
    ) -> None:
        self.message = message
        self.details = details or {}
        super().__init__(self.message)

class ValidationError(SunaivaException):
    """Business logic validation failure."""
    pass

class TelnyxAPIError(SunaivaException):
    """Third-party API integration failure."""

    def __init__(
        self,
        message: str,
        status_code: int | None = None,
        response_body: dict | None = None
    ) -> None:
        super().__init__(message)
        self.status_code = status_code
        self.response_body = response_body

class ScheduleError(SunaivaException):
    """Job scheduling failure."""
    pass

class NotFoundError(SunaivaException):
    """Resource not found."""
    pass

# FastAPI exception handlers
async def sunaiva_exception_handler(request, exc: SunaivaException) -> HTTPException:
    """Convert internal exceptions to HTTP responses."""
    if isinstance(exc, ValidationError):
        raise HTTPException(
            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
            detail={"message": exc.message, "errors": exc.details}
        )
    elif isinstance(exc, NotFoundError):
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail={"message": exc.message}
        )
    elif isinstance(exc, TelnyxAPIError):
        raise HTTPException(
            status_code=status.HTTP_502_BAD_GATEWAY,
            detail={
                "message": "External service error",
                "details": exc.message
            }
        )
    raise HTTPException(
        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
        detail={"message": "Internal server error"}
    )
```

## 2. Database Layer (asyncpg)

```python
# app/db/session.py
import contextlib
from typing import AsyncGenerator

import asyncpg
from app.core.config import get_settings

class Database:
    """Connection pool manager for PostgreSQL."""

    def __init__(self) -> None:
        self.pool: asyncpg.Pool | None = None

    async def connect(self) -> None:
        """Initialize connection pool."""
        settings = get_settings()
        self.pool = await asyncpg.create_pool(
            dsn=settings.database_url,
            min_size=10,
            max_size=20,
            command_timeout=60,
            server_settings={
                'jit': 'off',  # Disable JIT for short queries
                'application_name': 'sunaiva_api'
            }
        )

    async def disconnect(self) -> None:
        """Close pool gracefully."""
        if self.pool:
            await self.pool.close()

    async def fetch_one(self, query: str, *args) -> dict | None:
        """Fetch single row as dict."""
        if not self.pool:
            raise RuntimeError("Database not connected")
        async with self.pool.acquire() as conn:
            row = await conn.fetchrow(query, *args)
            return dict(row) if row else None

    async def fetch_many(self, query: str, *args) -> list[dict]:
        """Fetch multiple rows as list of dicts."""
        if not self.pool:
            raise RuntimeError("Database not connected")
        async with self.pool.acquire() as conn:
            rows = await conn.fetch(query, *args)
            return [dict(row) for row in rows]

    async def execute(self, query: str, *args) -> str:
        """Execute command returning status."""
        if not self.pool:
            raise RuntimeError("Database not connected")
        async with self.pool.acquire() as conn:
            return await conn.execute(query, *args)

    @contextlib.asynccontextmanager
    async def transaction(self) -> AsyncGenerator[asyncpg.Connection, None]:
        """Transaction context manager."""
        if not self.pool:
            raise RuntimeError("Database not connected")
        async with self.pool.acquire() as conn:
            async with conn.transaction():
                yield conn

db = Database()
```

```python
# app/db/repositories/call.py
from datetime import datetime
from typing import Any
from uuid import UUID

import asyncpg
from app.core.exceptions import NotFoundError
from app.db.session import db
from app.models.call import CallStatus

class CallRepository:
    """Data access layer for scheduled calls."""

    async def create(
        self,
        business_id: UUID,
        lead_phone: str,
        from_number: str,
        scheduled_at: datetime,
        script_config: dict[str, Any],
        timezone: str,
        metadata: dict[str, Any] | None = None
    ) -> dict[str, Any]:
        """Insert new scheduled call."""
        query = """
            INSERT INTO scheduled_calls (
                business_id, lead_phone, from_number, scheduled_at,
                script_config, timezone, status, metadata, created_at, updated_at
            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW(), NOW())
            RETURNING
                id, business_id, lead_phone, from_number, scheduled_at,
                script_config, timezone, status, telnyx_call_id, metadata,
                created_at, updated_at
        """
        try:
            row = await db.fetch_one(
                query,
                business_id,
                lead_phone,
                from_number,
                scheduled_at,
                script_config,
                timezone,
                CallStatus.SCHEDULED.value,
                metadata or {}
            )
            return row
        except asyncpg.UniqueViolationError as e:
            raise ValueError(f"Call already exists: {e}")

    async def get_by_id(self, call_id: UUID) -> dict[str, Any]:
        """Retrieve call by ID."""
        query = """
            SELECT * FROM scheduled_calls
            WHERE id = $1
        """
        row = await db.fetch_one(query, call_id)
        if not row:
            raise NotFoundError(f"Call {call_id} not found")
        return row

    async def get_due_calls(
        self,
        before: datetime,
        limit: int = 100
    ) -> list[dict[str, Any]]:
        """Fetch calls ready for execution."""
        query = """
            SELECT * FROM scheduled_calls
            WHERE status = $1
            AND scheduled_at <= $2
            ORDER BY scheduled_at ASC
            LIMIT $3
            FOR UPDATE SKIP LOCKED
        """
        return await db.fetch_many(
            query,
            CallStatus.SCHEDULED.value,
            before,
            limit
        )

    async def update_status(
        self,
        call_id: UUID,
        status: CallStatus,
        telnyx_call_id: str | None = None,
        error_message: str | None = None,
        extra_data: dict | None = None
    ) -> dict[str, Any]:
        """Atomic status transition."""
        query = """
            UPDATE scheduled_calls
            SET
                status = $2,
                telnyx_call_id = COALESCE($3, telnyx_call_id),
                error_message = COALESCE($4, error_message),
                extra_data = extra_data || COALESCE($5, '{}'::jsonb),
                updated_at = NOW()
            WHERE id = $1
            RETURNING *
        """
        row = await db.fetch_one(
            query,
            call_id,
            status.value,
            telnyx_call_id,
            error_message,
            extra_data
        )
        if not row:
            raise NotFoundError(f"Call {call_id} not found")
        return row

    async def get_by_telnyx_id(self, telnyx_call_id: str) -> dict[str, Any] | None:
        """Lookup by external provider ID."""
        query = """
            SELECT * FROM scheduled_calls
            WHERE telnyx_call_id = $1
        """
        return await db.fetch_one(query, telnyx_call_id)

# SQL Schema (run during migrations):
"""
CREATE TYPE call_status AS ENUM (
    'scheduled', 'in_progress', 'completed', 'failed', 'cancelled'
);

CREATE TABLE scheduled_calls (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    business_id UUID NOT NULL REFERENCES businesses(id),
    lead_phone VARCHAR(20) NOT NULL,
    from_number VARCHAR(20) NOT NULL,
    scheduled_at TIMESTAMPTZ NOT NULL,
    timezone VARCHAR(50) NOT NULL DEFAULT 'UTC',
    script_config JSONB NOT NULL,  -- AI voice configuration
    status call_status NOT NULL DEFAULT 'scheduled',
    telnyx_call_id VARCHAR(100),
    metadata JSONB DEFAULT '{}',
    extra_data JSONB DEFAULT '{}',  -- Call recordings, transcripts, etc.
    error_message TEXT,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

CREATE INDEX idx_scheduled_calls_status_time
    ON scheduled_calls(status, scheduled_at)
    WHERE status = 'scheduled';

CREATE INDEX idx_scheduled_calls_business
    ON scheduled_calls(business_id, created_at DESC);

CREATE INDEX idx_scheduled_calls_telnyx_id
    ON scheduled_calls(telnyx_call_id)
    WHERE telnyx_call_id IS NOT NULL;
"""
```

## 3. Domain Models & Schemas

```python
# app/models/call.py
from enum import Enum
from datetime import datetime
from uuid import UUID
from typing import Any
from pydantic import BaseModel, Field, ConfigDict

class CallStatus(str, Enum):
    """Finite state machine for call lifecycle."""
    SCHEDULED = "scheduled"
    IN_PROGRESS = "in_progress"
    COMPLETED = "completed"
    FAILED = "failed"
    CANCELLED = "cancelled"

class ScriptConfig(BaseModel):
    """AI voice script configuration."""
    model_config = ConfigDict(extra="allow")

    greeting: str = Field(..., min_length=1, max_length=500)
    purpose: str = Field(..., description="Call objective/context")
    questions: list[str] = Field(default_factory=list)
    voice_profile: str = Field(default="female-1", pattern=r"^(male|female)-\d$")
    language: str = Field(default="en-US", pattern=r"^[a-z]{2}-[A-Z]{2}$")
    max_duration_seconds: int = Field(default=300, ge=30, le=1800)
    record_call: bool = Field(default=True)

    # Telnyx specific
    connection_id: str | None = None  # SIP connection or app ID override

class ScheduledCall(BaseModel):
    """Domain model for scheduled calls."""
    model_config = ConfigDict(from_attributes=True)

    id: UUID
    business_id: UUID
    lead_phone: str = Field(..., pattern=r"^\+[1-9]\d{1,14}$")
    from_number: str = Field(..., pattern=r"^\+[1-9]\d{1,14}$")
    scheduled_at: datetime
    timezone: str
    script_config: ScriptConfig
    status: CallStatus
    telnyx_call_id: str | None = None
    metadata: dict[str, Any]
    created_at: datetime
    updated_at: datetime
```

```python
# app/schemas/call.py
from datetime import datetime
from typing import Any
from uuid import UUID
from pydantic import BaseModel, Field, field_validator
from zoneinfo import ZoneInfo, available_timezones

from app.models.call import CallStatus, ScriptConfig

class ScheduleCallRequest(BaseModel):
    """Inbound request to schedule an outbound call."""

    lead_phone: str = Field(
        ...,
        description="E.164 formatted phone number",
        examples=["+14155552671"]
    )
    from_number: str = Field(
        ...,
        description="Business phone number (must be verified in Telnyx)",
        examples=["+14155551234"]
    )
    scheduled_at: datetime = Field(
        ...,
        description="ISO8601 datetime with timezone"
    )
    script_config: ScriptConfig
    metadata: dict[str, Any] = Field(
        default_factory=dict,
        description="Business-specific data passed through to webhooks"
    )

    @field_validator("scheduled_at")
    @classmethod
    def validate_future_datetime(cls, v: datetime) -> datetime:
        """Ensure call is scheduled in the future."""
        if v.tzinfo is None:
            raise ValueError("scheduled_at must include timezone info")

        now = datetime.now(v.tzinfo)
        if v <= now:
            raise ValueError("scheduled_at must be in the future")

        # Ensure minimum 1 minute buffer for processing
        min_buffer = now.replace(second=0, microsecond=0)
        if v < min_buffer:
            raise ValueError("scheduled_at must be at least 1 minute in the future")

        return v

    @field_validator("scheduled_at")
    @classmethod
    def normalize_timezone(cls, v: datetime) -> datetime:
        """Convert to UTC for storage while preserving original zone info."""
        return v.astimezone(ZoneInfo("UTC"))

class ScheduleCallResponse(BaseModel):
    """Confirmation of scheduled call."""

    id: UUID
    status: CallStatus
    scheduled_at: datetime
    timezone: str
    estimated_cost_cents: int | None = Field(
        None,
        description="Estimated cost based on max duration"
    )
    message: str = "Call scheduled successfully"

class CallStatusUpdate(BaseModel):
    """Webhook payload from Telnyx."""

    event_type: str = Field(..., alias="event_type")
    call_id: str = Field(..., alias="call_id")  # Telnyx call ID
    call_control_id: str | None = Field(None, alias="call_control_id")
    from_number: str | None = Field(None, alias="from")
    to_number: str | None = Field(None, alias="to")
    direction: str | None = None
    status: str | None = None  # answered, hungup, etc.
    duration_seconds: int | None = None
    recording_url: str | None = None
    transcription: str | None = None
    raw_payload: dict[str, Any] = Field(default_factory=dict)

class CallDetailResponse(BaseModel):
    """Full call record for API consumers."""

    id: UUID
    business_id: UUID
    lead_phone: str
    from_number: str
    status: CallStatus
    scheduled_at: datetime
    started_at: datetime | None = None
    completed_at: datetime | None = None
    duration_seconds: int | None = None
    script_config: ScriptConfig
    telnyx_call_id: str | None = None
    recording_url: str | None = None
    transcript: str | None = None
    error_message: str | None = None
    metadata: dict[str, Any]
```

## 4. External Service Integration

```python
# app/services/telnyx_client.py
import asyncio
import json
from typing import Any

import aiohttp
from tenacity import (
    retry,
    retry_if_exception_type,
    stop_after_attempt,
    wait_exponential
)

from app.core.config import get_settings
from app.core.exceptions import TelnyxAPIError

class TelnyxClient:
    """Async HTTP client for Telnyx API with circuit breaker pattern."""

    def __init__(self) -> None:
        settings = get_settings()
        self.api_key = settings.telnyx_api_key
        self.app_id = settings.telnyx_app_id
        self.base_url = "https://api.telnyx.com/v2"
        self.webhook_url = settings.telnyx_webhook_url
        self._session: aiohttp.ClientSession | None = None

    async def _get_session(self) -> aiohttp.ClientSession:
        """Lazy session initialization with connection pooling."""
        if self._session is None or self._session.closed:
            timeout = aiohttp.ClientTimeout(total=30, connect=10)
            headers = {
                "Authorization": f"Bearer {self.api_key}",
                "Content-Type": "application/json",
                "Accept": "application/json"
            }
            self._session = aiohttp.ClientSession(
                timeout=timeout,
                headers=headers,
                raise_for_status=False
            )
        return self._session

    @retry(
        retry=retry_if_exception_type((aiohttp.ClientError, asyncio.TimeoutError)),
        stop=stop_after_attempt(3),
        wait=wait_exponential(multiplier=1, min=2, max=10),
        reraise=True
    )
    async def initiate_call(
        self,
        to_number: str,
        from_number: str,
        connection_id: str | None = None,
        custom_headers: dict[str, str] | None = None
    ) -> dict[str, Any]:
        """
        Initiate outbound call via Telnyx.

        Returns Telnyx call object with 'call_control_id' for further control.
        """
        session = await self._get_session()

        payload = {
            "to": to_number,
            "from": from_number,
            "connection_id": connection_id or self.app_id,
            "webhook_url": self.webhook_url,
            "webhook_url_method": "POST",
            "audio_url": None,  # We use call control for AI voice
            "time_limit": 1800,  # 30 min max
            "answer_on_bridge": True,
            "enable_recording": True,
            "custom_headers": custom_headers or {}
        }

        try:
            async with session.post(
                f"{self.base_url}/calls",
                json={"call": payload}
            ) as response:
                body = await response.json()

                if response.status >= 400:
                    raise TelnyxAPIError(
                        f"Telnyx API error: {body.get('errors', 'Unknown error')}",
                        status_code=response.status,
                        response_body=body
                    )

                return body.get("data", {})

        except aiohttp.ClientResponseError as e:
            raise TelnyxAPIError(
                f"HTTP error initiating call: {e.message}",
                status_code=e.status
            )
        except asyncio.TimeoutError:
            raise TelnyxAPIError("Timeout connecting to Telnyx API")

    async def speak_text(
        self,
        call_control_id: str,
        text: str,
        voice: str = "female",
        language: str = "en-US"
    ) -> dict[str, Any]:
        """Send AI-generated voice text to active call."""
        session = await self._get_session()

        payload = {
            "call_control_id": call_control_id,
            "payload": text,
            "voice": voice,
            "language": language,
            "client_state": json.dumps({"type": "ai_speak"})
        }

        async with session.post(
            f"{self.base_url}/calls/actions/speak",
            json={"payload": payload}
        ) as response:
            body = await response.json()
            if response.status >= 400:
                raise TelnyxAPIError(f"Speak command failed: {body}")
            return body.get("data", {})

    async def close(self) -> None:
        """Clean up session."""
        if self._session and not self._session.closed:
            await self._session.close