Response:
I'll implement a production-grade feature flag system for Sunaiva with consistent hashing for percentage rollouts, Redis caching, and a lightweight widget client.

## 1. Database Schema & Models

```python
# models/feature_flags.py
from enum import Enum
from typing import Optional, List, Dict, Any
from datetime import datetime
from pydantic import BaseModel, Field, ConfigDict

class FlagType(str, Enum):
    BOOLEAN = "boolean"
    PERCENTAGE = "percentage"
    USER_LIST = "user_list"

class FeatureFlagBase(BaseModel):
    """Base feature flag model."""
    name: str = Field(..., min_length=1, max_length=100, pattern=r"^[a-z0-9_]+$")
    description: Optional[str] = Field(None, max_length=500)
    flag_type: FlagType
    enabled: bool = True

    # Type-specific configuration
    # boolean: {"value": true/false}
    # percentage: {"percentage": 50, "seed": "v1"}
    # user_list: {"user_ids": ["user_123", "user_456"]}
    config: Dict[str, Any] = Field(default_factory=dict)

    # Metadata for targeting (e.g., {"plan": "pro", "region": "us"})
    targeting_rules: Optional[Dict[str, Any]] = None

class FeatureFlagCreate(FeatureFlagBase):
    """Model for creating a new feature flag."""
    pass

class FeatureFlagUpdate(BaseModel):
    """Model for updating an existing feature flag."""
    description: Optional[str] = Field(None, max_length=500)
    enabled: Optional[bool] = None
    config: Optional[Dict[str, Any]] = None
    targeting_rules: Optional[Dict[str, Any]] = None

class FeatureFlagInDB(FeatureFlagBase):
    """Model representing a flag as stored in the database."""
    model_config = ConfigDict(from_attributes=True)

    id: int
    created_at: datetime
    updated_at: datetime
    created_by: Optional[str] = None

class FlagEvaluationRequest(BaseModel):
    """Request model for flag evaluation."""
    user_id: Optional[str] = None
    context: Dict[str, Any] = Field(default_factory=dict)  # e.g., {"plan": "pro"}

class FlagEvaluationResponse(BaseModel):
    """Response model for flag evaluation."""
    flag_name: str
    enabled: bool
    reason: str  # "boolean_true", "percentage_match", "user_in_list", "default_false", etc.
    metadata: Optional[Dict[str, Any]] = None
```

## 2. Database Repository Layer

```python
# db/repository.py
import asyncpg
import json
from typing import Optional, List
from models.feature_flags import FeatureFlagInDB, FeatureFlagCreate, FeatureFlagUpdate

class FeatureFlagRepository:
    """Repository for feature flag database operations."""

    def __init__(self, pool: asyncpg.Pool):
        self.pool = pool

    async def get_by_name(self, name: str) -> Optional[FeatureFlagInDB]:
        """Retrieve a flag by its unique name."""
        async with self.pool.acquire() as conn:
            row = await conn.fetchrow(
                "SELECT * FROM feature_flags WHERE name = $1", name
            )
            if row:
                return FeatureFlagInDB(**dict(row))
            return None

    async def get_all(self, enabled_only: bool = False) -> List[FeatureFlagInDB]:
        """Retrieve all feature flags."""
        async with self.pool.acquire() as conn:
            query = "SELECT * FROM feature_flags"
            if enabled_only:
                query += " WHERE enabled = true"
            query += " ORDER BY created_at DESC"

            rows = await conn.fetch(query)
            return [FeatureFlagInDB(**dict(row)) for row in rows]

    async def create(self, flag: FeatureFlagCreate, created_by: Optional[str] = None) -> FeatureFlagInDB:
        """Create a new feature flag."""
        async with self.pool.acquire() as conn:
            row = await conn.fetchrow(
                """
                INSERT INTO feature_flags
                (name, description, flag_type, enabled, config, targeting_rules, created_by)
                VALUES ($1, $2, $3, $4, $5, $6, $7)
                RETURNING *
                """,
                flag.name,
                flag.description,
                flag.flag_type.value,
                flag.enabled,
                json.dumps(flag.config),
                json.dumps(flag.targeting_rules) if flag.targeting_rules else None,
                created_by
            )
            return FeatureFlagInDB(**dict(row))

    async def update(self, name: str, flag_update: FeatureFlagUpdate) -> Optional[FeatureFlagInDB]:
        """Update an existing feature flag."""
        updates = []
        values = []
        param_idx = 1

        if flag_update.description is not None:
            updates.append(f"description = ${param_idx}")
            values.append(flag_update.description)
            param_idx += 1

        if flag_update.enabled is not None:
            updates.append(f"enabled = ${param_idx}")
            values.append(flag_update.enabled)
            param_idx += 1

        if flag_update.config is not None:
            updates.append(f"config = ${param_idx}")
            values.append(json.dumps(flag_update.config))
            param_idx += 1

        if flag_update.targeting_rules is not None:
            updates.append(f"targeting_rules = ${param_idx}")
            values.append(json.dumps(flag_update.targeting_rules))
            param_idx += 1

        if not updates:
            return await self.get_by_name(name)

        updates.append("updated_at = NOW()")
        values.append(name)

        query = f"""
            UPDATE feature_flags
            SET {', '.join(updates)}
            WHERE name = ${param_idx}
            RETURNING *
        """

        async with self.pool.acquire() as conn:
            row = await conn.fetchrow(query, *values)
            if row:
                return FeatureFlagInDB(**dict(row))
            return None

    async def delete(self, name: str) -> bool:
        """Delete a feature flag."""
        async with self.pool.acquire() as conn:
            result = await conn.execute(
                "DELETE FROM feature_flags WHERE name = $1", name
            )
            # asyncpg returns 'DELETE <count>'
            return int(result.split()[-1]) > 0

# SQL Migration (run this in your migrations):
"""
CREATE TABLE IF NOT EXISTS feature_flags (
    id SERIAL PRIMARY KEY,
    name VARCHAR(100) UNIQUE NOT NULL,
    description TEXT,
    flag_type VARCHAR(20) NOT NULL CHECK (flag_type IN ('boolean', 'percentage', 'user_list')),
    enabled BOOLEAN DEFAULT true,
    config JSONB NOT NULL DEFAULT '{}',
    targeting_rules JSONB,
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    created_by VARCHAR(100)
);

CREATE INDEX idx_feature_flags_name ON feature_flags(name);
CREATE INDEX idx_feature_flags_enabled ON feature_flags(enabled);

-- Trigger to update updated_at timestamp
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
    NEW.updated_at = NOW();
    RETURN NEW;
END;
$$ language 'plpgsql';

CREATE TRIGGER update_feature_flags_updated_at
    BEFORE UPDATE ON feature_flags
    FOR EACH ROW
    EXECUTE FUNCTION update_updated_at_column();
"""
```

## 3. Service Layer with Caching & Evaluation Logic

```python
# services/feature_flags.py
import json
import hashlib
import structlog
from typing import Optional, Dict, Any, List
from datetime import timedelta
import redis.asyncio as redis
from db.repository import FeatureFlagRepository
from models.feature_flags import (
    FeatureFlagInDB,
    FlagEvaluationResponse,
    FeatureFlagCreate,
    FeatureFlagUpdate,
    FlagType
)

logger = structlog.get_logger()

class FeatureFlagService:
    """Service for feature flag business logic with Redis caching."""

    CACHE_PREFIX = "sunaiva:flags:"
    CACHE_TTL = timedelta(minutes=5)

    def __init__(self, repository: FeatureFlagRepository, redis_client: redis.Redis):
        self.repo = repository
        self.redis = redis_client

    def _get_cache_key(self, flag_name: str) -> str:
        """Generate Redis cache key for a flag."""
        return f"{self.CACHE_PREFIX}{flag_name}"

    async def _get_cached_flag(self, name: str) -> Optional[FeatureFlagInDB]:
        """Retrieve flag from Redis cache."""
        try:
            cached = await self.redis.get(self._get_cache_key(name))
            if cached:
                data = json.loads(cached)
                return FeatureFlagInDB(**data)
        except Exception as e:
            logger.error("redis_cache_error", error=str(e), flag_name=name)
        return None

    async def _cache_flag(self, flag: FeatureFlagInDB):
        """Store flag in Redis cache."""
        try:
            await self.redis.setex(
                self._get_cache_key(flag.name),
                self.CACHE_TTL,
                flag.model_dump_json()
            )
        except Exception as e:
            logger.error("redis_cache_set_error", error=str(e), flag_name=flag.name)

    async def _invalidate_cache(self, name: str):
        """Remove flag from cache (used on updates)."""
        try:
            await self.redis.delete(self._get_cache_key(name))
        except Exception as e:
            logger.error("redis_cache_invalidate_error", error=str(e), flag_name=name)

    def _evaluate_percentage(self, user_id: str, percentage: int, seed: str = "v1") -> bool:
        """
        Deterministically evaluate percentage-based flag using consistent hashing.
        Same user_id + flag combo always returns same result.
        """
        # Create hash from user_id + seed for consistency
        hash_input = f"{user_id}:{seed}"
        hash_value = hashlib.md5(hash_input.encode()).digest()
        # Convert first 4 bytes to integer (0-2^32)
        int_value = struct.unpack(">I", hash_value[:4])[0]
        # Normalize to 0-100
        user_percentage = (int_value / (2**32 - 1)) * 100
        return user_percentage < percentage

    def _matches_targeting_rules(self, flag: FeatureFlagInDB, context: Dict[str, Any]) -> bool:
        """Check if user context matches flag targeting rules."""
        if not flag.targeting_rules:
            return True

        for key, value in flag.targeting_rules.items():
            if key not in context:
                return False
            if isinstance(value, list):
                if context[key] not in value:
                    return False
            elif context[key] != value:
                return False
        return True

    async def evaluate_flag(
        self,
        name: str,
        user_id: Optional[str] = None,
        context: Dict[str, Any] = None
    ) -> FlagEvaluationResponse:
        """
        Evaluate a feature flag for a specific user.

        Returns enabled=False with reason on any error (fail-safe).
        """
        context = context or {}

        # Try cache first, then database
        flag = await self._get_cached_flag(name)
        if not flag:
            flag = await self.repo.get_by_name(name)
            if flag:
                await self._cache_flag(flag)

        if not flag:
            return FlagEvaluationResponse(
                flag_name=name,
                enabled=False,
                reason="flag_not_found",
                metadata={"error": "Feature flag does not exist"}
            )

        # Check if flag is globally disabled
        if not flag.enabled:
            return FlagEvaluationResponse(
                flag_name=name,
                enabled=False,
                reason="globally_disabled",
                metadata={"flag_type": flag.flag_type}
            )

        # Check targeting rules
        if not self._matches_targeting_rules(flag, context):
            return FlagEvaluationResponse(
                flag_name=name,
                enabled=False,
                reason="targeting_mismatch",
                metadata={"targeting_rules": flag.targeting_rules}
            )

        # Evaluate based on flag type
        try:
            if flag.flag_type == FlagType.BOOLEAN:
                value = flag.config.get("value", False)
                return FlagEvaluationResponse(
                    flag_name=name,
                    enabled=bool(value),
                    reason="boolean_evaluated",
                    metadata={"config_value": value}
                )

            elif flag.flag_type == FlagType.PERCENTAGE:
                if not user_id:
                    return FlagEvaluationResponse(
                        flag_name=name,
                        enabled=False,
                        reason="user_id_required",
                        metadata={"error": "Percentage rollout requires user_id"}
                    )

                percentage = flag.config.get("percentage", 0)
                seed = flag.config.get("seed", "v1")
                is_enabled = self._evaluate_percentage(user_id, percentage, seed)

                return FlagEvaluationResponse(
                    flag_name=name,
                    enabled=is_enabled,
                    reason="percentage_match" if is_enabled else "percentage_mismatch",
                    metadata={"percentage": percentage, "user_id": user_id}
                )

            elif flag.flag_type == FlagType.USER_LIST:
                allowed_users = flag.config.get("user_ids", [])
                if not user_id:
                    return FlagEvaluationResponse(
                        flag_name=name,
                        enabled=False,
                        reason="user_id_required",
                        metadata={"error": "User list requires user_id"}
                    )

                is_enabled = user_id in allowed_users
                return FlagEvaluationResponse(
                    flag_name=name,
                    enabled=is_enabled,
                    reason="user_in_list" if is_enabled else "user_not_in_list",
                    metadata={"list_size": len(allowed_users)}
                )

            else:
                return FlagEvaluationResponse(
                    flag_name=name,
                    enabled=False,
                    reason="unknown_flag_type",
                    metadata={"flag_type": flag.flag_type}
                )

        except Exception as e:
            logger.error("flag_evaluation_error", error=str(e), flag_name=name)
            return FlagEvaluationResponse(
                flag_name=name,
                enabled=False,
                reason="evaluation_error",
                metadata={"error": str(e)}
            )

    # Admin methods
    async def create_flag(self, flag: FeatureFlagCreate, created_by: Optional[str] = None) -> FeatureFlagInDB:
        """Create a new feature flag."""
        # Validate config based on type
        self._validate_config(flag.flag_type, flag.config)
        return await self.repo.create(flag, created_by)

    async def update_flag(self, name: str, update: FeatureFlagUpdate) -> Optional[FeatureFlagInDB]:
        """Update a feature flag and invalidate cache."""
        if update.config:
            # Need to get current flag to know type for validation
            current = await self.repo.get_by_name(name)
            if not current:
                return None
            self._validate_config(current.flag_type, update.config)

        updated = await self.repo.update(name, update)
        if updated:
            await self._invalidate_cache(name)
            await self._cache_flag(updated)  # Cache new value
        return updated

    async def delete_flag(self, name: str) -> bool:
        """Delete a feature flag and invalidate cache."""
        success = await self.repo.delete(name)
        if success:
            await self._invalidate_cache(name)
        return success

    async def list_flags(self, enabled_only: bool = False) -> List[FeatureFlagInDB]:
        """List all feature flags."""
        return await self.repo.get_all(enabled_only)

    def _validate_config(self, flag_type: FlagType, config: Dict[str, Any]):
        """Validate configuration based on flag type."""
        if flag_type == FlagType.BOOLEAN:
            if "value" not in config:
                raise ValueError("Boolean flags require 'value' in config")
        elif flag_type == FlagType.PERCENTAGE:
            pct = config.get("percentage")
            if not isinstance(pct, (int, float)) or not (0 <= pct <= 100):
                raise ValueError("Percentage flags require 'percentage' between 0-100")
        elif flag_type == FlagType.USER_LIST:
            user_ids = config.get("user_ids")
            if not isinstance(user_ids, list):
                raise ValueError("User list flags require 'user_ids' array in config")
```

## 4. FastAPI Routes

```python
# api/routes.py
from fastapi import APIRouter, Depends, HTTPException, Query, status
from typing import List, Optional
import asyncpg
import redis.asyncio as redis

from models.feature_flags import (
    FeatureFlagCreate,
    FeatureFlagUpdate,
    FeatureFlagInDB,
    FlagEvaluationRequest,
    FlagEvaluationResponse
)
from services.feature_flags import FeatureFlagService
from db.repository import FeatureFlagRepository

router = APIRouter(prefix="/api/v1/flags", tags=["feature-flags"])

# Dependency injection (simplified - use your actual dependency injection)
async def get_db_pool() -> asyncpg.Pool:
    # Your existing database pool
    pass

async def get_redis_client() -> redis.Redis:
    # Your existing Redis client
    pass

async def get_flag_service(
    pool: asyncpg.Pool = Depends(get_db_pool),
    redis_client: redis.Redis = Depends(get_redis_client)
) -> FeatureFlagService:
    repo = FeatureFlagRepository(pool)
    return FeatureFlagService(repo, redis_client)

# Public Evaluation Endpoint
@router.get("/{name}", response_model=FlagEvaluationResponse)
async def evaluate_flag(
    name: str,
    user_id: Optional[str] = Query(None, description="Unique user identifier"),
    context: Optional[str] = Query(None, description="JSON string of context"),
    service: FeatureFlagService = Depends(get_flag_service)
):
    """
    Evaluate a feature flag for a specific user.

    - **name**: Flag name (e.g., "new_widget_design")
    - **user_id**: Optional user identifier for percentage/user_list flags
    - **context**: Optional JSON string for targeting (e.g., '{"plan":"pro"}')
    """
    import json
    ctx = {}
    if context:
        try:
            ctx = json.loads(context)
        except json.JSONDecodeError:
            raise HTTPException(
                status_code=400,
                detail="Invalid JSON in context parameter"
            )

    result = await service.evaluate_flag(name, user_id, ctx)
    return result

@router.post("/evaluate", response_model=FlagEvaluationResponse)
async def evaluate_flag_post(
    name: str,
    request: FlagEvaluationRequest,
    service: FeatureFlagService = Depends(get_flag_service)
):
    """
    POST endpoint for flag evaluation (preferred for large contexts).
    """
    result = await service.evaluate_flag(name, request.user_id, request.context)
    return result

# Admin Endpoints (In production, add authentication dependencies)
@router.post("", response_model=FeatureFlagInDB, status_code=status.HTTP_201_CREATED)
async def create_flag(
    flag: FeatureFlagCreate,
    service: FeatureFlagService = Depends(get_flag_service),
    current_user: str = Depends(get_current_admin_user)  # Implement this
):
    """Create a new feature flag."""
    try:
        return await service.create_flag(flag, created_by=current_user)
    except ValueError as e:
        raise HTTPException(status_code=400, detail=str(e))
    except asyncpg.UniqueViolationError:
        raise HTTPException(
            status_code=409,
            detail=f"Flag with name '{flag.name}' already exists"
        )

@router.get("", response_model=List[FeatureFlagInDB])
async def list_flags(
    enabled_only: bool = Query(False, description="Filter to enabled flags only"),
    service: FeatureFlagService = Depends(get_flag_service),
    current_user: str = Depends(get_current_admin_user)
):
    """List all feature flags."""
    return await service.list_flags(enabled_only)

@router.get("/admin/{name}", response_model=FeatureFlagInDB)
async def get_flag(
    name: str,
    service: FeatureFlagService = Depends(get_flag_service),
    current_user: str = Depends(get_current_admin_user)
):
    """Get a specific flag configuration."""
    flag = await service.repo.get_by_name(name)
    if not flag:
        raise HTTPException(status_code=404, detail="Flag not found")
    return flag

@router.patch("/{name}", response_model=FeatureFlagInDB)
async def update_flag(
    name: str


Bulk Import/Export for Widget Configurations

Response:
 Here's the production-ready implementation for **TW-API-31** following Sunaiva's architecture standards:

## 1. Domain Models & Schemas (`src/models/widget_bulk.py`)

```python
"""
Pydantic models for bulk widget import/export operations.
Validates CSV row-level data and provides structured error reporting.
"""

from datetime import datetime
from enum import Enum
from typing import Optional, List, Dict, Any
from pydantic import BaseModel, Field, field_validator, HttpUrl, ConfigDict
import re

class WidgetPosition(str, Enum):
    """Valid widget positions for CSV import."""
    BOTTOM_RIGHT = "bottom_right"
    BOTTOM_LEFT = "bottom_left"
    TOP_RIGHT = "top_right"
    TOP_LEFT = "top_left"

class VoiceType(str, Enum):
    """Available AI voice types."""
    NEUTRAL = "neutral"
    FRIENDLY = "friendly"
    PROFESSIONAL = "professional"
    ENERGETIC = "energetic"

class WidgetCSVRow(BaseModel):
    """
    Represents a single row in the import CSV.
    Line number is injected during parsing, not part of CSV data.
    """
    model_config = ConfigDict(str_strip_whitespace=True)

    # Required fields
    name: str = Field(..., min_length=1, max_length=100, description="Widget display name")
    website_url: HttpUrl = Field(..., description="Target website URL")
    phone_number: str = Field(..., description="E.164 formatted phone number")

    # Optional fields with defaults
    greeting_message: Optional[str] = Field(None, max_length=500)
    voice_type: VoiceType = Field(default=VoiceType.NEUTRAL)
    position: WidgetPosition = Field(default=WidgetPosition.BOTTOM_RIGHT)
    primary_color: Optional[str] = Field(None, pattern=r"^#[0-9A-Fa-f]{6}$")
    agency_client_id: Optional[str] = Field(None, max_length=50, description="External reference ID")
    auto_open_delay: Optional[int] = Field(None, ge=0, le=60, description="Seconds until auto-open")

    # Metadata (injected by parser)
    line_number: int = Field(default=0, exclude=True)
    raw_data: Dict[str, Any] = Field(default_factory=dict, exclude=True)

    @field_validator("phone_number")
    @classmethod
    def validate_e164(cls, v: str) -> str:
        """Validate E.164 phone number format."""
        if not re.match(r"^\+[1-9]\d{1,14}$", v):
            raise ValueError("Phone number must be in E.164 format (e.g., +1234567890)")
        return v

    @field_validator("name")
    @classmethod
    def validate_name_not_empty(cls, v: str) -> str:
        """Ensure name is not just whitespace."""
        if not v.strip():
            raise ValueError("Widget name cannot be empty")
        return v.strip()

class RowValidationError(BaseModel):
    """Detailed error information for a specific CSV row."""
    line_number: int = Field(..., description="1-based line number in CSV")
    field: Optional[str] = Field(None, description="Field name that failed validation")
    error_message: str = Field(..., description="Human-readable error description")
    raw_value: Optional[str] = Field(None, description="Original value from CSV")

class BulkImportResult(BaseModel):
    """Response model for bulk import operations."""
    success: bool = Field(..., description="True if all rows processed without errors")
    processed_count: int = Field(..., ge=0)
    created_count: int = Field(..., ge=0)
    failed_count: int = Field(..., ge=0)
    dry_run: bool = Field(..., description="Whether this was a validation-only run")
    errors: List[RowValidationError] = Field(default_factory=list)
    created_widget_ids: List[str] = Field(default_factory=list, description="UUIDs of created widgets")
    processing_time_ms: float = Field(..., description="Total processing time")

class ExportFilter(BaseModel):
    """Query parameters for widget export filtering."""
    agency_id: Optional[str] = None
    created_after: Optional[datetime] = None
    created_before: Optional[datetime] = None
    active_only: bool = True
```

## 2. CSV Processing Engine (`src/services/csv_processor.py`)

```python
"""
High-performance CSV processing with async support, validation,
and detailed error reporting for bulk operations.
"""

import csv
import io
from typing import List, Tuple, Dict, Any, Optional, AsyncGenerator
from asyncio import to_thread
from datetime import datetime
import logging

from src.models.widget_bulk import WidgetCSVRow, RowValidationError, WidgetPosition, VoiceType

logger = logging.getLogger(__name__)

class CSVProcessingError(Exception):
    """Raised when CSV structure is invalid (headers, encoding, etc.)."""
    pass

class AsyncCSVProcessor:
    """
    Handles CSV parsing and validation for widget bulk operations.
    Memory-efficient for large files using streaming parsing.
    """

    REQUIRED_HEADERS = {"name", "website_url", "phone_number"}
    OPTIONAL_HEADERS = {
        "greeting_message", "voice_type", "position",
        "primary_color", "agency_client_id", "auto_open_delay"
    }
    ALL_HEADERS = REQUIRED_HEADERS | OPTIONAL_HEADERS

    def __init__(self, max_file_size_mb: int = 10):
        self.max_file_size = max_file_size_mb * 1024 * 1024

    async def parse_upload(
        self,
        file_content: bytes,
        filename: str = "upload.csv"
    ) -> Tuple[List[WidgetCSVRow], List[RowValidationError]]:
        """
        Parse uploaded CSV bytes into validated Widget rows.

        Returns:
            Tuple of (valid_rows, validation_errors)
        """
        # Size check
        if len(file_content) > self.max_file_size:
            raise CSVProcessingError(
                f"File size exceeds maximum of {self.max_file_size // (1024*1024)}MB"
            )

        # Run CPU-bound CSV parsing in thread pool
        return await to_thread(self._parse_sync, file_content, filename)

    def _parse_sync(
        self,
        content: bytes,
        filename: str
    ) -> Tuple[List[WidgetCSVRow], List[RowValidationError]]:
        """Synchronous CSV parsing logic (runs in thread pool)."""
        valid_rows: List[WidgetCSVRow] = []
        errors: List[RowValidationError] = []

        try:
            # Handle BOM and encoding
            text_content = content.decode("utf-8-sig")
        except UnicodeDecodeError as e:
            raise CSVProcessingError(f"File encoding must be UTF-8: {str(e)}")

        reader = csv.DictReader(io.StringIO(text_content))

        # Validate headers
        if not reader.fieldnames:
            raise CSVProcessingError("CSV file is empty or has no headers")

        headers = set(reader.fieldnames)
        missing_required = self.REQUIRED_HEADERS - headers
        if missing_required:
            raise CSVProcessingError(
                f"Missing required columns: {', '.join(missing_required)}"
            )

        # Warn about unknown columns (non-blocking)
        unknown_columns = headers - self.ALL_HEADERS
        if unknown_columns:
            logger.warning(f"CSV contains unknown columns: {unknown_columns}")

        # Process rows
        for line_num, row in enumerate(reader, start=2):  # Start at 2 (1 is header)
            row_errors, widget_row = self._validate_row(row, line_num)
            if row_errors:
                errors.extend(row_errors)
            else:
                valid_rows.append(widget_row)

        return valid_rows, errors

    def _validate_row(
        self,
        raw_row: Dict[str, Any],
        line_number: int
    ) -> Tuple[List[RowValidationError], Optional[WidgetCSVRow]]:
        """
        Validate a single CSV row.
        Returns tuple of (errors, valid_row_or_none).
        """
        errors = []

        # Pre-process: clean empty strings to None for optional fields
        cleaned_data = {}
        for key, value in raw_row.items():
            if value is not None and str(value).strip() == "":
                cleaned_data[key] = None
            else:
                cleaned_data[key] = value

        # Attempt Pydantic validation
        try:
            # Inject metadata
            cleaned_data["line_number"] = line_number
            cleaned_data["raw_data"] = dict(raw_row)

            widget_row = WidgetCSVRow.model_validate(cleaned_data)
            return [], widget_row

        except Exception as e:
            # Extract field-specific errors from Pydantic ValidationError
            error_details = []
            try:
                # Try to parse structured error
                for error in e.errors():
                    field = ".".join(str(x) for x in error["loc"]) if error["loc"] else "general"
                    msg = error["msg"]
                    raw_val = cleaned_data.get(field, "N/A")

                    error_details.append(RowValidationError(
                        line_number=line_number,
                        field=field,
                        error_message=msg,
                        raw_value=str(raw_val) if raw_val is not None else None
                    ))
            except AttributeError:
                # Fallback for non-Pydantic exceptions
                error_details.append(RowValidationError(
                    line_number=line_number,
                    field=None,
                    error_message=str(e),
                    raw_value=None
                ))

            return error_details, None

    def generate_export_csv(self, widgets: List[Dict[str, Any]]) -> str:
        """
        Generate CSV string from widget records for export.
        """
        if not widgets:
            return ""

        output = io.StringIO()
        writer = csv.DictWriter(output, fieldnames=sorted(self.ALL_HEADERS))
        writer.writeheader()

        for widget in widgets:
            # Map DB record to CSV format
            row = {
                "name": widget.get("name"),
                "website_url": widget.get("website_url"),
                "phone_number": widget.get("phone_number"),
                "greeting_message": widget.get("greeting_message"),
                "voice_type": widget.get("voice_type", VoiceType.NEUTRAL),
                "position": widget.get("position", WidgetPosition.BOTTOM_RIGHT),
                "primary_color": widget.get("primary_color"),
                "agency_client_id": widget.get("agency_client_id"),
                "auto_open_delay": widget.get("auto_open_delay")
            }
            writer.writerow(row)

        return output.getvalue()
```

## 3. Database Repository (`src/repositories/widget_repo.py`)

```python
"""
Asyncpg repository for widget bulk operations with transaction support.
"""

from typing import List, Dict, Any, Optional
import asyncpg
import logging
from datetime import datetime

from src.models.widget_bulk import WidgetCSVRow

logger = logging.getLogger(__name__)

class WidgetRepository:
    """PostgreSQL repository for widget CRUD and bulk operations."""

    def __init__(self, pool: asyncpg.Pool):
        self.pool = pool

    async def bulk_create(
        self,
        widgets: List[WidgetCSVRow],
        agency_id: str,
        created_by: str
    ) -> List[str]:
        """
        Insert multiple widgets in a single transaction.
        Returns list of created widget UUIDs.

        Raises:
            asyncpg.UniqueViolationError: If website_url conflicts with existing widget
            asyncpg.ForeignKeyViolationError: If agency_id invalid
        """
        if not widgets:
            return []

        created_ids = []

        # Use transaction for atomicity
        async with self.pool.acquire() as conn:
            async with conn.transaction():
                # Prepare statement for performance
                stmt = await conn.prepare("""
                    INSERT INTO widgets (
                        id, agency_id, name, website_url, phone_number,
                        greeting_message, voice_type, position, primary_color,
                        agency_client_id, auto_open_delay, created_by, created_at
                    ) VALUES (
                        gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, NOW()
                    )
                    RETURNING id
                """)

                for widget in widgets:
                    try:
                        result = await stmt.fetchval(
                            agency_id,
                            widget.name,
                            str(widget.website_url),
                            widget.phone_number,
                            widget.greeting_message,
                            widget.voice_type.value,
                            widget.position.value,
                            widget.primary_color,
                            widget.agency_client_id,
                            widget.auto_open_delay
                        )
                        created_ids.append(str(result))
                    except asyncpg.UniqueViolationError as e:
                        # Re-raise with context for the service layer to handle
                        raise DuplicateWidgetError(
                            f"Widget with URL {widget.website_url} already exists",
                            line_number=widget.line_number
                        ) from e

        return created_ids

    async def get_for_export(
        self,
        agency_id: Optional[str] = None,
        filters: Optional[Dict[str, Any]] = None
    ) -> List[Dict[str, Any]]:
        """
        Fetch widgets for CSV export with optional filtering.
        """
        conditions = ["1=1"]
        params = []
        param_idx = 1

        if agency_id:
            conditions.append(f"agency_id = ${param_idx}")
            params.append(agency_id)
            param_idx += 1

        if filters:
            if filters.get("active_only"):
                conditions.append("is_active = true")
            if filters.get("created_after"):
                conditions.append(f"created_at >= ${param_idx}")
                params.append(filters["created_after"])
                param_idx += 1
            if filters.get("created_before"):
                conditions.append(f"created_at <= ${param_idx}")
                params.append(filters["created_before"])
                param_idx += 1

        query = f"""
            SELECT
                name, website_url, phone_number, greeting_message,
                voice_type, position, primary_color, agency_client_id,
                auto_open_delay, created_at
            FROM widgets
            WHERE {" AND ".join(conditions)}
            ORDER BY created_at DESC
        """

        async with self.pool.acquire() as conn:
            rows = await conn.fetch(query, *params)
            return [dict(row) for row in rows]

class DuplicateWidgetError(Exception):
    """Raised when attempting to create a widget with duplicate unique key."""
    def __init__(self, message: str, line_number: int):
        super().__init__(message)
        self.line_number = line_number
```

## 4. Business Logic Service (`src/services/widget_bulk_service.py`)

```python
"""
Business logic for bulk widget operations.
Coordinates between CSV processing, validation, and persistence.
"""

import time
from typing import List, Tuple
import logging

from src.models.widget_bulk import (
    WidgetCSVRow, BulkImportResult, RowValidationError,
    ExportFilter, CSVProcessingError
)
from src.repositories.widget_repo import WidgetRepository, DuplicateWidgetError
from src.services.csv_processor import AsyncCSVProcessor

logger = logging.getLogger(__name__)

class WidgetBulkService:
    """
    Service layer for bulk widget import/export.
    Handles dry-run logic, partial failure management, and audit logging.
    """

    def __init__(
        self,
        repository: WidgetRepository,
        csv_processor: AsyncCSVProcessor
    ):
        self.repo = repository
        self.processor = csv_processor

    async def import_widgets(
        self,
        file_content: bytes,
        agency_id: str,
        user_id: str,
        dry_run: bool = False
    ) -> BulkImportResult:
        """
        Process CSV upload and optionally create widgets.

        Strategy:
        1. Parse CSV with row-level validation (collects all syntax errors)
        2. If syntax errors exist, return early (fail fast on bad CSV)
        3. If dry_run, return validation results without DB write
        4. Otherwise, attempt DB insertion with transaction per batch
        5. Collect DB-level errors (duplicates, FK violations) with row refs
        """
        start_time = time.time()

        try:
            # Step 1: Parse and validate CSV structure
            valid_rows, validation_errors = await self.processor.parse_upload(
                file_content
            )

            # Fail fast on CSV syntax/validation errors
            if validation_errors:
                return BulkImportResult(
                    success=False,
                    processed_count=len(valid_rows) + len(validation_errors),
                    created_count=0,
                    failed_count=len(validation_errors),
                    dry_run=dry_run,
                    errors=validation_errors,
                    created_widget_ids=[],
                    processing_time_ms=(time.time() - start_time) * 1000
                )

            if dry_run:
                # Validation-only mode
                return BulkImportResult(
                    success=True,
                    processed_count=len(valid_rows),
                    created_count=0,  # Nothing actually created
                    failed_count=0,
                    dry_run=True,
                    errors=[],
                    created_widget_ids=[],
                    processing_time_ms=(time.time() - start_time) * 1000
                )

            # Step 2: Database insertion
            db_errors: List[RowValidationError] = []
            created_ids: List[str] = []

            try:
                created_ids = await self.repo.bulk_create(
                    valid_rows, agency_id, user_id
                )
            except DuplicateWidgetError as e:
                # Handle duplicate URL errors mapped back to CSV lines
                db_errors.append(RowValidationError(
                    line_number=e.line_number,
                    field="website_url",
                    error_message=str(e),
                    raw_value=None
                ))
            except Exception as e:
                # Catch-all for DB errors (FK violations, etc.)
                logger.error(f"Database error during bulk import: {e}", exc_info=True)
                db_errors.append(RowValidationError(
                    line_number=0,  # Unknown line
                    field=None,
                    error_message=f"Database error: {str(e)}",
                    raw_value=None
                ))

            processing_time = (time.time() - start_time) * 1000

            return BulkImportResult(
                success=len(db_errors) == 0,
                processed_count=len(valid_rows),
                created_count=len(created_ids),
                failed_count=len(db_errors),
                dry_run=False,
                errors=db_errors,
                created_widget_ids=created_ids,
                processing_time_ms=processing_time
            )

        except CSVProcessingError as e:
            # Fatal CSV structure errors (missing headers, encoding, etc.)
            logger.error(f"CSV processing failed: {e}")
            return BulkImportResult(
                success=False,
                processed_count=0,
                created_count=0,
                failed_count=1,
                dry_run=dry_run,
                errors=[RowValidationError(
                    line_number=0,
                    field=None,
                    error_message=str(e),
                    raw_value=None
                )],
                created_widget_ids=[],
                processing_time_ms=(time.time() - start_time) * 1000
            )

    async def export_widgets(
        self,
        agency_id: Optional[str],
        filters: ExportFilter
    ) -> Tuple[str, int]:
        """
        Generate CSV export of widgets.

        Returns:
            Tuple of (csv_content, record_count)


Response:
 ```python
"""
Sunaiva Onboarding Flow API
Production-ready implementation for TW-API-32
"""

from __future__ import annotations

import json
import logging
from datetime import datetime
from enum import IntEnum
from typing import Any, Dict, List, Optional, Protocol
from uuid import UUID, uuid4

import asyncpg
import redis.asyncio as redis
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.security import HTTPBearer
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator

# Configure logging
logger = logging.getLogger(__name__)

# =============================================================================
# Exceptions
# =============================================================================

class OnboardingError(Exception):
    """Base exception for onboarding flow errors."""
    pass

class OnboardingNotFoundError(OnboardingError):
    """Raised when onboarding session is not found."""
    pass

class InvalidStepError(OnboardingError):
    """Raised when invalid step number or data is provided."""
    pass

class ValidationError(OnboardingError):
    """Raised when step data validation fails."""
    pass

class DatabaseTransactionError(OnboardingError):
    """Raised when database transaction fails."""
    pass

# =============================================================================
# Enums and Constants
# =============================================================================

class OnboardingStep(IntEnum):
    """Enumeration of onboarding steps with their completion weights."""
    BUSINESS_INFO = 1   # 25%
    WIDGET_CONFIG = 2   # 50%
    EMBED_CODE = 3      # 75%
    TEST_CALL = 4       # 100%

    @classmethod
    def get_progress_percentage(cls, step: int) -> int:
        """Calculate completion percentage based on current step."""
        return min(step * 25, 100)

# =============================================================================
# Pydantic Models (Request/Response)
# =============================================================================

class BusinessInfoData(BaseModel):
    """Schema for business information step."""
    model_config = ConfigDict(strict=True)

    business_name: str = Field(..., min_length=1, max_length=255)
    industry: str = Field(..., min_length=1, max_length=100)
    phone_number: str = Field(..., pattern=r'^\+?1?\d{9,15}$')
    timezone: str = Field(default="UTC", max_length=50)
    description: Optional[str] = Field(default=None, max_length=1000)

    @field_validator('phone_number')
    @classmethod
    def validate_phone(cls, v: str) -> str:
        """Normalize phone number format."""
        return v.strip().replace('-', '').replace(' ', '')

class WidgetConfigData(BaseModel):
    """Schema for widget configuration step."""
    model_config = ConfigDict(strict=True)

    voice_id: str = Field(..., min_length=1, max_length=100)
    greeting_message: str = Field(..., min_length=1, max_length=500)
    color_scheme: str = Field(default="#007bff", pattern=r'^#[0-9A-Fa-f]{6}$')
    position: str = Field(default="bottom-right",
                         pattern=r'^(bottom-right|bottom-left|top-right|top-left)$')
    language: str = Field(default="en", max_length=10)

    @field_validator('greeting_message')
    @classmethod
    def validate_greeting(cls, v: str) -> str:
        """Ensure greeting message is not empty after stripping."""
        if not v.strip():
            raise ValueError("Greeting message cannot be empty")
        return v.strip()

class EmbedCodeData(BaseModel):
    """Schema for embed code configuration step."""
    model_config = ConfigDict(strict=True)

    domain: str = Field(..., min_length=1, max_length=255)
    page_patterns: List[str] = Field(default_factory=list, max_length=10)
    auto_open: bool = Field(default=False)
    open_delay_seconds: int = Field(default=0, ge=0, le=300)

    @field_validator('domain')
    @classmethod
    def validate_domain(cls, v: str) -> str:
        """Normalize domain format."""
        v = v.strip().lower()
        if not v.startswith(('http://', 'https://')):
            v = f"https://{v}"
        return v

class TestCallData(BaseModel):
    """Schema for test call step."""
    model_config = ConfigDict(strict=True)

    test_phone_number: str = Field(..., pattern=r'^\+?1?\d{9,15}$')
    completed_call_id: Optional[str] = Field(default=None, max_length=100)
    call_successful: bool = Field(default=False)
    notes: Optional[str] = Field(default=None, max_length=500)

class OnboardingStatusResponse(BaseModel):
    """Response schema for onboarding status."""
    model_config = ConfigDict(strict=True)

    session_id: UUID
    current_step: int
    completion_percentage: int
    steps_completed: List[int]
    is_completed: bool
    widget_id: Optional[UUID] = None
    created_at: datetime
    updated_at: datetime

class StepUpdateResponse(BaseModel):
    """Response schema for step update."""
    model_config = ConfigDict(strict=True)

    success: bool
    current_step: int
    completion_percentage: int
    next_step: Optional[int] = None
    message: str

class OnboardingSessionCreate(BaseModel):
    """Response schema for onboarding start."""
    model_config = ConfigDict(strict=True)

    session_id: UUID
    widget_id: UUID
    message: str = "Onboarding started successfully"

# =============================================================================
# Database Repository Pattern
# =============================================================================

class OnboardingRepository:
    """Repository for onboarding session database operations."""

    def __init__(self, db_pool: asyncpg.Pool):
        self.db_pool = db_pool

    async def create_session(
        self,
        user_id: UUID,
        widget_id: UUID
    ) -> Dict[str, Any]:
        """
        Create a new onboarding session and widget skeleton.

        Uses a transaction to ensure atomicity between session and widget creation.
        """
        async with self.db_pool.acquire() as conn:
            async with conn.transaction():
                try:
                    # Create onboarding session
                    session_id = uuid4()
                    row = await conn.fetchrow(
                        """
                        INSERT INTO onboarding_sessions
                            (id, user_id, widget_id, current_step, completed_steps, created_at, updated_at)
                        VALUES
                            ($1, $2, $3, 0, $4, NOW(), NOW())
                        RETURNING id, current_step, completed_steps, created_at, updated_at
                        """,
                        session_id,
                        user_id,
                        widget_id,
                        []
                    )

                    logger.info(f"Created onboarding session {session_id} for user {user_id}")
                    return dict(row)

                except asyncpg.PostgresError as e:
                    logger.error(f"Database error creating onboarding session: {e}")
                    raise DatabaseTransactionError(f"Failed to create onboarding session: {e}")

    async def get_session(self, session_id: UUID) -> Optional[Dict[str, Any]]:
        """Retrieve onboarding session by ID."""
        async with self.db_pool.acquire() as conn:
            row = await conn.fetchrow(
                """
                SELECT
                    id, user_id, widget_id, current_step, completed_steps,
                    business_info, widget_config, embed_code, test_call,
                    created_at, updated_at, completed_at
                FROM onboarding_sessions
                WHERE id = $1
                """,
                session_id
            )
            return dict(row) if row else None

    async def update_step(
        self,
        session_id: UUID,
        step: int,
        step_data: Dict[str, Any],
        widget_config_update: Optional[Dict[str, Any]] = None
    ) -> Dict[str, Any]:
        """
        Update specific step data and mark step as completed.

        Args:
            session_id: The onboarding session UUID
            step: Step number (1-4)
            step_data: JSONB data for the step
            widget_config_update: Optional widget configuration updates
        """
        # Map step numbers to column names
        step_columns = {
            1: "business_info",
            2: "widget_config",
            3: "embed_code",
            4: "test_call"
        }

        if step not in step_columns:
            raise InvalidStepError(f"Invalid step number: {step}")

        column_name = step_columns[step]

        async with self.db_pool.acquire() as conn:
            async with conn.transaction():
                try:
                    # Update step data and add to completed_steps array if not present
                    row = await conn.fetchrow(
                        f"""
                        UPDATE onboarding_sessions
                        SET
                            {column_name} = $1,
                            current_step = GREATEST(current_step, $2),
                            completed_steps = (
                                SELECT array_agg(DISTINCT x)
                                FROM unnest(array_append(completed_steps, $2)) AS x
                            ),
                            updated_at = NOW(),
                            completed_at = CASE WHEN $2 = 4 THEN NOW() ELSE completed_at END
                        WHERE id = $3
                        RETURNING id, current_step, completed_steps, widget_id, updated_at
                        """,
                        json.dumps(step_data),
                        step,
                        session_id
                    )

                    if not row:
                        raise OnboardingNotFoundError(f"Session {session_id} not found")

                    # If widget config update provided, update widget table
                    if widget_config_update:
                        await conn.execute(
                            """
                            UPDATE widgets
                            SET config = config || $1::jsonb,
                                status = CASE
                                    WHEN $2 = 4 THEN 'active'
                                    ELSE 'draft'
                                END,
                                updated_at = NOW()
                            WHERE id = $3
                            """,
                            json.dumps(widget_config_update),
                            step,
                            row['widget_id']
                        )

                    logger.info(f"Updated step {step} for session {session_id}")
                    return dict(row)

                except asyncpg.PostgresError as e:
                    logger.error(f"Database error updating step: {e}")
                    raise DatabaseTransactionError(f"Failed to update step: {e}")

    async def get_status(self, session_id: UUID) -> Optional[Dict[str, Any]]:
        """Get current onboarding status with completion percentage."""
        session = await self.get_session(session_id)
        if not session:
            return None

        current_step = session['current_step']
        completed_steps = session['completed_steps'] or []

        # Calculate percentage based on highest completed step or current step
        max_completed = max(completed_steps) if completed_steps else 0
        effective_step = max(current_step, max_completed)

        session['completion_percentage'] = OnboardingStep.get_progress_percentage(effective_step)
        session['steps_completed'] = completed_steps
        session['is_completed'] = 4 in completed_steps or effective_step >= 4

        return session

class WidgetRepository:
    """Repository for widget operations."""

    def __init__(self, db_pool: asyncpg.Pool):
        self.db_pool = db_pool

    async def create_skeleton(self, user_id: UUID, name: str = "My First Widget") -> UUID:
        """Create a widget skeleton in draft status."""
        widget_id = uuid4()

        async with self.db_pool.acquire() as conn:
            try:
                await conn.execute(
                    """
                    INSERT INTO widgets (id, user_id, name, config, status, created_at, updated_at)
                    VALUES ($1, $2, $3, $4::jsonb, 'draft', NOW(), NOW())
                    """,
                    widget_id,
                    user_id,
                    name,
                    json.dumps({
                        "version": "1.0",
                        "source": "onboarding_wizard",
                        "initialized_at": datetime.utcnow().isoformat()
                    })
                )
                logger.info(f"Created widget skeleton {widget_id} for user {user_id}")
                return widget_id
            except asyncpg.PostgresError as e:
                logger.error(f"Failed to create widget: {e}")
                raise DatabaseTransactionError(f"Failed to create widget: {e}")

# =============================================================================
# Service Layer
# =============================================================================

class OnboardingService:
    """Business logic service for onboarding flow."""

    def __init__(
        self,
        onboarding_repo: OnboardingRepository,
        widget_repo: WidgetRepository,
        redis_client: Optional[redis.Redis] = None
    ):
        self.onboarding_repo = onboarding_repo
        self.widget_repo = widget_repo
        self.redis = redis_client
        self.cache_ttl = 3600  # 1 hour

    def _get_cache_key(self, session_id: UUID) -> str:
        """Generate Redis cache key for session status."""
        return f"onboarding:status:{session_id}"

    async def _invalidate_cache(self, session_id: UUID) -> None:
        """Invalidate cached session status."""
        if self.redis:
            await self.redis.delete(self._get_cache_key(session_id))

    async def start_onboarding(self, user_id: UUID) -> OnboardingSessionCreate:
        """
        Initialize onboarding flow for a new user.

        Creates:
        1. Widget skeleton in draft mode
        2. Onboarding session tracking record
        """
        try:
            # Create widget skeleton first
            widget_id = await self.widget_repo.create_skeleton(user_id)

            # Create onboarding session
            session = await self.onboarding_repo.create_session(user_id, widget_id)

            return OnboardingSessionCreate(
                session_id=session['id'],
                widget_id=widget_id,
                message="Onboarding started successfully. Proceed to step 1: Business Info."
            )

        except DatabaseTransactionError:
            raise
        except Exception as e:
            logger.error(f"Unexpected error starting onboarding: {e}")
            raise OnboardingError(f"Failed to start onboarding: {str(e)}")

    async def save_step(
        self,
        session_id: UUID,
        step: int,
        data: BaseModel
    ) -> StepUpdateResponse:
        """
        Save data for a specific onboarding step.

        Validates step sequence and updates widget configuration incrementally.
        """
        # Validate step number
        if not 1 <= step <= 4:
            raise InvalidStepError(f"Step must be between 1 and 4, got {step}")

        # Get current session to validate sequence (optional: enforce linear progression)
        session = await self.onboarding_repo.get_session(session_id)
        if not session:
            raise OnboardingNotFoundError(f"Session {session_id} not found")

        # Prepare step-specific widget updates
        widget_update = None
        step_data = data.model_dump()

        if step == 1:  # Business Info
            widget_update = {
                "business_name": step_data.get("business_name"),
                "business_phone": step_data.get("phone_number"),
                "timezone": step_data.get("timezone")
            }
        elif step == 2:  # Widget Config
            widget_update = {
                "voice_id": step_data.get("voice_id"),
                "greeting_message": step_data.get("greeting_message"),
                "appearance": {
                    "color_scheme": step_data.get("color_scheme"),
                    "position": step_data.get("position")
                },
                "language": step_data.get("language")
            }
        elif step == 3:  # Embed Code
            widget_update = {
                "domain": step_data.get("domain"),
                "page_patterns": step_data.get("page_patterns"),
                "behavior": {
                    "auto_open": step_data.get("auto_open"),
                    "open_delay_seconds": step_data.get("open_delay_seconds")
                }
            }
        elif step == 4:  # Test Call
            # Just mark as tested, no widget config update needed
            pass

        # Update database
        updated = await self.onboarding_repo.update_step(
            session_id=session_id,
            step=step,
            step_data=step_data,
            widget_config_update=widget_update
        )

        # Invalidate cache
        await self._invalidate_cache(session_id)

        # Determine next step
        next_step = step + 1 if step < 4 else None

        return StepUpdateResponse(
            success=True,
            current_step=updated['current_step'],
            completion_percentage=OnboardingStep.get_progress_percentage(updated['current_step']),
            next_step=next_step,
            message=f"Step {step} completed successfully."
        )

    async def get_status(self, session_id: UUID) -> OnboardingStatusResponse:
        """Retrieve current onboarding status with caching."""
        # Try cache first
        if self.redis:
            cached = await self.redis.get(self._get_cache_key(session_id))
            if cached:
                data = json.loads(cached)
                return OnboardingStatusResponse(**data)

        # Get from database
        status = await self.onboarding_repo.get_status(session_id)
        if not status:
            raise OnboardingNotFoundError(f"Session {session_id} not found")

        response = OnboardingStatusResponse(
            session_id=status['id'],
            current_step=status['current_step'],
            completion_percentage=status['completion_percentage'],
            steps_completed=status['steps_completed'],
            is_completed=status['is_completed'],
            widget_id=status['widget_id'],
            created_at=status['created_at'],
            updated_at=status['updated_at']
        )

        # Cache the result
        if self.redis:
            await self.redis.setex(
                self._get_cache_key(session_id),
                self.cache_ttl,
                response.model_dump_json()
            )

        return response

# =============================================================================
# FastAPI Dependencies
# =============================================================================

# Dependency injection stubs - replace with actual implementations
async def get_db_pool() -> asyncpg.P


Response:
I'll implement the i18n system for Sunaiva's widget.js with a comprehensive backend (FastAPI) and frontend (TypeScript) solution. This includes translation management, RTL support, and browser language detection.

## 1. Database Schema

```sql
-- backend/db/migrations/004_i18n_support.sql
-- Enable UUID extension if not already enabled
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

-- Translations table: stores all UI strings
CREATE TABLE translations (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    locale VARCHAR(10) NOT NULL, -- e.g., 'en-US', 'es', 'zh-CN'
    namespace VARCHAR(50) NOT NULL DEFAULT 'common', -- e.g., 'widget', 'voice', 'errors'
    key VARCHAR(100) NOT NULL,
    value TEXT NOT NULL,
    metadata JSONB DEFAULT '{}', -- plural forms, context, etc.
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    UNIQUE(locale, namespace, key)
);

-- Widget configuration for i18n settings
CREATE TABLE widget_i18n_config (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    widget_id UUID NOT NULL REFERENCES widgets(id) ON DELETE CASCADE,
    default_locale VARCHAR(10) NOT NULL DEFAULT 'en-US',
    supported_locales TEXT[] NOT NULL DEFAULT ARRAY['en-US'],
    auto_detect BOOLEAN NOT NULL DEFAULT true,
    rtl_enabled BOOLEAN NOT NULL DEFAULT true,
    created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
    UNIQUE(widget_id)
);

-- Indexes for performance
CREATE INDEX idx_translations_locale_namespace ON translations(locale, namespace);
CREATE INDEX idx_translations_key ON translations(key);
CREATE INDEX idx_widget_i18n_widget_id ON widget_i18n_config(widget_id);

-- Trigger to update updated_at timestamp
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
    NEW.updated_at = NOW();
    RETURN NEW;
END;
$$ language 'plpgsql';

CREATE TRIGGER update_translations_updated_at BEFORE UPDATE ON translations
    FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();

CREATE TRIGGER update_widget_i18n_updated_at BEFORE UPDATE ON widget_i18n_config
    FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();

-- Seed data for common languages
INSERT INTO translations (locale, namespace, key, value) VALUES
-- English (US)
('en-US', 'widget', 'greeting', 'Hello! How can I help you today?'),
('en-US', 'widget', 'placeholder', 'Type your message...'),
('en-US', 'widget', 'send_button', 'Send'),
('en-US', 'widget', 'voice_button', 'Hold to speak'),
('en-US', 'widget', 'close_button', 'Close'),
('en-US', 'widget', 'powered_by', 'Powered by Sunaiva'),
('en-US', 'errors', 'mic_permission', 'Please allow microphone access to use voice features.'),
('en-US', 'errors', 'connection_error', 'Connection lost. Retrying...'),

-- Spanish
('es', 'widget', 'greeting', '¡Hola! ¿Cómo puedo ayudarte hoy?'),
('es', 'widget', 'placeholder', 'Escribe tu mensaje...'),
('es', 'widget', 'send_button', 'Enviar'),
('es', 'widget', 'voice_button', 'Mantén para hablar'),
('es', 'widget', 'close_button', 'Cerrar'),
('es', 'widget', 'powered_by', 'Desarrollado por Sunaiva'),
('es', 'errors', 'mic_permission', 'Por favor permite el acceso al micrófono para usar funciones de voz.'),
('es', 'errors', 'connection_error', 'Conexión perdida. Reintentando...'),

-- French
('fr', 'widget', 'greeting', 'Bonjour ! Comment puis-je vous aider aujourd''hui ?'),
('fr', 'widget', 'placeholder', 'Tapez votre message...'),
('fr', 'widget', 'send_button', 'Envoyer'),
('fr', 'widget', 'voice_button', 'Maintenez pour parler'),
('fr', 'widget', 'close_button', 'Fermer'),
('fr', 'widget', 'powered_by', 'Propulsé par Sunaiva'),

-- German
('de', 'widget', 'greeting', 'Hallo! Wie kann ich Ihnen heute helfen?'),
('de', 'widget', 'placeholder', 'Nachricht eingeben...'),
('de', 'widget', 'send_button', 'Senden'),
('de', 'widget', 'voice_button', 'Zum Sprechen halten'),
('de', 'widget', 'close_button', 'Schließen'),
('de', 'widget', 'powered_by', 'Powered by Sunaiva'),

-- Chinese (Simplified)
('zh', 'widget', 'greeting', '您好！今天我能为您提供什么帮助？'),
('zh', 'widget', 'placeholder', '输入您的消息...'),
('zh', 'widget', 'send_button', '发送'),
('zh', 'widget', 'voice_button', '按住说话'),
('zh', 'widget', 'close_button', '关闭'),
('zh', 'widget', 'powered_by', '由 Sunaiva 提供技术支持'),

-- Japanese
('ja', 'widget', 'greeting', 'こんにちは！今日はどのようにお手伝いできますか？'),
('ja', 'widget', 'placeholder', 'メッセージを入力...'),
('ja', 'widget', 'send_button', '送信'),
('ja', 'widget', 'voice_button', '話すために押し続ける'),
('ja', 'widget', 'close_button', '閉じる'),
('ja', 'widget', 'powered_by', 'Sunaiva 提供');
```

## 2. Backend Models (Pydantic v2)

```python
# backend/models/i18n.py
"""
Internationalization models for Sunaiva platform.
Handles translations, locale configurations, and RTL support.
"""

from datetime import datetime
from enum import Enum
from typing import Any, Dict, List, Optional, Set
from uuid import UUID

from pydantic import BaseModel, ConfigDict, Field, field_validator

class Direction(str, Enum):
    """Text direction for layout support."""
    LTR = "ltr"
    RTL = "rtl"

class Locale(str, Enum):
    """Supported locales for Sunaiva widgets."""
    EN_US = "en-US"
    EN_AU = "en-AU"
    ES = "es"
    FR = "fr"
    DE = "de"
    ZH = "zh"
    JA = "ja"
    AR = "ar"  # Arabic for RTL testing
    HE = "he"  # Hebrew for RTL testing

# RTL languages set
RTL_LOCALES: Set[str] = {"ar", "he", "fa", "ur", "yi"}

class TranslationMetadata(BaseModel):
    """Metadata for translation entries (plural forms, context, etc.)."""
    context: Optional[str] = None
    plural_form: Optional[str] = None
    gender: Optional[str] = None
    custom_data: Dict[str, Any] = Field(default_factory=dict)

    model_config = ConfigDict(extra="allow")

class TranslationBase(BaseModel):
    """Base translation model."""
    locale: Locale
    namespace: str = Field(default="common", max_length=50)
    key: str = Field(..., max_length=100)
    value: str
    metadata: TranslationMetadata = Field(default_factory=TranslationMetadata)

class TranslationCreate(TranslationBase):
    """Model for creating translations."""
    pass

class TranslationUpdate(BaseModel):
    """Model for updating translations."""
    value: Optional[str] = None
    metadata: Optional[TranslationMetadata] = None

class TranslationInDB(TranslationBase):
    """Translation model as stored in database."""
    model_config = ConfigDict(from_attributes=True)

    id: UUID
    created_at: datetime
    updated_at: datetime

class WidgetI18nConfigBase(BaseModel):
    """Base internationalization configuration for widgets."""
    default_locale: Locale = Locale.EN_US
    supported_locales: List[Locale] = Field(default_factory=lambda: [Locale.EN_US])
    auto_detect: bool = True
    rtl_enabled: bool = True

    @field_validator("supported_locales")
    @classmethod
    def validate_supported_locales(cls, v: List[Locale]) -> List[Locale]:
        """Ensure supported_locales is not empty."""
        if not v:
            raise ValueError("At least one supported locale must be provided")
        return v

class WidgetI18nConfigCreate(WidgetI18nConfigBase):
    """Model for creating widget i18n configuration."""
    widget_id: UUID

class WidgetI18nConfigUpdate(BaseModel):
    """Model for updating widget i18n configuration."""
    default_locale: Optional[Locale] = None
    supported_locales: Optional[List[Locale]] = None
    auto_detect: Optional[bool] = None
    rtl_enabled: Optional[bool] = None

class WidgetI18nConfigInDB(WidgetI18nConfigBase):
    """Widget i18n configuration as stored in database."""
    model_config = ConfigDict(from_attributes=True)

    id: UUID
    widget_id: UUID
    created_at: datetime
    updated_at: datetime

class LocaleInfo(BaseModel):
    """Locale metadata for frontend consumption."""
    code: str
    name: str
    native_name: str
    direction: Direction
    is_rtl: bool

class TranslationBundle(BaseModel):
    """Complete translation bundle for a widget."""
    locale: str
    namespace: str
    translations: Dict[str, str]
    direction: Direction
    fallback_locale: Optional[str] = None

class BrowserLocaleDetection(BaseModel):
    """Result of browser locale detection."""
    detected_locale: str
    confidence: float
    available_locales: List[str]
    matched_locale: Optional[str] = None
```

## 3. Backend Services

```python
# backend/services/i18n.py
"""
Internationalization service for managing translations and locale detection.
Handles caching, fallback chains, and RTL support.
"""

import json
import re
from typing import Dict, List, Optional, Set, Tuple
from uuid import UUID

import asyncpg
from fastapi import HTTPException, status

from backend.models.i18n import (
    BrowserLocaleDetection,
    Direction,
    Locale,
    LocaleInfo,
    RTL_LOCALES,
    TranslationBundle,
    TranslationCreate,
    TranslationInDB,
    TranslationUpdate,
    WidgetI18nConfigCreate,
    WidgetI18nConfigInDB,
    WidgetI18nConfigUpdate,
)
from backend.services.cache import CacheService

class I18nService:
    """Service for internationalization operations."""

    # Locale metadata mapping
    LOCALE_INFO: Dict[str, LocaleInfo] = {
        "en-US": LocaleInfo(code="en-US", name="English (US)", native_name="English", direction=Direction.LTR, is_rtl=False),
        "en-AU": LocaleInfo(code="en-AU", name="English (Australia)", native_name="English", direction=Direction.LTR, is_rtl=False),
        "es": LocaleInfo(code="es", name="Spanish", native_name="Español", direction=Direction.LTR, is_rtl=False),
        "fr": LocaleInfo(code="fr", name="French", native_name="Français", direction=Direction.LTR, is_rtl=False),
        "de": LocaleInfo(code="de", name="German", native_name="Deutsch", direction=Direction.LTR, is_rtl=False),
        "zh": LocaleInfo(code="zh", name="Chinese (Simplified)", native_name="中文", direction=Direction.LTR, is_rtl=False),
        "ja": LocaleInfo(code="ja", name="Japanese", native_name="日本語", direction=Direction.LTR, is_rtl=False),
        "ar": LocaleInfo(code="ar", name="Arabic", native_name="العربية", direction=Direction.RTL, is_rtl=True),
        "he": LocaleInfo(code="he", name="Hebrew", native_name="עברית", direction=Direction.RTL, is_rtl=True),
    }

    def __init__(self, db_pool: asyncpg.Pool, cache_service: CacheService):
        self.db_pool = db_pool
        self.cache = cache_service
        self.cache_ttl = 3600  # 1 hour

    def _get_cache_key(self, locale: str, namespace: str, widget_id: Optional[str] = None) -> str:
        """Generate cache key for translations."""
        if widget_id:
            return f"i18n:{widget_id}:{locale}:{namespace}"
        return f"i18n:global:{locale}:{namespace}"

    async def get_translation(
        self,
        locale: str,
        namespace: str,
        key: str,
        widget_id: Optional[UUID] = None
    ) -> Optional[str]:
        """
        Retrieve a single translation string.

        Args:
            locale: Locale code (e.g., 'en-US', 'es')
            namespace: Translation namespace (e.g., 'widget', 'errors')
            key: Translation key
            widget_id: Optional widget-specific override

        Returns:
            Translated string or None if not found
        """
        cache_key = self._get_cache_key(locale, namespace, str(widget_id) if widget_id else None)

        # Try cache first
        cached = await self.cache.get(cache_key)
        if cached:
            translations = json.loads(cached)
            return translations.get(key)

        # Fetch from database
        async with self.db_pool.acquire() as conn:
            # Check for widget-specific override first
            if widget_id:
                row = await conn.fetchrow(
                    """
                    SELECT value FROM translations
                    WHERE widget_id = $1 AND locale = $2 AND namespace = $3 AND key = $4
                    """,
                    widget_id, locale, namespace, key
                )
                if row:
                    return row['value']

            # Fall back to global translation
            row = await conn.fetchrow(
                """
                SELECT value FROM translations
                WHERE widget_id IS NULL AND locale = $1 AND namespace = $2 AND key = $3
                """,
                locale, namespace, key
            )

            return row['value'] if row else None

    async def get_translation_bundle(
        self,
        locale: str,
        namespace: str = "widget",
        widget_id: Optional[UUID] = None,
        fallback_locale: Optional[str] = "en-US"
    ) -> TranslationBundle:
        """
        Retrieve a complete translation bundle for a widget.

        Args:
            locale: Target locale
            namespace: Translation namespace
            widget_id: Widget-specific configuration
            fallback_locale: Fallback locale if translations missing

        Returns:
            TranslationBundle with all strings and metadata
        """
        cache_key = self._get_cache_key(locale, namespace, str(widget_id) if widget_id else None)

        # Try cache
        cached = await self.cache.get(cache_key)
        if cached:
            data = json.loads(cached)
            return TranslationBundle(**data)

        # Build bundle from database
        translations: Dict[str, str] = {}

        async with self.db_pool.acquire() as conn:
            # Get widget config to determine fallbacks
            config_row = None
            if widget_id:
                config_row = await conn.fetchrow(
                    "SELECT * FROM widget_i18n_config WHERE widget_id = $1",
                    widget_id
                )

            # Fetch translations with fallback logic
            locales_to_try = [locale]
            if fallback_locale and fallback_locale != locale:
                locales_to_try.append(fallback_locale)
            if "en-US" not in locales_to_try:
                locales_to_try.append("en-US")

            for try_locale in locales_to_try:
                # Check for widget-specific first
                if widget_id:
                    rows = await conn.fetch(
                        """
                        SELECT key, value FROM translations
                        WHERE widget_id = $1 AND locale = $2 AND namespace = $3
                        AND key NOT IN (SELECT unnest($4::text[]))
                        """,
                        widget_id, try_locale, namespace, list(translations.keys())
                    )
                    for row in rows:
                        translations[row['key']] = row['value']

                # Then global
                rows = await conn.fetch(
                    """
                    SELECT key, value FROM translations
                    WHERE widget_id IS NULL AND locale = $1 AND namespace = $2
                    AND key NOT IN (SELECT unnest($3::text[]))
                    """,
                    try_locale, namespace, list(translations.keys())
                )
                for row in rows:
                    translations[row['key']] = row['value']

        # Determine direction
        is_rtl = self._is_rtl(locale)
        direction = Direction.RTL if is_rtl else Direction.LTR

        bundle = TranslationBundle(
            locale=locale,
            namespace=namespace,
            translations=translations,
            direction=direction,
            fallback_locale=fallback_locale if locale != fallback_locale else None
        )

        # Cache the bundle
        await self.cache.set(cache_key, bundle.model_dump_json(), ttl=self.cache_ttl)

        return bundle

    def _is_rtl(self, locale: str) -> bool:
        """Check if locale requires RTL layout."""
        # Extract base language code (e.g., 'ar-SA' -> 'ar')
        base_lang = locale.split('-')[0].lower()
        return base_lang in RTL_LOCALES

    async def detect_locale(
        self,
        accept_language: Optional[str],
        widget_id: UUID
    ) -> BrowserLocaleDetection:
        """
        Detect best locale from browser headers and widget configuration.

        Args:
            accept_language: HTTP Accept-Language header
            widget_id: Widget to check supported locales

        Returns:
            Detection result with matched locale
        """
        # Get widget config
        async with self.db_pool.acquire() as conn:
            config = await conn.fetchrow(
                "SELECT supported_locales, default_locale FROM widget_i18n_config WHERE widget_id = $1",
                widget_id
            )

            if not config:
                # Return default if no config exists
                return BrowserLocaleDetection(
                    detected_locale="en-US",
                    confidence=1.0,
                    available_locales=["en-US"],
                    matched_locale="en-US"
                )

            supported = config['supported_locales']
            default = config['default_locale']

        if not accept_language:
            return BrowserLocaleDetection(
                detected_locale=default,
                confidence=1.0,
                available_locales=supported,
                matched_locale=default
            )

        # Parse Accept-Language header
        # Format: "en-US,en;q=0.9,es;q=0.8,fr;q=0.7"
        locales = []
        for item in accept_language.split(','):
            item = item.strip()
            if ';' in item:
                locale, q = item.split(';')
                q = float(q.replace('q=', ''))
            else:
                locale = item
                q = 1.0
            locales.append((locale, q))

        # Sort by quality score
        locales.sort(key=lambda x: x[1], reverse=True)

        # Find best match
        matched_locale = None
        confidence = 0.0

        for locale, q in locales:
            # Exact match
            if locale in supported:
                matched_locale = locale
                confidence = q
                break

            # Language match (e.g., "es" matches "es-ES")
            lang = locale.split('-')[0]
            for