Response:
 Here is the production-ready implementation for the Multi-Tenant Widget Configuration story.

## 1. Database Schema

```sql
-- migrations/001_create_widget_tenants.sql
CREATE TABLE widget_tenants (
    widget_id VARCHAR(64) PRIMARY KEY,
    config JSONB NOT NULL DEFAULT '{}',
    api_key_hash VARCHAR(256),  -- Internal: never exposed publicly
    plan_type VARCHAR(50) DEFAULT 'basic',  -- Internal: billing tier
    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
);

CREATE INDEX idx_widget_tenants_widget_id ON widget_tenants(widget_id);
CREATE INDEX idx_widget_tenants_config_gin ON widget_tenants USING GIN (config);

-- Trigger to auto-update updated_at
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
    NEW.updated_at = CURRENT_TIMESTAMP;
    RETURN NEW;
END;
$$ language 'plpgsql';

CREATE TRIGGER update_widget_tenants_updated_at
    BEFORE UPDATE ON widget_tenants
    FOR EACH ROW
    EXECUTE FUNCTION update_updated_at_column();
```

## 2. Main Implementation

```python
# widget/api/tenant_config.py
"""
Multi-Tenant Widget Configuration API

This module provides public and administrative endpoints for managing
widget configurations with Redis caching and PostgreSQL persistence.
"""

import json
import logging
import re
from datetime import datetime, timedelta
from enum import Enum
from typing import Any, Dict, List, Optional, Union

import aioredis
from fastapi import APIRouter, Depends, HTTPException, Path, Security, status
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from pydantic import BaseModel, ConfigDict, Field, field_validator
from sqlalchemy import Column, DateTime, String, select, update
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import declarative_base

# Configure logging
logger = logging.getLogger(__name__)

# -----------------------------------------------------------------------------
# Enums and Constants
# -----------------------------------------------------------------------------

class VoiceType(str, Enum):
    """Supported voice types for the widget."""
    MALE = "male"
    FEMALE = "female"
    ACCENT = "accent"

class PositionType(str, Enum):
    """Widget positioning options."""
    BOTTOM_RIGHT = "bottom-right"
    BOTTOM_LEFT = "bottom-left"

class CacheConfig:
    """Cache configuration constants."""
    PREFIX = "widget:config"
    TTL_SECONDS = 300  # 5 minutes
    LOCK_TIMEOUT = 10  # Prevent cache stampede

# -----------------------------------------------------------------------------
# Pydantic Schemas
# -----------------------------------------------------------------------------

class ColorsSchema(BaseModel):
    """Color configuration with hex validation."""
    primary: str = Field(..., description="Primary brand color")
    secondary: str = Field(..., description="Secondary brand color")
    text: str = Field(..., description="Text color")

    @field_validator('primary', 'secondary', 'text')
    @classmethod
    def validate_hex_color(cls, v: str) -> str:
        """Validate 6-digit hex color codes."""
        if not re.match(r'^#[0-9A-Fa-f]{6}$', v):
            raise ValueError(f"Invalid hex color format: {v}. Expected format: #RRGGBB")
        return v.lower()

class FAQItem(BaseModel):
    """FAQ entry structure."""
    question: str = Field(..., min_length=1, max_length=500)
    answer: str = Field(..., min_length=1, max_length=2000)

class ServiceItem(BaseModel):
    """Service offering structure."""
    name: str = Field(..., min_length=1, max_length=100)
    description: Optional[str] = Field(None, max_length=500)
    price_range: Optional[str] = Field(None, max_length=50)

class HoursSchedule(BaseModel):
    """Business hours configuration."""
    timezone: str = Field(
        ...,
        pattern=r'^[A-Za-z_]+/[A-Za-z_]+$',
        description="IANA timezone (e.g., America/New_York)"
    )
    schedule: Dict[str, str] = Field(
        ...,
        description="Day: hours mapping (e.g., {'monday': '09:00-17:00', 'sunday': 'closed'})"
    )

    @field_validator('schedule')
    @classmethod
    def validate_schedule(cls, v: Dict[str, str]) -> Dict[str, str]:
        """Validate day names and time formats."""
        valid_days = {'monday', 'tuesday', 'wednesday', 'thursday',
                     'friday', 'saturday', 'sunday'}

        for day, hours in v.items():
            day_lower = day.lower()
            if day_lower not in valid_days:
                raise ValueError(f"Invalid day: {day}")

            if hours.lower() != 'closed':
                # Validate time range format HH:MM-HH:MM
                if not re.match(r'^\d{2}:\d{2}-\d{2}:\d{2}$', hours):
                    raise ValueError(f"Invalid time format for {day}: {hours}. Use HH:MM-HH:MM or 'closed'")

        return {k.lower(): v for k, v in v.items()}

class WidgetConfigBase(BaseModel):
    """Base configuration shared across all endpoints."""
    business_name: str = Field(..., min_length=1, max_length=200)
    greeting: str = Field(..., min_length=1, max_length=1000)
    voice: VoiceType
    colors: ColorsSchema
    position: PositionType
    language: str = Field(
        ...,
        pattern=r'^[a-z]{2}(-[A-Z]{2})?$',
        description="ISO 639-1 language code (e.g., 'en', 'en-US')"
    )
    faq: List[FAQItem] = Field(default_factory=list, max_length=50)
    services: List[ServiceItem] = Field(default_factory=list, max_length=20)
    hours: HoursSchedule

class WidgetConfigUpdate(WidgetConfigBase):
    """Schema for configuration updates (admin)."""
    pass

class WidgetConfigPublic(WidgetConfigBase):
    """
    Public-facing configuration.
    Excludes any internal-only fields that might be added to the database.
    """
    model_config = ConfigDict(from_attributes=True)

class WidgetConfigInternal(BaseModel):
    """Internal representation including DB metadata."""
    widget_id: str
    config: Dict[str, Any]
    plan_type: str  # Internal field: not exposed in public endpoints
    updated_at: datetime
    created_at: datetime

# -----------------------------------------------------------------------------
# Database Model
# -----------------------------------------------------------------------------

Base = declarative_base()

class WidgetTenant(Base):
    """SQLAlchemy model for widget_tenants table."""
    __tablename__ = "widget_tenants"

    widget_id = Column(String(64), primary_key=True, index=True)
    config = Column(JSONB, nullable=False, default=dict)
    api_key_hash = Column(String(256), nullable=True)  # Internal only
    plan_type = Column(String(50), default="basic")    # Internal only
    created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
    updated_at = Column(DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow)

# -----------------------------------------------------------------------------
# Dependencies
# -----------------------------------------------------------------------------

async def get_db_session() -> AsyncSession:
    """
    Dependency to get async database session.
    In production, this comes from your connection pool.
    """
    # Implementation depends on your setup (e.g., asyncpg, databases)
    raise NotImplementedError("Configure with your actual DB session provider")

async def get_redis_client() -> aioredis.Redis:
    """
    Dependency to get Redis client.
    """
    # Implementation depends on your Redis setup
    raise NotImplementedError("Configure with your actual Redis client")

async def verify_admin_token(
    credentials: HTTPAuthorizationCredentials = Security(HTTPBearer())
) -> Dict[str, Any]:
    """
    Verify JWT admin token and return payload.

    Raises:
        HTTPException: 401 if token invalid, 403 if insufficient permissions
    """
    token = credentials.credentials

    try:
        # In production: use python-jose or similar to decode JWT
        # payload = jwt.decode(token, SECRET_KEY, algorithms=["HS256"])

        # Mock validation for example:
        if token != "valid_admin_token":
            raise HTTPException(
                status_code=status.HTTP_401_UNAUTHORIZED,
                detail="Invalid authentication credentials",
                headers={"WWW-Authenticate": "Bearer"},
            )

        # Verify admin role
        payload = {"sub": "admin_user_123", "role": "admin"}
        if payload.get("role") != "admin":
            raise HTTPException(
                status_code=status.HTTP_403_FORBIDDEN,
                detail="Insufficient permissions"
            )

        return payload

    except Exception as e:
        logger.error(f"Token validation error: {e}")
        raise HTTPException(
            status_code=status.HTTP_401_UNAUTHORIZED,
            detail="Invalid token"
        )

# -----------------------------------------------------------------------------
# Service Layer
# -----------------------------------------------------------------------------

class WidgetConfigService:
    """
    Business logic for widget configuration management.
    Handles caching, validation, and database operations.
    """

    def __init__(self, db: AsyncSession, redis: aioredis.Redis):
        self.db = db
        self.redis = redis

    def _cache_key(self, widget_id: str) -> str:
        """Generate Redis cache key."""
        return f"{CacheConfig.PREFIX}:{widget_id}"

    async def get_config(self, widget_id: str) -> Optional[WidgetConfigPublic]:
        """
        Retrieve widget configuration with caching strategy.

        Strategy:
        1. Check Redis cache
        2. On cache miss: query PostgreSQL
        3. Validate and cache result
        4. Handle cache failures gracefully (cache-aside pattern)
        """
        cache_key = self._cache_key(widget_id)

        # Attempt cache retrieval
        try:
            cached_data = await self.redis.get(cache_key)
            if cached_data:
                logger.debug(f"Cache hit for widget {widget_id}")
                config_dict = json.loads(cached_data)
                return WidgetConfigPublic(**config_dict)
        except aioredis.RedisError as e:
            logger.error(f"Redis read error: {e}")
            # Continue to database on cache failure

        # Cache miss or failure - query database
        result = await self.db.execute(
            select(WidgetTenant).where(WidgetTenant.widget_id == widget_id)
        )
        tenant = result.scalar_one_or_none()

        if not tenant:
            logger.warning(f"Widget {widget_id} not found")
            return None

        if not tenant.config:
            raise HTTPException(
                status_code=status.HTTP_404_NOT_FOUND,
                detail="Configuration not initialized for this widget"
            )

        # Validate stored config against schema
        try:
            public_config = WidgetConfigPublic(**tenant.config)
        except Exception as e:
            logger.error(f"Data integrity error for widget {widget_id}: {e}")
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail="Invalid configuration stored in database"
            )

        # Write to cache (fire and forget - don't block response)
        try:
            await self.redis.setex(
                cache_key,
                CacheConfig.TTL_SECONDS,
                json.dumps(public_config.model_dump())
            )
            logger.debug(f"Cached config for widget {widget_id}")
        except aioredis.RedisError as e:
            logger.error(f"Redis write error: {e}")
            # Non-critical: continue without caching

        return public_config

    async def update_config(
        self,
        widget_id: str,
        config_update: WidgetConfigUpdate,
        admin_id: str
    ) -> WidgetConfigPublic:
        """
        Update widget configuration with transaction safety and cache invalidation.

        Args:
            widget_id: Target widget identifier
            config_update: New configuration data
            admin_id: Administrator making the change (for audit)

        Returns:
            Updated public configuration

        Raises:
            HTTPException: 404 if widget not found, 422 if validation fails
        """
        # Additional business logic validation
        self._validate_business_rules(config_update)

        # Verify widget exists
        result = await self.db.execute(
            select(WidgetTenant).where(


Widget Security - CORS and Domain Verification


Response:
 ```python
#!/usr/bin/env python3
"""
GHL Location Update Script (S-K103)

Production-ready Python script to update GoHighLevel (GHL) location details.
Updates location name to "Bunker FNQ" and address to "Kuranda QLD 4881".

Features:
    - Type-hinted with Pydantic validation
    - Comprehensive error handling with custom exceptions
    - Exponential backoff retry logic for transient failures
    - Structured logging
    - Environment-based configuration
    - Context manager for resource cleanup

Environment Variables:
    GHL_API_KEY (required): Bearer token for GHL API authentication
    GHL_LOCATION_ID (required): UUID of the location to update
    GHL_BASE_URL (optional): API base URL (default: https://rest.gohighlevel.com/v1)
    LOG_LEVEL (optional): DEBUG, INFO, WARNING, ERROR (default: INFO)

Usage:
    export GHL_API_KEY="your-api-key"
    export GHL_LOCATION_ID="your-location-id"
    python ghl_location_update.py

Exit Codes:
    0: Success
    1: Configuration error (missing env vars, invalid config)
    2: API error (authentication, not found, validation errors)
    3: Unexpected error
"""

import os
import sys
import json
import logging
from typing import Dict, Any, Optional, Final
from dataclasses import dataclass
from enum import IntEnum
import httpx
from pydantic import BaseModel, Field, ValidationError, validator
from tenacity import (
    retry,
    stop_after_attempt,
    wait_exponential,
    retry_if_exception_type,
    before_sleep_log,
    RetryError
)

# Constants
DEFAULT_BASE_URL: Final[str] = "https://rest.gohighlevel.com/v1"
MAX_RETRIES: Final[int] = 3
REQUIRED_ENV_VARS: Final[list] = ["GHL_API_KEY", "GHL_LOCATION_ID"]

# Logging Configuration
def configure_logging(log_level: Optional[str] = None) -> logging.Logger:
    """
    Configure structured logging with timestamps and log levels.

    Args:
        log_level: Optional override for log level (defaults to env var LOG_LEVEL or INFO)

    Returns:
        Configured logger instance
    """
    level = log_level or os.getenv("LOG_LEVEL", "INFO").upper()

    logging.basicConfig(
        level=getattr(logging, level, logging.INFO),
        format="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S"
    )
    return logging.getLogger("ghl_location_updater")

logger = configure_logging()

class ExitCode(IntEnum):
    """Standardized exit codes for shell integration and CI/CD pipelines."""
    SUCCESS = 0
    CONFIG_ERROR = 1
    API_ERROR = 2
    UNEXPECTED_ERROR = 3

class GHLAPIError(Exception):
    """
    Custom exception for GHL API errors.

    Attributes:
        status_code: HTTP status code returned by API
        message: Error message from API or exception
        response_body: Raw response body for debugging
    """

    def __init__(self, status_code: int, message: str, response_body: Optional[str] = None):
        self.status_code = status_code
        self.message = message
        self.response_body = response_body
        super().__init__(f"GHL API Error [{status_code}]: {message}")

class ConfigurationError(Exception):
    """Raised when required configuration is missing or invalid."""
    pass

class LocationUpdatePayload(BaseModel):
    """
    Pydantic model for GHL Location Update API payload.

    Validates the request structure before sending to API.
    """
    name: str = Field(default="Bunker FNQ", description="Location business name")
    address: str = Field(default="Kuranda QLD 4881", description="Formatted business address")

    @validator('name')
    def name_must_not_be_empty(cls, v):
        if not v or not v.strip():
            raise ValueError('Location name cannot be empty')
        return v.strip()

    @validator('address')
    def address_must_not_be_empty(cls, v):
        if not v or not v.strip():
            raise ValueError('Address cannot be empty')
        return v.strip()

    class Config:
        """Pydantic configuration."""
        schema_extra = {
            "example": {
                "name": "Bunker FNQ",
                "address": "Kuranda QLD 4881"
            }
        }

class GHLConfig(BaseModel):
    """
    Configuration model for GHL API connection.

    Validates environment variables and provides typed configuration access.
    """
    api_key: str = Field(..., description="GHL API Bearer token")
    location_id: str = Field(..., description="UUID of location to update")
    base_url: str = Field(default=DEFAULT_BASE_URL, description="GHL API base URL")

    @validator('api_key')
    def api_key_must_be_valid(cls, v):
        if not v or len(v.strip()) < 10:  # Basic length check for API keys
            raise ValueError('API key appears invalid (too short or empty)')
        return v.strip()

    @validator('location_id')
    def location_id_must_be_valid(cls, v):
        if not v or not v.strip():
            raise ValueError('Location ID cannot be empty')
        return v.strip()

    @validator('base_url')
    def base_url_must_be_http(cls, v):
        if not v.startswith(('http://', 'https://')):
            raise ValueError('Base URL must start with http:// or https://')
        return v.rstrip('/')  # Remove trailing slash

    class Config:
        """Allow environment variable mapping."""
        env_prefix = 'GHL_'

class GHLLocationClient:
    """
    HTTP client for GHL Location API operations.

    Implements context manager protocol for proper resource cleanup.
    Includes automatic retry logic for transient failures (5xx errors, timeouts).
    """

    def __init__(self, config: GHLConfig):
        """
        Initialize client with configuration.

        Args:
            config: Validated GHLConfig instance

        Raises:
            ConfigurationError: If config is invalid
        """
        self.config = config
        self._client: Optional[httpx.Client] = None
        self._setup_client()

    def _setup_client(self) -> None:
        """Configure HTTP client with authentication headers."""
        headers = {
            "Authorization": f"Bearer {self.config.api_key}",
            "Content-Type": "application/json",
            "Accept": "application/json",
            "User-Agent": "GHL-Location-Updater/1.0"
        }

        self._client = httpx.Client(
            base_url=self.config.base_url,
            headers=headers,
            timeout=httpx.Timeout(30.0, connect=10.0),
            follow_redirects=True
        )
        logger.debug(f"HTTP client initialized for {self.config.base_url}")

    @retry(
        retry=retry_if_exception_type((httpx.HTTPStatusError, httpx.TimeoutException, httpx.ConnectError)),
        stop=stop_after_attempt(MAX_RETRIES),
        wait=wait_exponential(multiplier=1, min=2, max=10),
        before_sleep=before_sleep_log(logger, logging.WARNING),
        reraise=True
    )
    def update_location(self, payload: LocationUpdatePayload) -> Dict[str, Any]:
        """
        Update GHL location details via API.

        Args:
            payload: LocationUpdatePayload with name and address

        Returns:
            Dict containing API response data

        Raises:
            GHLAPIError: When API returns 4xx/5xx errors
            RetryError: When max retries exceeded for transient errors
        """
        if not self._client:
            raise RuntimeError("HTTP client not initialized")

        endpoint = f"/locations/{self.config.location_id}"

        try:
            logger.info(f"Updating location {self.config.location_id} with payload: {payload.json()}")

            response = self._client.put(
                endpoint,
                json=payload.dict()
            )

            # Raise for 4xx/5xx status codes (triggers retry if applicable)
            response.raise_for_status()

            data = response.json()
            logger.info(f"Successfully updated location {self.config.location_id}")
            logger.debug(f"API Response: {json.dumps(data, indent=2)}")

            return data

        except httpx.HTTPStatusError as e:
            # Handle specific HTTP error codes
            status_code = e.response.status_code
            try:
                error_body = e.response.json()
                error_msg = error_body.get('message', str(e))
            except json.JSONDecodeError:
                error_body = e.response.text
                error_msg = str(e)

            # Don't retry client errors (4xx) except 429 (rate limit)
            if status_code == 401:
                raise GHLAPIError(401, "Authentication failed - invalid API key", error_body)
            elif status_code == 403:
                raise GHLAPIError(403, "Forbidden - insufficient permissions", error_body)
            elif status_code == 404:
                raise GHLAPIError(404, f"Location {self.config.location_id} not found", error_body)
            elif status_code == 422:
                raise GHLAPIError(422, f"Validation error: {error_msg}", error_body)
            elif status_code == 429:
                # Rate limit - allow retry
                logger.warning("Rate limit hit, retrying...")
                raise
            else:
                # Server errors (5xx) will be retried by decorator
                raise GHLAPIError(status_code, f"HTTP Error: {error_msg}", error_body)

        except httpx.TimeoutException:
            logger.error("Request timed out")
            raise
        except httpx.ConnectError:
            logger.error("Connection error")
            raise

    def close(self) -> None:
        """Close HTTP client connections."""
        if self._client:
            self._client.close()
            self._client = None
            logger.debug("HTTP client closed")

    def __enter__(self) -> 'GHLLocationClient':
        """Context manager entry."""
        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        """Context manager exit with cleanup."""
        self.close()

def load_configuration() -> GHLConfig:
    """
    Load and validate configuration from environment variables.

    Returns:
        Validated GHLConfig instance

    Raises:
        ConfigurationError: If required env vars missing or invalid


Response:
 ```python
"""
Widget Security Module - CORS, Domain Verification, and Rate Limiting

This module provides comprehensive security controls for widget serving:
- DNS-based domain ownership verification with caching
- Dynamic CORS validation against registered tenant domains
- Referer header validation for script loading
- Redis-backed sliding window rate limiting

Usage:
    from widget.api.security import WidgetSecurityMiddleware, require_domain_verified

    app.add_middleware(WidgetSecurityMiddleware, ...)
"""

import hashlib
import logging
import re
import time
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from enum import Enum
from typing import Callable, Dict, List, Optional, Set, Union
from urllib.parse import urlparse

import dns.resolver
import redis.asyncio as redis
from fastapi import FastAPI, HTTPException, Request, Response, Depends
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from pydantic import BaseModel, Field, validator
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.types import ASGIApp

# Configure structured logging
logger = logging.getLogger(__name__)

# ============================================================================
# Configuration
# ============================================================================

class SecurityConfig(BaseModel):
    """Security configuration settings."""

    redis_url: str = Field(default="redis://localhost:6379/0", env="REDIS_URL")
    rate_limit_requests: int = Field(default=100, description="Max requests per window")
    rate_limit_window: int = Field(default=3600, description="Window size in seconds")
    dns_cache_ttl: int = Field(default=86400, description="DNS verification cache TTL (24h)")
    dns_txt_prefix: str = Field(default="widget-verify", description="TXT record prefix")
    cors_max_age: int = Field(default=600, description="CORS preflight cache max age")
    allow_missing_referer: bool = Field(default=False, description="Allow empty referer headers")

    class Config:
        env_prefix = "WIDGET_SECURITY_"

# ============================================================================
# Exceptions
# ============================================================================

class SecurityError(Exception):
    """Base security exception."""
    pass

class DomainNotVerifiedError(SecurityError):
    """Raised when domain ownership cannot be verified."""
    pass

class RateLimitExceededError(SecurityError):
    """Raised when rate limit is exceeded."""
    pass

class InvalidOriginError(SecurityError):
    """Raised when origin is not in allowed domains."""
    pass

class InvalidRefererError(SecurityError):
    """Raised when referer header is invalid or missing."""
    pass

# ============================================================================
# Models
# ============================================================================

class Tenant(BaseModel):
    """Tenant data model."""
    id: str
    name: str
    verified_domains: Set[str] = Field(default_factory=set)
    is_active: bool = True

class WidgetMetadata(BaseModel):
    """Widget registration metadata."""
    widget_id: str
    tenant_id: str
    allowed_origins: Set[str] = Field(default_factory=set)

    @validator('allowed_origins', pre=True, each_item=True)
    def normalize_domains(cls, v):
        """Normalize domain to lowercase."""
        return v.lower().strip()

class RateLimitStatus(BaseModel):
    """Rate limit status response."""
    limit: int
    remaining: int
    reset_time: int
    window: str = "sliding"

# ============================================================================
# Services
# ============================================================================

class TenantRegistry(ABC):
    """Abstract tenant registry for domain lookups."""

    @abstractmethod
    async def get_tenant_by_domain(self, domain: str) -> Optional[Tenant]:
        """Fetch tenant by domain."""
        pass

    @abstractmethod
    async def get_widget_metadata(self, widget_id: str) -> Optional[WidgetMetadata]:
        """Fetch widget metadata including allowed origins."""
        pass

    @abstractmethod
    async def is_domain_registered(self, widget_id: str, domain: str) -> bool:
        """Check if domain is registered for widget."""
        pass

class RedisTenantRegistry(TenantRegistry):
    """Redis-backed tenant registry implementation."""

    def __init__(self, redis_client: redis.Redis):
        self.redis = redis_client

    async def get_tenant_by_domain(self, domain: str) -> Optional[Tenant]:
        # In production, query your database
        # This is a mock implementation
        data = await self.redis.get(f"tenant:domain:{domain}")
        if data:
            return Tenant.parse_raw(data)
        return None

    async def get_widget_metadata(self, widget_id: str) -> Optional[WidgetMetadata]:
        data = await self.redis.get(f"widget:{widget_id}")
        if data:
            return WidgetMetadata.parse_raw(data)
        return None

    async def is_domain_registered(self, widget_id: str, domain: str) -> bool:
        metadata = await self.get_widget_metadata(widget_id)
        if not metadata:
            return False
        return domain.lower() in metadata.allowed_origins

class DNSVerificationService:
    """
    DNS TXT record verification service with Redis caching.

    Verifies domain ownership by checking for TXT records in format:
    widget-verify=<tenant_id>
    """

    def __init__(
        self,
        redis_client: redis.Redis,
        config: SecurityConfig
    ):
        self.redis = redis_client
        self.config = config
        self.resolver = dns.resolver.Resolver()
        self.resolver.timeout = 5
        self.resolver.lifetime = 5

    async def verify_domain(
        self,
        domain: str,
        tenant_id: str
    ) -> bool:
        """
        Verify domain ownership via DNS TXT record.

        Args:
            domain: Domain to verify (e.g., "example.com")
            tenant_id: Expected tenant ID in TXT record

        Returns:
            bool: True if verified, False otherwise

        Caches result in Redis for 24 hours to avoid repeated DNS lookups.
        """
        cache_key = f"dns_verify:{domain}:{tenant_id}"

        # Check cache first
        cached = await self.redis.get(cache_key)
        if cached is not None:
            logger.debug(f"DNS cache hit for {domain}")
            return cached == "1"

        # Perform DNS lookup
        is_verified = await self._check_txt_record(domain, tenant_id)

        # Cache result for 24 hours
        cache_value = "1" if is_verified else "0"
        await self.redis.setex(cache_key, self.config.dns_cache_ttl, cache_value)

        logger.info(
            f"DNS verification for {domain} (tenant {tenant_id}): {is_verified}"
        )
        return is_verified

    async def _check_txt_record(self, domain: str, tenant_id: str) -> bool:
        """Perform actual DNS TXT record lookup."""
        expected_record = f"{self.config.dns_txt_prefix}={tenant_id}"

        try:
            # Use thread pool for blocking DNS operation
            loop = asyncio.get_event_loop()
            answers = await loop.run_in_executor(
                None,
                lambda: self.resolver.resolve(domain, 'TXT')
            )

            for rdata in answers:
                for txt_string in rdata.strings:
                    try:
                        decoded = txt_string.decode('utf-8')
                        if decoded == expected_record:
                            return True
                    except UnicodeDecodeError:
                        continue

        except dns.resolver.NXDOMAIN:
            logger.warning(f"Domain {domain} does not exist")
        except dns.resolver.NoAnswer:
            logger.warning(f"No TXT records found for {domain}")
        except dns.exception.Timeout:
            logger.error(f"DNS timeout for {domain}")
        except Exception as e:
            logger.error(f"DNS lookup error for {domain}: {e}")

        return False

    async def invalidate_cache(self, domain: str, tenant_id: str) -> None:
        """Invalidate DNS verification cache."""
        cache_key = f"dns_verify:{domain}:{tenant_id}"
        await self.redis.delete(cache_key)

class RateLimiter:
    """
    Sliding window rate limiter using Redis Sorted Sets.

    Algorithm:
    1. Remove entries older than window from sorted set
    2. Count remaining entries (current requests in window)
    3. If count < limit, add current timestamp and allow
    4. Else, deny request

    Key format: rate_limit:{widget_id}
    Score: timestamp (for range queries)
    Member: unique request identifier (timestamp + hash)
    """

    def __init__(self, redis_client: redis.Redis, config: SecurityConfig):
        self.redis = redis_client
        self.config = config

    async def is_allowed(self, widget_id: str) -> tuple[bool, RateLimitStatus]:
        """
        Check if request is allowed under rate limit.

        Returns:
            Tuple of (is_allowed, rate_limit_status)
        """
        now = time.time()
        window_start = now - self.config.rate_limit_window
        key = f"rate_limit:{widget_id}"

        # Create pipeline for atomic operations
        pipe = self.redis.pipeline()

        # Remove old entries outside window
        pipe.zremrangebyscore(key, 0, window_start)

        # Count current entries
        pipe.zcard(key)

        results = await pipe.execute()
        current_count = results[1]

        if current_count >= self.config.rate_limit_requests:
            # Calculate reset time (oldest entry in window + window size)
            oldest = await self.redis.zrange(key, 0, 0, withscores=True)
            reset_time = int(oldest[0][1] + self.config.rate_limit_window) if oldest else int(now + self.config.rate_limit_window)

            status = RateLimitStatus(
                limit=self.config.rate_limit_requests,
                remaining=0,
                reset_time=reset_time,
                window="sliding"
            )
            return False, status

        # Add current request
        member = f"{now}:{hashlib.md5(str(now).encode()).hexdigest()[:8]}"
        await self.redis.zadd(key, {member: now})
        await self.redis.expire(key, self.config.rate_limit_window)

        status = RateLimitStatus(
            limit=self.config.rate_limit_requests,
            remaining=self.config.rate_limit_requests - current_count - 1,
            reset_time=int(now + self.config.rate_limit_window),
            window="sliding"
        )

        return True, status

    async def get_status(self, widget_id: str) -> RateLimitStatus:
        """Get current rate limit status without incrementing."""
        now = time.time()
        window_start = now - self.config.rate_limit_window
        key = f"rate_limit:{widget_id}"

        pipe = self.redis.pipeline()
        pipe.zremrangebyscore(key, 0, window_start)
        pipe.zcard(key)
        results = await pipe.execute()

        current_count = results[1]
        remaining = max(0, self.config.rate_limit_requests - current_count)

        # Estimate reset time
        oldest = await self.redis.zrange(key, 0, 0, withscores=True)
        reset_time = int(oldest[0][1] + self.config.rate_limit_window) if oldest else int(now)

        return RateLimitStatus(
            limit=self.config.rate_limit_requests,
            remaining=remaining,
            reset_time=reset_time,
            window="sliding"
        )

# ============================================================================
# Security Middleware & Dependencies
# ============================================================================

class WidgetSecurityMiddleware(BaseHTTPMiddleware):
    """
    Main security middleware for widget API.

    Validates:
    - CORS origins against registered domains
    - Referer header for script requests
    - Rate limiting per widget_id
    """

    def __init__(
        self,
        app: ASGIApp,
        redis_client: redis.Redis,
        tenant_registry: TenantRegistry,
        config: Optional[SecurityConfig] = None
    ):
        super().__init__(app)
        self.redis = redis_client
        self.registry = tenant_registry
        self.config = config or SecurityConfig()
        self.rate_limiter = RateLimiter(redis_client, self.config)
        self.dns_verifier = DNSVerificationService(redis_client, self.config)

    async def dispatch(self, request: Request, call_next: Callable) -> Response:
        """
        Process request through security checks.

        Order of operations:
        1. Extract widget_id from path or query
        2. Check rate limit
        3. Validate Origin header (CORS)
        4. Validate Referer header (for widget.js)
        5. Proceed or return error
        """
        try:
            # Extract widget ID from path (e.g., /widget/{id}/script.js) or query param
            widget_id = self._extract_widget_id(request)
            if not widget_id:
                return await call_next(request)

            # Check rate limit first (most restrictive)
            is_allowed, rate_status = await self.rate_limiter.is_allowed(widget_id)
            if not is_allowed:
                logger.warning(f"Rate limit exceeded for widget {widget_id}")
                return JSONResponse(
                    status_code=429,
                    content={
                        "error": "Rate limit exceeded",
                        "limit": rate_status.limit,
                        "reset_time": rate_status.reset_time
                    },
                    headers={
                        "X-RateLimit-Limit": str(rate_status.limit),
                        "X-RateLimit-Remaining": "0",
                        "X-RateLimit-Reset": str(rate_status.reset_time),
                        "Retry-After": str(rate_status.reset_time - int(time.time()))
                    }
                )

            # Add rate limit headers to successful responses
            response = await self._process_security_checks(request, call_next, widget_id)

            # Inject rate limit headers
            if isinstance(response, Response):
                response.headers["X-RateLimit-Limit"] = str(rate_status.limit)
                response.headers["X-RateLimit-Remaining"] = str(rate_status.remaining)

            return response

        except Exception as e:
            logger.error(f"Security middleware error: {e}", exc_info=True)
            # Fail secure: return 500 for unexpected errors
            return JSONResponse(
                status_code=500,
                content={"error": "Internal security error"}
            )

    def _extract_widget_id(self, request: Request) -> Optional[str]:
        """Extract widget ID from URL path or query parameters."""
        # Try path parameter first
        path_params = request.path_params
        if "widget_id" in path_params:
            return path_params["widget_id"]

        # Try query parameter
        return request.query_params.get("widget_id")

    async def _process_security_checks(
        self,
        request: Request,
        call_next: Callable,
        widget_id: str
    ) -> Response:
        """Execute CORS and Referer validations."""

        origin = request.headers.get("origin")
        referer = request.headers.get("referer")

        # Get widget metadata
        metadata = await self.registry.get_widget_metadata(widget_id)
        if not metadata:
            logger.warning(f"Widget not found: {widget_id}")
            raise HTTPException(status_code=404, detail="Widget not found")

        # CORS Check: Validate Origin header if present
        if origin:
            origin_domain = self._extract_domain(origin)
            if not await self._is_origin_allowed(origin_domain, metadata):
                logger.warning(
                    f"CORS violation: {origin_domain} not allowed for {widget_id}"
                )
                return JSONResponse(
                    status_code=403,
                    content={"error": "Origin not allowed"}
                )

        # Referer Check: For GET requests to widget.js, verify referer
        if request.method == "GET" and "widget.js" in request.url.path:
            if not referer and not self.config.allow_missing_referer:
                return JSONResponse(
                    status_code=403,
                    content={"error": "Referer header required"}
                )

            if referer:
                referer_domain = self._extract_domain(referer)
                if not await self._is_domain_registered(referer_domain, metadata):
                    logger.warning(
                        f"Referer check failed: {referer_domain} for {widget_id}"
                    )
                    return JSONResponse(
                        status_code=403,
                        content={"error": "Domain not authorized"}
                    )

        # Process request
        response = await call_next(request)

        # Add CORS headers if origin was validated
        if origin and await self._is_origin_allowed(self._extract_domain(origin), metadata):
            response.headers["Access-Control-Allow-Origin"] = origin
            response.headers["Access-Control-Allow-Credentials"] = "true"
            response.headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"
            response.headers["Access-Control-Allow-Headers"] = "Content-Type, Authorization"
            response.headers["Access-Control-Max-Age"] = str(self.config.cors_max_age)

        return response

    def _extract_domain(self, url: str) -> str:
        """Extract normalized domain from URL."""
        if not url.startswith(('http://', 'https://')):
            url = 'https://' + url
        parsed = urlparse(url)
        return parsed.netloc.lower().split(':')[0]  # Remove port if present

    async def _is_origin_allowed(
        self,
        domain: str,
        metadata: WidgetMetadata
    ) -> bool:
        """Check if domain is in allowed origins or verified via DNS."""
        # Check static allowed list
        if domain in metadata.allowed_origins:
            return True

        # Could add dynamic DNS verification here for additional flexibility
        return False

    async def _is_domain_registered(
        self,
        domain: str,
        metadata: WidgetMetadata
    ) -> bool:
        """Verify domain is registered for the widget."""
        return domain in metadata.allowed_origins

# ============================================================================
# Dependency Injectors for FastAPI
# ============================================================================

async def get_security_config() -> SecurityConfig:
    """Dependency provider for security configuration."""
    return SecurityConfig()

async def get_redis_client(config: SecurityConfig = Depends(get_security_config)) -> redis.Redis:
    """Dependency provider for Redis client."""
    return redis.from_url(config.redis_url, encoding="utf-8", decode_responses=True)

async def verify_domain_dependency(
    domain: str,
    tenant_id: str,
    redis_client: redis.Redis = Depends(get_redis_client),
    config: SecurityConfig = Depends(get_security_config)
) -> bool:
    """
    FastAPI dependency for domain verification.

    Usage:
        @app.post("/verify-domain")
        async def verify(domain: str, tenant_id: str, verified: bool = Depends(verify_domain_dependency)):
            ...
    """
    verifier = DNSVerificationService(redis_client, config)
    return await verifier.verify_domain(domain, tenant_id)

async def check_rate_limit(
    widget_id: str,
    redis_client: redis.Redis = Depends(get_redis_client),
    config: SecurityConfig = Depends(get_security_config)
) -> RateLimitStatus:
    """
    FastAPI dependency for rate limit checking.

    Raises HTTPException 429 if limit exceeded.
    """
    limiter = RateLimiter(redis_client, config)
    is_allowed, status = await limiter.is_allowed(widget_id)

    if not is_allowed:
        raise HTTPException(
            status_code=429,
            detail="Rate limit exceeded",
            headers={
                "X-RateLimit-Limit": str(status.limit),
                "X-RateLimit-Remaining": "0",
                "X-RateLimit-Reset": str(status.reset_time),
                "Retry-After": str(status.reset_time - int(time.time()))
            }
        )

    return status

# ============================================================================
# Example Usage & Test Scenarios
# ============================================================================

def create_app() -> FastAPI:
    """
    Factory function to create FastAPI application with security middleware.

    This demonstrates production setup.
    """
    app = FastAPI(title="Widget API", version="1.0.0")

    # Configuration
    config = SecurityConfig()

    @app.on_event("startup")
    async def startup():
        app.state.redis = redis.from_url(
            config.redis_url,
            encoding="utf-8",
            decode_responses=True
        )
        app.state.registry = RedisTenantRegistry(app.state.redis)

    @app.on_event("shutdown")
    async def shutdown():
        await app.state.redis.close()

    # Add security middleware
    app.add_middleware(
        WidgetSecurityMiddleware,
        redis_client=app.state.redis,
        tenant_registry=app.state.registry,
        config=config
    )

    @app.get("/widget/{widget_id}/widget.js")
    async def serve_widget(
        widget_id: str,
        request: Request,
        rate_status: RateLimitStatus = Depends(check_rate_limit)
    ):
        """
        Serve widget.js with full security stack:
        - Rate limiting (100/hour per widget_id)
        - Referer validation (must be registered domain)
        - CORS headers (if origin matches registered domain)
        """
        # In production, serve actual widget JavaScript
        return {
            "widget_id": widget_id,
            "script": "// Widget code here",
            "rate_limit_remaining": rate_status.remaining
        }

    @app.post("/admin/verify-domain")
    async def verify_domain_endpoint(


Response:
```python
"""
clawdtalk/surprise_engine.py

Surprise Detection Engine for memory storage optimization.
Implements novelty detection using incremental cluster centroids.
"""

import logging
import threading
from dataclasses import dataclass, field
from typing import List, Optional, Tuple

import numpy as np
from numpy.typing import NDArray

logger = logging.getLogger(__name__)

@dataclass
class MemoryCluster:
    """
    Represents a cluster of similar memories with incremental centroid updates.

    Attributes:
        centroid: The cluster center vector (mean of all members)
        count: Number of memories in this cluster
        id: Unique cluster identifier
    """
    centroid: NDArray[np.float64]
    count: int = 0
    id: int = field(default_factory=lambda: id(MemoryCluster))

    def update_incremental(self, new_vector: NDArray[np.float64]) -> None:
        """
        Update centroid incrementally using running mean formula.

        Formula: new_centroid = (old_centroid * n + new_vector) / (n + 1)
        Complexity: O(1) vs O(n) for full recompute.

        Args:
            new_vector: New memory embedding to incorporate

        Raises:
            ValueError: If vector dimensions don't match centroid
        """
        if new_vector.shape != self.centroid.shape:
            raise ValueError(
                f"Dimension mismatch: cluster has {self.centroid.shape}, "
                f"got {new_vector.shape}"
            )

        # White box requirement: Incremental update, not full recompute
        self.centroid = (self.centroid * self.count + new_vector) / (self.count + 1)
        self.count += 1

class SurpriseEngine:
    """
    Engine for calculating surprise scores of new memories.

    Uses cluster-based novelty detection with incremental centroid updates.
    Surprise score ranges from 0.0 (completely expected) to 1.0 (highly surprising).

    Attributes:
        threshold: Minimum surprise score to store a memory (default: 0.3)
        max_clusters: Maximum number of clusters to maintain
        dimension: Expected embedding dimension
    """

    def __init__(
        self,
        threshold: float = 0.3,
        max_clusters: int = 1000,
        dimension: Optional[int] = None,
        distance_metric: str = "euclidean"
    ):
        """
        Initialize the Surprise Engine.

        Args:
            threshold: Surprise threshold for memory storage (0.0-1.0)
            max_clusters: Maximum clusters before forced merging
            dimension: Expected embedding dimension (optional)
            distance_metric: Distance metric ('euclidean' or 'cosine')

        Raises:
            ValueError: If threshold not in [0, 1] or invalid metric
        """
        if not 0.0 <= threshold <= 1.0:
            raise ValueError(f"Threshold must be in [0, 1], got {threshold}")

        if distance_metric not in ("euclidean", "cosine"):
            raise ValueError(f"Unknown metric: {distance_metric}")

        self.threshold = threshold
        self.max_clusters = max_clusters
        self.dimension = dimension
        self.distance_metric = distance_metric

        self._clusters: List[MemoryCluster] = []
        self._lock = threading.RLock()  # Thread-safe cluster updates
        self._total_memories_processed = 0

        logger.info(
            f"SurpriseEngine initialized: threshold={threshold}, "
            f"max_clusters={max_clusters}, metric={distance_metric}"
        )

    def calculate_surprise(self, embedding: NDArray[np.float64]) -> float:
        """
        Calculate surprise score for a new memory embedding.

        Algorithm:
        1. If no existing clusters, everything is surprising (1.0)
        2. Find distance to nearest cluster centroid
        3. Normalize distance to [0, 1] range using adaptive scaling

        Args:
            embedding: Vector representation of memory (1-D array)

        Returns:
            float: Surprise score in range [0.0, 1.0]

        Raises:
            ValueError: If embedding dimensions are invalid
            TypeError: If embedding is not a numpy array
        """
        # Validate input
        self._validate_embedding(embedding)

        with self._lock:
            # White box requirement: Empty memory set returns 1.0
            if not self._clusters:
                logger.debug("No existing clusters, returning max surprise")
                return 1.0

            # Find nearest cluster
            distances = self._compute_distances(embedding)
            min_distance = float(np.min(distances))

            # White box requirement: Score normalization
            surprise = self._normalize_distance(min_distance)

            logger.debug(
                f"Calculated surprise: {surprise:.4f} "
                f"(min_distance: {min_distance:.4f})"
            )

            return surprise

    def process_memory(
        self,
        embedding: NDArray[np.float64],
        force_store: bool = False
    ) -> Tuple[bool, float]:
        """
        Process a new memory and decide whether to store it.

        Only stores memories with surprise > threshold (default 0.3).

        Args:
            embedding: Memory embedding vector
            force_store: If True, store regardless of surprise score

        Returns:
            Tuple of (stored: bool, surprise_score: float)
        """
        surprise = self.calculate_surprise(embedding)
        should_store = force_store or surprise > self.threshold

        if should_store:
            self._integrate_memory(embedding)
            logger.info(
                f"Memory stored (surprise={surprise:.4f}, "
                f"threshold={self.threshold})"
            )
        else:
            logger.debug(
                f"Memory rejected (surprise={surprise:.4f} <= "
                f"threshold={self.threshold})"
            )

        self._total_memories_processed += 1
        return should_store, surprise

    def get_cluster_stats(self) -> dict:
        """Return current cluster statistics for monitoring."""
        with self._lock:
            return {
                "num_clusters": len(self._clusters),
                "total_memories": sum(c.count for c in self._clusters),
                "avg_cluster_size": (
                    np.mean([c.count for c in self._clusters])
                    if self._clusters else 0.0
                ),
                "threshold": self.threshold,
                "total_processed": self._total_memories_processed
            }

    def _validate_embedding(self, embedding: NDArray[np.float64]) -> None:
        """Validate embedding format and dimensions."""
        if not isinstance(embedding, np.ndarray):
            raise TypeError(f"Expected numpy array, got {type(embedding)}")

        if embedding.ndim != 1:
            raise ValueError(f"Expected 1-D array, got shape {embedding.shape}")

        if self.dimension is not None and embedding.shape[0] != self.dimension:
            raise ValueError(
                f"Expected dimension {self.dimension}, got {embedding.shape[0]}"
            )

        if not np.isfinite(embedding).all():
            raise ValueError("Embedding contains NaN or infinite values")

    def _compute_distances(self, embedding: NDArray[np.float64]) -> NDArray[np.float64]:
        """Compute distances from embedding to all cluster centroids."""
        if self.distance_metric == "euclidean":
            # L2 norm
            return np.array([
                np.linalg.norm(embedding - cluster.centroid)
                for cluster in self._clusters
            ])
        else:  # cosine
            # Cosine distance = 1 - cosine similarity
            return np.array([
                1.0 - np.dot(embedding, cluster.centroid) / (
                    np.linalg.norm(embedding) * np.linalg.norm(cluster.centroid) + 1e-10
                )
                for cluster in self._clusters
            ])

    def _normalize_distance(self, distance: float) -> float:
        """
        Normalize distance to surprise score [0, 1].

        Uses exponential decay for Euclidean distance to map
        [0, infinity) -> [0, 1).
        """
        if self.distance_metric == "cosine":
            # Cosine distance is naturally in [0, 2], normalize to [0, 1]
            return min(distance / 2.0, 1.0)
        else:
            # For Euclidean, use adaptive normalization
            if self.dimension:
                max_expected = np.sqrt(self.dimension)
            else:
                # Estimate from existing clusters
                if self._clusters:
                    centroid_norms = [
                        np.linalg.norm(c.centroid) for c in self._clusters
                    ]
                    avg_norm = np.mean(centroid_norms)
                    max_expected = 2.0 * avg_norm  # heuristic scaling
                else:
                    max_expected = 1.0

            # Normalize and apply sigmoid-like compression
            normalized = distance / max_expected
            surprise = 1.0 - np.exp(-normalized)  # Maps [0, inf) -> [0, 1)
            return float(min(surprise, 1.0))

    def _integrate_memory(self, embedding: NDArray[np.float64]) -> None:
        """
        Integrate new memory into cluster structure.

        White box requirement: Uses incremental update (not full recompute).
        Assigns to nearest cluster or creates new cluster if novel enough.
        """
        with self._lock:
            if not self._clusters:
                # First memory creates first cluster
                new_cluster = MemoryCluster(
                    centroid=embedding.copy(),
                    count=1
                )
                self._clusters.append(new_cluster)
                return

            # Find nearest cluster
            distances = self._compute_distances(embedding)
            nearest_idx = int(np.argmin(distances))
            nearest_cluster = self._clusters[nearest_idx]

            # Incremental update of existing cluster
            nearest_cluster.update_incremental(embedding)

            # Create new cluster if this is very novel and we have room
            min_dist = distances[nearest_idx]
            if (min_dist > 0.8 and
                len(self._clusters) < self.max_clusters and
                min_dist > np.mean(distances) * 1.5):

                new_cluster = MemoryCluster(
                    centroid=embedding.copy(),
                    count=1
                )
                self._clusters.append(new_cluster)
                logger.debug(f"Created new cluster {new_cluster.id}")

# Example usage and test scenarios
if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)

    print("=" * 60)
    print("Surprise Engine - Black Box & White Box Tests")
    print("=" * 60)

    # Initialize engine with 128-dimensional embeddings
    engine = SurpriseEngine(threshold=0.3, dimension=128)

    # Test 1: White Box - Empty memory set returns 1.0
    print("\n1. White Box: Empty set behavior")
    novel_embedding = np.random.randn(128).astype(np.float64)
    novel_embedding = novel_embedding / np.linalg.norm(novel_embedding)
    score = engine.calculate_surprise(novel_embedding)
    print(f"   Empty set surprise: {score:.4f} (expected: 1.0)")
    assert score == 1.0, "Empty set should return max surprise"

    # Test 2: Black Box - Seed with known facts (similar vectors)
    print("\n2. Black Box: Seeding with known facts...")
    base_concept = np.random.randn(128).astype(np.float64)
    base_concept = base_concept / np.linalg.norm(base_concept)

    for i in range(10):
        # Add small noise to base concept (simulating similar memories)
        noise = np.random.randn(128) * 0.1
        similar_vec = base_concept + noise
        similar_vec = similar_vec / np.linalg.norm(similar_vec)
        engine.process_memory(similar_vec)

    # Test 3: Black Box - Known fact should have low surprise
    print("\n3. Black Box: Known fact detection")
    similar_fact = base_concept + np.random.randn(128) * 0.05
    similar_fact = similar_fact / np.linalg.norm(similar_fact)
    surprise_score = engine.calculate_surprise(similar_fact)
    print(f"   Known fact surprise: {surprise_score:.4f} (expected: < 0.3)")
    assert surprise_score < 0.3, f"Similar fact should have low surprise, got {surprise_score}"

    # Test 4: Black Box - Novel fact should have high surprise
    print("\n4. Black Box: Novel fact detection")
    # Create orthogonal vector (maximally different)
    novel_fact = np.random.randn(128).astype(np.float64)
    novel_fact = novel_fact / np.linalg.norm(novel_fact)
    # Project out component in direction of base_concept
    novel_fact = novel_fact - np.dot(novel_fact, base_concept) * base_concept
    novel_fact = novel_fact / np.linalg.norm(novel_fact)

    surprise_score = engine.calculate_surprise(novel_fact)
    print(f"   Novel fact surprise: {surprise_score:.4f} (expected: > 0.3)")
    assert surprise_score > 0.3, f"Novel fact should have high surprise, got {surprise_score}"

    # Test 5: Black Box - Threshold filtering
    print("\n5. Black Box: Threshold filtering (threshold=0.3)")
    stored_similar, score_similar = engine.process_memory(similar_fact)
    stored_novel, score_novel = engine.process_memory(novel_fact)

    print(f"   Similar fact stored: {stored_similar} (expected: False)")
    print(f"   Novel fact stored: {stored_novel} (expected: True)")
    assert not stored_similar, "Similar fact should be filtered out"
    assert stored_novel, "Novel fact should be stored"

    # Test 6: White Box - Verify incremental updates
    print("\n6. White Box: Verify incremental centroid updates")
    stats = engine.get_cluster_stats()
    print(f"   Cluster stats: {stats}")
    assert stats['num_clusters'] >= 1, "Should have at least one cluster"
    assert stats['total_memories'] >= 11, "Should have stored memories"

    # Verify normalization bounds
    print("\n7. White Box: Score normalization bounds")
    test_cases = [
        (base_concept, "identical"),  # Should be ~0
        (novel_fact, "orthogonal"),   # Should be high
    ]
    for vec, desc in test_cases:
        score = engine.calculate_surprise(vec)
        assert 0.0 <= score <= 1.0, f"Score {score} out of bounds for {desc}"
        print(f"   {desc}: {score:.4f} (in [0, 1])")

    print("\n" + "=" * 60)
    print("All tests passed!")
    print("=" * 60)
```