Call Recording Manager Tests

Response:
```python
"""
Comprehensive test suite for Recording Storage Manager.

This module tests S3 integration, signed URL generation, retention policies,
storage tracking, and metadata indexing using moto for AWS service mocking.
"""

import pytest
import boto3
import hashlib
import json
from datetime import datetime, timedelta, timezone
from unittest.mock import patch, MagicMock, call
from moto import mock_s3, mock_dynamodb, mock_secretsmanager
from botocore.exceptions import ClientError
from freezegun import freeze_time
import aioboto3
import asyncio
from typing import Dict, Any, Optional

# Assuming these imports match your project structure
# from src.recording_manager import RecordingStorageManager
# from src.models import RecordingMetadata, StorageStats

@pytest.fixture
def aws_credentials():
    """Mocked AWS Credentials for moto."""
    return {
        "AWS_ACCESS_KEY_ID": "testing",
        "AWS_SECRET_ACCESS_KEY": "testing",
        "AWS_SECURITY_TOKEN": "testing",
        "AWS_SESSION_TOKEN": "testing",
        "AWS_DEFAULT_REGION": "us-east-1",
    }

@pytest.fixture
def s3_client(aws_credentials, monkeypatch):
    """Create a mocked S3 client."""
    for key, value in aws_credentials.items():
        monkeypatch.setenv(key, value)

    with mock_s3():
        client = boto3.client("s3", region_name="us-east-1")
        yield client

@pytest.fixture
def dynamodb_resource(aws_credentials):
    """Create a mocked DynamoDB resource."""
    with mock_dynamodb():
        resource = boto3.resource("dynamodb", region_name="us-east-1")
        yield resource

@pytest.fixture
def test_bucket(s3_client):
    """Create and yield a test S3 bucket."""
    bucket_name = "test-recordings-bucket"
    s3_client.create_bucket(Bucket=bucket_name)
    s3_client.put_bucket_versioning(
        Bucket=bucket_name,
        VersioningConfiguration={"Status": "Enabled"}
    )
    return bucket_name

@pytest.fixture
def metadata_table(dynamodb_resource):
    """Create DynamoDB table for metadata indexing."""
    table = dynamodb_resource.create_table(
        TableName="RecordingMetadata",
        KeySchema=[{"AttributeName": "recording_id", "KeyType": "HASH"}],
        AttributeDefinitions=[{"AttributeName": "recording_id", "AttributeType": "S"}],
        BillingMode="PAY_PER_REQUEST",
    )
    table.wait_until_exists()
    return table

@pytest.fixture
def sample_recording_metadata():
    """Generate sample recording metadata for testing."""
    return {
        "recording_id": "rec-12345",
        "session_id": "sess-67890",
        "user_id": "user-abcde",
        "filename": "meeting_recording.mp4",
        "content_type": "video/mp4",
        "duration_seconds": 3600,
        "created_at": datetime.now(timezone.utc).isoformat(),
        "file_size": 1024 * 1024 * 100,  # 100MB
        "checksum": hashlib.sha256(b"test_content").hexdigest(),
        "tags": ["meeting", "team-a", "q1-2024"],
        "retention_days": 90,
    }

@pytest.fixture
def storage_manager(s3_client, test_bucket, metadata_table):
    """
    Create an instance of RecordingStorageManager with mocked dependencies.

    In a real scenario, this would be imported from your module.
    """
    # Mock implementation for demonstration
    class RecordingStorageManager:
        def __init__(self, bucket_name, table_name, region="us-east-1"):
            self.bucket_name = bucket_name
            self.table_name = table_name
            self.s3 = boto3.client("s3", region_name=region)
            self.dynamodb = boto3.resource("dynamodb", region_name=region)
            self.table = self.dynamodb.Table(table_name)

        async def upload_recording(
            self,
            recording_id: str,
            file_content: bytes,
            metadata: Dict[str, Any]
        ) -> Dict[str, str]:
            """Upload recording to S3 and index metadata."""
            key = f"recordings/{recording_id}/{metadata['filename']}"

            # Upload to S3
            self.s3.put_object(
                Bucket=self.bucket_name,
                Key=key,
                Body=file_content,
                ContentType=metadata.get("content_type", "application/octet-stream"),
                Metadata={
                    "recording-id": recording_id,
                    "user-id": metadata.get("user_id", ""),
                    "checksum": metadata.get("checksum", ""),
                }
            )

            # Index in DynamoDB
            item = {
                "recording_id": recording_id,
                "s3_key": key,
                "bucket": self.bucket_name,
                "uploaded_at": datetime.now(timezone.utc).isoformat(),
                "file_size": len(file_content),
                **metadata
            }
            self.table.put_item(Item=item)

            return {"s3_key": key, "bucket": self.bucket_name, "recording_id": recording_id}

        def generate_signed_url(
            self,
            recording_id: str,
            expiration: int = 3600,
            method: str = "get_object"
        ) -> str:
            """Generate presigned URL for recording access."""
            # Query DynamoDB to get S3 key
            response = self.table.get_item(Key={"recording_id": recording_id})
            if "Item" not in response:
                raise ValueError(f"Recording {recording_id} not found")

            s3_key = response["Item"]["s3_key"]

            url = self.s3.generate_presigned_url(
                method,
                Params={"Bucket": self.bucket_name, "Key": s3_key},
                ExpiresIn=expiration,
            )
            return url

        async def enforce_retention_policy(self, max_age_days: int = 90):
            """Delete recordings older than retention period."""
            cutoff_date = datetime.now(timezone.utc) - timedelta(days=max_age_days)
            cutoff_iso = cutoff_date.isoformat()

            # Scan for old recordings (in production, use GSI on uploaded_at)
            response = self.table.scan(
                FilterExpression="uploaded_at < :cutoff",
                ExpressionAttributeValues={":cutoff": cutoff_iso}
            )

            deleted_count = 0
            for item in response.get("Items", []):
                try:
                    # Delete from S3
                    self.s3.delete_object(
                        Bucket=item["bucket"],
                        Key=item["s3_key"]
                    )

                    # Delete from DynamoDB
                    self.table.delete_item(
                        Key={"recording_id": item["recording_id"]}
                    )
                    deleted_count += 1
                except ClientError as e:
                    print(f"Failed to delete {item['recording_id']}: {e}")

            return deleted_count

        def get_storage_stats(self, user_id: Optional[str] = None) -> Dict[str, Any]:
            """Calculate storage statistics."""
            if user_id:
                # Query by user (would need GSI in production)
                response = self.table.scan(
                    FilterExpression="user_id = :uid",
                    ExpressionAttributeValues={":uid": user_id}
                )
            else:
                response = self.table.scan()

            items = response.get("Items", [])
            total_size = sum(item.get("file_size", 0) for item in items)
            count = len(items)

            return {
                "total_bytes": total_size,
                "total_recordings": count,
                "average_file_size": total_size // count if count > 0 else 0,
                "user_id": user_id,
            }

        async def multipart_upload(
            self,
            recording_id: str,
            file_parts: list,
            metadata: Dict[str, Any]
        ) -> Dict[str, str]:
            """Handle multipart upload for large files."""
            key = f"recordings/{recording_id}/{metadata['filename']}"

            # Initiate multipart upload
            mpu = self.s3.create_multipart_upload(
                Bucket=self.bucket_name,
                Key=key,
                Metadata={"recording-id": recording_id}
            )
            upload_id = mpu["UploadId"]

            try:
                parts = []
                for i, part_data in enumerate(file_parts, 1):
                    response = self.s3.upload_part(
                        Bucket=self.bucket_name,
                        Key=key,
                        UploadId=upload_id,
                        PartNumber=i,
                        Body=part_data,
                    )
                    parts.append({
                        "PartNumber": i,
                        "ETag": response["ETag"]
                    })

                # Complete multipart upload
                self.s3.complete_multipart_upload(
                    Bucket=self.bucket_name,
                    Key=key,
                    UploadId=upload_id,
                    MultipartUpload={"Parts": parts}
                )

                # Index metadata
                total_size = sum(len(part) for part in file_parts)
                self.table.put_item(Item={
                    "recording_id": recording_id,
                    "s3_key": key,
                    "bucket": self.bucket_name,
                    "file_size": total_size,
                    "multipart": True,
                    "parts_count": len(parts),
                    **metadata
                })

                return {"s3_key": key, "parts": len(parts)}

            except Exception as e:
                # Abort on failure
                self.s3.abort_multipart_upload(
                    Bucket=self.bucket_name,
                    Key=key,
                    UploadId=upload_id
                )
                raise

    return RecordingStorageManager(test_bucket, "RecordingMetadata")

class TestS3Upload:
    """Tests for S3 upload functionality."""

    @pytest.mark.asyncio
    async def test_upload_single_file_success(
        self, storage_manager, sample_recording_metadata, s3_client, test_bucket
    ):
        """Test successful upload of a single recording file."""
        recording_id = sample_recording_metadata["recording_id"]
        file_content = b"Test recording content"

        result = await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=file_content,
            metadata=sample_recording_metadata
        )

        assert result["recording_id"] == recording_id
        assert result["bucket"] == test_bucket
        assert recording_id in result["s3_key"]

        # Verify S3 object exists
        s3_response = s3_client.get_object(Bucket=test_bucket, Key=result["s3_key"])
        assert s3_response["Body"].read() == file_content

    @pytest.mark.asyncio
    async def test_upload_with_metadata_headers(
        self, storage_manager, sample_recording_metadata, s3_client, test_bucket
    ):
        """Test that metadata is correctly stored in S3 object headers."""
        recording_id = sample_recording_metadata["recording_id"]
        file_content = b"Content with metadata"

        await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=file_content,
            metadata=sample_recording_metadata
        )

        # List objects and check metadata
        objects = s3_client.list_objects_v2(Bucket=test_bucket)
        obj_key = objects["Contents"][0]["Key"]
        head_response = s3_client.head_object(Bucket=test_bucket, Key=obj_key)

        assert head_response["Metadata"]["recording-id"] == recording_id
        assert head_response["Metadata"]["user-id"] == sample_recording_metadata["user_id"]
        assert "checksum" in head_response["Metadata"]

    @pytest.mark.asyncio
    @pytest.mark.parametrize("file_size_mb", [1, 10, 100, 500])
    async def test_upload_various_file_sizes(
        self, storage_manager, sample_recording_metadata, file_size_mb
    ):
        """Test upload handling for various file sizes."""
        recording_id = f"rec-size-{file_size_mb}"
        file_content = b"x" * (file_size_mb * 1024 * 1024)
        sample_recording_metadata["file_size"] = len(file_content)

        result = await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=file_content,
            metadata=sample_recording_metadata
        )

        assert result["recording_id"] == recording_id

    @pytest.mark.asyncio
    async def test_upload_unicode_filename(
        self, storage_manager, sample_recording_metadata
    ):
        """Test handling of unicode characters in filenames."""
        unicode_filename = "会议记录_🎥_ réunion.mp4"
        sample_recording_metadata["filename"] = unicode_filename
        recording_id = "rec-unicode-001"

        result = await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=b"unicode test",
            metadata=sample_recording_metadata
        )

        assert unicode_filename in result["s3_key"]

    @pytest.mark.asyncio
    async def test_upload_empty_file(
        self, storage_manager, sample_recording_metadata
    ):
        """Test handling of empty file uploads."""
        recording_id = "rec-empty"

        result = await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=b"",
            metadata=sample_recording_metadata
        )

        assert result["recording_id"] == recording_id

    @pytest.mark.asyncio
    async def test_upload_duplicate_recording_id(
        self, storage_manager, sample_recording_metadata
    ):
        """Test behavior when uploading with duplicate recording ID."""
        recording_id = "rec-duplicate"
        file_content = b"original"

        await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=file_content,
            metadata=sample_recording_metadata
        )

        # Upload again with same ID but different content
        new_content = b"updated"
        result = await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=new_content,
            metadata=sample_recording_metadata
        )

        # Should overwrite or create new version depending on implementation
        assert result["recording_id"] == recording_id

class TestMultipartUpload:
    """Tests for multipart upload functionality."""

    @pytest.mark.asyncio
    async def test_multipart_upload_success(
        self, storage_manager, sample_recording_metadata, s3_client, test_bucket
    ):
        """Test successful multipart upload of large file."""
        recording_id = "rec-multipart-001"
        parts = [b"part1_data", b"part2_data", b"part3_data"]

        result = await storage_manager.multipart_upload(
            recording_id=recording_id,
            file_parts=parts,
            metadata=sample_recording_metadata
        )

        assert result["parts"] == 3
        assert "s3_key" in result

        # Verify concatenated content
        s3_obj = s3_client.get_object(Bucket=test_bucket, Key=result["s3_key"])
        content = s3_obj["Body"].read()
        assert content == b"part1_datapart2_datapart3_data"

    @pytest.mark.asyncio
    async def test_multipart_upload_failure_rollback(
        self, storage_manager, sample_recording_metadata, s3_client, test_bucket, mocker
    ):
        """Test that multipart upload aborts on failure."""
        recording_id = "rec-fail-001"
        parts = [b"part1", b"part2"]

        # Mock upload_part to fail on second part
        original_upload_part = s3_client.upload_part
        call_count = 0

        def fail_on_second(*args, **kwargs):
            nonlocal call_count
            call_count += 1
            if call_count == 2:
                raise ClientError(
                    {"Error": {"Code": "500", "Message": "Internal Error"}},
                    "UploadPart"
                )
            return original_upload_part(*args, **kwargs)

        mocker.patch.object(storage_manager.s3, "upload_part", side_effect=fail_on_second)

        with pytest.raises(ClientError):
            await storage_manager.multipart_upload(
                recording_id=recording_id,
                file_parts=parts,
                metadata=sample_recording_metadata
            )

        # Verify no incomplete multipart uploads left
        # Note: In real implementation, we'd check ListMultipartUploads

class TestSignedUrlGeneration:
    """Tests for presigned URL generation and expiry."""

    @pytest.mark.asyncio
    async def test_generate_get_url_success(
        self, storage_manager, sample_recording_metadata
    ):
        """Test successful generation of GET presigned URL."""
        recording_id = sample_recording_metadata["recording_id"]

        # First upload a file
        await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=b"test content",
            metadata=sample_recording_metadata
        )

        url = storage_manager.generate_signed_url(recording_id, expiration=3600)

        assert "https://" in url or "http://" in url
        assert recording_id in url or "recordings" in url
        assert "X-Amz-Expires=3600" in url or "Expires=" in url

    @pytest.mark.asyncio
    async def test_generate_url_for_nonexistent_recording(self, storage_manager):
        """Test error handling for URL generation of non-existent recording."""
        with pytest.raises(ValueError, match="not found"):
            storage_manager.generate_signed_url("nonexistent-id")

    @pytest.mark.asyncio
    @pytest.mark.parametrize("expiration,valid", [
        (60, True),      # 1 minute
        (3600, True),    # 1 hour
        (86400, True),   # 1 day
        (604800, True),  # 7 days (max for presigned URLs)
        (604801, False), # Over 7 days (should fail or cap)
    ])
    async def test_url_expiry_variations(
        self, storage_manager, sample_recording_metadata, expiration, valid
    ):
        """Test URL generation with various expiration times."""
        recording_id = sample_recording_metadata["recording_id"]
        await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=b"test",
            metadata=sample_recording_metadata
        )

        if valid:
            url = storage_manager.generate_signed_url(recording_id, expiration=expiration)
            assert url is not None
            # Verify expiry parameter in URL
            assert str(expiration) in url or "Expires=" in url
        else:
            # Should either raise error or cap at max
            with pytest.raises((ValueError, ClientError)):
                storage_manager.generate_signed_url(recording_id, expiration=expiration)

    @pytest.mark.asyncio
    async def test_url_content_type_preservation(
        self, storage_manager, sample_recording_metadata, s3_client, test_bucket
    ):
        """Test that signed URL respects original content type."""
        recording_id = "rec-content-type"
        sample_recording_metadata["content_type"] = "video/webm"

        await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=b"video data",
            metadata=sample_recording_metadata
        )

        url = storage_manager.generate_signed_url(recording_id)

        # In real scenario, we'd make HTTP request to URL and check Content-Type header
        # Here we verify the S3 object has correct type
        key = f"recordings/{recording_id}/{sample_recording_metadata['filename']}"
        head = s3_client.head_object(Bucket=test_bucket, Key=key)
        assert head["ContentType"] == "video/webm"

class TestRetentionPolicy:
    """Tests for retention policy enforcement."""

    @pytest.mark.asyncio
    @freeze_time("2024-01-15 12:00:00")
    async def test_retention_deletion_of_old_files(
        self, storage_manager, sample_recording_metadata, s3_client, test_bucket, metadata_table
    ):
        """Test deletion of recordings past retention period."""
        # Upload with old date
        old_date = (datetime.now(timezone.utc) - timedelta(days=100)).isoformat()
        sample_recording_metadata["uploaded_at"] = old_date
        sample_recording_metadata["retention_days"] = 30

        recording_id = "rec-old-001"
        await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=b"old content",
            metadata=sample_recording_metadata
        )

        # Enforce 90-day retention
        deleted = await storage_manager.enforce_retention_policy(max_age_days=90)

        assert deleted == 1

        # Verify deletion from S3
        objects = s3_client.list_objects_v2(Bucket=test_bucket)
        assert objects.get("KeyCount", 0) == 0

        # Verify deletion from DynamoDB
        response = metadata_table.get_item(Key={"recording_id": recording_id})
        assert "Item" not in response

    @pytest.mark.asyncio
    @freeze_time("2024-01-15 12:00:00")
    async def test_retention_preservation_of_recent_files(
        self, storage_manager, sample_recording_metadata, s3_client, test_bucket
    ):
        """Test that recent recordings are not deleted."""
        # Upload with recent date
        recent_date = (datetime.now(timezone.utc) - timedelta(days=30)).isoformat()
        sample_recording_metadata["uploaded_at"] = recent_date

        recording_id = "rec-recent-001"
        await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=b"recent content",
            metadata=sample_recording_metadata
        )

        # Enforce 90-day retention
        deleted = await storage_manager.enforce_retention_policy(max_age_days=90)

        assert deleted == 0

        # Verify still in S3
        objects = s3_client.list_objects_v2(Bucket=test_bucket)
        assert objects["KeyCount"] == 1

    @pytest.mark.asyncio
    async def test_custom_retention_per_recording(
        self, storage_manager, sample_recording_metadata, metadata_table
    ):
        """Test that individual recording retention settings are respected."""
        # This test assumes retention is stored per-recording
        recording_id = "rec-custom-retention"
        sample_recording_metadata["retention_days"] = 365

        await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=b"long term storage",
            metadata=sample_recording_metadata
        )

        item = metadata_table.get_item(Key={"recording_id": recording_id})["Item"]
        assert item["retention_days"] == 365

class TestStorageTracking:
    """Tests for storage usage tracking and quotas."""

    @pytest.mark.asyncio
    async def test_storage_stats_accuracy(
        self, storage_manager, sample_recording_metadata
    ):
        """Test accurate calculation of storage statistics."""
        # Upload multiple recordings
        sizes = [1024, 2048, 4096]
        for i, size in enumerate(sizes):
            meta = sample_recording_metadata.copy()
            meta["recording_id"] = f"rec-stats-{i}"
            meta["user_id"] = "user-1"

            await storage_manager.upload_recording(
                recording_id=meta["recording_id"],
                file_content=b"x" * size,
                metadata=meta
            )

        stats = storage_manager.get_storage_stats()

        assert stats["total_recordings"] == 3
        assert stats["total_bytes"] == sum(sizes)
        assert stats["average_file_size"] == sum(sizes) // 3

    @pytest.mark.asyncio
    async def test_storage_stats_per_user(
        self, storage_manager, sample_recording_metadata
    ):
        """Test storage statistics filtered by user."""
        # Upload for different users
        for user in ["user-a", "user-b", "user-a"]:
            meta = sample_recording_metadata.copy()
            meta["recording_id"] = f"rec-{user}-{datetime.now().timestamp()}"
            meta["user_id"] = user

            await storage_manager.upload_recording(
                recording_id=meta["recording_id"],
                file_content=b"x" * 1000,
                metadata=meta
            )

        stats_a = storage_manager.get_storage_stats(user_id="user-a")
        stats_b = storage_manager.get_storage_stats(user_id="user-b")

        assert stats_a["total_recordings"] == 2
        assert stats_b["total_recordings"] == 1
        assert stats_a["user_id"] == "user-a"

    @pytest.mark.asyncio
    async def test_storage_quota_enforcement(
        self, storage_manager, sample_recording_metadata, mocker
    ):
        """Test that storage quotas are enforced."""
        # Mock quota limit
        QUOTA_LIMIT = 5000

        # First upload under quota
        meta = sample_recording_metadata.copy()
        meta["recording_id"] = "rec-quota-1"

        await storage_manager.upload_recording(
            recording_id=meta["recording_id"],
            file_content=b"x" * 3000,
            metadata=meta
        )

        # Check current usage
        stats = storage_manager.get_storage_stats()
        current_usage = stats["total_bytes"]

        # Simulate quota check
        if current_usage > QUOTA_LIMIT:
            pytest.fail("Should not exceed quota")

        # In real implementation, would test that upload fails when quota exceeded

class TestMetadataIndexing:
    """Tests for metadata indexing in DynamoDB."""

    @pytest.mark.asyncio
    async def test_metadata_indexing_on_upload(
        self, storage_manager, sample_recording_metadata, metadata_table
    ):
        """Test that metadata is correctly indexed after upload."""
        recording_id = sample_recording_metadata["recording_id"]

        await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=b"content",
            metadata=sample_recording_metadata
        )

        # Verify DynamoDB entry
        response = metadata_table.get_item(Key={"recording_id": recording_id})
        assert "Item" in response

        item = response["Item"]
        assert item["recording_id"] == recording_id
        assert item["user_id"] == sample_recording_metadata["user_id"]
        assert item["filename"] == sample_recording_metadata["filename"]
        assert "s3_key" in item
        assert "uploaded_at" in item

    @pytest.mark.asyncio
    async def test_metadata_search_by_tags(
        self, storage_manager, sample_recording_metadata, metadata_table
    ):
        """Test searching recordings by tags."""
        # Upload with specific tags
        sample_recording_metadata["tags"] = ["important", "meeting", "archive"]

        await storage_manager.upload_recording(
            recording_id="rec-tags-001",
            file_content=b"tagged content",
            metadata=sample_recording_metadata
        )

        # In production, this would use a GSI on tags
        # Here we verify the tags are stored
        response = metadata_table.get_item(Key={"recording_id": "rec-tags-001"})
        assert "important" in response["Item"]["tags"]
        assert "meeting" in response["Item"]["tags"]

    @pytest.mark.asyncio
    async def test_metadata_consistency_after_delete(
        self, storage_manager, sample_recording_metadata, metadata_table, s3_client, test_bucket
    ):
        """Test that metadata is removed when recording is deleted."""
        recording_id = "rec-delete-consistency"

        await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=b"to be deleted",
            metadata=sample_recording_metadata
        )

        # Delete from S3 directly (simulating retention cleanup)
        key = f"recordings/{recording_id}/{sample_recording_metadata['filename']}"
        s3_client.delete_object(Bucket=test_bucket, Key=key)

        # In real implementation, should have cascade delete or orphan detection
        # Verify metadata still exists (to be cleaned up by separate process)
        response = metadata_table.get_item(Key={"recording_id": recording_id})
        assert "Item" in response  # Or assert not exists if cascade delete implemented

class TestAsyncConcurrency:
    """Tests for concurrent operations."""

    @pytest.mark.asyncio
    async def test_concurrent_uploads(
        self, storage_manager, sample_recording_metadata
    ):
        """Test handling of concurrent uploads."""
        async def upload_batch(batch_id: int):
            meta = sample_recording_metadata.copy()
            meta["recording_id"] = f"rec-concurrent-{batch_id}"
            return await storage_manager.upload_recording(
                recording_id=meta["recording_id"],
                file_content=f"content-{batch_id}".encode(),
                metadata=meta
            )

        # Run 10 concurrent uploads
        tasks = [upload_batch(i) for i in range(10)]
        results = await asyncio.gather(*tasks)

        assert len(results) == 10
        # Verify all unique
        ids = [r["recording_id"] for r in results]
        assert len(set(ids)) == 10

    @pytest.mark.asyncio
    async def test_concurrent_signed_url_generation(
        self, storage_manager, sample_recording_metadata
    ):
        """Test concurrent URL generation doesn't cause race conditions."""
        recording_id = sample_recording_metadata["recording_id"]

        await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=b"concurrent test",
            metadata=sample_recording_metadata
        )

        async def generate_url_task():
            return storage_manager.generate_signed_url(recording_id, expiration=3600)

        # Generate 50 URLs concurrently
        tasks = [generate_url_task() for _ in range(50)]
        urls = await asyncio.gather(*tasks)

        assert len(urls) == 50
        # All should be valid URLs
        for url in urls:
            assert url.startswith("http")
            assert "X-Amz-Signature" in url or "Signature=" in url

class TestErrorHandling:
    """Tests for error handling and edge cases."""

    @pytest.mark.asyncio
    async def test_s3_permission_denied(
        self, storage_manager, sample_recording_metadata, mocker
    ):
        """Test handling of S3 permission errors."""
        mocker.patch.object(
            storage_manager.s3,
            "put_object",
            side_effect=ClientError(
                {"Error": {"Code": "AccessDenied", "Message": "Access Denied"}},
                "PutObject"
            )
        )

        with pytest.raises(ClientError) as exc_info:
            await storage_manager.upload_recording(
                recording_id="rec-perm-denied",
                file_content=b"content",
                metadata=sample_recording_metadata
            )

        assert exc_info.value.response["Error"]["Code"] == "AccessDenied"

    @pytest.mark.asyncio
    async def test_dynamodb_throttling(
        self, storage_manager, sample_recording_metadata, mocker
    ):
        """Test handling of DynamoDB throttling."""
        # First S3 succeeds, then DynamoDB fails
        call_count = 0

        def fail_on_second_call(*args, **kwargs):
            nonlocal call_count
            call_count += 1
            if call_count > 1:
                raise ClientError(
                    {"Error": {"Code": "ProvisionedThroughputExceededException"}},
                    "PutItem"
                )
            return {"ETag": '"mock-etag"'}

        mocker.patch.object(storage_manager.s3, "put_object", side_effect=fail_on_second_call)

        with pytest.raises(ClientError):
            await storage_manager.upload_recording(
                recording_id="rec-dynamo-fail",
                file_content=b"content",
                metadata=sample_recording_metadata
            )

    def test_invalid_recording_id_format(self, storage_manager):
        """Test validation of recording ID formats."""
        # Test various invalid formats
        invalid_ids = ["", "   ", "rec/with/slashes", "rec..dots", "a" * 1025]

        for invalid_id in invalid_ids:
            # Implementation should validate and raise ValueError
            pass  # Add validation logic based on implementation

# Integration-style test that combines multiple operations
class TestEndToEndWorkflow:
    """End-to-end workflow tests."""

    @pytest.mark.asyncio
    async def test_full_lifecycle(
        self, storage_manager, sample_recording_metadata, s3_client, metadata_table
    ):
        """Test complete lifecycle: upload, access, retention."""
        recording_id = "rec-lifecycle-001"

        # 1. Upload
        upload_result = await storage_manager.upload_recording(
            recording_id=recording_id,
            file_content=b"lifecycle test content",
            metadata=sample_recording_metadata
        )
        assert upload_result["recording_id"] == recording_id

        # 2. Generate access URL
        url = storage_manager.generate_signed_url(recording_id)
        assert url is not None

        # 3. Check storage stats
        stats = storage_manager.get_storage_stats()
        assert stats["total_recordings"] == 1

        # 4. Verify metadata indexed
        item = metadata_table.get_item(Key={"recording_id": recording_id})["Item"]
        assert item["recording_id"] == recording_id

        # 5. Simulate time passing and enforce retention
        with freeze_time(datetime.now(timezone.utc) + timedelta(days=100)):
            deleted = await storage_manager.enforce_retention_policy(max_age_days=90)
            assert deleted == 1

        # 6. Verify cleanup
        objects = s3_client.list_objects_v2(Bucket=upload_result["bucket"])
        assert objects.get("KeyCount", 0) == 0

        response = metadata_table.get_item(Key={"recording_id": recording_id})
        assert "Item" not in response
```


A/B Testing Framework Tests


Transcript Search Tests

Response:
 ```python
"""
Comprehensive test suite for the Transcript Search Engine.

This module covers full-text search functionality, filtering, sorting,
highlighting, and export capabilities with extensive edge case handling.
"""

import pytest
import pytest_asyncio
import csv
import io
import re
from datetime import datetime, timedelta
from unittest.mock import Mock, patch, AsyncMock, MagicMock
from typing import List, Dict, Any, Optional

# Assuming the following module structure (adjust imports as needed)
# from transcript_search.engine import TranscriptSearchEngine, SearchResult
# from transcript_search.models import Transcript, SearchQuery
# from transcript_search.exceptions import SearchError, InvalidQueryError

class TestTranscriptSearchEngine:
    """Test suite for Transcript Search Engine core functionality."""

    @pytest.fixture
    def mock_search_client(self):
        """Fixture providing a mocked search client (e.g., Elasticsearch, Whoosh)."""
        client = Mock()
        client.search = Mock(return_value={
            "hits": {
                "total": {"value": 0},
                "hits": []
            }
        })
        return client

    @pytest.fixture
    def search_engine(self, mock_search_client):
        """Fixture providing a configured search engine instance."""
        # Adjust import path based on actual implementation
        from transcript_search.engine import TranscriptSearchEngine

        engine = TranscriptSearchEngine(client=mock_search_client)
        return engine

    @pytest.fixture
    def sample_transcripts(self) -> List[Dict[str, Any]]:
        """Fixture providing sample transcript data for testing."""
        base_date = datetime(2024, 1, 15, 10, 30, 0)

        return [
            {
                "id": "trans_001",
                "content": "The quick brown fox jumps over the lazy dog. "
                          "This meeting discussed quarterly earnings and revenue growth.",
                "speaker": "John Doe",
                "timestamp": base_date,
                "meeting_id": "meet_123",
                "confidence_score": 0.95
            },
            {
                "id": "trans_002",
                "content": "Revenue increased by 15% compared to last quarter. "
                          "The fox was quick indeed.",  # Partial match for "quick"
                "speaker": "Jane Smith",
                "timestamp": base_date + timedelta(days=1),
                "meeting_id": "meet_124",
                "confidence_score": 0.88
            },
            {
                "id": "trans_003",
                "content": "Special characters test: $100, €50, 日本語 text, "
                          "and <script>alert('xss')</script> attempts.",
                "speaker": "Bob Wilson",
                "timestamp": base_date - timedelta(days=5),
                "meeting_id": "meet_125",
                "confidence_score": 0.92
            },
            {
                "id": "trans_004",
                "content": "",  # Empty content edge case
                "speaker": "Silent Alice",
                "timestamp": base_date + timedelta(days=10),
                "meeting_id": "meet_126",
                "confidence_score": 0.0
            }
        ]

    @pytest.fixture
    def populated_engine(self, search_engine, sample_transcripts):
        """Fixture with pre-populated transcript data."""
        search_engine.index_transcripts(sample_transcripts)
        return search_engine

    # =========================================================================
    # Full-Text Search Accuracy Tests
    # =========================================================================

    @pytest.mark.parametrize("query,expected_ids,description", [
        ("quick brown fox", ["trans_001", "trans_002"],
         "Multi-term search with partial matches"),
        ("quarterly earnings", ["trans_001"],
         "Exact phrase matching"),
        ("revenue growth quarterly", ["trans_001", "trans_002"],
         "Term order independence"),
        ("QUICK BROWN", ["trans_001", "trans_002"],
         "Case insensitivity"),
        ("nonexistent term", [],
         "No matches returns empty list"),
        ("fox revenue", ["trans_001", "trans_002"],
         "OR logic between terms"),
        ('"quick brown fox"', ["trans_001"],
         "Literal phrase search with quotes"),
    ])
    def test_full_text_search_accuracy(
        self, populated_engine, query, expected_ids, description
    ):
        """
        Verify search accuracy across different query patterns.

        Tests exact matching, partial matching, case sensitivity,
        phrase queries, and boolean logic.
        """
        results = populated_engine.search(query=query)
        result_ids = [r["id"] for r in results]

        assert set(result_ids) == set(expected_ids), \
            f"Failed for case: {description}"

    @pytest.mark.asyncio
    async def test_async_full_text_search(self, search_engine, sample_transcripts):
        """
        Test async search operations for non-blocking I/O.

        Ensures the search method properly handles async execution
        and returns results consistent with sync version.
        """
        search_engine.index_transcripts_async = AsyncMock(
            return_value=None
        )
        search_engine.search_async = AsyncMock(
            return_value=[sample_transcripts[0]]
        )

        await search_engine.index_transcripts_async(sample_transcripts)
        results = await search_engine.search_async(query="quick")

        assert len(results) == 1
        assert results[0]["id"] == "trans_001"
        search_engine.search_async.assert_called_once()

    def test_search_with_wildcards(self, populated_engine):
        """
        Test wildcard and prefix search capabilities.

        Covers * and ? wildcards, prefix matching, and regex patterns
        if supported by the underlying engine.
        """
        # Assuming the engine supports wildcards
        results = populated_engine.search(query="quar*")
        assert len(results) >= 1
        assert any("quarterly" in r["content"].lower() for r in results)

    def test_fuzzy_search(self, populated_engine):
        """
        Test fuzzy matching for typo tolerance.

        Verifies that queries with minor typos still match intended terms
        using Levenshtein distance or similar algorithms.
        """
        # "qwuick" instead of "quick" - 1 character difference
        results = populated_engine.search(query="qwuick~")
        assert len(results) > 0, "Fuzzy search should tolerate 1-char typos"

    # =========================================================================
    # Date Range Filtering Tests
    # =========================================================================

    @pytest.mark.parametrize("start_date,end_date,expected_count,description", [
        (datetime(2024, 1, 14), datetime(2024, 1, 16), 2,
         "Exact 2-day range"),
        (datetime(2024, 1, 10), datetime(2024, 1, 20), 4,
         "Wide range covering all"),
        (datetime(2024, 1, 16), datetime(2024, 1, 25), 2,
         "Future dates only"),
        (datetime(2024, 1, 10), datetime(2024, 1, 10), 0,
         "Single day with no data"),
        (None, datetime(2024, 1, 16), 3,
         "Open start date"),
        (datetime(2024, 1, 16), None, 2,
         "Open end date"),
        (None, None, 4,
         "No date filter"),
    ])
    def test_date_range_filtering(
        self, populated_engine, start_date, end_date, expected_count, description
    ):
        """
        Verify date range filtering with various boundary conditions.

        Tests inclusive/exclusive boundaries, open-ended ranges,
        and timezone handling.
        """
        results = populated_engine.search(
            query="*",
            date_range={"start": start_date, "end": end_date}
        )

        assert len(results) == expected_count, \
            f"Failed for case: {description}"

    def test_invalid_date_range(self, populated_engine):
        """
        Test handling of invalid date ranges (start > end).

        Should raise an appropriate exception or return empty results.
        """
        with pytest.raises((ValueError, Exception)) as exc_info:
            populated_engine.search(
                query="test",
                date_range={
                    "start": datetime(2024, 1, 20),
                    "end": datetime(2024, 1, 10)
                }
            )

        assert "date" in str(exc_info.value).lower() or \
               "range" in str(exc_info.value).lower()

    def test_timezone_aware_date_filtering(self, populated_engine):
        """
        Test proper handling of timezone-aware datetime objects.

        Ensures UTC conversion and comparison work correctly across zones.
        """
        from datetime import timezone

        utc_date = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc)

        # Should handle timezone conversion without errors
        results = populated_engine.search(
            query="*",
            date_range={"start": utc_date, "end": None}
        )

        assert isinstance(results, list)

    # =========================================================================
    # Relevance Sorting Tests
    # =========================================================================

    def test_relevance_sorting_default(self, populated_engine):
        """
        Verify default sorting by relevance score (descending).

        Higher relevance scores should appear first.
        """
        results = populated_engine.search(query="revenue quick")

        # trans_001 has both "revenue" and "quick", should rank higher
        # trans_002 has "revenue" and "quick" but in different context
        if len(results) >= 2:
            assert results[0]["_score"] >= results[1]["_score"]

    def test_custom_sorting_options(self, populated_engine):
        """
        Test sorting by different fields (date, confidence, speaker).

        Verifies sort parameter acceptance and correct ordering.
        """
        # Sort by date descending
        results = populated_engine.search(
            query="*",
            sort_by="timestamp",
            sort_order="desc"
        )

        timestamps = [r["timestamp


CRM Connector Tests - HubSpot

Response:
 ```python
"""
Comprehensive pytest suite for HubSpot CRM Connector.

Covers contact management, call logging, deal creation, OAuth2 flows,
webhook handling, and API error scenarios including rate limiting.
"""

import json
import hmac
import hashlib
import base64
from datetime import datetime, timedelta
from typing import Dict, Any, Optional
from unittest.mock import Mock, patch, AsyncMock, MagicMock

import pytest
import pytest_asyncio
from pytest import raises, mark
import httpx
from httpx import Response, HTTPStatusError, RequestError

# Assuming the connector implementation follows this interface
class HubSpotCRMConnector:
    """Placeholder for actual implementation structure."""
    pass

# Fixtures
@pytest.fixture
def mock_hubspot_client():
    """Fixture providing a mocked HubSpot API client."""
    client = Mock()
    client.crm = Mock()
    client.crm.contacts = Mock()
    client.crm.deals = Mock()
    client.crm.activities = Mock()
    client.auth = Mock()
    return client

@pytest.fixture
def sample_contact_data():
    """Fixture providing valid contact data."""
    return {
        "email": "test@example.com",
        "firstname": "John",
        "lastname": "Doe",
        "phone": "+1234567890",
        "company": "TestCorp",
        "lifecyclestage": "lead"
    }

@pytest.fixture
def sample_deal_data():
    """Fixture providing valid deal data."""
    return {
        "dealname": "Enterprise License 2024",
        "amount": 50000,
        "dealstage": "qualifiedtobuy",
        "pipeline": "default",
        "closedate": "2024-12-31"
    }

@pytest.fixture
def sample_call_data():
    """Fixture providing valid call activity data."""
    return {
        "timestamp": datetime.now().isoformat(),
        "duration_ms": 300000,
        "disposition": "CONNECTED",
        "notes": "Discussed implementation timeline",
        "outcome": "Interested"
    }

@pytest.fixture
def oauth_tokens():
    """Fixture providing OAuth2 token response."""
    return {
        "access_token": "test_access_token_123",
        "refresh_token": "test_refresh_token_456",
        "expires_in": 3600,
        "token_type": "Bearer"
    }

@pytest.fixture
def rate_limit_headers():
    """Fixture providing HubSpot rate limit headers."""
    return {
        "X-HubSpot-RateLimit-Remaining": "0",
        "X-HubSpot-RateLimit-Max": "100",
        "Retry-After": "60"
    }

@pytest.fixture
def webhook_payload():
    """Fixture providing sample HubSpot webhook payload."""
    return {
        "eventId": 12345,
        "subscriptionId": 67890,
        "portalId": 123456,
        "occurredAt": datetime.now().isoformat(),
        "subscriptionType": "contact.creation",
        "attemptNumber": 0,
        "objectId": 98765,
        "changeSource": "CRM",
        "propertyName": "lifecyclestage",
        "propertyValue": "customer"
    }

@pytest.fixture
def webhook_signature(webhook_payload):
    """Generate test webhook signature."""
    client_secret = "test_webhook_secret"
    payload_str = json.dumps(webhook_payload, sort_keys=True)
    signature = base64.b64encode(
        hmac.new(
            client_secret.encode(),
            payload_str.encode(),
            hashlib.sha256
        ).digest()
    ).decode()
    return signature, client_secret

class TestContactOperations:
    """Test suite for contact creation and update operations."""

    @pytest.mark.asyncio
    async def test_create_contact_success(self, mock_hubspot_client, sample_contact_data):
        """Test successful contact creation with valid data."""
        # Arrange
        expected_response = {
            "id": "12345",
            "properties": sample_contact_data,
            "createdAt": datetime.now().isoformat()
        }
        mock_hubspot_client.crm.contacts.basic_api.create = AsyncMock(
            return_value=Mock(to_dict=lambda: expected_response))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act
        result = await connector.create_contact(sample_contact_data)

        # Assert
        assert result["id"] == "12345"
        assert result["properties"]["email"] == "test@example.com"
        mock_hubspot_client.crm.contacts.basic_api.create.assert_called_once()

    @pytest.mark.asyncio
    @mark.parametrize("invalid_data,expected_error", [
        ({"email": "invalid-email"}, "Invalid email format"),
        ({"email": ""}, "Email is required"),
        ({"email": "test@test.com", "phone": "invalid-phone"}, "Invalid phone format"),
        ({}, "Missing required fields"),
    ])
    async def test_create_contact_validation_errors(
        self, mock_hubspot_client, invalid_data, expected_error
    ):
        """Test contact creation validation with various invalid inputs."""
        # Arrange
        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act & Assert
        with raises(ValueError, match=expected_error):
            await connector.create_contact(invalid_data)

    @pytest.mark.asyncio
    async def test_update_contact_success(self, mock_hubspot_client):
        """Test successful contact update."""
        # Arrange
        contact_id = "12345"
        update_data = {"firstname": "Jane", "lastname": "Smith"}
        expected_response = {
            "id": contact_id,
            "properties": update_data,
            "updatedAt": datetime.now().isoformat()
        }
        mock_hubspot_client.crm.contacts.basic_api.update = AsyncMock(
            return_value=Mock(to_dict=lambda: expected_response))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act
        result = await connector.update_contact(contact_id, update_data)

        # Assert
        assert result["id"] == contact_id
        assert result["properties"]["firstname"] == "Jane"
        mock_hubspot_client.crm.contacts.basic_api.update.assert_called_once()

    @pytest.mark.asyncio
    async def test_update_contact_not_found(self, mock_hubspot_client):
        """Test update operation for non-existent contact."""
        # Arrange
        contact_id = "nonexistent"
        mock_hubspot_client.crm.contacts.basic_api.update = AsyncMock(
            side_effect=HTTPStatusError(
                "Contact not found",
                request=Mock(),
                response=Mock(status_code=404)
            ))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act & Assert
        with raises(HTTPStatusError) as exc_info:
            await connector.update_contact(contact_id, {"firstname": "Test"})
        assert exc_info.value.response.status_code == 404

    @pytest.mark.asyncio
    async def test_create_or_update_contact_upsert(self, mock_hubspot_client, sample_contact_data):
        """Test upsert operation - update if exists, create if not."""
        # Arrange
        email = sample_contact_data["email"]

        # First call raises 409 (conflict), second succeeds
        mock_hubspot_client.crm.contacts.basic_api.create = AsyncMock(
            side_effect=[
                HTTPStatusError("Conflict", request=Mock(), response=Mock(status_code=409)),
                Mock(to_dict=lambda: {"id": "12345", "properties": sample_contact_data})
            ])
        mock_hubspot_client.crm.contacts.search_api.do_search = AsyncMock(
            return_value=Mock(results=[Mock(id="existing_123")]))
        mock_hubspot_client.crm.contacts.basic_api.update = AsyncMock(
            return_value=Mock(to_dict=lambda: {"id": "existing_123", "updated": True}))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act
        result = await connector.create_or_update_contact(sample_contact_data)

        # Assert
        assert result is not None
        mock_hubspot_client.crm.contacts.search_api.do_search.assert_called_once()

class TestCallActivityLogging:
    """Test suite for call activity logging functionality."""

    @pytest.mark.asyncio
    async def test_log_call_activity_success(
        self, mock_hubspot_client, sample_contact_data, sample_call_data
    ):
        """Test successful logging of call activity to contact."""
        # Arrange
        contact_id = "12345"
        expected_engagement = {
            "id": "eng_123",
            "engagement": {
                "type": "CALL",
                "timestamp": sample_call_data["timestamp"]
            },
            "metadata": {
                "durationMilliseconds": sample_call_data["duration_ms"],
                "disposition": sample_call_data["disposition"],
                "body": sample_call_data["notes"]
            }
        }
        mock_hubspot_client.crm.engagements.basic_api.create = AsyncMock(
            return_value=Mock(to_dict=lambda: expected_engagement))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act
        result = await connector.log_call_activity(contact_id, sample_call_data)

        # Assert
        assert result["engagement"]["type"] == "CALL"
        assert result["metadata"]["durationMilliseconds"] == 300000
        mock_hubspot_client.crm.engagements.basic_api.create.assert_called_once()

    @pytest.mark.asyncio
    @mark.parametrize("duration,expected_status", [
        (0, "NO_ANSWER"),
        (60000, "CONNECTED"),
        (3600000, "CONNECTED"),  # 1 hour
        (None, "FAILED"),
    ])
    async def test_call_duration_disposition_mapping(
        self, mock_hubspot_client, duration, expected_status
    ):
        """Test that call durations map to correct dispositions."""
        # Arrange
        call_data = {
            "duration_ms": duration,
            "timestamp": datetime.now().isoformat()
        }
        mock_hubspot_client.crm.engagements.basic_api.create = AsyncMock(
            return_value=Mock(to_dict=lambda: {"id": "eng_123"}))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act
        await connector.log_call_activity("123", call_data)

        # Assert
        call_args = mock_hubspot_client.crm.engagements.basic_api.create.call_args
        metadata = call_args[1]["engagement_data"]["metadata"]
        assert metadata["durationMilliseconds"] == (duration or 0)

    @pytest.mark.asyncio
    async def test_log_call_with_recording_url(self, mock_hubspot_client):
        """Test logging call with recording attachment."""
        # Arrange
        call_data = {
            "duration_ms": 300000,
            "recording_url": "https://cdn.example.com/call_123.mp3",
            "timestamp": datetime.now().isoformat()
        }
        mock_hubspot_client.crm.engagements.basic_api.create = AsyncMock(
            return_value=Mock(to_dict=lambda: {"id": "eng_123"}))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act
        result = await connector.log_call_activity("123", call_data)

        # Assert
        call_args = mock_hubspot_client.crm.engagements.basic_api.create.call_args
        metadata = call_args[1]["engagement_data"]["metadata"]
        assert "recording_url" in metadata["body"]

class TestDealCreation:
    """Test suite for deal creation from leads."""

    @pytest.mark.asyncio
    async def test_create_deal_from_lead_success(
        self, mock_hubspot_client, sample_deal_data, sample_contact_data
    ):
        """Test successful deal creation associated with a lead."""
        # Arrange
        contact_id = "contact_123"
        company_id = "company_456"

        mock_deal_response = {
            "id": "deal_789",
            "properties": sample_deal_data
        }
        mock_hubspot_client.crm.deals.basic_api.create = AsyncMock(
            return_value=Mock(to_dict=lambda: mock_deal_response))

        # Mock associations API
        mock_hubspot_client.crm.associations.basic_api.create = AsyncMock(
            return_value=Mock(status="COMPLETE"))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act
        result = await connector.create_deal_from_lead(
            contact_id=contact_id,
            company_id=company_id,
            deal_data=sample_deal_data
        )

        # Assert
        assert result["id"] == "deal_789"
        assert result["properties"]["amount"] == 50000

        # Verify associations were created
        assert mock_hubspot_client.crm.associations.basic_api.create.call_count == 2

    @pytest.mark.asyncio
    async def test_create_deal_with_line_items(self, mock_hubspot_client):
        """Test deal creation with associated line items/products."""
        # Arrange
        deal_data = {
            "dealname": "Product Bundle",
            "amount": 10000,
            "line_items": [
                {"product_id": "prod_1", "quantity": 2, "price": 3000},
                {"product_id": "prod_2", "quantity": 1, "price": 4000}
            ]
        }

        mock_hubspot_client.crm.deals.basic_api.create = AsyncMock(
            return_value=Mock(to_dict=lambda: {"id": "deal_123", "properties": deal_data}))
        mock_hubspot_client.crm.line_items.basic_api.create = AsyncMock(
            return_value=Mock(id="line_item_1"))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act
        result = await connector.create_deal_from_lead(
            contact_id="contact_1",
            deal_data=deal_data
        )

        # Assert
        assert result["id"] == "deal_123"
        # Verify line items were created
        assert mock_hubspot_client.crm.line_items.basic_api.create.call_count == 2

    @pytest.mark.asyncio
    async def test_create_deal_invalid_pipeline_stage(self, mock_hubspot_client):
        """Test deal creation with invalid pipeline stage."""
        # Arrange
        invalid_deal_data = {
            "dealname": "Test Deal",
            "dealstage": "invalid_stage_id"
        }
        mock_hubspot_client.crm.deals.basic_api.create = AsyncMock(
            side_effect=HTTPStatusError(
                "Invalid dealstage",
                request=Mock(),
                response=Mock(status_code=400)
            ))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act & Assert
        with raises(HTTPStatusError) as exc_info:
            await connector.create_deal_from_lead(
                contact_id="contact_1",
                deal_data=invalid_deal_data
            )
        assert exc_info.value.response.status_code == 400

class TestOAuth2TokenRefresh:
    """Test suite for OAuth2 authentication and token refresh."""

    @pytest.mark.asyncio
    async def test_refresh_access_token_success(self, oauth_tokens):
        """Test successful token refresh."""
        # Arrange
        with patch('httpx.AsyncClient.post') as mock_post:
            mock_post.return_value = Mock(
                status_code=200,
                json=lambda: oauth_tokens,
                raise_for_status=Mock()
            )

            connector = HubSpotCRMConnector(
                client_id="test_client_id",
                client_secret="test_client_secret",
                refresh_token="old_refresh_token"
            )

            # Act
            new_token = await connector.refresh_access_token()

            # Assert
            assert new_token == oauth_tokens["access_token"]
            assert connector.access_token == oauth_tokens["access_token"]
            mock_post.assert_called_once()

    @pytest.mark.asyncio
    async def test_token_expiration_handling(self, oauth_tokens):
        """Test automatic token refresh when token is expired."""
        # Arrange
        connector = HubSpotCRMConnector(
            access_token="expired_token",
            refresh_token="valid_refresh_token",
            token_expires_at=datetime.now() - timedelta(hours=1)
        )

        with patch.object(connector, 'refresh_access_token', new_callable=AsyncMock) as mock_refresh:
            mock_refresh.return_value = "new_valid_token"
            mock_hubspot_client = Mock()
            mock_hubspot_client.crm.contacts.basic_api.get_by_id = AsyncMock(
                return_value=Mock(to_dict=lambda: {"id": "123"}))
            connector.client = mock_hubspot_client

            # Act
            await connector.get_contact("123")

            # Assert
            mock_refresh.assert_called_once()

    @pytest.mark.asyncio
    async def test_refresh_token_invalid_grant(self):
        """Test handling of invalid refresh token."""
        # Arrange
        with patch('httpx.AsyncClient.post') as mock_post:
            mock_post.return_value = Mock(
                status_code=400,
                json=lambda: {"error": "invalid_grant", "error_description": "Refresh token is invalid"},
                raise_for_status=Mock(side_effect=HTTPStatusError(
                    "Bad Request", request=Mock(), response=Mock(status_code=400)
                ))
            )

            connector = HubSpotCRMConnector(refresh_token="invalid_token")

            # Act & Assert
            with raises(HTTPStatusError):
                await connector.refresh_access_token()

    @pytest.mark.asyncio
    async def test_concurrent_token_refresh(self, oauth_tokens):
        """Test that concurrent requests don't trigger multiple token refreshes."""
        # Arrange
        connector = HubSpotCRMConnector(
            access_token="expired_token",
            token_expires_at=datetime.now() - timedelta(hours=1)
        )

        with patch.object(connector, 'refresh_access_token', new_callable=AsyncMock) as mock_refresh:
            mock_refresh.return_value = "new_token"

            # Act - Simulate concurrent requests
            import asyncio
            tasks = [
                connector.ensure_valid_token(),
                connector.ensure_valid_token(),
                connector.ensure_valid_token()
            ]
            await asyncio.gather(*tasks)

            # Assert - Refresh should only be called once despite concurrent requests
            mock_refresh.assert_called_once()

class TestWebhookSync:
    """Test suite for webhook handling and synchronization."""

    @pytest.mark.asyncio
    async def test_webhook_signature_verification(self, webhook_payload, webhook_signature):
        """Test webhook signature validation."""
        # Arrange
        signature, secret = webhook_signature
        connector = HubSpotCRMConnector(webhook_secret=secret)

        # Act
        is_valid = connector.verify_webhook_signature(
            json.dumps(webhook_payload),
            signature
        )

        # Assert
        assert is_valid is True

    @pytest.mark.asyncio
    async def test_webhook_invalid_signature(self, webhook_payload):
        """Test rejection of webhook with invalid signature."""
        # Arrange
        connector = HubSpotCRMConnector(webhook_secret="correct_secret")

        # Act
        is_valid = connector.verify_webhook_signature(
            json.dumps(webhook_payload),
            "invalid_signature"
        )

        # Assert
        assert is_valid is False

    @pytest.mark.asyncio
    async def test_handle_contact_creation_webhook(self, mock_hubspot_client, webhook_payload):
        """Test processing of contact creation webhook."""
        # Arrange
        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        with patch.object(connector, 'verify_webhook_signature', return_value=True):
            # Act
            result = await connector.handle_webhook(webhook_payload, "valid_signature")

            # Assert
            assert result["processed"] is True
            assert result["object_id"] == webhook_payload["objectId"]

    @pytest.mark.asyncio
    @mark.parametrize("event_type", [
        "contact.creation",
        "contact.propertyChange",
        "deal.creation",
        "deal.propertyChange",
        "contact.deletion"
    ])
    async def test_webhook_event_type_routing(self, mock_hubspot_client, event_type):
        """Test that different webhook types are routed to correct handlers."""
        # Arrange
        payload = {
            "subscriptionType": event_type,
            "objectId": 12345
        }
        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        with patch.object(connector, 'verify_webhook_signature', return_value=True):
            with patch.object(connector, f'_handle_{event_type.replace(".", "_")}',
                            new_callable=AsyncMock) as mock_handler:
                # Act
                await connector.handle_webhook(payload, "sig")

                # Assert
                mock_handler.assert_called_once()

    @pytest.mark.asyncio
    async def test_webhook_retry_logic(self, mock_hubspot_client):
        """Test webhook processing with retry on failure."""
        # Arrange
        payload = {"subscriptionType": "contact.creation", "objectId": 123}
        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        with patch.object(connector, 'verify_webhook_signature', return_value=True):
            with patch.object(connector, '_process_webhook',
                            new_callable=AsyncMock) as mock_process:
                # Fail twice, succeed on third
                mock_process.side_effect = [Exception("DB Error"),
                                          Exception("Network Error"),
                                          None]

                # Act
                result = await connector.handle_webhook(payload, "sig", max_retries=3)

                # Assert
                assert mock_process.call_count == 3
                assert result["processed"] is True

class TestErrorHandling:
    """Test suite for API error handling and rate limiting."""

    @pytest.mark.asyncio
    async def test_rate_limit_retry_with_backoff(self, mock_hubspot_client, rate_limit_headers):
        """Test exponential backoff on rate limit (429) errors."""
        # Arrange
        from httpx import Response

        mock_hubspot_client.crm.contacts.basic_api.get_by_id = AsyncMock(
            side_effect=[
                HTTPStatusError(
                    "Rate limit exceeded",
                    request=Mock(),
                    response=Mock(status_code=429, headers=rate_limit_headers)
                ),
                Mock(to_dict=lambda: {"id": "123", "email": "test@test.com"})
            ])

        connector = HubSpotCRMConnector(
            client=mock_hubspot_client,
            max_retries=3,
            base_delay=0.1  # Short delay for testing
        )

        with patch('asyncio.sleep', new_callable=AsyncMock) as mock_sleep:
            # Act
            result = await connector.get_contact("123")

            # Assert
            assert result["id"] == "123"
            mock_sleep.assert_called_once()
            # Verify exponential backoff (60 seconds from Retry-After header)
            assert mock_sleep.call_args[0][0] >= 60

    @pytest.mark.asyncio
    async def test_rate_limit_max_retries_exceeded(self, mock_hubspot_client, rate_limit_headers):
        """Test failure after max retry attempts."""
        # Arrange
        mock_hubspot_client.crm.contacts.basic_api.get_by_id = AsyncMock(
            side_effect=HTTPStatusError(
                "Rate limit exceeded",
                request=Mock(),
                response=Mock(status_code=429, headers=rate_limit_headers)
            ))

        connector = HubSpotCRMConnector(
            client=mock_hubspot_client,
            max_retries=2,
            base_delay=0.01
        )

        with patch('asyncio.sleep', new_callable=AsyncMock):
            # Act & Assert
            with raises(HTTPStatusError) as exc_info:
                await connector.get_contact("123")
            assert exc_info.value.response.status_code == 429

    @pytest.mark.asyncio
    async def test_server_error_retry(self, mock_hubspot_client):
        """Test retry on 5xx server errors."""
        # Arrange
        mock_hubspot_client.crm.deals.basic_api.create = AsyncMock(
            side_effect=[
                HTTPStatusError(
                    "Internal Server Error",
                    request=Mock(),
                    response=Mock(status_code=500)
                ),
                HTTPStatusError(
                    "Service Unavailable",
                    request=Mock(),
                    response=Mock(status_code=503)
                ),
                Mock(to_dict=lambda: {"id": "deal_123"})
            ])

        connector = HubSpotCRMConnector(
            client=mock_hubspot_client,
            max_retries=3,
            base_delay=0.01
        )

        with patch('asyncio.sleep', new_callable=AsyncMock):
            # Act
            result = await connector.create_deal_from_lead("contact_1", {})

            # Assert
            assert result["id"] == "deal_123"
            assert mock_hubspot_client.crm.deals.basic_api.create.call_count == 3

    @pytest.mark.asyncio
    async def test_network_timeout_handling(self, mock_hubspot_client):
        """Test handling of network timeouts."""
        # Arrange
        mock_hubspot_client.crm.contacts.basic_api.create = AsyncMock(
            side_effect=RequestError("Connection timeout"))

        connector = HubSpotCRMConnector(
            client=mock_hubspot_client,
            timeout=5.0,
            max_retries=1
        )

        # Act & Assert
        with raises(RequestError):
            await connector.create_contact({"email": "test@test.com"})

    @pytest.mark.asyncio
    async def test_batch_operation_partial_failure(self, mock_hubspot_client):
        """Test handling of partial failures in batch operations."""
        # Arrange
        contacts = [
            {"email": "valid1@test.com"},
            {"email": "valid2@test.com"},
            {"email": "invalid"}  # This one will fail
        ]

        mock_hubspot_client.crm.contacts.batch_api.create = AsyncMock(
            return_value=Mock(
                results=[Mock(id="1"), Mock(id="2")],
                errors=[Mock(message="Invalid email", context={"email": "invalid"})]
            ))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act
        results = await connector.batch_create_contacts(contacts)

        # Assert
        assert len(results["successful"]) == 2
        assert len(results["failed"]) == 1
        assert results["failed"][0]["error"] == "Invalid email"

    @pytest.mark.asyncio
    async def test_quota_exceeded_handling(self, mock_hubspot_client):
        """Test handling of daily API quota exceeded."""
        # Arrange
        mock_hubspot_client.crm.contacts.basic_api.get_by_id = AsyncMock(
            side_effect=HTTPStatusError(
                "Daily quota exceeded",
                request=Mock(),
                response=Mock(
                    status_code=429,
                    headers={
                        "X-HubSpot-RateLimit-Remaining": "0",
                        "X-HubSpot-RateLimit-Secondly-Remaining": "0"
                    }
                )
            ))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act & Assert
        with raises(Exception) as exc_info:
            await connector.get_contact("123")
        assert "quota" in str(exc_info.value).lower() or exc_info.value.response.status_code == 429

class TestEdgeCases:
    """Test suite for edge cases and boundary conditions."""

    @pytest.mark.asyncio
    async def test_empty_response_handling(self, mock_hubspot_client):
        """Test handling of empty API responses."""
        # Arrange
        mock_hubspot_client.crm.contacts.basic_api.get_by_id = AsyncMock(
            return_value=Mock(to_dict=lambda: None))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act
        result = await connector.get_contact("123")

        # Assert
        assert result is None

    @pytest.mark.asyncio
    async def test_unicode_character_handling(self, mock_hubspot_client):
        """Test handling of unicode characters in contact data."""
        # Arrange
        unicode_data = {
            "firstname": "José",
            "lastname": "García",
            "company": "测试公司",
            "email": "jose@example.com"
        }
        mock_hubspot_client.crm.contacts.basic_api.create = AsyncMock(
            return_value=Mock(to_dict=lambda: {"id": "123", "properties": unicode_data}))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act
        result = await connector.create_contact(unicode_data)

        # Assert
        assert result["properties"]["firstname"] == "José"
        assert result["properties"]["company"] == "测试公司"

    @pytest.mark.asyncio
    async def test_very_long_string_handling(self, mock_hubspot_client):
        """Test handling of strings exceeding HubSpot limits."""
        # Arrange
        long_note = "A" * 100000  # HubSpot typically has 65,536 char limits
        call_data = {
            "notes": long_note,
            "duration_ms": 1000,
            "timestamp": datetime.now().isoformat()
        }
        mock_hubspot_client.crm.engagements.basic_api.create = AsyncMock(
            side_effect=HTTPStatusError(
                "Property value too long",
                request=Mock(),
                response=Mock(status_code=400)
            ))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act & Assert
        with raises(HTTPStatusError):
            await connector.log_call_activity("123", call_data)

    @pytest.mark.asyncio
    async def test_concurrent_request_handling(self, mock_hubspot_client):
        """Test handling of concurrent API requests."""
        # Arrange
        import asyncio

        mock_hubspot_client.crm.contacts.basic_api.create = AsyncMock(
            side_effect=lambda **kwargs: Mock(
                to_dict=lambda: {"id": f"contact_{datetime.now().timestamp()}"}
            ))

        connector = HubSpotCRMConnector(client=mock_hubspot_client)

        # Act
        tasks = [
            connector.create_contact({"email": f"user{i}@test.com"})
            for i in range(10)
        ]
        results = await asyncio.gather(*tasks, return_exceptions=True)

        # Assert
        successful = [r for r in results if not isinstance(r, Exception)]
        assert len(successful) == 10
        assert mock_hubspot_client.crm.contacts.basic_api.create.call_count == 10

# Integration-style tests with mocked HTTP client
@pytest.mark.asyncio
async def test_full_lead_conversion_workflow():
    """Test complete workflow from lead to deal with call logging."""
    # Arrange
    with patch('httpx.AsyncClient') as mock_client_class:
        mock_client = AsyncMock()
        mock_client_class.return_value = mock_client

        # Mock responses for sequence: Contact -> Call -> Deal
        responses = [
            Mock(status_code=200, json=lambda: {"id": "contact_123"}),  # Create contact
            Mock(status_code=200, json=lambda: {"id": "call_456"}),     # Log call
            Mock(status_code=200, json=lambda: {"id": "deal_789"}),     # Create deal
            Mock(status_code=200, json=lambda: {"status": "COMPLETE"})  # Association
        ]
        mock_client.post = AsyncMock(side_effect=responses)
        mock_client.get = AsyncMock(return_value=Mock(status_code=200, json=lambda: {}))

        connector = HubSpotCRMConnector(http_client=mock_client)

        # Act
        contact = await connector.create_contact({
            "email": "lead@example.com",
            "firstname": "New",
            "lastname": "Lead"
        })

        call = await connector.log_call_activity(contact["id"], {
            "duration_ms": 60000,
            "notes": "Qualified lead, ready for proposal"
        })

        deal = await connector.create_deal_from_lead(
            contact_id=contact["id"],
            deal_data={
                "dealname": "Qualified Lead Deal",
                "amount": 10000
            }
        )

        # Assert
        assert contact["id"] == "contact_123"
        assert call["id"] == "call_456"
        assert deal["id"] == "deal_789"
        assert mock_client.post.call_count == 4
```