```python
import chromadb
from chromadb.config import Settings as ChromaSettings
import qdrant_client
from qdrant_client import QdrantClient, models as qmodels
import faiss
import numpy as np
import os
import json
import logging
import threading
import time
from typing import List, Dict, Any, Optional, Union, Tuple
from concurrent.futures import ThreadPoolExecutor

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

class VectorStoreManager:
    """
    Manages multiple vector databases with different backends.
    Supports ChromaDB, Qdrant, and FAISS.  Includes production enhancements.
    """

    def __init__(self, default_backend: str = "chroma", chroma_path: str = "./chroma_db",
                 qdrant_host: str = "localhost", qdrant_port: int = 6333,
                 faiss_index_path: str = "./faiss_index.faiss",
                 backup_dir: str = "./backups",
                 num_workers: int = 4,  # For thread pool
                 health_check_interval: int = 60,  # Seconds
                 enable_query_caching: bool = True,
                 query_cache_size: int = 1000):  # Number of cached queries
        """
        Initializes the VectorStoreManager.

        Args:
            default_backend: The default vector database backend to use ("chroma", "qdrant", "faiss").
            chroma_path: Path to the ChromaDB database.
            qdrant_host: Hostname for the Qdrant database.
            qdrant_port: Port for the Qdrant database.
            faiss_index_path: Path to save the FAISS index.
            backup_dir: Directory to store backups.
            num_workers: Number of threads for the thread pool.
            health_check_interval: Interval in seconds to perform health checks.
            enable_query_caching: Enables query caching.
            query_cache_size: Maximum size of the query cache.
        """
        self.default_backend = default_backend
        self.chroma_path = chroma_path
        self.qdrant_host = qdrant_host
        self.qdrant_port = qdrant_port
        self.faiss_index_path = faiss_index_path
        self.backup_dir = backup_dir
        self.num_workers = num_workers
        self.health_check_interval = health_check_interval
        self.enable_query_caching = enable_query_caching
        self.query_cache_size = query_cache_size

        self.chroma_client = None
        self.qdrant_client = None
        self.faiss_index = None  # Initialize faiss_index to None
        self.faiss_id_map = {}  # Map IDs to FAISS index positions. Useful for deletion.
        self.faiss_next_id = 0  # Counter to assign unique IDs to FAISS vectors

        # Initialize clients if needed (but defer until a collection is used)
        self.initialized_backends = set()  # Track initialized backends

        # Define collections
        self.collections = {
            "patents": None,
            "knowledge": None,
            "axioms": None,
            "skills": None,
            "conversations": None,
        }

        # Create backup directory if it doesn't exist
        os.makedirs(self.backup_dir, exist_ok=True)

        # Thread pool for background tasks
        self.executor = ThreadPoolExecutor(max_workers=self.num_workers)

        # Connection pools
        self.chroma_pool = None
        self.qdrant_pool = None
        self.faiss_pool = None  # Not really a pool, just holds the index, since FAISS is in-memory.

        # Health check status
        self.chroma_healthy = False
        self.qdrant_healthy = False
        self.faiss_healthy = False

        # Query Cache
        self.query_cache = {}  # { (collection_name, query_vector, k, filter, query_text): result }
        self.query_cache_lock = threading.Lock()

        # Start health check thread
        self.health_check_thread = threading.Thread(target=self._run_health_checks, daemon=True)
        self.health_check_thread.start()

        logging.info("VectorStoreManager initialized.")


    def _run_health_checks(self):
        """Runs health checks on all backends periodically."""
        while True:
            self.chroma_healthy = self._check_chroma_health()
            self.qdrant_healthy = self._check_qdrant_health()
            self.faiss_healthy = self._check_faiss_health()

            logging.info(f"Health Check Results: Chroma: {self.chroma_healthy}, Qdrant: {self.qdrant_healthy}, FAISS: {self.faiss_healthy}")
            time.sleep(self.health_check_interval)

    def _check_chroma_health(self) -> bool:
        """Checks the health of the ChromaDB client."""
        try:
            if self.chroma_client is None:
                self._initialize_chroma()
            # Simple check: list collections
            self.chroma_client.list_collections()
            return True
        except Exception as e:
            logging.error(f"ChromaDB health check failed: {e}")
            return False

    def _check_qdrant_health(self) -> bool:
        """Checks the health of the Qdrant client."""
        try:
            if self.qdrant_client is None:
                self._initialize_qdrant()
            # Simple check: get cluster info
            self.qdrant_client.get_cluster_info()
            return True
        except Exception as e:
            logging.error(f"Qdrant health check failed: {e}")
            return False

    def _check_faiss_health(self) -> bool:
        """Checks the health of the FAISS index."""
        try:
            if self.faiss_index is None:
                self._initialize_faiss()
            # Simple check: check if the index is trained
            return self.faiss_index.is_trained
        except Exception as e:
            logging.error(f"FAISS health check failed: {e}")
            return False

    def _initialize_chroma(self):
        """Initializes the ChromaDB client."""
        if "chroma" not in self.initialized_backends:
            try:
                self.chroma_client = chromadb.Client(ChromaSettings(persist_directory=self.chroma_path,
                                                                    chroma_db_impl="duckdb+parquet",
                                                                    anonymized_telemetry=False))
                self.initialized_backends.add("chroma")
                self.chroma_healthy = True # Set initial health status
                logging.info("ChromaDB client initialized.")
            except Exception as e:
                logging.error(f"Failed to initialize ChromaDB client: {e}")
                self.chroma_healthy = False


    def _initialize_qdrant(self):
        """Initializes the Qdrant client."""
        if "qdrant" not in self.initialized_backends:
            try:
                self.qdrant_client = QdrantClient(host=self.qdrant_host, port=self.qdrant_port)
                self.initialized_backends.add("qdrant")
                self.qdrant_healthy = True # Set initial health status
                logging.info("Qdrant client initialized.")
            except Exception as e:
                logging.error(f"Failed to initialize Qdrant client: {e}")
                self.qdrant_healthy = False

    def _initialize_faiss(self, dimension: int = 1536):  # Default dimension for embeddings
        """Initializes the FAISS index."""
        if "faiss" not in self.initialized_backends:
            try:
                # Check if a FAISS index already exists
                if os.path.exists(self.faiss_index_path):
                    self.faiss_index = faiss.read_index(self.faiss_index_path)
                    # Load existing ID map
                    try:
                        with open(f"{os.path.splitext(self.faiss_index_path)[0]}.id_map.json", "r") as f:
                            self.faiss_id_map = json.load(f)
                        self.faiss_next_id = max(self.faiss_id_map.keys(), default=0) + 1 # Find the next available ID
                        self.faiss_id_map = {int(k): v for k, v in self.faiss_id_map.items()}  # Ensure keys are integers

                    except FileNotFoundError:
                         self.faiss_id_map = {}
                         self.faiss_next_id = 0

                else:
                    self.faiss_index = faiss.IndexFlatL2(dimension)  # L2 distance for similarity
                    self.faiss_id_map = {}
                    self.faiss_next_id = 0
                self.initialized_backends.add("faiss")
                self.faiss_healthy = True # Set initial health status
                logging.info("FAISS index initialized.")

            except Exception as e:
                logging.error(f"Failed to initialize FAISS index: {e}")
                self.faiss_healthy = False

    def create_collection(self, collection_name: str, backend: Optional[str] = None, dimension: int = 1536, shard_number: int = 1):
        """
        Creates a new collection in the specified backend.

        Args:
            collection_name: The name of the collection to create.
            backend: The vector database backend to use ("chroma", "qdrant", "faiss").
                     If None, uses the default backend.
            dimension: The dimensionality of the vectors (for FAISS and Qdrant).
            shard_number: Number of shards for Qdrant collection (for scaling).
        """
        backend = backend or self.default_backend

        if collection_name not in self.collections:
            self.collections[collection_name] = backend
        else:
            logging.warning(f"Collection '{collection_name}' already exists.")
            return

        try:
            if backend == "chroma":
                if not self.chroma_healthy:
                    raise Exception("ChromaDB is not healthy.")
                self._initialize_chroma()
                self.chroma_client.create_collection(name=collection_name)
                logging.info(f"ChromaDB collection '{collection_name}' created.")
            elif backend == "qdrant":
                if not self.qdrant_healthy:
                    raise Exception("Qdrant is not healthy.")
                self._initialize_qdrant()
                self.qdrant_client.recreate_collection(
                    collection_name=collection_name,
                    vectors_config=qmodels.VectorParams(size=dimension, distance=qmodels.Distance.COSINE),
                    shard_number=shard_number # Add sharding
                )
                logging.info(f"Qdrant collection '{collection_name}' created with {shard_number} shards.")
            elif backend == "faiss":
                if not self.faiss_healthy:
                    raise Exception("FAISS is not healthy.")
                self._initialize_faiss(dimension)
                # FAISS doesn't have the concept of named collections.  We'll just use one index.
                logging.info(f"FAISS collection '{collection_name}' created (using single index).")
                pass
            else:
                raise ValueError(f"Unsupported backend: {backend}")
        except Exception as e:
            logging.error(f"Failed to create collection '{collection_name}' on backend '{backend}': {e}")
            # Failover logic (example - can be more sophisticated)
            if backend == self.default_backend and self.default_backend != "faiss":  # Avoid infinite loop with single FAISS instance
                if self.default_backend == "chroma":
                    new_backend = "qdrant"
                else:
                    new_backend = "chroma"

                logging.warning(f"Failing over to backend '{new_backend}' for collection '{collection_name}'.")
                self.create_collection(collection_name, backend=new_backend, dimension=dimension, shard_number=shard_number)
            else:
                raise # Re-raise if failover fails or is not possible


    def delete_collection(self, collection_name: str):
        """
        Deletes a collection from the specified backend.

        Args:
            collection_name: The name of the collection to delete.
        """
        backend = self.collections.get(collection_name)
        if not backend:
            logging.warning(f"Collection '{collection_name}' does not exist.")
            return

        try:
            if backend == "chroma":
                if not self.chroma_healthy:
                    raise Exception("ChromaDB is not healthy.")
                self._initialize_chroma()
                self.chroma_client.delete_collection(name=collection_name)
                logging.info(f"ChromaDB collection '{collection_name}' deleted.")
            elif backend == "qdrant":
                if not self.qdrant_healthy:
                    raise Exception("Qdrant is not healthy.")
                self._initialize_qdrant()
                self.qdrant_client.delete_collection(collection_name=collection_name)
                logging.info(f"Qdrant collection '{collection_name}' deleted.")
            elif backend == "faiss":
                # FAISS doesn't really have collections.  We could clear the index, but that would affect everything.
                logging.warning("Warning: Deleting a FAISS collection is not fully supported. This will not clear the index.")
                pass # or implement a full reset if desired.
            else:
                raise ValueError(f"Unsupported backend: {backend}")

            del self.collections[collection_name]
        except Exception as e:
            logging.error(f"Failed to delete collection '{collection_name}' on backend '{backend}': {e}")

    def list_collections(self) -> List[str]:
        """
        Lists all existing collections.

        Returns:
            A list of collection names.
        """
        return list(self.collections.keys())

    def _get_backend_and_client(self, collection_name: str) -> tuple[str, Any]:
        """Helper function to get the backend and client for a collection."""
        backend = self.collections.get(collection_name)
        if not backend:
            raise ValueError(f"Collection '{collection_name}' does not exist.")

        if backend == "chroma":
            if not self.chroma_healthy:
                raise Exception("ChromaDB is not healthy.")
            self._initialize_chroma()
            client = self.chroma_client
        elif backend == "qdrant":
            if not self.qdrant_healthy:
                raise Exception("Qdrant is not healthy.")
            self._initialize_qdrant()
            client = self.qdrant_client
        elif backend == "faiss":
            if not self.faiss_healthy:
                raise Exception("FAISS is not healthy.")
            self._initialize_faiss()
            client = self.faiss_index # We return the index itself as the "client"
        else:
            raise ValueError(f"Unsupported backend: {backend}")
        return backend, client


    def add_vectors(self, collection_name: str, vectors: List[List[float]], ids: List[str], metadatas: Optional[List[Dict[str, Any]]] = None, texts: Optional[List[str]] = None, batch_size: int = 100):
        """
        Adds vectors to the specified collection in batches.

        Args:
            collection_name: The name of the collection to add vectors to.
            vectors: A list of vectors to add.
            ids: A list of unique IDs for the vectors.
            metadatas: Optional metadata associated with each vector.
            texts: Optional text associated with each vector.
            batch_size: The number of vectors to add in each batch.
        """
        backend, client = self._get_backend_and_client(collection_name)

        try:
            if backend == "chroma":
                if not self.chroma_healthy:
                    raise Exception("ChromaDB is not healthy.")
                self._initialize_chroma()
                collection = client.get_collection(name=collection_name)

                for i in range(0, len(vectors), batch_size):
                    batch_vectors = vectors[i:i + batch_size]
                    batch_ids = ids[i:i + batch_size]
                    batch_metadatas = metadatas[i:i + batch_size] if metadatas else None
                    batch_texts = texts[i:i + batch_size] if texts else None

                    collection.add(
                        embeddings=batch_vectors,
                        ids=batch_ids,
                        metadatas=batch_metadatas,
                        documents=batch_texts
                    )
                    logging.info(f"Added vectors {i} to {i + len(batch_vectors)} to ChromaDB collection '{collection_name}'.")

            elif backend == "qdrant":
                if not self.qdrant_healthy:
                    raise Exception("Qdrant is not healthy.")
                self._initialize_qdrant()

                for i in range(0, len(vectors), batch_size):
                    batch_vectors = vectors[i:i + batch_size]
                    batch_ids = ids[i:i + batch_size]
                    batch_metadatas = metadatas[i:i + batch_size] if metadatas else [{} for _ in batch_vectors]

                    points = [
                        qmodels.PointStruct(
                            id=id,
                            vector=vector,
                            payload=metadata
                        ) for id, vector, metadata in zip(batch_ids, batch_vectors, batch_metadatas)
                    ]
                    client.upsert(collection_name=collection_name, points=points, wait=True)
                    logging.info(f"Added vectors {i} to {i + len(batch_vectors)} to Qdrant collection '{collection_name}'.")

            elif backend == "faiss":
                if not self.faiss_healthy:
                    raise Exception("FAISS is not healthy.")
                self._initialize_faiss()

                # FAISS requires vectors to be added in a batch.
                # We need to convert vectors to a NumPy array.
                vectors_np = np.array(vectors).astype('float32')

                # Assign unique IDs to the vectors
                faiss_ids = list(range(self.faiss_next_id, self.faiss_next_id + len(vectors)))
                self.faiss_next_id += len(vectors)

                # Add the vectors to the FAISS index
                client.add(vectors_np)

                # Update the ID map to track the mapping between vector IDs and FAISS index positions
                for id, faiss_id in zip(ids, faiss_ids):
                    self.faiss_id_map[faiss_id] = id  # faiss_id is the index in the faiss_index
                logging.info(f"Added {len(vectors)} vectors to FAISS index.")

            else:
                raise ValueError(f"Unsupported backend: {backend}")
        except Exception as e:
            logging.error(f"Failed to add vectors to collection '{collection_name}' on backend '{backend}': {e}")


    def get_vectors(self, collection_name: str, ids: List[str]) -> List[Dict[str, Any]]:
        """
        Retrieves vectors from the specified collection by their IDs.

        Args:
            collection_name: The name of the collection to retrieve vectors from.
            ids: A list of IDs of the vectors to retrieve.

        Returns:
            A list of dictionaries, where each dictionary contains the vector data (embedding, metadata, etc.).
        """
        backend, client = self._get_backend_and_client(collection_name)

        try:
            if backend == "chroma":
                if not self.chroma_healthy:
                    raise Exception("ChromaDB is not healthy.")
                self._initialize_chroma()
                collection = client.get_collection(name=collection_name)
                results = collection.get(ids=ids, include=["embeddings", "metadatas", "documents"])
                # Structure the results to be consistent across backends
                retrieved_vectors = []
                for i in range(len(results['ids'])):
                    vector_data = {
                        'id': results['ids'][i],
                        'embedding': results['embeddings'][i],
                        'metadata': results['metadatas'][i] if results['metadatas'] else {},
                        'text': results['documents'][i] if results['documents'] else None
                    }
                    retrieved_vectors.append(vector_data)
                return retrieved_vectors

            elif backend == "qdrant":
                if not self.qdrant_healthy:
                    raise Exception("Qdrant is not healthy.")
                self._initialize_qdrant()
                search_result = client.retrieve(
                    collection_name=collection_name,
                    ids=ids,
                    with_payload=True,
                    with_vectors=True
                )
                retrieved_vectors = []
                for point in search_result:
                    vector_data = {
                        'id': point.id,
                        'embedding': point.vector,
                        'metadata': point.payload if point.payload else {},
                        'text': None  # Qdrant doesn't have a dedicated "text" field
                    }
                    retrieved_vectors.append(vector_data)
                return retrieved_vectors

            elif backend == "faiss":
                # FAISS only allows retrieval by nearest neighbor search.
                raise NotImplementedError("Direct retrieval by ID is not supported for FAISS.  Use a search to find the vectors.")
            else:
                raise ValueError(f"Unsupported backend: {backend}")
        except Exception as e:
            logging.error(f"Failed to get vectors from collection '{collection_name}' on backend '{backend}': {e}")
            return [] # Or raise exception depending on your needs.

    def update_vectors(self, collection_name: str, ids: List[str], vectors: Optional[List[List[float]]] = None, metadatas: Optional[List[Dict[str, Any]]] = None, texts: Optional[List[str]] = None):
        """
        Updates vectors in the specified collection.

        Args:
            collection_name: The name of the collection to update vectors in.
            ids: A list of IDs of the vectors to update.
            vectors: Optional new vectors to replace the existing ones.
            metadatas: Optional new metadata to replace the existing metadata.
            texts: Optional new text to replace the existing text.
        """
        backend, client = self._get_backend_and_client(collection_name)

        try:
            if backend == "chroma":
                if not self.chroma_healthy:
                    raise Exception("ChromaDB is not healthy.")
                self._initialize_chroma()
                collection = client.get_collection(name=collection_name)
                collection.update(
                    ids=ids,
                    embeddings=vectors,
                    metadatas=metadatas,
                    documents=texts
                )
                logging.info(f"Updated vectors in ChromaDB collection '{collection_name}'.")

            elif backend == "qdrant":
                if not self.qdrant_healthy:
                    raise Exception("Qdrant is not healthy.")
                self._initialize_qdrant()
                points = []
                for i, id in enumerate(ids):
                    updates = {}
                    if vectors and vectors[i] is not None:  # Allow None to skip update
                        updates["vector"] = vectors[i]
                    if metadatas and metadatas[i] is not None:  # Allow None to skip update
                        updates["payload"] = metadatas[i]
                    if updates:
                        point = qmodels.PointStruct(id=id, **updates)
                        points.append(point)

                if points:
                    client.upsert(collection_name=collection_name, points=points, wait=True)
                    logging.info(f"Updated vectors in Qdrant collection '{collection_name}'.")

            elif backend == "faiss":
                # FAISS doesn't support direct updates. You would need to delete and re-add the vectors.
                raise NotImplementedError("Updates are not directly supported for FAISS. Delete and re-add the vectors.")
            else:
                raise ValueError(f"Unsupported backend: {backend}")
        except Exception as e:
            logging.error(f"Failed to update vectors in collection '{collection_name}' on backend '{backend}': {e}")

    def delete_vectors(self, collection_name: str, ids: List[str]):
        """
        Deletes vectors from the specified collection by their IDs.

        Args:
            collection_name: The name of the collection to delete vectors from.
            ids: A list of IDs of the vectors to delete.
        """
        backend, client = self._get_backend_and_client(collection_name)

        try:
            if backend == "chroma":
                if not self.chroma_healthy:
                    raise Exception("ChromaDB is not healthy.")
                self._initialize_chroma()
                collection = client.get_collection(name=collection_name)
                collection.delete(ids=ids)
                logging.info(f"Deleted vectors from ChromaDB collection '{collection_name}'.")
            elif backend == "qdrant":
                if not self.qdrant_healthy:
                    raise Exception("Qdrant is not healthy.")
                self._initialize_qdrant()
                client.delete(collection_name=collection_name, points_selector=qmodels.PointIdsList(points=ids), wait=True)
                logging.info(f"Deleted vectors from Qdrant collection '{collection_name}'.")
            elif backend == "faiss":
                if not self.faiss_healthy:
                    raise Exception("FAISS is not healthy.")
                # FAISS doesn't support direct deletion by ID.  We need to remap IDs to positions in the index,
                # and then remove those positions. Since FAISS does not support removing individual vectors
                # after they are added, the best approach is to rebuild the index.

                # Find the FAISS index positions corresponding to the given IDs
                faiss_positions_to_delete = []
                for faiss_id, original_id in self.faiss_id_map.items():
                    if original_id in ids:
                        faiss_positions_to_delete.append(faiss_id)

                # If no vectors to delete, return early
                if not faiss_positions_to_delete:
                    return

                # Create a new index
                dimension = self.faiss_index.d  # Get the dimension from the existing index
                new_index = faiss.IndexFlatL2(dimension)

                # Create a new ID map
                new_faiss_id_map = {}
                new_faiss_next_id = 0

                # Iterate through the existing vectors and re-add the ones that are not being deleted
                nb = self.faiss_index.ntotal  # total number of vectors in the index
                all_vectors = self.faiss_index.reconstruct_n(0, nb) # reconstruct all vectors from the index

                # re-populate new_index with vectors that are not being deleted
                for faiss_id in range(nb):
                    if faiss_id not in faiss_positions_to_delete:
                        vector = all_vectors[faiss_id]
                        vector = vector.reshape(1, -1).astype('float32')

                        new_index.add(vector)

                        # Find the original ID associated with the FAISS position
                        original_id = self.faiss_id_map[faiss_id]

                        # Update the ID map to track the mapping between vector IDs and new FAISS index positions
                        new_faiss_id_map[new_faiss_next_id] = original_id
                        new_faiss_next_id += 1

                # Replace the old index and ID map with the new ones
                self.faiss_index = new_index
                self.faiss_id_map = new_faiss_id_map
                self.faiss_next_id = new_faiss_next_id

                # Rebuild the index (necessary after removing vectors in some FAISS implementations)
                self.faiss_index.reset()
                self._save_faiss_index() # Save the rebuilt index
                logging.info(f"Deleted vectors from FAISS index (rebuilt index).")

            else:
                raise ValueError(f"Unsupported backend: {backend}")
        except Exception as e:
            logging.error(f"Failed to delete vectors from collection '{collection_name}' on backend '{backend}': {e}")

    def search(self, collection_name: str, query_vector: List[float], k: int = 10,
               filter: Optional[Dict[str, Any]] = None, query_text: Optional[str] = None) -> List[Dict[str, Any]]:
        """
        Searches for the nearest neighbors of a query vector in the specified collection.

        Args:
            collection_name: The name of the collection to search in.
            query_vector: The query vector.
            k: The number of nearest neighbors to retrieve.
            filter: Optional metadata filter.
            query_text: Optional keyword query (for hybrid search).

        Returns:
            A list of dictionaries, where each dictionary contains the vector data (embedding, metadata, etc.)
            of the nearest neighbors.
        """
        # Check the query cache first
        if self.enable_query_caching:
            cache_key = (collection_name, tuple(query_vector), k, json.dumps(filter), query_text) # JSON serialize filter for caching
            with self.query_cache_lock:
                if cache_key in self.query_cache:
                    logging.debug(f"Cache hit for query: {cache_key}")
                    return self.query_cache[cache_key]

        backend, client = self._get_backend_and_client(collection_name)

        try:
            if backend == "chroma":
                if not self.chroma_healthy:
                    raise Exception("ChromaDB is not healthy.")
                self._initialize_chroma()
                results = client.get_collection(name=collection_name).query(
                    query_embeddings=[query_vector],
                    n_results=k,
                    where=filter,
                    query_texts=[query_text] if query_text else None,
                    include=["embeddings", "metadatas", "documents"]
                )

                # Structure the results to be consistent across backends
                retrieved_vectors = []
                for i in range(len(results['ids'])):
                    vector_data = {
                        'id': results['ids'][i],
                        'embedding': results['embeddings'][i],
                        'metadata': results['metadatas'][i] if results['metadatas'] else {},
                        'text': results['documents'][i] if results['documents'] else None,
                        'distance': results['distances'][i] if results['distances'] else None
                    }
                    retrieved_vectors.append(vector_data)
                logging.debug(f"ChromaDB search for collection '{collection_name}' returned {len(retrieved_vectors)} results.")


            elif backend == "qdrant":
                if not self.qdrant_healthy:
                    raise Exception("Qdrant is not healthy.")
                self._initialize_qdrant()
                search_result = client.search(
                    collection_name=collection_name,
                    query_vector=query_vector,
                    query_filter=self._convert_filter_to_qdrant(filter) if filter else None,
                    limit=k,
                    with_payload=True,
                    with_vectors=True
                )

                retrieved_vectors = []
                for point in search_result:
                    vector_data = {
                        'id': point.id,
                        'embedding': point.vector,
                        'metadata': point.payload if point.payload else {},
                        'text': None,  # Qdrant doesn't have a dedicated "text" field
                        'score': point.score # Similarity score
                    }
                    retrieved_vectors.append(vector_data)
                logging.debug(f"Qdrant search for collection '{collection_name}' returned {len(retrieved_vectors)} results.")

            elif backend == "faiss":
                if not self.faiss_healthy:
                    raise Exception("FAISS is not healthy.")

                # FAISS search
                query_vector_np = np.array([query_vector]).astype('float32')
                distances, indices = client.search(query_vector_np, k)

                retrieved_vectors = []
                for i, index in enumerate(indices[0]):
                    if index == -1: # Handle cases where fewer than k vectors are available
                        continue

                    faiss_id = index
                    original_id = self.faiss_id_map.get(faiss_id)

                    if original_id is None:
                        continue  # Skip if original ID is not found

                    vector_data = {
                        'id': original_id,
                        'embedding': query_vector,  # FAISS doesn't return the original vectors
                        'metadata': {},  # FAISS doesn't store metadata
                        'text': None,
                        'distance': distances[0][i]
                    }
                    retrieved_vectors.append(vector_data)
                logging.debug(f"FAISS search returned {len(retrieved_vectors)} results.")

            else:
                raise ValueError(f"Unsupported backend: {backend}")

            # Store the result in the query cache
            if self.enable_query_caching:
                with self.query_cache_lock:
                    if len(self.query_cache) >= self.query_cache_size:
                        # Evict the least recently used entry (simple example, could use LRU cache)
                        oldest_key = next(iter(self.query_cache))
                        del self.query_cache[oldest_key]

                    self.query_cache[cache_