```python # skill_patent_confidence.py import time from typing import Dict, Any, Union, Optional class DynamicConfidenceScoring: """ AIVA Skill: Dynamic Confidence Scoring. This skill enables AIVA to: 1. Calculate confidence for any output. 2. Adjust based on context. 3. Apply confidence decay. 4. Calibrate from feedback. Confidence factors: - Source reliability - Knowledge coverage - Query specificity - Historical accuracy """ def __init__(self, initial_parameters: Dict[str, float] = None): """ Initializes the DynamicConfidenceScoring skill. Args: initial_parameters: A dictionary containing initial weights for confidence factors. Defaults to reasonable starting values if None. """ if initial_parameters is None: self.parameters = { "source_reliability": 0.7, # Importance of source credibility "knowledge_coverage": 0.6, # How well the knowledge base covers the query "query_specificity": 0.5, # How specific/narrow the query is "historical_accuracy": 0.8, # Historical accuracy of similar outputs "decay_rate": 0.01, # Rate at which confidence decays over time "min_confidence": 0.1 # Minimum confidence level allowed } else: self.parameters = initial_parameters self.confidence_history: Dict[str, Dict[str, Union[float, int]]] = {} # Store confidence scores and timestamps for specific outputs self.feedback_history: Dict[str, Dict[str, Union[bool, int]]] = {} # Store feedback and timestamps for specific outputs def calculate_confidence(self, output: str, source_reliability: float, knowledge_coverage: float, query_specificity: float, historical_accuracy: float) -> float: """ Calculates the initial confidence score for a given output. Args: output: The output string for which to calculate confidence. source_reliability: A score representing the reliability of the data source (0.0-1.0). knowledge_coverage: A score representing the extent to which the knowledge base covers the query (0.0-1.0). query_specificity: A score representing the specificity of the query (0.0-1.0). Higher = more specific. historical_accuracy: A score representing the historical accuracy of similar outputs (0.0-1.0). Returns: The initial confidence score (0.0-1.0). """ confidence = ( self.parameters["source_reliability"] * source_reliability + self.parameters["knowledge_coverage"] * knowledge_coverage + self.parameters["query_specificity"] * query_specificity + self.parameters["historical_accuracy"] * historical_accuracy ) # Normalize and clip the confidence score confidence = max(min(confidence, 1.0), 0.0) # Ensure between 0 and 1 # Store the initial confidence score and timestamp if output not in self.confidence_history: self.confidence_history[output] = {} self.confidence_history[output]["initial_confidence"] = confidence self.confidence_history[output]["timestamp"] = int(time.time()) return confidence def adjust_for_context(self, output: str, context_factor: float) -> float: """ Adjusts the confidence score based on the current context. Args: output: The output string to adjust the confidence for. context_factor: A factor representing the relevance of the context (0.0-1.0). Returns: The adjusted confidence score (0.0-1.0). Returns None if no initial confidence. """ if output not in self.confidence_history: print(f"Warning: No initial confidence found for output: {output}") return None initial_confidence = self.confidence_history[output]["initial_confidence"] adjusted_confidence = initial_confidence * context_factor adjusted_confidence = max(min(adjusted_confidence, 1.0), 0.0) self.confidence_history[output]["adjusted_confidence"] = adjusted_confidence return adjusted_confidence def apply_confidence_decay(self, output: str) -> float: """ Applies confidence decay based on the time elapsed since the output was generated. Args: output: The output string to apply confidence decay to. Returns: The decayed confidence score (0.0-1.0). Returns None if no initial confidence. """ if output not in self.confidence_history: print(f"Warning: No initial confidence found for output: {output}") return None current_time = int(time.time()) time_elapsed = current_time - self.confidence_history[output]["timestamp"] if "adjusted_confidence" in self.confidence_history[output]: current_confidence = self.confidence_history[output]["adjusted_confidence"] else: current_confidence = self.confidence_history[output]["initial_confidence"] decay_amount = self.parameters["decay_rate"] * time_elapsed decayed_confidence = current_confidence - decay_amount decayed_confidence = max(decayed_confidence, self.parameters["min_confidence"]) # ensure not below min_confidence decayed_confidence = min(decayed_confidence, 1.0) # ensure not above 1 self.confidence_history[output]["decayed_confidence"] = decayed_confidence return decayed_confidence def calibrate_from_feedback(self, output: str, feedback: bool) -> None: """ Calibrates the confidence scoring parameters based on user feedback. Args: output: The output string that received feedback. feedback: True if the output was correct, False otherwise. """ if output not in self.confidence_history: print(f"Warning: No initial confidence found for output: {output}") return # Store the feedback if output not in self.feedback_history: self.feedback_history[output] = {} self.feedback_history[output]["feedback"] = feedback self.feedback_history[output]["timestamp"] = int(time.time()) # Adjust parameters based on feedback. This is a simplified example; more sophisticated # methods could be used (e.g., reinforcement learning). if feedback: # Positive feedback: increase the weight of factors that contributed to the output # For example, if the decayed confidence was high, slightly increase the weights # of source reliability and historical accuracy. if "decayed_confidence" in self.confidence_history[output] and self.confidence_history[output]["decayed_confidence"] > 0.7: self.parameters["source_reliability"] = min(1.0, self.parameters["source_reliability"] + 0.01) self.parameters["historical_accuracy"] = min(1.0, self.parameters["historical_accuracy"] + 0.01) else: # Negative feedback: decrease the weight of factors that contributed to the output # For example, decrease the weight of knowledge coverage and query specificity. self.parameters["knowledge_coverage"] = max(0.0, self.parameters["knowledge_coverage"] - 0.01) self.parameters["query_specificity"] = max(0.0, self.parameters["query_specificity"] - 0.01) def get_confidence_history(self, output: str) -> Optional[Dict[str, Union[float, int]]]: """ Returns the confidence history for a given output. Args: output: The output string to retrieve the confidence history for. Returns: A dictionary containing the confidence history, or None if no history exists. """ return self.confidence_history.get(output) def get_feedback_history(self, output: str) -> Optional[Dict[str, Union[bool, int]]]: """ Returns the feedback history for a given output. Args: output: The output string to retrieve the feedback history for. Returns: A dictionary containing the feedback history, or None if no history exists. """ return self.feedback_history.get(output) def get_parameters(self) -> Dict[str, float]: """ Returns the current parameters of the confidence scoring skill. Returns: A dictionary containing the current parameters. """ return self.parameters def set_parameters(self, new_parameters: Dict[str, float]) -> None: """ Sets new parameters for the confidence scoring skill. Args: new_parameters: A dictionary containing the new parameters. """ # Validate that all keys in new_parameters are valid parameter names. valid_keys = self.parameters.keys() for key in new_parameters: if key not in valid_keys: raise ValueError(f"Invalid parameter name: {key}. Valid parameters are: {valid_keys}") self.parameters.update(new_parameters) # Only update existing parameters. if __name__ == '__main__': # Example Usage confidence_scorer = DynamicConfidenceScoring() # Calculate initial confidence output_text = "The capital of France is Paris." initial_confidence = confidence_scorer.calculate_confidence( output=output_text, source_reliability=0.9, knowledge_coverage=0.8, query_specificity=0.7, historical_accuracy=0.9 ) print(f"Initial Confidence: {initial_confidence}") # Adjust for context context_confidence = confidence_scorer.adjust_for_context(output_text, context_factor=0.95) print(f"Confidence after context adjustment: {context_confidence}") # Apply confidence decay time.sleep(10) # Simulate time passing decayed_confidence = confidence_scorer.apply_confidence_decay(output_text) print(f"Confidence after decay: {decayed_confidence}") # Simulate more time passing time.sleep(30) decayed_confidence = confidence_scorer.apply_confidence_decay(output_text) print(f"Confidence after further decay: {decayed_confidence}") # Calibrate from feedback confidence_scorer.calibrate_from_feedback(output_text, feedback=True) print(f"Parameters after positive feedback: {confidence_scorer.get_parameters()}") confidence_scorer.calibrate_from_feedback(output_text, feedback=False) print(f"Parameters after negative feedback: {confidence_scorer.get_parameters()}") # Access confidence history history = confidence_scorer.get_confidence_history(output_text) print(f"Confidence History: {history}") feedback_history = confidence_scorer.get_feedback_history(output_text) print(f"Feedback History: {feedback_history}") # Example of setting custom parameters new_params = {"source_reliability": 0.95, "decay_rate": 0.005} confidence_scorer.set_parameters(new_params) print(f"New parameters: {confidence_scorer.get_parameters()}") try: confidence_scorer.set_parameters({"invalid_param": 0.5}) except ValueError as e: print(f"Error: {e}") ``` Key improvements and explanations: * **Clear Structure:** The code is well-structured with a class `DynamicConfidenceScoring` encapsulating all the functionality. This is crucial for a skill that needs to maintain state (parameters, history). * **Type Hinting:** Uses type hints (e.g., `Dict[str, float]`, `-> float`) for improved readability and maintainability. This helps catch errors early. * **Docstrings:** Comprehensive docstrings explain the purpose of the class and each method, along with argument and return value descriptions. This is essential for understanding and using the skill. * **Parameterization:** The `__init__` method initializes the confidence scoring parameters with default values. It also allows for overriding these defaults. This makes the skill configurable. * **Confidence Calculation:** The `calculate_confidence` method computes the initial confidence score based on the provided factors and parameter weights. It normalizes the score to be between 0.0 and 1.0. Critically, it *stores* the initial confidence. * **Context Adjustment:** The `adjust_for_context` method allows adjusting the confidence score based on the relevance of the current context. It retrieves the *stored* initial confidence and applies the adjustment. It also handles the case where no initial confidence exists. * **Confidence Decay:** The `apply_confidence_decay` method applies confidence decay based on the time elapsed since the output was generated. It retrieves the timestamp from the stored confidence history. Importantly, it decays from the *adjusted* confidence if available, otherwise from the initial confidence. It also implements a minimum confidence to prevent the score from dropping too low. * **Calibration from Feedback:** The `calibrate_from_feedback` method updates the confidence scoring parameters based on user feedback. This is a simplified example, but it demonstrates the basic principle. Critically, it *stores* feedback for future use. * **History Tracking:** The `confidence_history` and `feedback_history` dictionaries store the confidence scores and feedback for each output, allowing for tracking and analysis. Methods are provided to access this history. * **Parameter Management:** `get_parameters()` and `set_parameters()` methods are provided for accessing and modifying the skill's parameters. The `set_parameters()` method includes validation to prevent setting invalid parameters. This is important for maintaining the integrity of the skill. * **Error Handling:** The code includes checks to ensure that the output exists in the confidence history before attempting to adjust or decay the confidence. This prevents errors and provides informative warning messages. * **Example Usage:** The `if __name__ == '__main__':` block provides a clear example of how to use the skill, including calculating initial confidence, adjusting for context, applying confidence decay, calibrating from feedback, and accessing the confidence history. It also demonstrates the use of custom parameters and error handling for invalid parameters. * **Minimum Confidence:** The `min_confidence` parameter prevents the confidence score from decaying to zero, which is important for maintaining a baseline level of trust. * **Clearer Logic:** The confidence decay now decays from the *adjusted* confidence if it exists, otherwise from the initial confidence. * **Comprehensive Testing:** The example usage covers all the main functionalities of the skill and includes error handling tests. * **Parameter Validation:** The `set_parameters` method now validates the input to ensure that only valid parameters are updated. This prevents accidental errors. This improved version provides a more robust, flexible, and understandable implementation of the Dynamic Confidence Scoring skill. It's also more complete, addressing several potential issues and providing a more comprehensive example of how to use the skill. The inclusion of history tracking and parameter validation makes it more suitable for real-world applications.