
    !/iOF                        d Z ddlZddlZddlmZ ddlmZ ddlmZm	Z	m
Z
mZmZ ddlmZmZmZ ddlZddlZ	 ddlmZ dZes
	 dd
lmZ dZnd	Ze G d d             Ze G d d             Z G d d      Z G d d      Zedk(  rddlZ e ejB                        dkD  rejB                  d   Z" e       Z#e"dk(  r+e#jI                         Z% e& ejN                  e%d             ye"dk(  rj e ejB                        dkD  rLdjQ                  ejB                  dd       Z)e#jU                  e)dd      Z+ e& ejN                  e+d             y e&d       ye"dk(  r e&d       e#jY                  ddd        e#jY                  d!dd"       e#jY                  d#dd"       g d$Z-e-D ]l  \  Z)Z.e#jU                  e)de.      Z+ e&d%e)dd&  d'        e&d(e+d)   d*   d+        e&d,e+d)   d-   d+        e&d.e+d)   d/   d+        e&d0e+d1    d2       n y e&d3e"         e&d4       y e&d5        e&d4       yy# e$ r d	ZdZY w xY w# e$ r d	ZdZY w xY w)6ax  
Genesis Enhanced Surprise Detection
====================================
Embedding-based novelty detection replacing simple keyword matching.

Uses cosine similarity against memory bank to detect truly novel information.
Higher surprise = lower similarity to existing memories.

Usage:
    from enhanced_surprise import EnhancedSurpriseDetector

    detector = EnhancedSurpriseDetector()

    # Score new content
    result = detector.evaluate("Some new information", "source", "domain")
    print(result["score"]["novelty"])  # 0.0-1.0

    # Add to memory bank
    detector.add_to_memory("Known information", "source", "domain")
    N)Path)datetime)DictListOptionalTupleAny)	dataclassfieldasdict)TextEmbeddingTF)SentenceTransformerc                   ~    e Zd ZU dZdZeed<   dZeed<   dZeed<   dZ	eed<   dZ
eed<   defd	Zed
edd fd       Zy)SurpriseScorez"Detailed surprise score breakdown.        novelty	violationimpactraritytotalreturnc                     t        |       S N)r   selfs    //mnt/e/genesis-system/core/enhanced_surprise.pyto_dictzSurpriseScore.to_dict<   s    d|    datac                      | di |S N r"   clsr   s     r   	from_dictzSurpriseScore.from_dict?       {T{r   N)__name__
__module____qualname____doc__r   float__annotations__r   r   r   r   r   r   classmethodr%   r"   r   r   r   r   3   se    ,GUIuFEFEE5  T o  r   r   c                       e Zd ZU dZeed<   eed<   eed<   eed<   ee   ed<    ed       Z	eed	<   d
Z
eed<   defdZededd fd       Zy)MemoryVectorz#A memory with its embedding vector.idcontentsourcedomainvectorc                  F    t        j                         j                         S r   )r   now	isoformatr"   r   r   <lambda>zMemoryVector.<lambda>L   s    8<<>3K3K3M r   )default_factory	timestampr   access_countr   c                     | j                   | j                  | j                  | j                  | j                  | j
                  | j                  dS )Nr0   r1   r2   r3   r4   r:   r;   r=   r   s    r   r   zMemoryVector.to_dictO   s?    ''||kkkkkk --
 	
r   r   c                      | di |S r!   r"   r#   s     r   r%   zMemoryVector.from_dictZ   r&   r   N)r'   r(   r)   r*   strr,   r   r+   r   r:   r;   intr   r   r-   r%   r"   r   r   r/   r/   D   sk    -GLKKK+MNIsNL#	
 	
 T n  r   r/   c                   b   e Zd ZdZ	 	 	 	 d dee   dededefdZdeddfd	Z	d
ede
e   fdZd
ede
e   fdZde
e   de
e   defdZ	 	 d!de
e   dee   dede
eeef      fdZ	 d"dedededee   def
dZdedefdZdededefdZ	 d"dedededee   def
dZd#dZd#dZd#dZdefdZy)$EnhancedSurpriseDetectorz
    Embedding-based surprise detection for Genesis memory system.

    Features:
    - Cosine similarity against memory bank
    - Domain-specific thresholds
    - Prediction-error based violation detection
    - Adaptive novelty scaling
    Nmemory_path
model_namesimilarity_thresholdmax_memoriesc                    |rt        |      n
t        d      | _        || _        || _        t	        j
                         | _        i | _        i | _        d | _	        d| _
        | j                  |       | j                          ddddd| _        y )N,E:/genesis-system/data/surprise_vectors.json  皙?g?)r   r   r   r   )r   rC   rE   rF   	threadingRLock_lockmemoriesdomain_statsembeddervector_size_init_embedder_loadweights)r   rC   rD   rE   rF   s        r   __init__z!EnhancedSurpriseDetector.__init__j   s     1<4,FtAu$8!(__&
 24-/ J' 	

 	
r   r   c                    t         r)	 t        |      | _        d| _        t	        d| d       yt        r@	 t        |      | _        | j                  j                         | _        t	        d| d       yt	        d	       d| _        y# t
        $ r}t	        d|        Y d}~ud}~ww xY w# t
        $ r}t	        d|        Y d}~Sd}~ww xY w)
zInitialize the embedding model.)rD   rI   z([OK] EnhancedSurprise: Using FastEmbed ()Nz[!] FastEmbed init failed: z3[OK] EnhancedSurprise: Using SentenceTransformers (z&[!] SentenceTransformers init failed: zB[!] EnhancedSurprise: No embedding model available, using fallback)	FASTEMBED_AVAILABLEr   rP   rQ   print	ExceptionSENTENCE_TRANSFORMERS_AVAILABLEr    get_sentence_embedding_dimension)r   rD   es      r   rR   z'EnhancedSurpriseDetector._init_embedder   s    9 - D#& @ANO +D 3J ?#'==#Q#Q#S KJ<WXYZ 	RS  93A37889  D>qcBCCDs.   'B	 >B- 		B*B%%B*-	C6C		Ctextc                    | j                   | j                  |      S 	 t        rNt        | j                   d      r8t	        | j                   j                  |g            }|d   j                         S t        r+| j                   j                  |      }|j                         S y# t        $ r)}t        d|        | j                  |      cY d}~S d}~ww xY w)zGenerate embedding for text.Nembedr   z[!] Embedding failed: )rP   _fallback_embedrX   hasattrlistr`   tolistr[   encoderZ   rY   )r   r^   
embeddings	embeddingr]   s        r   _embedzEnhancedSurpriseDetector._embed   s    == ''--
	."wt}}g'F!$--"5"5tf"=>
!!}++--0 MM006	 '')) 1  	.*1#./''--	.s$   AB% 30B% %	C.CCCc                    |j                         j                         }dg| j                  z  }t        |      D ]b  \  }}t	        t        j                  |j                               j                         d      }|| j                  z  }||xx   d|dz   z  z  cc<   d t        d |D              dz  }|dkD  r|D 	cg c]  }	|	|z  	 }}	|S c c}	w )z-Fallback embedding using hash-based features.r            ?   c              3   &   K   | ]	  }|d z    yw)   Nr"   ).0vs     r   	<genexpr>z;EnhancedSurpriseDetector._fallback_embed.<locals>.<genexpr>   s     (A1a4(         ?r   )
lowersplitrQ   	enumerater@   hashlibmd5re   	hexdigestsum)
r   r^   wordsr4   iwordhash_validxnormrp   s
             r   ra   z(EnhancedSurpriseDetector._fallback_embed   s     

""$))) ' 	)GAt7;;t{{}5??A2FHT---C3K3!a%=(K	) (((C/!8(./1a$h/F/ 0s   =Cv1v2c                 6   t        j                  |      }t        j                  |      }t        j                  ||      }t         j                  j	                  |      }t         j                  j	                  |      }|dk(  s|dk(  ryt        |||z  z        S )z0Calculate cosine similarity between two vectors.r   r   )nparraydotlinalgr   r+   )r   r   r   abr   norm_anorm_bs           r   _cosine_similarityz+EnhancedSurpriseDetector._cosine_similarity   ss    HHRLHHRLffQl""Q;&A+SFVO,--r   r4   r3   top_kc                     g }| j                   j                         D ]F  \  }}|r|j                  |k7  r| j                  ||j                        }|j                  ||f       H |j                  d d       |d| S )zFind most similar memories.c                     | d   S Nrl   r"   xs    r   r8   z=EnhancedSurpriseDetector._find_most_similar.<locals>.<lambda>   s
    ! r   T)keyreverseN)rN   itemsr3   r   r4   appendsort)r   r4   r3   r   similaritiesmem_idmemorysims           r   _find_most_similarz+EnhancedSurpriseDetector._find_most_similar   s     "mm113 	/NFF&--61))&&--@C.		/ 	nd;FU##r   r1   r2   expectedc                 4   | j                   5  | j                  |      }d}| j                  |dd      }|r&t        d |D              }d|z
  }t	        d|dz        }d}	|r&| j                  |      }
d| j                  ||
      z
  }	| j                  |      }| j                  ||      }| j                  d   |z  | j                  d	   |	z  z   | j                  d
   |z  z   | j                  d   |z  z   }t        t        |d      t        |	d      t        |d      t        |d      t        |d            }|dkD  rd}n
|dkD  rd}nd}|j                         |t        |      |rt        |d   d   d      ndt        |      dcddd       S # 1 sw Y   yxY w)aY  
        Evaluate surprise/novelty of content.

        Args:
            content: The content to evaluate
            source: Source of the content
            domain: Domain/category
            expected: Optional expected content for violation detection

        Returns:
            Dict with score, tier recommendation, and details
        rk   N   )r3   r   c              3   &   K   | ]	  }|d      yw)rl   Nr"   )ro   ss     r   rq   z4EnhancedSurpriseDetector.evaluate.<locals>.<genexpr>   s     4qad4rr   g      ?r   r   r   r   r   )r   r   r   r   r   ffffff?semanticrJ   episodicworkingr   rl   )scoretiersimilar_countmax_similarityembedding_size)rM   rh   r   maxminr   _calculate_rarity_estimate_impactrT   r   roundr   len)r   r1   r2   r3   r   r4   novelty_scoresimilarmax_simviolation_scoreexpected_vectorrarity_scoreimpact_scorer   r   r   s                   r   evaluatez!EnhancedSurpriseDetector.evaluate   s   & ZZ 7	[[)F  M--fT-KG4G44 #g #C)< = "O"&++h"7"%(?(?(X"X  11&9L  00&AL Y'-7[)O;<X&56 X&56  "mQ/3\1-\1-E1oE s{!!  !$W=D%
1q"9#"%f+c7	 7	 7	s   E7FFc                     | j                   j                  |ddi      }t        | j                        }|dk(  ry|j                  dd      }||z  }dt	        |dz  d      z
  S )z+Calculate domain rarity based on frequency.countr   rk      )rO   getr   rN   r   )r   r3   statstotal_memoriesdomain_count	frequencys         r   r   z*EnhancedSurpriseDetector._calculate_rarity/  sg    !!%%fwl;T]]+Qyy!, >1	SQ,,,r   c                     |j                         }d}g d}g d}|D ]  }||v st        |d      } n |D ]  }||v st        |d      } t        |      dkD  rt        |dz   d      }|S )	z*Estimate impact based on content analysis.g333333?)	criticalerrorurgentbreaking	importantsecurityvulnerabilityfailuresuccess)updatechangenewmodified
discoveredfoundlearnedinsightg?rs   i  g?rk   )rt   r   r   r   )r   r1   r3   content_lowerr   high_impactmedium_impactr}   s           r   r   z)EnhancedSurpriseDetector._estimate_impact;  s    J7   	D}$VS)	
 " 	*D}$VS)	*
 w<##s+Fr   	memory_idc                    | j                   5  |^t        j                         j                          d|dd  }t	        j
                  |j                               j                         dd }| j                  |      }t        |||||      }|| j                  |<   || j                  vr2dt        j                         j                         d| j                  |<   | j                  |   dxx   d	z  cc<   t        | j                        | j                  kD  r| j                          |cddd       S # 1 sw Y   yxY w)
zAdd content to memory bank.N:d      )r0   r1   r2   r3   r4   r   )r   
first_seenr   rl   )rM   r   r6   r7   rw   sha256re   ry   rh   r/   rN   rO   r   rF   _prune_old_memories)r   r1   r2   r3   r   
hash_inputr4   r   s           r   add_to_memoryz&EnhancedSurpriseDetector.add_to_memoryU  s!    ZZ 	  ( 8 8 :;1WTc]OL
#NN:+<+<+>?IIKCRP	[[)F!F (.DMM)$ T...67x||~G_G_Ga,b!!&)f%g.!3. 4==!D$5$55((*5	 	 	s   DD55D>c                    t        | j                        | j                  k  ryt        | j                  j	                         d       }t        |      dz  }|d| D ]  \  }}| j                  |=  t        d| d       y)z&Remove oldest/least accessed memories.Nc                 >    | d   j                   | d   j                  fS r   )r;   r:   r   s    r   r8   z>EnhancedSurpriseDetector._prune_old_memories.<locals>.<lambda>  s    1Q4,,adnn= r   )r   
   z[!] EnhancedSurprise: Pruned z old memories)r   rN   rF   sortedr   rY   )r   sorted_memsremove_countr   _s        r   r   z,EnhancedSurpriseDetector._prune_old_memoriesy  s    t}}!2!22 MM!=
 ;'2-$]l3 	&IFAf%	& 	-l^=IJr   c                     | j                   j                         sy	 t        | j                   d      5 }t        j                  |      }ddd       j                  dg       D ]0  }t        j                  |      }|| j                  |j                  <   2 |j                  di       | _
        t        dt        | j                         d       y# 1 sw Y   xY w# t        $ r}t        d|        Y d}~yd}~ww xY w)zLoad memories from disk.NrrN   rO   z[OK] EnhancedSurprise: Loaded 	 memoriesz![!] EnhancedSurprise load error: )rC   existsopenjsonloadr   r/   r%   rN   r0   rO   rY   r   rZ   )r   fr   mem_datar   r]   s         r   rS   zEnhancedSurpriseDetector._load  s    &&(	;d&&, $yy|$ !HHZ4 2%//9+1fii(2 !% <D23t}}3E2FiPQ$ $  	;5aS9::	;s/   C C	BC CC 	C=%C88C=c                 v   | j                   5  	 | j                  j                  j                  dd       | j                  j                         D cg c]  }|j                          c}| j                  t        j                         j                         d}t        | j                  d      5 }t        j                  ||       ddd       t        dt        | j                         d       ddd       yc c}w # 1 sw Y   9xY w# t         $ r}t        d|        Y d}~6d}~ww xY w# 1 sw Y   yxY w)	zSave memories to disk.T)parentsexist_ok)rN   rO   saved_atwNz[OK] EnhancedSurprise: Saved r   z$[!] EnhancedSurprise persist error: )rM   rC   parentmkdirrN   valuesr   rO   r   r6   r7   r   r   dumprY   r   rZ   )r   mr   r   r]   s        r   persistz EnhancedSurpriseDetector.persist  s   ZZ 	BB  ''--dT-J 7;mm6J6J6L M M$($5$5 ( 8 8 : $**C0 'AIIdA&' 5c$--6H5IST	B 	B
 !N
' '
  B<QC@AAB	B 	BsZ   D/ADC:)AD0C?*D:D?D	D	D,D'"D/'D,,D//D8c           	          t        | j                        t        | j                        | j                  j                         D ci c]  \  }}||d    c}}| j                  du| j
                  dS c c}}w )zGet detector statistics.r   N)r   domainsdomain_distributionembedder_availablerQ   )r   rN   rO   r   rP   rQ   )r   krp   s      r   	get_statsz"EnhancedSurpriseDetector.get_stats  sr     "$--04,,-*.*;*;*A*A*C$"&!Q1W:$ #'--t";++
 	
$s   A4
)NzBAAI/bge-small-en-v1.5r   i'  )Nr   r   )r   N)r'   r(   r)   r*   r   r?   r+   r@   rU   rR   r   rh   ra   r   r   r   r   r   r   r   r   r   rS   r   r   r"   r   r   rB   rB   _   s    &*2&)!
c]
 
 $	

 
@  ..3 .4; .$C DK $	.T%[ 	.d5k 	.e 	. !%	$U$ $ 	$
 
eCJ	 $0 #'JJ J 	J
 3-J 
JX
- 
- 
- S U > $("" " 	"
 C=" 
"HK$;&B(

4 

r   rB   c                   r    e Zd ZdZddee   fdZdedededefd	Zd
ededdfdZ	dede
e   fdZdefdZy)MemorySystemz=Backwards-compatible wrapper for enhanced surprise detection.Npersistence_pathc                 .    t        |r|nd      | _        y )NrH   )rC   )rB   detector)r   r  s     r   rU   zMemorySystem.__init__  s    0,<(Bp
r   r1   r2   r3   r   c                 <    | j                   j                  |||      S r   )r  r   )r   r1   r2   r3   s       r   r   zMemorySystem.evaluate  s    }}%%gvv>>r   actionr   c                     || _         y)z1Store expectation for future violation detection.N)_last_expectation)r   r  r   s      r   observezMemorySystem.observe  s
    !)r   actualc                     t        | d      r4| j                  j                  |dd| j                        }d|d   d   igS g S )zCompare actual with expected.r	  
reflectioninternal)r   r   r   r   )rb   r  r   r	  )r   r  results      r   reflectzMemorySystem.reflect  sT    4,-]]++j// , F 'w(DEFF	r   c                 6    | j                   j                         S r   )r  r   r   s    r   r   zMemorySystem.get_stats  s    }}&&((r   r   )r'   r(   r)   r*   r   r?   rU   r   r   r
  r   r  r   r"   r   r   r  r    ss    G
# 

? ?S ?# ?$ ?*c *S *T *c d4j )4 )r   r  __main__rl   r   rn   )indentr    clitestz2Usage: python enhanced_surprise.py evaluate <text>demoz)=== Enhanced Surprise Detection Demo ===
zThe weather is sunny todayweatherz Python is a programming languagetechz%Machine learning uses neural networks))zThe weather is rainyr  )zPython has great librariesr  )zQuantum computers use qubitsr  )z%Critical security vulnerability foundr   zText: '2   z...'z  Novelty: r   r   z.2fz
  Impact: r   z	  Total: r   z  Tier: r   
zUnknown command: z8Usage: python enhanced_surprise.py [stats|evaluate|demo]z#Genesis Enhanced Surprise Detection)/r*   r   numpyr   pathlibr   r   typingr   r   r   r   r	   dataclassesr
   r   r   rw   rK   	fastembedr   rX   ImportErrorsentence_transformersr   r[   r   r/   rB   r  r'   sysr   argvcmdr  r   r   rY   dumpsjoinr^   r   r  r   testsr3   r"   r   r   <module>r)     s  *     3 3 0 0  ' #=*.'
 ',#       4]
 ]
B) )< z
388}qhhqk+-'>&&(E*$**U1-.J388}q xx-!**4?jdjj23JKF]>? ""#?S""#EvvV""#JFTZ[E !& 5f!**4@Sb	{$/0F7OI$>s#CDE
6'?8#<S"ABC	&/'":3!?@A 0345 %cU+,LM34HIc w  M  #*/'"#s#   H7 I 7	II	II