
    miY$                     T   d dl Z d dlZd dlZd dlZd dlmZ d dlZd dlm	Z	m
Z
mZmZ d dlmZmZ d dlZd dlZd dlmZ  ej&                  ej(                         dZdZd	Z e	d
eej2                  d      Z e	deee   ej2                  d      Zej<                  Zeege f   Z! ed      Z"dej2                  defdZ#efdedej2                  fdZ$deedf   fdZ%dej2                  defdZ&dej2                  dej2                  fdZ'e G d d             Z(e G d d             Z)y)    N)uuid4)	TypedDictLiteralUnionCallable)	dataclassasdict)	getLogger)level__id__
__vector____metrics__Data)r   r   DataBaseembedding_dimdatamatrixznano-vectordbarrayreturnc                 d    t        j                  | j                               j                         S N)base64	b64encodetobytesdecode)r   s    M/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/nano_vectordb/dbs.pyarray_to_buffer_stringr      s"    EMMO,3355    
base64_strc                 V    t        j                  t        j                  |       |      S )Ndtype)np
frombufferr   	b64decode)r    r#   s     r   buffer_string_to_arrayr'      s    ==))*5UCCr   c                 H   t         j                  j                  |       sy t        | d      5 }t	        j
                  |      }d d d        t        d         j                  d|d         |d<   t        j                  d|d   j                   d       |S # 1 sw Y   UxY w)Nutf-8encodingr   r   zLoad  data)ospathexistsopenjsonloadr'   reshapeloggerinfoshape)	file_namefr   s      r   load_storager:   #   s    77>>)$	i'	* ayy|+DN;CC
D!DN KK%X,,-U34K s   BB!ac                 d    t        j                  | j                               j                         S r   )hashlibmd5r   	hexdigestr;   s    r   hash_ndarrayrA   /   s     ;;qyy{#--//r   c                 L    | t         j                  j                  | dd      z  S )Nr,   T)axiskeepdims)r$   linalgnormr@   s    r   	normalizerG   3   s     ryy~~ab4~888r   c                      e Zd ZU eed<   dZed   ed<   dZeed<   d Z	d Z
d Zd	 Zd
ee   fdZdee   fdZdee   fdZd Zd Z	 	 	 ddej*                  dedededee   f
dZ	 ddej*                  dededefdZy)NanoVectorDBr   cosinemetricznano-vectordb.jsonstorage_filec                 n    | j                   dk(  r&t        | j                  d         | j                  d<   y y )NrJ   r   )rK   rG   _NanoVectorDB__storageselfs    r   pre_processzNanoVectorDB.pre_process=   s/    ;;("'01I'JDNN8$ #r   c           	      8   | j                   g t        j                  g t              j	                  d| j                         d}t        | j                        xs |}|d   | j                   k(  sJ d| j                    d|d           || _        d| j                  i| _	        | j                  | j                  v sJ d| j                   d	       | j                          t        j                  d
t        |        dt        | j                  d          d       y )Nr"   r   r   r   z"Embedding dim mismatch, expected: z, but loaded: rJ   zMetric z not supportedzInit  r   r-   )r   r$   r   Floatr4   r:   rL   rN   _cosine_queryusable_metricsrK   rQ   r5   r6   r	   len)rP   default_storagestorages      r   __post_init__zNanoVectorDB.__post_init__A   s   !//hhr/774;M;MN

 )):):;NO$(:(::	m/0B0B/C>RYZiRjQkl	m: d((
 {{d111XWT[[M3XX1eF4L>3t~~f/E+F*GuMNr   c                 :    | j                   j                  di       S Nadditional_data)rN   getrO   s    r   get_additional_dataz NanoVectorDB.get_additional_dataS   s    ~~!!"3R88r   c                 "    || j                   d<   y r\   )rN   )rP   kwargss     r   store_additional_dataz"NanoVectorDB.store_additional_dataV   s    ,2()r   datasc           
      T   |D ci c])  }|j                  t        t        |t                       |+ }}| j                  dk(  r.|j                         D ]  }t        |t                 |t        <    g g d}t        | j                  d         D ]  \  }}|t           |v s|j                  |t                 }|t           j                  t              | j                  d   |<   |t        = || j                  d   |<   |d   j                  |t                   t        |      dk(  r|S |d   j                  t        |j!                                      t#        j$                  |j                         D cg c]  }|t            c}t              }	g }
|j'                         D ]&  \  }}|t        = ||t        <   |
j                  |       ( | j                  d   j                  |
       t#        j(                  | j                  d   |	g      | j                  d<   |S c c}w c c}w )	NrJ   )updateinsertr   r   re   r   rf   r"   )r^   f_IDrA   f_VECTORrK   valuesrG   	enumeraterN   popastyperT   appendrW   extendlistkeysr$   r   itemsvstack)rP   rc   r   _index_datasvreport_returnialready_dataupdate_d
new_matrix	new_datasnew_knew_ds                r   upsertzNanoVectorDB.upsertY   s   KP
CGDHHT<X78$>
 
 ;;("!((* 5'(4(5#%4()?@ 	COA|D!\1'++L,>?.6x.@.G.G.Nx(+X&,4v&q)h'..|D/AB	C |!  h&&tL,=,=,?'@AXX(4(;(;(=>T(^>e

 	(..0 	$LE5hE$KU#	$ 	v%%i0#%99dnnX.F
-S#Tx 7
$ ?s   .H 7H%idsc                 ^    | j                   d   D cg c]  }|t           |v s| c}S c c}w Nr   )rN   rg   )rP   r~   r   s      r   r^   zNanoVectorDB.getw   s*    !%!7M4:;LMMMs   **c                 p   t        |      }g }g }t        | j                  d         D ]K  \  }}|t           |v r*|j	                  |       |j                  |t                  ;|j	                  |       M || j                  d<   t        j                  | j                  d   |d      | j                  d<   y )Nr   r   r   )rC   )setrj   rN   rg   rm   remover$   delete)rP   r~   	left_datadelete_indexrv   r   s         r   r   zNanoVectorDB.deletez   s    #h	 !78 	'GAtDzS ##A&

4:&  &	' "+v#%99NN8$l$
x r   c                     i | j                   dt        | j                   d         i}t        | j                  dd      5 }t	        j
                  ||d       d d d        y # 1 sw Y   y xY w)Nr   wr)   r*   F)ensure_ascii)rN   r   r1   rL   r2   dump)rP   rY   r9   s      r   savezNanoVectorDB.save   si    
nn
,T^^H-EF
 $##S7; 	6qIIgqu5	6 	6 	6s   A##A,c                 2    t        | j                  d         S r   )rW   rN   rO   s    r   __len__zNanoVectorDB.__len__   s    4>>&)**r   Nquerytop_kbetter_than_thresholdfilter_lambdar   c                 H     | j                   | j                     ||||      S )N)r   )rV   rK   )rP   r   r   r   r   s        r   r   zNanoVectorDB.query   s,     0t""4;;/5/}
 	
r   c           	      l   t        |      }|;| j                  d   }t        j                  t	        | j                  d               }nXt        j
                  t        | j                  d         D cg c]  \  }} ||      r| c}}      }| j                  d   |   }t        j                  ||      }	t        j                  |	      | d  }
|
d d d   }
||
   }g }t        ||
      D ]@  \  }}||	|   |k  r |S |j                  i | j                  d   |   t        |	|   i       B |S c c}}w )Nr   r   r,   )rG   rN   r$   arangerW   r   rj   dotargsortziprm   	f_METRICS)rP   r   r   r   r   
use_matrixfilter_indexrv   r   scores
sort_indexsort_abs_indexresultsabs_irel_is                  r   rU   zNanoVectorDB._cosine_query   sG    %  1J99S)?%@AL88 $-T^^F-C#D4$T* L 1,?J
E*ZZ'0
"%
%j1
; 	XLE5%15M$99 NNVdnnV4U;VYuVW	X 's   3D0
)
   NNr   )__name__
__module____qualname__int__annotations__rK   r   rL   strrQ   rZ   r_   rb   ro   r   r}   r^   r   r   r   r$   ndarrayfloatConditionLambdadictr   rU    r   r   rI   rI   7   s     (FGH(,L#,KO$93DJ <NtCy N
$s) 
6+ '+)-	
zz	
 	
  %		

 '	
 
d	
  *. zz     %	 
 ' r   rI   c                       e Zd ZU eed<   dZed   ed<   dZeed<   dZe	ed<   e
d        Zd	 Zd
e	defdZd
e	dedefdZd
e	defdZde	fdZd
e	fdZd
e	defdZd Zy)MultiTenantNanoVDBr   rJ   rK   i  max_capacityz./nano_multi_tenant_storagestorage_dirc                     d|  dS )Nnanovdb_z.jsonr   )	tenant_ids    r   jsonfile_from_idz#MultiTenantNanoVDB.jsonfile_from_id   s    )E**r   c                 T    | j                   dk  rt        d      i | _        g | _        y )N   z%max_capacity should be greater than 0)r   
ValueError_MultiTenantNanoVDB__storage _MultiTenantNanoVDB__cache_queuerO   s    r   rZ   z MultiTenantNanoVDB.__post_init__   s+    q DEE24(*r   r   r   c                     || j                   v xs= t        j                  j                  | j                   d| j                  |             S N/)r   r.   r/   r0   r   r   rP   r   s     r   contain_tenantz!MultiTenantNanoVDB.contain_tenant   sH    DNN* 
bggnn $"7"7	"B!CD/
 	
r   in_memory_tenantc                    t        t        | j                        | j                         t        | j                        | j                  k\  r| j                  j	                  | j
                  j	                  d            }t        j                  j                  | j                        st        j                  | j                         |j                          || j                  |<   | j
                  j                  |       y )Nr   )printrW   r   r   rk   r   r.   r/   r0   r   makedirsr   rm   )rP   r   r   vdbs       r   __load_tenant_in_cachez)MultiTenantNanoVDB.__load_tenant_in_cache   s     	c$..!4#4#45t~~$"3"33..$$T%7%7%;%;A%>?C77>>$"2"23D,,-HHJ$4y!!!),r   c           	      $   || j                   v r| j                   |   S | j                  |      st        d| d      t        | j                  | j
                  | j                   d| j                  |             }| j                  ||       |S )NzTenant z not in storager   rK   rL   )	r   r   r   rI   r   rK   r   r   )_MultiTenantNanoVDB__load_tenant_in_cacherP   r   r   s      r   __load_tenantz MultiTenantNanoVDB.__load_tenant   s    &>>),,""9-wykABB';; ,,-Qt/D/DY/O.PQ

 	##I/?@r   c           	          t        t                     }t        | j                  | j                  | j
                   d| j                  |             }| j                  ||       |S )Nr   r   )r   r   rI   r   rK   r   r   r   r   s      r   create_tenantz MultiTenantNanoVDB.create_tenant   sb    L	';; ,,-Qt/D/DY/O.PQ

 	##I/?@r   c                 n   || j                   v r6| j                   j                  |       | j                  j                  |       t        j
                  j                  | j                   d| j                  |             r4t	        j                  | j                   d| j                  |              y y r   )	r   rk   r   r   r.   r/   r0   r   r   r   s     r   delete_tenantz MultiTenantNanoVDB.delete_tenant   s    &NNy)%%i077>>T--.a0E0Ei0P/QRSII))*!D,A,A),L+MNO Tr   c                 $    | j                  |      S r   ) _MultiTenantNanoVDB__load_tenantr   s     r   
get_tenantzMultiTenantNanoVDB.get_tenant  s    !!),,r   c                     t         j                  j                  | j                        st        j                  | j                         | j
                  j                         D ]  }|j                           y r   )r.   r/   r0   r   r   r   ri   r   )rP   dbs     r   r   zMultiTenantNanoVDB.save  sO    ww~~d../KK(()..'') 	BGGI	r   N)r   r   r   r   r   rK   r   r   r   r   staticmethodr   rZ   boolr   rI   r   r   r   r   r   r   r   r   r   r   r      s     (FGH(L#4K4+ ++
 
 

0<	 s  |  s Ps P-C -L -r   r   )*r.   r2   r   r=   uuidr   numpyr$   typingr   r   r   r   dataclassesr   r	   sqlite3loggingr
   basicConfigINFOrg   rh   r   r   r   r   r   ro   r   float32rT   r   r   r5   r   r'   r:   rA   rG   rI   r   r   r   r   <module>r      sZ   	      6 6 )      ',, '	CrzzBC#tDzRZZP 	

D64<(	?	#6"** 6 6 38 Ds DBJJ D	uXt^4 	0BJJ 03 09 9

 9 G G GT H H Hr   