[i$ddlmZmZmZddlmZddlmZmZddl Z ddl Z ddl m Z ddl ZddlmZdZdZe Gd d ZGd d Zy) )DictListSet)SnowballStemmer)get_all_punctuationremove_non_alphanumericN) dataclass)SparseEmbeddingi}icLeZdZUeed<eeed<eed<eed<eeed<y) WordEmbeddingwordformscountword_id embeddingN)__name__ __module__ __qualname__str__annotations__rintfloatk/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/fastembed/sparse/utils/sparse_vectors_converter.pyr r s% I 9 J LE{rr c eZdZ ddeededededef dZedede d e fd Z d e d e d efd Z ede ed e efdZ ddeeefde d eeeffdZdeeefde de d efdZdeeefde de d efdZy)SparseVectorConverter stopwordsstemmerkbavg_lenctt}hd}||_||z|z|_||_||_||_y)N>[CLS][PAD][SEP][UNK][MASK])setrrunwanted_tokensr r!r")selfrrr r!r" punctuationspecial_tokenss r__init__zSparseVectorConverter.__init__sE-/0 G *^;iG rr shiftreturncdttj|}t|z }|||zz}|S)N)absmmh3hash INT32_MAX)clsr r/ token_hash range_size remapped_hashs runkn_word_token_idz(SparseVectorConverter.unkn_word_token_id,s54) & j!89 rnum_occurrences sentence_lenc||jdzz}|||jd|jz |j|z|jz zzzz}|S)N)r r!r")r+r;r<ress rbm25_tfzSparseVectorConverter.bm25_tf7sS!, 1tvv:8MPTP\P\8\+\!]]] rvectorct|Dcgc]}|dz c}dz}|dkr|S|Dcgc]}||z  c}Scc}wcc}w)Ng?g:0yE>)sum)r6rAxnorms rnormalize_vectorz&SparseVectorConverter.normalize_vector<sK&)QAqD)*c1 $;M"()QD))**s : ?sentence_embeddingtoken_max_lengthcHi}|jD] \}}|jdkDr|||<||jvr+t|j }t |dkDsS|j D]}|jj|}t ||ks-||jvs<||vr#tj|||<|||_ c||xj|jz c_ ||xj|jz c_ |S)u Clean miniCOIL-produced sentence_embedding, as unknown to the miniCOIL's stemmer tokens should fully resemble our BM25 token representation. sentence_embedding = {"9°": {"word": "9°", "word_id": -1, "count": 2, "embedding": [1], "forms": ["9°"]}, "9": {"word": "9", "word_id": -1, "count": 2, "embedding": [1], "forms": ["9"]}, "bat": {"word": "bat", "word_id": 2, "count": 3, "embedding": [0.2, 0.1, -0.2, -0.2], "forms": ["bats", "bat"]}, "9°9": {"word": "9°9", "word_id": -1, "count": 1, "embedding": [1], "forms": ["9°9"]}, "screech": {"word": "screech", "word_id": -1, "count": 1, "embedding": [1], "forms": ["screech"]}, "screeched": {"word": "screeched", "word_id": -1, "count": 1, "embedding": [1], "forms": ["screeched"]} } cleaned_embedding_ground_truth = { "9": {"word": "9", "word_id": -1, "count": 6, "embedding": [1], "forms": ["9°", "9", "9°9", "9°9"]}, "bat": {"word": "bat", "word_id": 2, "count": 3, "embedding": [0.2, 0.1, -0.2, -0.2], "forms": ["bats", "bat"]}, "screech": {"word": "screech", "word_id": -1, "count": 2, "embedding": [1], "forms": ["screech", "screeched"]} } r)itemsrr*rstriplensplitr stem_wordcopydeepcopyr rr) r+rHrInew_sentence_embeddingr r word_cleanedsubwordstemmed_subwords r clean_wordsz!SparseVectorConverter.clean_wordsCs3*<>1779" aOD)  1$/8&t,4/// 7t<BBD |$q(#/#5#5#7 a/3||/E/Eg/N04DD /t7K7K K.6LLJN--XaJb 6 GO^ 6 G L 6 G M MQZQ`Q` ` M 6 G M MQZQ`Q` ` M a/" aH&%rembedding_size vocab_sizecg}g}||ztzdztz}|j|}d}|jD]} || jz }|jD]} | j} | j} |j | |} | dkDr\| j } |j| }t|D]0\}}|j| |z|z|j|| z2|j|j| j||j| ttj|tjtj|tj S)a Convert miniCOIL sentence embedding to Qdrant sparse vector Example input: ``` { "vector": WordEmbedding({ // Vocabulary word, encoded with miniCOIL normally "word": "vector", "forms": ["vector", "vectors"], "count": 2, "word_id": 1231, "embedding": [0.1, 0.2, 0.3, 0.4] }), "axiotic": WordEmbedding({ // Out-of-vocabulary word, fallback to BM25 "word": "axiotic", "forms": ["axiotics"], "count": 1, "word_id": -1, }) } ``` rCrdtypeindicesvalues)GAPrVr^rrr@rrG enumerateappendr:r r nparrayint32float32)r+rHrWrXr]r^unknown_words_shiftsentence_embedding_cleanedr<rrr;tfembedding_valuesnormalized_embeddingval_idvalues rembedding_to_vectorz)SparseVectorConverter.embedding_to_vectorsv>  . (S 01 4 &*%5%56H%I" 3::< ,I IOO +L ,4::< "I''G'ooOo|rxsO"",O !B     ^ ^ r