iF- ddlZddlZddlZddlZddlZddlZddlmZmZddlmZddl m Z m Z m Z m Z ddlmZddlZddlZddlmZddlmZdZeZd ed efd Zd ed efd Zd ed e ej>effd Z de eejBfded efdZ"d3defdZ#d3dejHde e e%e&ffdZ'dejPd efdZ)dejTd e&fdZ+dejPdede e de efdZ,edZ-e-j]edefd Z/e-j]e0de0fd!Z/e-j]e%de%fd"Z/e-j]e1de1fd#Z/e-j]e&de&fd$Z/e-j]e2dde2dfd%Z/e-j]edefd&Z/e-j]edefd'Z/e-j]e3de3fd(Z/e-j]ejhdejhfd)Z/d*Z5d+efd,Zd-e6d.efd/Z7dejHfd0Z8d1ejrfd2Z:y)4N)datedatetime)singledispatch)TupleUnionOptionalAny)urlparse)validate_table_namec. ddl}|S#t$rYywxYw)Nr)adlfs ImportError)rs H/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/lancedb/util.pysafe_import_adlfsrs! s  urireturncpt|}|j}|sd}|S|dvrd}|St|dk(rd}|S)z Get the scheme of a URI. If the URI does not have a scheme, assume it is a file URI. Parameters ---------- uri : str The URI to parse. Returns ------- str: The scheme of the URI. file)s3as3ns3r )r schemelen)rparsedrs rget_uri_schemer#sUc]F ]]F  M > ! M V  Mct|}t|jdk(r|S|js |jS|j|jzS)z Get the location of a URI. If the parameter is not a url, assumes it is just a path Parameters ---------- uri : str The URI to parse. Returns ------- str: Location part of the URL, without scheme r )r rrnetlocpath)rrs rget_uri_locationr!>sIc]F 6==Q =={{}}v{{**rct|dk(rDtjtjj ddd}t |}||fSt|dk(rttjtjj dtjj d}tjtj|}t |}||fStjj|S) zT Get a PyArrow FileSystem from a URI, handling extra environment variables. r AWS_ENDPOINT)endpoint_overriderequest_timeoutconnect_timeoutazAZURE_STORAGE_ACCOUNT_NAMEAZURE_STORAGE_ACCOUNT_KEY) account_name account_key) rpa_fs S3FileSystemosenvirongetr!rAzureBlobFileSystem PyFileSystem FSSpecHandler FileSystemfrom_uri)rfsr az_blob_fss r fs_from_urir9Xscd"    jjnn^<   $4x   $):..(DE 'BC/    3 3J ? @$4x    $ $S ))rbasepartsct|tjr|j|St |}t |dk(rt tj|g|St |}dj|jg|Dcgc]}|jdc}}|j|jScc}w)a Join a URI with multiple parts, handles both local and remote paths Parameters ---------- base : str The base URI parts : str The parts to join to the base URI, each separated by the appropriate path separator for the URI scheme and OS r/)r ) isinstancepathlibPathjoinpathstrrr joinr rstrip_replacegeturl)r:r;urlpnew_paths rjoin_urirJss$ %t}}e$$ t9Ddv%7<<-u-..tn88SXX4F4FGqQXXc]GH|||*1133Hs Cmodulecl tj|S#t$rtd|xs|wxYw)ac Import the specified module. If the module is not installed, raise an ImportError with a helpful message. Parameters ---------- module : str The name of the module to import mitigation : Optional[str] The package(s) to install to mitigate the error. If not provided then the module name will be used. zPlease install ) importlib import_moduler)rK mitigations rattempt_import_or_raiserPsBD&&v.. DOJ,@&+ABCCDs3tblflattenc|dur1 |j}td|jDr. |St|tr0|dkr t d|dkDr|j}|dz}|dkDr|S)aJ Flatten all struct columns in a table. Parameters ---------- flatten: Optional[Union[int, bool]] If flatten is True, flatten all nested columns. If flatten is an integer, flatten the nested columns up to the specified depth. If unspecified, do not flatten the nested columns. Tc3nK|]-}tjj|j/ywN)patypes is_structtype).0cols r z"flatten_columns..s#FC288%%chh/Fs35rzIPlease specify a positive integer for flatten or the boolean value `True`r )rRanyschemar>int ValueError)rQrRs rflatten_columnsras$++-CF3::FF J GS ! a< k++-C qLGk Jrr^cd}d}|jD]F}|j|}t|js*|dz }|dkDr t d|dk(sE|}H|dk(r t d|S)z Get the vector column name Parameters ---------- schema : pa.Schema The schema of the vector column. Returns ------- str: the vector column name. rr z_Schema has more than one vector column. Please specify the vector column name for vector searchz^There is no vector column in the data. Please specify the vector column name for vector search)namesfieldis_vector_columnrYr`)r^vector_col_namevector_col_count field_nameres rinf_vector_column_queryrjsOll -  Z( EJJ '  ! !# ( "Q&", -1 F   r data_typecPtjj|rStjj|js)tjj |jrytjj |rt|jSy)z Check if the column is a vector column. Parameters ---------- data_type : pa.DataType The data type of the column. Returns ------- bool: True if the column is a vector column. TF)rVrWis_fixed_size_list is_floating value_typeis_uint8is_listrf)rks rrfrfso xx""9- Y112 88  Y11 2   ) $ 4 455 r query_typequeryvector_column_namecj||S|dk(ry||dk(r t|}|S|S#t$r}|d}~wwxYw)Nftshybrid)rj Exception)r^rrrsrtes rinfer_vector_column_namerzs] %!!U J(2 !8!@    G s " 2-2ctd)Nz/SQL conversion is not implemented for this type)NotImplementedErrorvalues r value_to_sqlrs O PPrr~c2|jdd}d|dS)N'z'')replacer}s r_rs MM#t $E ugQ<rcPdtj|jdS)z~Convert bytes to a hex string literal. See https://datafusion.apache.org/user-guide/sql/data_types.html#binary-types zX'r)binasciihexlifydecoder}s rrrs(   '..01 33rct|SrUrBr}s rrr$ u:rct|SrUrr}s rrr)rrc4t|jSrU)rBupperr}s rrr.s u:   rcy)NNULLr}s rrr3s rc*d|jdSNr isoformatr}s rrr8 u ! ##rc*d|jdSrrr}s rrr=rrcLddjtt|zdzS)N[z, ])rCmaprr}s rrrBs" 3|U34 4s ::rc4t|jSrU)rtolistr}s rrrGs   ''rcBtjfd}|S)zThis is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.ctjdttjdjdtdtjdt|i|S)Nalwaysz Function z6 is deprecated and will be removed in a future version)category stackleveldefault)warnings simplefilterDeprecationWarningwarn__name__)argskwargsfuncs rnew_funczdeprecated..new_funcQs`h(:; DMM?+..(   i);<T$V$$r) functoolswraps)rrs` r deprecatedrLs' __T % % Ornamect|y)zVerify the table name is valid.N)native_validate_table_name)rs rr r bs t$rbase_exceptionnotect|dr|j|yt|jdtr,|jddz|zg|jdd|_yt d)Nadd_noter r zCannot add note to exception)hasattrrr>rrBr`)rrs rrrgst~z*% N''*C 0    "T )D 0   $  788rc tdddtjffd }jt |j Dcgc]}||j |c}Scc}w)a& Convert a PyArrow Table to a PyTorch Tensor. Each column is converted to a tensor (using zero-copy via DLPack) and the columns are then stacked into a single tensor. Fails if torch is not installed. Fails if any column is more than one chunk. Fails if a column's data type is not supported by PyTorch. Parameters ---------- tbl : pa.Table or pa.RecordBatch The table or record batch to convert to a tensor. Returns ------- torch.Tensor: The tensor containing the columns of the table. torchr[cx|jdkDr tdj|jdS)Nr z8Single batch was too large to fit into a one-chunk tabler) num_chunksrx from_dlpackchunk)r[rs r to_tensorz tbl_to_tensor..to_tensors4 >>A VW W  1..r)rPrV ChunkedArraystackrange num_columnscolumn)rQrirs @r tbl_to_tensorrssV( $GW 5E/r/ ;;%:PQQ #**Q-0Q RRQsA+batchctdd}|j|jDcgc]}|j|c}Scc}w)a Convert a PyArrow RecordBatch to a PyTorch Tensor. Each column is converted to a tensor (using zero-copy via DLPack) and the columns are then stacked into a single tensor. Fails if torch is not installed. Fails if a column's data type is not supported by PyTorch. Parameters ---------- batch : pa.RecordBatch The record batch to convert to a tensor. Returns ------- torch.Tensor: The tensor containing the columns of the record batch. r)rPrcolumnsr)rrr[s rbatch_to_tensorrs<& $GW 5E ;;%--H3))#.H IIHsArU);rrrMr/r?rrrrtypingrrrr urllib.parser numpynppyarrowrV pyarrow.fsr7r-_lancedbr rrrrBrr!r5r9r@rJrPTabler_boolraSchemarjDataTyperfrzrregisterrbytesfloatrYlistndarrayr BaseExceptionrr RecordBatchrrrrrs6  #$..!G 6+#+#+4*S*U5#3#3S#89*645gll*+4S4S44DCD&HU395E,F@ BII # F . II C=! ,QQsS u4U44sSuUtTtDz"T$Z#x $X$!$t$T$$t;T;;rzz"(RZZ(#(,%c% 9] 9# 9SrxxS<J2>>Jr