ciHddlZddlZddlZddlZddlZddlmZmZmZddl m Z ddl m Z ddl mZddlmZddlmZddlZddlmZddlmZdd lmZmZ dd lmZd Zd Zej@d zZ!e jDe!de!Z#edZ$yGddeZ$y#e$rdZYDwxYw)N)AnyIterableOptional) client_info) exceptions)bigquery) initializer) TaskContext)cached_remote_fn)Block BlockAccessor)Datasink z +vertex_rayzray-on-vertex/) gapic_version user_agentc ^eZdZdedfdedeededeeddf dZd d Z d e e d e de fd Zy)_BigQueryDatasinkNTdataset project_id max_retry_cntoverwrite_tablereturnct||_|xstjj|_||_||_y)N)rr global_configprojectrrr)selfrrrrs C/tmp/pip-target-z3e9_cxr/lib/python/vertex_ray/bigquery_datasink.py__init__z_BigQueryDatasink.__init__<s3#DL(MK,E,E,M,MDO!.D #2D c&tj|jt}|jj ddd} |j ||jrHtd|jd z|j|jd|jd ytd |jd zdzy#tj$r3|j|jd|dtd|zYwxYw)Nrr.r)timeoutz$[Ray on Vertex AI]: Created dataset z/[Ray on Vertex AI]: Attempting to delete table z9 if it already exists since kwarg overwrite_table = True.T) not_found_okz3[Ray on Vertex AI]: The write will append to table z if it already exists z$since kwarg overwrite_table = False.) rClientrbq_inforsplit get_datasetrNotFoundcreate_datasetprintr delete_table)rclient dataset_ids ron_write_startz _BigQueryDatasink.on_write_startHs__T__'RF++C3A6J K"":. ##Edll^TQR##'q7d$I&<=><=&& K%%(9:,&GQS%T| tAjBtDYd} ~ nd} ~ wwxYw| j,kr3#1swYyxYw)Nr"T) autodetectblock_z.parquetSNAPPY) compressionrrb) job_configr$z-[Ray on Vertex AI]: A block write encounteredz a rate limit exceeded error z time(s).z Sleeping to try again.z[Ray on Vertex AI]: Maximum (z) retry count exceeded.z? Ray will attempt to retry the block write via fault tolerance.z\ For more information, see https://docs.ray.io/en/latest/ray-core/fault_tolerance/tasks.htmlz([Ray on Vertex AI]: Write failed due to z5 repeated API rate limit exceeded responses. Considerz9 specifiying the max_retry_cnt kwarg with a higher value.)$r for_blockto_arrowrr(r) LoadJobConfig SourceFormatPARQUET source_formatWriteDisposition WRITE_APPENDwrite_dispositiontempfileTemporaryDirectoryospathjoinuuiduuid4pq write_tableropenload_table_from_filelogginginforesultr Forbiddenr.debugtimesleepRATE_LIMIT_EXCEEDED_SLEEP_TIME RuntimeError) r6rrr0r=temp_dirfp retry_cnt source_filejobers r_write_single_blockz4_BigQueryDatasink.write.._write_single_blockgs &//6??A!Q%33tD +3+@+@+H+H (/7/H/H/U/U ,002$hh&h0OPBNN5"(C !I#t'9'99!"d^{"("="= +W#>#C G#LL6!!4#5#55;Dr}s&  **'&!/ ?9/5 !#((=8 + #.AR@S0T   fHf'HsBB! B!