
    ci[                         d dl Z d dlZd dlZd dlmZmZmZ d dlZd dlm	Z	 d dlm
Z
 d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZ d d	lmZ d d
lmZmZmZ d dlm Z  d dl!m"Z"m#Z#  ejH                         ddddd ejH                         gdddddddfdeejH                     dee%   dee%   dee%   dee%   dee%   deeejH                        deejL                     dee'   dee'   deejP                     deee%      deeej                         deee%e%f      de%fdZ)de%ddfd Z*de%dejV                  fd!Z,deejV                     fd"Z-de%deejH                     de%fd#Z.y)$    N)DictListOptional)initializer)utils)resource_manager_utils)persistent_resource_service)NfsMount)PersistentResourceRayLogsSpecRaySpecRayMetricSpecResourcePoolResourceRuntimeSpecServiceAccountSpec)PscInterfaceConfig)_gapic_utils_validation_utils	resources)field_mask_pb2)_V2_4_WARNING_MESSAGE_V2_9_WARNING_MESSAGEz2.47Thead_node_typepython_versionray_versionnetworkservice_accountcluster_nameworker_node_typescustom_imagesenable_metrics_collectionenable_loggingpsc_interface_configreserved_ip_ranges
nfs_mountslabelsreturnc           	         |t        j                  d       |dk(  rt        t              |dk(  r t	        j
                  t        t        d       t        j                         }||k7  rH|.| j                  "d}t        j                  d|d	|d
|d       nt        j                  d|z         |dt        j                         z   }| rW| j                  dk7  rt        d      | j                  t        d      | j                   | j"                  dkD  rt        d      i }t%               }d|_        | j                  |_        | j*                  |j,                  _        | j"                  |j,                  _        | j                   |j,                  _        | j.                  |j0                  _        | j2                  |j0                  _        | j"                  dkD  rdnd}| j                  | j                  }nJ|t        j4                  |||      }n0|j6                  |j8                  |j6                  }nt        d      |||j&                  <   g }d}|r?|D ]9  }|j                   |j"                  dkD  rt        d      t;        j<                  | |      }|j                  |j(                  |z   |_        nV|dkD  rQd|_        |j                  j>                  |j                  _        |j                  j@                  |j                  _         |dk(  rxt%               }d|dz    |_        |j                  |j                  |_        nJ|j                  j>                  |j                  _        |j                  j@                  |j                  _         |j*                  |j,                  _        |j"                  |j,                  _        |j                   |j,                  _        |j.                  |j0                  _        |j2                  |j0                  _        |jC                  |       |j"                  dkD  rdnd}|j                  |j                  }n&|t        j4                  |||      }n|j8                  }|||j&                  <   |dz  }< |g|z   }| }tE        |      }|	 }tG        |      }tI        |||      }|rKg }|D ]=  } |jC                  tK        | jL                  | jN                  | jP                               ? ||_)        |rtU        d|      }!tW        ||!      }"ntW        |      }"|
rtY        |
jZ                        }#nd}#t]        ||||"|#|      }$t^        j`                  jb                  }%t^        j`                  jd                  }&tg        jh                  |&      }'d|' d |% }(tk        jl                  |(|$|!      })to        jp                         }*	 |*js                  |)      }+|( d#| }-to        jv                  |-d$      }.|.jx                  S # tt        $ r},t        d"|,      |,d},~,ww xY w)%a  Create a ray cluster on the Vertex AI.

    Sample usage:

    from vertex_ray import Resources

    head_node_type = Resources(
        machine_type="n1-standard-8",
        node_count=1,
        accelerator_type="NVIDIA_TESLA_T4",
        accelerator_count=1,
        custom_image="us-docker.pkg.dev/my-project/ray-2-47-cpu-py3.11:latest",  # Optional
    )

    worker_node_types = [Resources(
        machine_type="n1-standard-8",
        node_count=2,
        accelerator_type="NVIDIA_TESLA_T4",
        accelerator_count=1,
        custom_image="us-docker.pkg.dev/my-project/ray-2-47-gpu-py3.11:latest",  # Optional
    )]

    cluster_resource_name = vertex_ray.create_ray_cluster(
        head_node_type=head_node_type,
        network="projects/my-project-number/global/networks/my-vpc-name",  # Optional
        service_account="my-service-account@my-project-number.iam.gserviceaccount.com",  # Optional
        cluster_name="my-cluster-name",  # Optional
        worker_node_types=worker_node_types,
        ray_version="2.47",
    )

    After a ray cluster is set up, you can call
    `ray.init(f"vertex_ray://{cluster_resource_name}", runtime_env=...)` without
    specifying ray cluster address to connect to the cluster. To shut down the
    cluster you can call `ray.delete_ray_cluster()`.
    Note: If the active ray cluster has not finished shutting down, you cannot
    create a new ray cluster with the same cluster_name.

    Args:
        head_node_type: The head node resource. Resources.node_count must be 1.
            If not set, default value of Resources() class will be used.
        python_version: Python version for the ray cluster.
        ray_version: Ray version for the ray cluster. Default is 2.47.1.
        network: Virtual private cloud (VPC) network. For Ray Client, VPC
            peering is required to connect to the Ray Cluster managed in the
            Vertex API service. For Ray Job API, VPC network is not required
            because Ray Cluster connection can be accessed through dashboard
            address.
        service_account: Service account to be used for running Ray programs on
            the cluster.
        cluster_name: This value may be up to 63 characters, and valid
            characters are `[a-z0-9_-]`. The first character cannot be a number
            or hyphen.
        worker_node_types: The list of Resources of the worker nodes. The same
            Resources object should not appear multiple times in the list.
        custom_images: The NodeImages which specifies head node and worker nodes
            images. All the workers will share the same image. If each Resource
            has a specific custom image, use `Resources.custom_image` for
            head/worker_node_type(s). Note that configuring `Resources.custom_image`
            will override `custom_images` here. Allowlist only.
        enable_metrics_collection: Enable Ray metrics collection for visualization.
        enable_logging: Enable exporting Ray logs to Cloud Logging.
        psc_interface_config: PSC-I config.
        reserved_ip_ranges: A list of names for the reserved IP ranges under
            the VPC network that can be used for this cluster. If set, we will
            deploy the cluster within the provided IP ranges. Otherwise, the
            cluster is deployed to any IP ranges under the provided VPC network.
            Example: ["vertex-ai-ip-range"].
        labels:
            The labels with user-defined metadata to organize Ray cluster.

            Label keys and values can be no longer than 64 characters (Unicode
            codepoints), can only contain lowercase letters, numeric characters,
            underscores and dashes. International characters are allowed.

            See https://goo.gl/xmQnxf for more information and examples of labels.

    Returns:
        The cluster_resource_name of the initiated Ray cluster on Vertex.
    Raise:
        ValueError: If the cluster is not created successfully.
        RuntimeError: If the ray_version is 2.4.
    NzQ[Ray on Vertex]: No VPC network configured. It is required for client connection.z2.4z2.9.3   )
stacklevelz2.47.1z/[Ray on Vertex]: Local runtime has Ray version z(, but the requested cluster runtime has z. Please ensure that the Ray versions match for client connectivity. You may "pip install --user --force-reinstall ray[default]==z0" and restart runtime before cluster connection.zt[Ray on Vertex]: Local runtime has Ray version %s.Please ensure that the Ray versions match for client connectivity.zray-cluster-zG[Ray on Vertex AI]: For head_node_type, Resources.node_count must be 1.zP[Ray on Vertex AI]: For head_node_type, Resources.autoscaling_spec must be None.r   zj[Ray on Vertex]: accelerator_type must be specified when accelerator_count is set to a value other than 0.z	head-nodeTFzl[Ray on Vertex AI]: custom_images.head and custom_images.worker must be specified when custom_images is set.zworker-pool)disabled)resource_pool_imagesray_metric_specray_logs_spec)serverpathmount_point)enable_custom_service_accountr   )ray_specservice_account_spec)r3   )network_attachment)resource_poolsr   r&   resource_runtime_specr#   r$   	projects//locations/)parentpersistent_resourcepersistent_resource_idz#Failed in cluster creation due to: z/persistentResources/)persistent_resource_name	tolerance)=logginginfoRuntimeErrorr   warningswarnr   DeprecationWarningr   get_local_ray_versioncustom_imager   timestamped_unique_name
node_count
ValueErrorautoscaling_specaccelerator_typeaccelerator_countr   idreplica_countmachine_typemachine_specboot_disk_type	disk_specboot_disk_size_gbget_image_uriheadworkerr   _check_machine_spec_identicalmin_replica_countmax_replica_countappendr   r   r   r
   r/   r0   r1   r%   r   r   r   r5   r   r   global_configlocationprojectr   get_project_numberr	   CreatePersistentResourceRequestr   !create_persistent_resource_clientcreate_persistent_resource	Exceptionget_persistent_resourcename)/r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   local_ray_verioninstall_ray_versionr,   resource_pool_0enable_cuda	image_uriworker_poolsiworker_node_typeadditional_replica_countresource_poolr6   metrics_collection_disabledr-   logging_disabledr.   r3   gapic_nfs_mounts	nfs_mountr4   r7   gapic_psc_interface_configr;   r\   
project_idproject_numberr:   requestclient_ecluster_resource_nameresponses/                                                  >/tmp/pip-target-z3e9_cxr/lib/python/vertex_ray/cluster_init.pycreate_ray_clusterr}   6   s   H _	
 e011g+-?AN(>>@&& ^%@%@%H"*LL $[2EG LLU"# %(E(E(GG$$)4  **6= 
 ++30014G 
  #nO$O$2$=$=O!0>0K0KO  -5C5U5UO  24B4S4SO  1/=/L/LO,2@2R2RO/(::Q>$EK""."//			%33
	 
			'M,@,@,L!&&	z
 	
 09++,L	A 1 B	 119$66: K  (1'N'N 0($  008 $114LL  - ,a/48O1(99KK $44F )99KK $44F (1, ,%01#6 #44<2B2M2MM/ )99KK "22D )99KK "22D ;K:W:W**7$66 **< %55 **; :J9X9X''6$66 ''9 ##M2&6&H&H1&LdRW#00< 0 = =I"* 1 ? ?#^[!I !. 4 4I9B$]%5%56FAEB	H &&5N&?"?#-HIO)))9:M1'#H
 # 	I##$++" ) 5 5	 /1*.+ 
 !4!5!

 !4X F%73FF&
" &*",%37- ((11H**22J+>>zJN(H:>F)II/+G ;;=FJ--g6
  &h&;L>J33!6H ==  J>BIJs   ,X' '	Y0X==Yrz   c                     t        j                         }t        j                  |       }	 |j	                  |       t        d       y# t        $ r}t        d|      |d}~ww xY w)zDelete Ray Cluster.

    Args:
        cluster_resource_name: Cluster resource name.
    Raises:
        FailedPrecondition: If the cluster is deleted already.
    rd   z5[Ray on Vertex AI]: Successfully deleted the cluster.z7[Ray on Vertex AI]: Failed in cluster deletion due to: N)r   r`   r	   DeletePersistentResourceRequestdelete_persistent_resourceprintrb   rI   )rz   rw   rv   ry   s       r|   delete_ray_clusterr   |  sf     ;;=F)II"G))'2EF Eq
	s   A	 		A$AA$c                     t        j                         }t        j                  |       }	 |j	                  |      }t        j                  |      }|r|S t        d      # t
        $ r}t        d|      |d}~ww xY w)z}Get Ray Cluster.

    Args:
        cluster_resource_name: Cluster resource name.
    Returns:
        A Cluster object.
    r   z:[Ray on Vertex AI]: Failed in getting the cluster due to: Nr;   z[Ray on Vertex AI]: Please delete and recreate the cluster (The cluster is not a Ray cluster or the cluster image is outdated).)r   r`   r	   GetPersistentResourceRequestrc   rb   rI   persistent_resource_to_cluster)rz   rw   rv   r{   ry   clusters         r|   get_ray_clusterr     s     ;;=F)FF"G11': 99hWG
 	J   H!
	s   A" "	A=+A88A=c                     t         j                  j                  } t         j                  j                  }t	        j
                  |      }d| d|  }t        j                  |      }t        j                         }	 |j                  |      }g }|D ],  }	t        j                  |	      }
|
s|j                  |
       . |S # t        $ r}t        d|      |d}~ww xY w)zList Ray Clusters under the currently authenticated project.

    Returns:
        List of Cluster objects that exists in the current authorized project.
    r8   r9   )r:   z;[Ray on Vertex AI]: Failed in listing the clusters due to: Nr   )r   r[   r\   r]   r   r^   r	   ListPersistentResourcesRequestr   r`   list_persistent_resourcesrb   rI   r   rZ   )r\   rt   ru   r:   rv   rw   r{   ry   ray_clustersr;   ray_clusters              r|   list_ray_clustersr     s     ((11H**22J+>>zJN(H:>F)HHG ;;=F33G< L' -"AA 3
 ,-   I1
	s   =C 	CCCc                 2   t        t        |            D ]`  }t        t        |            D ]G  }t        j                  ||   ||         }|dkD  s%||k7  s+t	        dd||    dz   d||    z          b t        j                  |       }t        j                  |      }d|j                  d   _
        t        |       }|j                  }|j                  }	t        |      t        |	      k7  rt	        dt        |      t        |	            d}
t        t        |            D ]  }t        j                  |||         }|dk7  s|dk(  r,||   j                  dk(  rd|z   |j                  d   _
        d}
R||   j                  |j                  ||
z      _
        ||   j                  dk(  st	        d	d
||    dz          t        j                   |t#        j$                  dg            }t        j&                         }	 |j)                  |      }t-        j,                         }|j/                         }t-        j,                         |z
  dz  }t1        dj3                  |             |j4                  S # t*        $ r}t	        d|      |d}~ww xY w)a  Update Ray Cluster (currently support resizing node counts for worker nodes).

    Sample usage:

    my_cluster = vertex_ray.get_ray_cluster(
            cluster_resource_name=my_existing_cluster_resource_name,
    )

    # Declaration to resize all the worker_node_type to node_count=1
    new_worker_node_types = []
    for worker_node_type in my_cluster.worker_node_types:
        worker_node_type.node_count = 1
        new_worker_node_types.append(worker_node_type)

    # Execution to update new node_count (block until complete)
    vertex_ray.update_ray_cluster(
            cluster_resource_name=my_cluster.cluster_resource_name,
            worker_node_types=new_worker_node_types,
    )

    Args:
        cluster_resource_name:
        worker_node_types: The list of Resources of the resized worker nodes.
            The same Resources object should not appear multiple times in the list.
    Returns:
        The cluster_resource_name of the Ray cluster on Vertex.
    r   z5[Ray on Vertex AI]: Worker_node_types have duplicate zmachine specs:  zand )r=   r)   z|[Ray on Vertex AI]: Desired number of worker_node_types (%i) does not match the number of the existing worker_node_type(%i).z%[Ray on Vertex AI]: Worker_node_type (z) must update to >= 1 nodeszresource_pools.replica_count)paths)r;   update_maskz;[Ray on Vertex AI]: Failed in updating the cluster due to: N<   zL[Ray on Vertex AI]: Successfully updated the cluster ({} mininutes elapsed).)rangelenr   rW   rI   r   rc   copydeepcopyr6   rN   r   r   r   rH   r	   UpdatePersistentResourceRequestr   	FieldMaskr`   update_persistent_resourcerb   timeresultr   formatrd   )rz   r   rk   jrm   r;   current_persistent_resourceprevious_ray_clusterr   previous_worker_node_types
not_mergedrv   rw   operation_futurery   
start_timer{   durations                     r|   update_ray_clusterr     s   > 3()* 
s,-. 		A'0'N'N!!$&7&:($ (!+Q K'(9!(<'=Q?@.q1234 		
 '>>!6 #'--0C"DBC..q1?*+@A)88N!5!G!G %?!@@/ !"*+
 	
 J3()* #,#J#J-a0$
  $q($).?.B.M.MQR.R ,, (66q9G J "!$// (66q:~FT
 !#..!3 ;+A.//JKL +4 *II7",,4R3STG ;;=F!<<WE J&&(H		j(R/H	V]]	

 ==  I1
	s   I; ;	JJJ)/r   r?   r   typingr   r   r   rB   google.cloud.aiplatformr   r   google.cloud.aiplatform.utilsr   %google.cloud.aiplatform_v1beta1.typesr	   7google.cloud.aiplatform_v1beta1.types.machine_resourcesr
   9google.cloud.aiplatform_v1beta1.types.persistent_resourcer   r   r   r   r   r   r   8google.cloud.aiplatform_v1beta1.types.service_networkingr   'google.cloud.aiplatform.vertex_ray.utilr   r   r   google.protobufr   9google.cloud.aiplatform.vertex_ray.util._validation_utilsr   r   	Resourcesstr
NodeImagesbool
PscIConfigr}   r   Clusterr   r   r        r|   <module>r      s!  $    ' '  / ) @ M L    + 5HI4G4G4I$(!'!%)"&>Qi>Q>Q>S=T4804%);?.259'+CY001CSMC #C c]	C
 c]C 3-C  Y%8%8 9:C I001C  (~C TNC #9#7#78C !c+C i0012C T#s(^$C 	CL
c d ,3 93D3D 64	 1 12 @rr37	8K8K3Lrrr   