
    cin                     4   d dl Z d dlmZ d dlZ	 d dlmZ d dlmc m	Z
 d dlmZmZmZmZmZ dZdZdZdZdZe j.                  j0                  e j.                  j                  gZ e j4                  d	
      d        Z e j4                  d	
      d        Z e j4                  d	
      d        Zd Zd Zd Z d Z!d Z"	 d1dZ#d Z$d Z%d Z&d Z'd Z(d Z)d Z*d Z+d Z,d Z-d Z.d  Z/d! Z0e j.                  jc                  d"#      d$        Z2d% Z3e j.                  ji                  d& e jj                  ddd'(       e jj                  d)dd*(       e jj                  dd)d+(       e jj                  d)d)d,(      g      d-        Z6d. Z7d/ Z8d0 Z9y# e$ r dZdZ
Y iw xY w)2    N)	timedelta)InMemoryKmsClientMockVersioningKmsClientverify_file_encryptedread_external_keys_to_dictparse_wrapped_keyzencrypted_table.in_mem.parquets   0123456789112345
footer_keys   1234567890123450col_keymodule)scopec                      t         j                  j                  t        j                  g d      t        j                  g d      t        j                  g d      d      } | S )N)         )abc)xyz)paTablefrom_pydictarray)
data_tables    L/tmp/pip-target-z3e9_cxr/lib/python/pyarrow/tests/parquet/test_encryption.pyr   r   2   sG    %%XXi XXo&XXo&' J
     c                  L    t        j                  t        t        ddgi      } | S )Nr   r   )r	   column_keyspeEncryptionConfigurationFOOTER_KEY_NAMECOL_KEY_NAME)basic_encryption_configs    r   r%   r%   <   s,     88"3*

 #"r   c                  N    t        j                  t        t        ddgid      } | S )Nr   r   F)r	   r   internal_key_materialr    )external_encryption_configs    r   r(   r(   F   s1    !#!;!;"3*
 $"% &%r   c                 f    t        j                  |       }d }t        j                  |      }||fS )z
    Sets up and returns the KMS connection configuration and crypto factory
    based on provided KMS configuration parameters.
    custom_kms_confc                     t        |       S Nr   kms_connection_configurations    r   kms_factoryz1setup_encryption_environment.<locals>.kms_factoryX        !=>>r   )r!   KmsConnectionConfigCryptoFactory)r+   kms_connection_configr1   crypto_factorys       r   setup_encryption_environmentr7   Q   s7    
 22?S? %%k2N .00r   c                     ||j                  d      ||j                  d      i}t        |      \  }}	t        | ||||	       ||	fS )zL
    Writes an encrypted parquet file based on the provided parameters.
    UTF-8)decoder7   write_encrypted_parquet)
pathr   footer_key_namecol_key_namer	   r
   encryption_configr+   r5   r6   s
             r   write_encrypted_filer@   a   sc     	**73gnnW-O -I-)> D*.?1>C !.00r   c           	      v   | t         z  }t        j                  t        t        ddgidt        d      d      }|j                  du sJ t        ||t        t        t        t        |      \  }}t        |       t        j                  t        d      	      }t        ||||      }|j                  |      sJ y
)DWrite an encrypted parquet, verify it's encrypted, and then read it.r   r   
AES_GCM_V1      @minutes   r	   r   encryption_algorithmcache_lifetimedata_key_length_bitsFrJ   N)PARQUET_NAMEr!   r"   r#   r$   r   uniform_encryptionr@   
FOOTER_KEYCOL_KEYr   DecryptionConfigurationread_encrypted_parquetequalstempdirr   r<   r?   r5   r6   decryption_configresult_tables           r   !test_encrypted_parquet_write_readrX   w   s    \!D
 22"3*
 * - " //5888,@j/<W-)> $ 22 -/)!6HL\***r   c           	      ^   | t         z  }t        j                  t        ddt	        d      d      }|j
                  du sJ t        ||t        t        t        d|      \  }}t        |       t        j                  t	        d            }t        ||||      }|j                  |      sJ y	)
rB   TrC   rD   rE   rG   )r	   rN   rI   rJ   rK   r   rL   N)rM   r!   r"   r#   r   rN   r@   r$   rO   r   rQ   rR   rS   rT   s           r   )test_uniform_encrypted_parquet_write_readrZ      s    \!D 22") - " //4777,@j/<S-)> $ 22 -/)!6HL\***r   c                     |j                   r|j                  ||      }n|j                  |||       }|J t        j                  | |j                  |      5 }|j                  |       d d d        y # 1 sw Y   y xY w)N)encryption_properties)r'   file_encryption_propertiespqParquetWriterschemawrite_table)r<   tabler?   r5   r6   r]   writers          r   r;   r;      s    ..%3%N%N!#4&6" &4%N%N!#4d&<"%111			%,,"<
> "AG5!" " "s   A44A=Tc                 ^   |r|j                  ||      }n|j                  |||       }|J t        j                  | |      }|j                  dk(  sJ t        j                  | |      }t        |j                        dk(  sJ t        j                  | |      }|j                  d      S )Ndecryption_propertiesr   Tuse_threads)	file_decryption_propertiesr^   read_metadatanum_columnsread_schemalennamesParquetFileread)	r<   rV   r5   r6   r'   ri   metar`   results	            r   rR   rR      s     %3%N%N!#4&6" &4%N%N!#4d&<" &111$>@Dq   ^^$>@Fv||!!!^^$>@F;;4;((r   c           	      
   | t         z  }t        j                  t        t        ddgidt        d      d      }t        ||t        t        t        t        |       t        |       t        t        t        j                  d      t        t        j                  d      i      \  }}t        j                  t        d      	      }t        j                  t        d
      5  t!        ||||       ddd       y# 1 sw Y   yxY w)zYWrite an encrypted parquet, verify it's encrypted,
    and then read it using wrong keys.r   r   rC   rD   rE   rG   rH   r9   rL   zIncorrect master key usedmatchN)rM   r!   r"   r#   r$   r   r@   rO   rP   r   r7   r:   rQ   pytestraises
ValueErrorrR   )rU   r   r<   r?   wrong_kms_connection_configwrong_crypto_factoryrV   s          r   +test_encrypted_parquet_write_read_wrong_keyr{      s     \!D
 22"3*
 * - " z?L#W.?A $8T0j''0V 95!5
 22 -/	z)E	F "#%@ 	"" " "s   !C99Dc                     t        | |       t        j                  t        d      5  t	        j
                  | t        z        j                          ddd       y# 1 sw Y   yxY w)zmWrite an encrypted parquet, verify it's encrypted,
    but then try to read it without decryption properties.no decryptionrt   N)rX   rv   rw   IOErrorr^   ro   rM   rp   rU   r   s     r   0test_encrypted_parquet_read_no_decryption_configr      sL     &gz:	w&6	7 6
w-.3356 6 6s   +AA%c                     t        | |       t        j                  t        d      5  t	        j
                  | t        z         ddd       y# 1 sw Y   yxY w)zwWrite an encrypted parquet, verify it's encrypted,
    but then try to read its metadata without decryption properties.r}   rt   N)rX   rv   rw   r~   r^   rj   rM   r   s     r   9test_encrypted_parquet_read_metadata_no_decryption_configr     sE     &gz:	w&6	7 1
</01 1 1   AAc                     t        | |       t        j                  t        d      5  t	        j
                  | t        z         ddd       y# 1 sw Y   yxY w)zuWrite an encrypted parquet, verify it's encrypted,
    but then try to read its schema without decryption properties.r}   rt   N)rX   rv   rw   r~   r^   rl   rM   r   s     r   7test_encrypted_parquet_read_schema_no_decryption_configr     sC     &gz:	w&6	7 /
w-./ / /r   c           
          | dz  }t        j                  t              }t        j                  t
        d      5  t        ||t        t        t        d|       ddd       y# 1 sw Y   yxY w)MWrite an encrypted parquet, but give only footer key,
    without column key.z)encrypted_table_no_col_key.in_mem.parquetr	   z4Either column_keys or uniform_encryption must be setrt   r   N)	r!   r"   r#   rv   rw   OSErrorr@   r$   rO   rU   r   r<   r?   s       r   'test_encrypted_parquet_write_no_col_keyr     sn     @@D 22"$ 
w%
& A 	T:'.?	A	A A As   A""A+c           
          | dz  }t        j                  t        t        ddgid      }t	        j
                  t        d      5  t        ||t        t        t        d|       d	d	d	       y	# 1 sw Y   y	xY w)
r   z=encrypted_table_col_key_and_uniform_encryption.in_mem.parquetr   r   T)r	   r   rN   z2Cannot set both column_keys and uniform_encryptionrt   r   N)	r!   r"   r#   r$   rv   rw   r   r@   rO   r   s       r   ;test_encrypted_parquet_write_col_key_and_uniform_encryptionr   '  s     TTD 22"3*
  ! 
wR
T A 	T:'.?	AA A As   A,,A5c                     | dz  }|}t        j                         }d }t        j                  |      }t        j                  t
        d      5  t        |||||       ddd       y# 1 sw Y   yxY w)<Write an encrypted parquet, but raise KeyError in KmsClient.(encrypted_table_kms_error.in_mem.parquetc                     t        |       S r-   r.   r/   s    r   r1   z;test_encrypted_parquet_write_kms_error.<locals>.kms_factoryD  s     !!=>>r   r	   rt   N)r!   r3   r4   rv   rw   KeyErrorr;   rU   r   r%   r<   r?   r5   r1   r6   s           r   &test_encrypted_parquet_write_kms_errorr   ;  sy     ??D/ 224?
 %%k2N	x|	4 Gj2C 5~	GG G G   A((A1c                     | dz  }|}t        j                         } G d dt         j                        fd}t        j                  |      }t	        j
                  t        d      5  t        |||||       ddd       y# 1 sw Y   yxY w)r   r   c                   "    e Zd ZdZd Zd Zd Zy)Jtest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClientzVA KmsClient implementation that throws exception in
        wrap/unwrap calls
        c                 P    t         j                  j                  |        || _        y)z%Create an InMemoryKmsClient instance.N)r!   	KmsClient__init__configselfr   s     r   r   zStest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.__init__^  s    LL!!$' DKr   c                     t        d      )NCannot Wrap Keyrx   r   	key_bytesmaster_key_identifiers      r   wrap_keyzStest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.wrap_keyc  s    .//r   c                     t        d      )NzCannot Unwrap Keyr   r   wrapped_keyr   s      r   
unwrap_keyzUtest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.unwrap_keyf  s    011r   N__name__
__module____qualname____doc__r   r   r    r   r   ThrowingKmsClientr   Y  s    		!
	0	2r   r   c                      |       S r-   r   )r0   r   s    r   r1   zDtest_encrypted_parquet_write_kms_specific_error.<locals>.kms_factoryi  s     !=>>r   r   rt   N)r!   r3   r   r4   rv   rw   rx   r;   )	rU   r   r%   r<   r?   r5   r1   r6   r   s	           @r   /test_encrypted_parquet_write_kms_specific_errorr   P  s     ??D/ 2242BLL 2 ? %%k2N	z):	; Gj2C 5~	GG G Gs   +BBc                     | dz  }|}t        j                         }d }t        j                  |      }t        j                  t
        d      5  t        |||||       ddd       y# 1 sw Y   yxY w)z@Write an encrypted parquet, but raise ValueError in kms_factory.0encrypted_table_kms_factory_error.in_mem.parquetc                     t        d      )NCannot create KmsClientr   r/   s    r   r1   zCtest_encrypted_parquet_write_kms_factory_error.<locals>.kms_factory}  s    233r   r   rt   N)r!   r3   r4   rv   rw   rx   r;   r   s           r   .test_encrypted_parquet_write_kms_factory_errorr   t  s~     GGD/ 2244 %%k2N	z6
8 G 	 j2C 5~	GG G Gr   c                     | dz  }|}t        j                         } G d d      fd}t        j                  |      }t        j                  t
              5  t        |||||       ddd       y# 1 sw Y   yxY w)z_Write an encrypted parquet, but use wrong KMS client type
    that doesn't implement KmsClient.r   c                   "    e Zd ZdZd Zd Zd Zy)Otest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClientz4This is not an implementation of KmsClient.
        c                 &    |j                   | _        y r-   )r+   master_keys_mapr   s     r   r   zXtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.__init__  s    #)#9#9D r   c                      y r-   r   r   s      r   r   zXtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.wrap_key      r   c                      y r-   r   r   s      r   r   zZtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.unwrap_key  r   r   Nr   r   r   r   WrongTypeKmsClientr     s    		:		r   r   c                      |       S r-   r   )r0   r   s    r   r1   zHtest_encrypted_parquet_write_kms_factory_type_error.<locals>.kms_factory  s    !">??r   N)r!   r3   r4   rv   rw   	TypeErrorr;   )	rU   r   r%   r<   r?   r5   r1   r6   r   s	           @r   3test_encrypted_parquet_write_kms_factory_type_errorr     s     GGD/ 224 @ %%k2N	y	! Gj2C 5~	GG G Gs   A33A<c            
      J   d } t        j                  t        t        ddgidddt	        d      dd	
      } | |       t        j                  t              }t        ddgi|_        d|_        d|_        d|_        t	        d      |_	        d|_
        d	|_         | |       y )Nc                 0   t         | j                  k(  sJ ddg| j                  t           k(  sJ d| j                  k(  sJ | j
                  sJ | j                  rJ t        d      | j                  k(  sJ | j                  rJ d| j                  k(  sJ y )Nr   r   AES_GCM_CTR_V1      $@rE      )r#   r	   r   r$   rI   plaintext_footerdouble_wrappingr   rJ   r'   rK   )r?   s    r   !validate_encryption_configurationzZtest_encrypted_parquet_encryption_configuration.<locals>.validate_encryption_configuration  s    "3">">>>>Sz.::<HHHH#4#I#IIII 1111$4444&*;*J*JJJJ$::::'<<<<<r   r   r   r   TFr   rE   r   )r	   r   rI   r   r   rJ   r'   rK   r   )r!   r"   r#   r$   r   r   rI   r   r   rJ   r'   rK   )r   r?   encryption_config_1s      r   /test_encrypted_parquet_encryption_configurationr     s    = 22"!C:0- .# 	 &&7844"$'3c3Z&B#/?,+/(*/')24)@&05-/2,%&9:r   c                      t        j                  t        d            } t        d      | j                  k(  sJ t        j                         }t        d      |_        t        d      |j                  k(  sJ y )Nr   rE   rL   )r!   rQ   r   rJ   )rV   decryption_config_1s     r   /test_encrypted_parquet_decryption_configurationr     sl    22 .0T"&7&F&FFFF446)24)@&T"&9&H&HHHHr   c                      d } t        j                  dddddd      } | |       t        j                         }d|_        d|_        d|_        ddd|_         | |       y )	Nc                     d| j                   k(  sJ d| j                  k(  sJ d| j                  k(  sJ ddd| j                  k(  sJ y )N	Instance1URL1MyTokenkey_material_1key_material_2key1key2kms_instance_idkms_instance_urlkey_access_tokenr+   )r5   s    r   validate_kms_connection_configzPtest_encrypted_parquet_kms_configuration.<locals>.validate_kms_connection_config  sb    3CCCCC.?????1BBBBB)3CD%556 	7 6r   r   r   r   r   r   r   r   )r!   r3   r   r   r   r+   )r   r5   kms_connection_config_1s      r   (test_encrypted_parquet_kms_configurationr     s    7 22#"$$
	 ##89 446.9+/5,/8,  /+ ##:;r   zNPlaintext footer - reading plaintext column subset reads encrypted columns too)reasonc                 >   | t         z  }t        j                  t        t        ddgidd      }t        j
                  t        t        j                  d      t        t        j                  d      i      }d }t        j                  |      }t        |||||       y	)
zWrite an encrypted parquet, with plaintext footer
    and with single wrapping,
    verify it's encrypted, and then read plaintext columns.r   r   TF)r	   r   r   r   r9   r*   c                     t        |       S r-   r.   r/   s    r   r1   zStest_encrypted_parquet_write_read_plain_footer_single_wrapping.<locals>.kms_factory  r2   r   N)rM   r!   r"   r#   r$   r3   rO   r:   rP   r4   r;   )rU   r   r<   r?   r5   r1   r6   s          r   >test_encrypted_parquet_write_read_plain_footer_single_wrappingr     s     \!D
 22"3*
  22Z..w7'..1
? %%k2ND*.?1>Cr   c           	         | t         z  }t        ||t        t        t        t
        |      \  }}t        |       t        j                         }t        ||||d      }t        j                  j                  j                  |      }t        |j                         x}	      t        |j                   t                 dz   k(  sJ t#        |	D 
cg c]  }
|j%                  |
      du c}
      sJ |j'                  |      sJ yc c}
w )zWrite an encrypted parquet file with external key material, verify
    it's encrypted, then read both the table and external store.
    Fr'   r   N)rM   r@   r#   r$   rO   rP   r   r!   rQ   rR   r   _parquet_encryptionFileSystemKeyMaterialStorefor_filerm   get_key_id_setr   allget_key_materialrS   )rU   r   r(   r<   r5   r6   rV   rW   storekey_idsks              r   *test_encrypted_parquet_write_read_externalr     s   
 \!D,@j/<W"-$)> $224)!6#%L ""==FFtLE%..00w1&22<@AAEG G GwG!&&q)5GHHH\*** Hs   D)double_wrap_initialdouble_wrap_rotatedzdouble wrapping)idFzsingle to double wrappedzdouble to singe wrappedzsingle wrappingc                 z   | t         z  }t        j                  t        t        ddgid      }t        j
                  d      }d }t        j                  |      }t        |||||       t        |      |j                  d       |j                  ||	       t        |      t        |       t        |t        j                         ||d
      }	t        v sJ t        v sJ t        v sJ t        v sJ dt        ddffd}
 |
t                |
t               |j                  |	      sJ y)a  Tests CryptoFactory.rotate_master_keys

    Note: The CryptoFactory.rotate_master_keys() double_wrapping keword arg
    may be either True (the default) or False regardless of whether
    EncryptionConfig.double_wrapping was set to true (also the default) when
    the external key material store was written. This means double wrapping may
    be set one way initially and then applied or removed during rotation.
    r   r   F)r	   r   r'   r   1)r   c                     t        |       S r-   )r   r/   s    r   r1   z8test_external_key_material_rotation.<locals>.kms_factoryR  s    &'CDDr   2)r   r   master_key_idreturnNc                     	|    }
r|j                   }n|j                  }t        |      \  }}}|    }r|j                   }n|j                  }t        |      \  }}}||k  sJ y r-   )wrapped_kekwrapped_dekr   )r   before_key_matbefore_key_wrapped_
before_verafter_key_matafter_key_wrapped	after_ver
after_keysbefore_keysr   r   s           r   check_rotated_external_keyszHtest_external_key_material_rotation.<locals>.check_rotated_external_keysr  s    $]3!/!;!;!/!;!;,-?@:q"=1 - 9 9 - 9 9+,=>9a I%%%r   )rM   r!   r"   r#   r$   r3   r4   r;   r   refresh_key_access_tokenrotate_master_keysr   rR   rQ   strrS   )reusable_tempdirr   r   r   r<   r?   r5   r1   r6   table_read_after_rotationr  r  r  s     ``       @@r   #test_external_key_material_rotationr  5  s]   & l*D22"!C:.#+	- 22CHE%%k2N -T2K 2237%%+ & -
 ,D1J$ 6
""$#!% k)));&&&j(((:%%%&3 &4 & &"  0-6777r   c           	      x   | t         z  }t        ||t        t        t        t
        |      \  }}t        |       t        j                  t        d            }t        d      D ]T  }|j                  ||      }|J t        j                  ||      }	|	j                  d      }
|j                  |
      rTJ  y)	z`Write an encrypted parquet, verify it's encrypted,
    and then read it multithreaded in a loop.rD   rE   rL   2   Nre   Trg   )rM   r@   r#   r$   rO   rP   r   r!   rQ   r   rangeri   r^   ro   rp   rS   )rU   r   r%   r<   r5   r6   rV   iri   rr   rW   s              r   test_encrypted_parquet_loopr    s     \!D
 -Aj/<W-!)> $22 -/ 2Y 	/%3%N%N!#4&6")555(BD{{t{4  ...	/r   c           	      R   | t         z  }t        ||t        t        t        t
        |      \  }}t        |       t        j                  t        d            }|j                  ||      }~t        j                  ||      }|j                  d      }	|j                  |	      sJ y)z^
    Test that decryption properties can be used if the crypto factory is no longer alive
    rD   rE   rL   re   Trg   N)rM   r@   r#   r$   rO   rP   r   r!   rQ   r   ri   r^   ro   rp   rS   )
rU   r   r%   r<   r5   r6   rV   ri   rr   rW   s
             r   %test_read_with_deleted_crypto_factoryr    s     \!D,@j/<W-!)> $ 22 -/!/!J!J0"2^^$>@F;;4;0L\***r   c           	      j   | t         z  }t        ||t        t        t        t
        |      \  }}t        j                  t        d            }|j                  ||      }t        j                  ||      }|j                  |      sJ t        j                  | |      }|j                  |      sJ y)z>Write an encrypted parquet then read it back using read_table.rD   rE   rL   re   N)rM   r@   r#   r$   rO   rP   r!   rQ   r   ri   r^   
read_tablerS   )	rU   r   r%   r<   r5   r6   rV   ri   rW   s	            r   !test_encrypted_parquet_read_tabler    s    \!D -Aj/<W-!)> 22 -/!/!J!J0"2 ===WXL \*** =='ACL\***r   )T):rv   datetimer   pyarrowr   pyarrow.parquetparquetr^   pyarrow.parquet.encryption
encryptionr!    pyarrow.tests.parquet.encryptionr   r   r   r   r   ImportErrorrM   rO   r#   rP   r$   markparquet_encryption
pytestmarkfixturer   r%   r(   r7   r@   rX   rZ   r;   rR   r{   r   r   r   r   r   r   r   r   r   r   r   r   xfailr   r   parametrizeparamr  r  r  r  r   r   r   <module>r'     s  "   E ++
E E 0 

 KK""
KK
 h   h#  # h&  &1 1,+>+6"" 26). "F61/A"A(G*!GHG(GB ;FI<:  2 3C3CN+2 2T4$56UD%?@T5%>?UE&78	5:;J8;J8Z/:+0+M  	B	Bs   F
 
	FF