
    U9j                         d dl Z d dlZd dlZd dlmZmZmZ d dlmZm	Z	 ddl
mZmZmZ d Z G d de	          Zd	 Z e            Z ed
          Zd Zd Zd ZddZg dZd Zd ZddZdS )    N)load_datasetAudioIterableDataset)EnglishTextNormalizerBasicMultilingualTextNormalizer   )read_manifestwrite_manifestnormalize_compound_pairsc                 f    |                                  dk    rdS |                                  dk    S )Nzignore time segment in scoringF )strip)refs    1ark_asr/space_ark_asr_3b/normalizer/data_utils.pyis_target_text_in_ranger      s.    
yy{{666uyy{{b      c                   *     e Zd ZdZd Zd fd	Z xZS )MultilingualNormalizerzBasicMultilingualTextNormalizer with optional number normalization.

    Call with just text for standard normalization (backward-compatible).
    Pass lang= to also convert digits to words via num2words.
    c                 f    t          j        dd|          }fd}t          j        d||          S )Nz(\d)\s+(\d{3})\bz\1\2c                     	 t          j         t          |                                                     S # t          $ r |                                 cY S w xY w)N)lang)	num2wordsintgroup	Exception)mr   s    r   _replacez;MultilingualNormalizer._normalize_numbers.<locals>._replace   s[    ! *3qwwyy>>EEEE ! ! !wwyy   !s   48 AAz\d+)resub)selftextr   r   s     ` r   _normalize_numbersz)MultilingualNormalizer._normalize_numbers   sF    v)7D99	! 	! 	! 	! 	!
 vfh---r   Nc                 z    t                                          |          }||                     ||          }|S N)super__call__r"   )r    sr   	__class__s      r   r&   zMultilingualNormalizer.__call__$   s:    GGQ''400Ar   r$   )__name__
__module____qualname____doc__r"   r&   __classcell__)r(   s   @r   r   r      sV         	. 	. 	.         r   r   c                     d| v r| d         S d| v r| d         S d| v r| d         S d| v r| d         S d| v r| d         S t          d          )Nr!   sentencenormalized_text
transcripttranscriptionzExpected transcript column of either 'text', 'sentence', 'normalized_text' or 'transcript'. Got sample of .join{sample.keys()}. Ensure a text column name is present in the dataset.)
ValueError)samples    r   get_textr5   +   s    f~	v		j!!	f	$	$'((			l##	F	"	"o&&Y
 
 	
r   F)remove_diacriticsc                 Z    t          |           | d<   t          | d                   | d<   | S )Noriginal_text	norm_text)r5   
normalizer)batchs    r   	normalizer<   A   s.    %e__E/#E/$:;;E+Lr   c                      t           j                            d          p t           j                            d          pd S )NHF_TOKENHUGGING_FACE_HUB_TOKEN)osenvironget r   r   get_hf_tokenrD   G   s0    :>>*%%Y8P)Q)QYUYYr   c           
          t          | j        | j        | j        t	          | dd          t                                }|S )N	streamingF)splitrF   token)r   dataset_pathdatasetrG   getattrrD   )argsrJ   s     r   	load_datarM   K   sC    j$U33nn  G Nr   >  c                     |                      dt          |                    } t          | t                    ri nddi} | j        t
          fi |} |                     t          dg          } | S )Naudio)sampling_rateload_from_cache_fileFr9   )input_columns)cast_columnr   
isinstancer   mapr<   filterr   )rJ   rQ   
map_kwargss      r   prepare_datarY   V   sw    !!'5}+M+M+MNNG!'?;;`BXZ_A`Jgk)22z22Gnn4[MnRRGNr   )id	file_namepathc                     | d S t          |                                           } | dk    rd S t          j                            |           S )Nr   )strr   r@   r\   basename)values    r   _basename_or_nonera   h   sG    }tJJE{{t7E"""r   c                 D   | d S t           D ]4}	 || v rt          | |                   }||c S ## t          $ r Y  d S w xY w	 |                     d          }n# t          $ r Y d S w xY wt          |t                    r"t          |                    d                    S d S )NrP   r\   )AUDIO_FILEPATH_METADATA_KEYSra   	TypeErrorrB   AttributeErrorrU   dict)r4   keyr_   rP   s       r   "extract_audio_filepath_from_samplerh   q   s    ~t+  	f}},VC[99'#OOO 	 	 	444	

7##   tt% 4 6!2!23334s   /
>>A 
A&%A&c                 T   |d| v rt          | d                   }net          |           dk    rRt          t          |                                                     }t	          |t
                    rt          |          }|g S t          D ]M}|                     |          }t	          |t
                    r!t          |          |k    rd |D             c S N|                     d          }t	          |t
                    rt          |          |k    rd |D             S d g|z  S )NrP   r   c                 ,    g | ]}t          |          S rC   )ra   ).0r`   s     r   
<listcomp>z6extract_audio_filepaths_from_batch.<locals>.<listcomp>   s!    AAA%e,,AAAr   c                     g | ];}t          |t                    r"t          |                    d                     nd<S )r\   N)rU   rf   ra   rB   )rk   rP   s     r   rl   z6extract_audio_filepaths_from_batch.<locals>.<listcomp>   sR     
 
 
 5?ud4K4KUeii//000QU
 
 
r   )lennextitervaluesrU   listrc   rB   )r;   
batch_sizefirst_valuerg   rq   audioss         r   "extract_audio_filepaths_from_batchrv      s=   eU7^,,JJZZ!^^tELLNN3344K+t,, . --
	+ B B3fd## 	BFz(A(AAA&AAAAAAYYwF&$ 
CKK:$=$=
 

 
 
 	
 6Jr   )rN   r$   )r   r@   r   datasetsr   r   r   r:   r   r   
eval_utilsr	   r
   r   r   r   r5   ml_normalizerr<   rD   rM   rY   rc   ra   rh   rv   rC   r   r   <module>rz      su   				 				     9 9 9 9 9 9 9 9 9 9 M M M M M M M M O O O O O O O O O O! ! !    <   2
 
 
" #"$$
&&???  Z Z Z	 	 	         # # #  ,     r   