
    U9jLG                         d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z
 d ZdefdZd Z	 	 	 dd	ed
edededededededefdZ	 	 	 	 ddededededef
dZdS )    N)SequenceMatcher)defaultdict)edit_distancec                 b   g g }}t          | |          D ]\  }}|                                }|                                }t          d||          }g g }
}	|                                D ]\  }}}}}|dk    r;|	                    |||                    |
                    |||                    Id                    |||                   }d                    |||                   }||k    r+|	                    |           |
                    |           |	                    |||                    |
                    |||                    |                    d                    |	                     |                    d                    |
                     ||fS )zAlign compound word boundaries between ref/pred pairs.

    When a mismatch region has identical characters ignoring whitespace,
    normalize both sides to the joined form.
    Nequal  )zipsplitr   get_opcodesextendjoinappend)refspredsnew_refs	new_predsref_text	pred_text	ref_words
pred_wordssmnew_rwnew_pwtagi1i2j1j2rcpcs                     1ark_asr/space_ark_asr_3b/normalizer/eval_utils.pynormalize_compound_pairsr#      s    biH"4// + +)NN$$	__&&
T9j99R#%>>#3#3 	5 	5CRRg~~i2.///jB/0000WWYr"u-..WWZ2.//88MM"%%%MM"%%%%MM)BrE"2333MM*RU"34444(()))&))****Y    manifest_pathc                     g }t          | dd          5 }|D ]>}t          |          dk    r)t          j        |          }|                    |           ?	 ddd           n# 1 swxY w Y   |S )ze
    Reads a manifest file (jsonl format) and returns a list of dictionaries containing samples.
    rutf-8encodingr   N)openlenjsonloadsr   )r%   dataflinedatums        r"   read_manifestr3   ,   s     D	mS7	3	3	3 #q 	# 	#D4yy1}}
4((E"""	## # # # # # # # # # # # # # #
 Ks   AA$$A(+A(c                  N   	 ddl m}  n# t          $ r t          j                            t          j                            t          j                            t                                        t          j                            t          j                            t                              fdt          j        D             t          _        t          j        	                    d           t          j
                            dd            ddl m}  Y nw xY w| S )Nr   )
data_utilsc                     g | ]p}t           j                            |pt          j                              k    r8t           j                            |pt          j                              k    n|qS  )ospathabspathgetcwd).0r9   normalizer_dir	repo_roots     r"   
<listcomp>z"get_data_utils.<locals>.<listcomp>?   sn     
 
 
wt2ry{{33~EE 3	44	AA  BAAr$   
normalizer)r@   r5   ImportErrorr8   r9   dirnamer:   __file__sysinsertmodulespop)r5   r=   r>   s    @@r"   get_data_utilsrH   9   s   *))))))) * * *GOOBGOOBGOOH4M4M$N$NOO	)B)BCC
 
 
 
 

 
 
 	9%%%d+++))))))))* s    DD"!D"
referencestranscriptionsmodel_iddataset_pathdataset_namer   audio_lengthtranscription_timeaudio_filepathsc	                    |                     dd          }|                     dd          }|                     dd          }t          |           t          |          k    r0t          dt          |            dt          |           d          |Pt          |          t          |           k    r0t          dt          |           dt          |            d          |Pt          |          t          |           k    r0t          d	t          |           dt          |            d          |Pt          |          t          |           k    r0t          d
t          |           dt          |            d          ||nt          |           dgz  }||nt          |           dgz  }||nt          |           dgz  }d}	t          j                            |	          st          j        |	           t          j                            |	d| d| d| d| d	          }
t          |
dd          5 }t          t          | ||||                    D ]F\  }\  }}}}}|r|nd| ||||d}|                    t          j        |d           d           G	 ddd           n# 1 swxY w Y   |
S )an  
    Writes a manifest file (jsonl format) and returns the path to the file.

    Args:
        references: Ground truth reference texts.
        transcriptions: Model predicted transcriptions.
        model_id: String identifier for the model.
        dataset_path: Path to the dataset.
        dataset_name: Name of the dataset.
        split: Dataset split name.
        audio_length: Length of each audio sample in seconds.
        transcription_time: Transcription time of each sample in seconds.
        audio_filepaths: List of file paths for each audio sample.

    Returns:
        Path to the manifest file.
    /-z'The number of samples in `references` (z) must match `transcriptions` (z).Nz)The number of samples in `audio_length` (z) must match `references` (z/The number of samples in `transcription_time` (z,The number of samples in `audio_filepaths` (z
./results/MODEL_	_DATASET__.jsonlwr(   r)   sample_)audio_filepathdurationtimetextr   F)ensure_ascii
)replacer,   
ValueErrorr8   r9   existsmakedirsr   r+   	enumerater
   writer-   dumps)rI   rJ   rK   rL   rM   r   rN   rO   rP   basedirr%   r0   idxr]   
transcriptrZ   r2   s                    r"   write_manifestrj   L   s   8 S))H''S11L''S11L
:#n----Dc*oo D D,/,?,?D D D
 
 	

 C$5$5Z$H$H<L8I8I < <(+J< < <
 
 	
 %#.@*A*AS__*T*T<cBT>U>U < <(+J< < <
 
 	
 "s?';';s:'N'N<3;O;O < <(+J< < <
 
 	
 %0c*oo6N 
 ) 	__v%  +6C
OOtf<T  G7>>'"" 
GGLLX(XX\XXLXX5XXX M 
mS7	3	3	3 BqYb
NL:Lo^^Z
 Z
 
	B 
	BUCU$
L2Dn 5C"W..RU(*' E GGtz%e<<<@@@AAAA
	BB B B B B B B B B B B B B B B s   A(J77J;>J;Fen	directorymultilingualcsv_onlylanguagec                   /01234567 |                      t          j                  r
| dd         } t          t	          j        |  dd                    }t          t          |                    }6:dk    r4t          d                               dd	          fd
|D             }t          |          dk    rt          d|            dt          f6fd}t                      3i 7d}|D ] }t          |          }	 ||          \  }
}dk    r3j        5n3fd55fd|	D             }5fd|	D             }d |	D             }d |	D             }t          |          ot          |          }|rFt          ||          \  }}|ddl}|                    d          }|                    ||          }ndx}x}x}}t'          ||          D ]\  }}|                                }|                                }|s|t          |          z  }Bt+          ||d          }||d         z  }||d         z  }||d         z  }||d         z  }||z   |z   }|dk    r||z  nd}t-          d|z  d          }|rLt/          |          }t/          |          }t-          t/          |          t/          |          z  d          }ndx}x}}|
 d | } |s|||d!ni }!||||d"|!7| <   |s|t          d#           t          d$           t          d#           7                                D ]:\  }"}#|" d%|#d         d&d'}$|#d(         |$d)|#d(         d&z  }$t          |$           ;t3          t4                    2t3          t4                    0t3          t4                    1t3          t6                    }%7                                D ]\  }"}#|"                    d*          d                                         }&2|&xx         |#d         z  cc<   |#d(         -0|&xx         |#d+         z  cc<   1|&xx         |#d,         z  cc<   n
dx0|&<   1|&<   |%|&xx         d-z  cc<   |st                       t          d#           t          d.           t          d#           2                                D ]&\  }"}#|#|%|"         z  }t          |" d%|d&d'           '0D ]0}"0|"         &0|"         1|"         z  }t          |" d/|d&           1t          d#           d0d0d1d2d3d4d5d6d7d8d9fd:d:d;d2d<d=d>d?fd@ddAdBdCdDdEdFdGdHdIfg}'dJ                    7                                          /7fdK4dN01246fdL	}(|'D ]Y\  })}*}+},|)                                }-|*+tA          /fdM|,D                       }.|.r |(|+|,|-           H|*/v r |(|+|,|-           Z27fS )Oa'  
    Scores all result files in a directory and returns a composite score over all evaluated datasets.

    Args:
        directory: Path to the result directory, containing one or more jsonl files.
        model_id: Optional, model name to filter out result files based on model name.
        multilingual: If True, apply compound word boundary normalization before
                      WER computation. Should only be enabled for non-English benchmarks.
        csv_only: If True, suppress all output except the CSV summary block.
        language: Language code used for normalization (e.g. 'en', 'de', 'fr').
                  When not 'en', ml_normalizer is used instead of the English normalizer.

    Returns:
        Composite score over all evaluated datasets and a dictionary of all results.
    Nz/**/*.jsonlT)	recursiver   zFiltering models by id:rR   rS   c                     g | ]}|v |	S r7   r7   )r<   fprK   s     r"   r?   z!score_results.<locals>.<listcomp>   s    DDDrX^^^^^r$   r   zNo result files found in rt   c                    |                      d          }| |d          } |                      d          }| d |                             dd                              d          }	dk    r}n0|                     d          }|d |         dz   ||dz   d          z   }| |d          }|                    dd                              d          }||fS )	NrT   DATASET_r   rV   rS   rR      rW   )findr`   rstripremovesuffix)rt   model_indexds_indexmodel_id_from_pathauthor_indexds_fp
dataset_idoriginal_model_ids          r"   parse_filepathz%score_results.<locals>.parse_filepath   s    ggh''77:&&		]228R@@GGLL(->"-D-D!2-22377L!3M\M!BS!HK]^jmn^n^p^pKq!q899]]:r22??II
!:--r$   rk   c                 2                         |           S )N)lang)ml_normalizer)r]   r5   ro   s    r"   <lambda>zscore_results.<locals>.<lambda>   s    Z%=%=d%=%R%R r$   c                 2    g | ]} |d                    S )r]   r7   r<   r2   	normalizes     r"   r?   z!score_results.<locals>.<listcomp>   s'    EEE5iif..EEEr$   c                 2    g | ]} |d                    S )r   r7   r   s     r"   r?   z!score_results.<locals>.<listcomp>   s(    KKKyy{!344KKKr$   c                     g | ]
}|d          S )r\   r7   r<   r2   s     r"   r?   z!score_results.<locals>.<listcomp>   s    444%f444r$   c                     g | ]
}|d          S )r[   r7   r   s     r"   r?   z!score_results.<locals>.<listcomp>   s    <<<%E*%<<<r$   wer)rI   predictions)merge_compoundsinsdelsubref_leng        d         z | )r   r   r   )r   rN   inference_timertfxP********************************************************************************zResults per dataset:z: WER = z0.2fz %r   z	, RTFx = |rN   r   rw   zComposite Results:z	: RTFx = appenzmodel,Avg Appen WER,Avg Scripted,Avg Conversational,Scripted-US,Scripted-AU,Scripted-CA,Scripted-IN,Conversational-US003,Conversational-US004,Conversational-IN)zScripted-USscripted)zScripted-AUr   )zScripted-CAr   )zScripted-INr   )zConversational-US003conversational)zConversational-US004r   )zConversational-INr   )!appen_scripted_filtered__american#appen_scripted_filtered__australian!appen_scripted_filtered__canadianappen_scripted_filtered__indian5appen_conversational_segmented_filtered__american_0035appen_conversational_segmented_filtered__american_004/appen_conversational_segmented_filtered__indian	dataoceanzsmodel,Avg DataOcean WER,Avg Scripted,Avg Conversational,Scripted-US,Scripted-GB,Conversational-US,Conversational-GB)zScripted-GBr   )zConversational-USr   )zConversational-GBr   )"dataocean_scripted_filtered__en_US"dataocean_scripted_filtered__en_GB2dataocean_conversational_segmented_filtered__en_US2dataocean_conversational_segmented_filtered__en_GBpubliczmodel,RTFx,License,Size (B),# Languages,Encoder,Decoder,AMI WER,Earnings22 WER,Gigaspeech WER,LS Clean WER,LS Other WER,SPGISpeech WER,Voxpopuli WER)zAMI WERN)zEarnings22 WERN)zGigaspeech WERN)zLS Clean WERN)zLS Other WERN)zSPGISpeech WERN)zVoxpopuli WERN)ami_testearnings22_testgigaspeech_testzlibrispeech_test.cleanzlibrispeech_test.otherspgispeech_testvoxpopuli_testr	   c                     |                                 D ]N\  }\  }}||k    r@                                 D ]+\  }}|                                 |v r||v r|d         c c S ,Od S )Nr   )itemsry   )		model_key	col_labelcol_map	ds_substrlabel_group
result_key
result_valresultss	           r"   find_wer_inz"score_results.<locals>.find_wer_in  s    *1--// 	1 	1&Iv	!!.5mmoo 1 1*J
 ''))Z77I<S<S)%000000tr$   c                    d                                  D             }t                      fd|D             }|rd| dnd}t                       t          d           t          |           t          d           D ]zfd|D             d D             rZt          t	                    t                    z  d	          }n                                }t          d
| d|            {t          |            D ]Њn}fd|D             fd|D             }t          d                                  D                       }	|	r&fd                                D             }
fd                                D             }d                                  D             }|r-t          t	          |          t          |          z  d	          nd}|
r-t          t	          |
          t          |
          z  d	          nd}|r-t          t	          |          t          |          z  d	          nd}t          | d| d| d| dd	                    |          z              {          t                            z  d	          }nd}t          | d| dd	                    |          z              t          d           d S )Nc                     g | ]\  }}|S r7   r7   )r<   r   r   s      r"   r?   z:score_results.<locals>.print_csv_block.<locals>.<listcomp>  s    CCCuCCCr$   c                 F    g | ]}|v                      |          |S r7   )add)r<   columnseens     r"   r?   z:score_results.<locals>.print_csv_block.<locals>.<listcomp>  s2    ddd&$RVRZRZ[aRbRbvr$   zCSV Summary (z):zCSV Summary:r   c                 *    g | ]} |          S r7   r7   r<   r   r   r   r   s     r"   r?   z:score_results.<locals>.print_csv_block.<locals>.<listcomp>  s'    ZZZFIvw??ZZZr$   c                     g | ]}||S Nr7   r<   values     r"   r?   z:score_results.<locals>.print_csv_block.<locals>.<listcomp>  s    III%u7H7H7H7Hr$   r   z	avg WER (z) = c                 ,    i | ]}| |          S r7   r7   r   s     r"   
<dictcomp>z:score_results.<locals>.print_csv_block.<locals>.<dictcomp>  s)    bbbFIvw G Gbbbr$   c                 N    g | ]!}|         t          |                   nd"S )Nr   )str)r<   r   wer_valss     r"   r?   z:score_results.<locals>.print_csv_block.<locals>.<listcomp>  s7    pppZ`&1A1MHV,---SUpppr$   c              3   $   K   | ]\  }}|d uV  d S r   r7   )r<   _labelgroups      r"   	<genexpr>z9score_results.<locals>.print_csv_block.<locals>.<genexpr>  s+      UU=65U$.UUUUUUr$   c                 Z    g | ]'\  }\  }}|d k                         |          x%(S )r   getr<   _dsr   r   r   r   s       r"   r?   z:score_results.<locals>.print_csv_block.<locals>.<listcomp>  sL     ! ! !+^eU
**e9L9L0L/Y /Y/Y/Yr$   c                 Z    g | ]'\  }\  }}|d k                         |          x%(S )r   r   r   s       r"   r?   z:score_results.<locals>.print_csv_block.<locals>.<listcomp>  sM     ' ' '+^eU 000x||E?R?R6Re5_ 5_5_5_r$   c                     g | ]}||S r   r7   r   s     r"   r?   z:score_results.<locals>.print_csv_block.<locals>.<listcomp>  s    VVVeEDUEDUDUDUr$   r   ,z,,,,,,)
valuessetprintroundsumr,   stripanyr   r   )headerr   family_namecsv_columnstitleavgr   csv_model_labelwer_cols
is_privatescripted_wersconversational_wersall_wersavg_overallavg_scriptedavg_convrtfx_valr   r   r   r   composite_audio_lengthcomposite_inference_timecomposite_werr   r   s    `               @@@@r"   print_csv_blockz&score_results.<locals>.print_csv_block  s   CC'..2B2BCCCuuddddKddd3>R/////Nheh& 	4 	4IZZZZZZkZZZHII8IIIH 4CMMCMM91==->-J))PYP_P_PaPa2%22S22333f& 	R 	RI3D3P//V_ObbbbbbVabbbHppppdopppHUUGNNDTDTUUUUUJ R! ! ! ! !/6}}! ! !
' ' ' ' '/6}}' ' '#
 WVx/@/@VVVIQYeCMMCMM$A1EEEWYTaiuS%7%7#m:L:L%LaPPPgi +E#122S9L5M5MMqQQQ 
 SS;SSSSSSSVYV^V^_gVhVhhiiii))4@$%;I%FIabkIl%lnoppHH!H;;8;;;chhx>P>PPQQQQhr$   c              3       K   | ]}|v V  	d S r   r7   )r<   r   all_dataset_idss     r"   r   z score_results.<locals>.<genexpr>  s(      SSiY/9SSSSSSr$   r   )!endswithr8   pathseplistglobsortedr   r`   r,   ra   r   rH   r3   r@   allr#   evaluateloadcomputer
   r   kaldi_edit_distancer   r   r   r   floatintr   r   keys
capitalizer   )8rl   rK   rm   rn   ro   result_filesr   
wer_metricresult_filemanifestmodel_id_of_filer   rI   r   r\   r[   compute_rtfxr   r   	total_ins	total_del	total_subtotal_ref_wordsrefpredr   r   resulttotal_errorsrN   r   r   r   extrakvmetricscount_entrieskeyFAMILY_CONFIGSr   
family_keypresence_substrr   r   r   
has_publicr   r   r   r   r5   r   r   r   r   s8    `  `                                          @@@@@@@@@r"   score_resultsr     s$   0 "*%% #crcN	 	Y";";";tLLLMML|,,--L !B'222##C--DDDD\DDD <A@Y@@AAA.3 . . . . . .   !!JGJ# 8
 8
 --'5~k'B'B$*t"-IIRRRRRIEEEEHEEE
KKKK(KKK448444<<8<<<4yy2S]] 	Q&>z;&W&W#J!%]]511
$$
$TTCCBCCIC	CI [99 
5 
5	TIIKK	!ZZ\\
  Z0I,Y
TXYYYVE]*	VE]*	VE]*	6)#44$y09<L4Ca4G4G,00SCC#Iq!! 	8x==L YYNXT2A66DD377L7>D(99Z99
NZb	)IFFF`b(,	
 

 

  	h$%%%hMMOO 	 	DAq55AeH5555Gy$7qy7777'NNNN  &&M(//*511$$M    1ggcll1o##%%cah&V9 "3'''1^+<<'''$S)))Q/?-@@))))JNN"3'*B3*Gca  h"###h!'')) 	. 	.DAqmA&&CQ,,,,,,----' 	2 	2A%a(4-a03KA3NN00T000111h J 6Q7R5P3NJJD 	
4 J 7R6QGG 	
& k .#;#;*@*@#;"9 	
Y;Nz hhw||~~..O    6 6 6 6 6 6 6 6 6 6p 9G : :4
OVW ++--"SSSS7SSSSSJ >===//OFG[999'!!r$   )NNN)NFFrk   )r8   r   r-   rD   difflibr   collectionsr   
kaldialignr   r   r#   r   r3   rH   r   rj   boolr  r7   r$   r"   <module>r!     s   				   



 # # # # # # # # # # # # ; ; ; ; ; ;  B
 
 
 
 
  4 # V VVV V 	V
 V V V V V V V Vv j" j"j"j" j" 	j"
 j" j" j" j" j" j"r$   