+
    UV-j,*  ã                   ó¼   € ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIH	t	 ^ RI
Ht ^ RIHt ^ RIHt ^ RIHt R tR	 R
 ltR tR t]R8X  d
   ]! 4        R# R# )é    N)Údeepcopy)Údump)Úload_dataset)Útqdm)Úload)Ú	inferencec           	     óà  € V P                  4       P                  RR4      P                  4       pVP                  4       p. ROp. ROp. ROp. pV Fl  p^V3^V33 F]  w  ršV
 FR  pVP                  V4      p\        P
                  ! WÂ4       F$  pVP                  VP                  4       W‰34       K&  	  KT  	  K_  	  Kn  	  V'       g   R# VP                  R R7       V^ ,          ^,          pWã8H  # )	zµ
Extracts the answer from the model's predictions.
predict: Model prediction text
answer: Ground truth answer (A, B, C, or D)
Returns: bool: True if answer matches, False otherwise
Ú
Ú Fc                 ó*   € V ^,          ) V ^ ,          ) 3# )é   © )Úms   &Úe/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/evals/mmstar.pyÚ<lambda>Ú extract_answer.<locals>.<lambda>C   s   €   1¥ ¨¨!­ u™~ó    )Úkey)z^{0}\bz^\({0}z^option {0}\bz\b{0}\s*[:\.\)]z(?:^|\.|\s)\s*{0}\.z\({0}\)zoption\s+{0}\bzchoice\s+{0}\b)z^the answer is {0}\bzanswer:\s*{0}\bzanswer\s+is\s+{0}\bz1correct\s+(?:answer|option|choice)\s+is:?\s+{0}\bzthe\s+answer\s+is\s+{0}\bzis\s+{0}\s*:zA(?:therefore|thus|hence)[,\s]+(?:the\s+)?(?:answer\s+is\s+)?{0}\bz(?:select|choose)\s+{0}\bzit\s+is\s+{0}\bzwould\s+be\s+{0}\bz&\*\*(?:revised\s+)?answer\*\*:\s*{0}\bzE(?:correct\s+)?category\s+(?:for\s+this\s+image\s+)?is\s+\*\*{0}[:\s])ÚaÚbÚcÚdÚe)	ÚlowerÚreplaceÚstripÚformatÚreÚfinditerÚappendÚendÚsort)ÚpredictÚanswerÚtextÚanswer_lowerÚgeneral_templatesÚconcluding_templatesÚpossible_answersÚmatchesÚansÚpriÚtemplate_listÚtemplateÚpatternÚmatchÚ
latest_anss   &&             r   Úextract_answerr2      sç   € ð �=‰=‹?×"Ñ" 4¨Ó-×3Ñ3Ó5€DØ—<‘<“>€Lò	ÐòÐò 1ÐØ€GãˆØ$%Ð';Ð#<¸qÐBSÐ>TÓ"UÑˆCÛ)�Ø"Ÿ/™/¨#Ó.�ÜŸ[š[¨Ö7�EØ—N‘N E§I¡I£K°Ð#:Ö;ó 8ó *ó #Vñ  ÷ Ùð ‡L�LÑ-€LÔ.Ø˜•˜A•€JàÑ%Ð%r   c                ó0   € V ^8„  d   QhR\         R\        /# )r   ÚdataÚ	eval_file)ÚlistÚstr)r   s   "r   Ú__annotate__r8   I   s   € ÷ A#ñ A#”dð A#¤sñ A#r   c                 ó	  € R R^ R^ R^ /RR^ R^ R^ /RR	^ R
^ R^ /RR^ R^ R^ /RR^ R^ R^ /RR^ R^ R^ //p\        V4      p\        V RR7       Fž  p\        VR,          4      p\        VR,          4      p\        VR,          4      p\        VR,          4      pW7,          V;;,          ^,          uu&   \        WV4      '       d#   W',          V;;,          ^,          uu&   ^VR&   K™  ^ VR&   K   	  / p	^ V	R&   ^ p
VP	                  4        FÈ  w  r¼\        W;,          P                  4       4      p^ pVP	                  4        F\  w  ppW;,          V,          pV^ 8”  d%   \        V4      \        V4      ,          W› R V R!2&   M
R"W› R V R!2&   VV,          pV
V,          p
K^  	  V^ 8”  d   \        V4      V,          MR"W›&   V	R;;,          V,          uu&   KÊ  	  \        V 4      ^ 8”  d/   \        V	R,          4      \        \        V 4      4      ,          V	R&   \        R>4       \        R$4       \        R=4       \        R%V
 R&\        V 4       R'V	R,          ^d,          R( R)24       \        R?4       \        R*4       \        R?4       R@ Fq  pWy9   g   K  \        W7,          P                  4       4      p\        W',          P                  4       4      p\        VR+ R,VR- R&VR- R'W—,          ^d,          R. R/24       Ks  	  \        RA4       \        R04       \        R?4       R@ F„  p\        R#VP                  4        R124       W',          P	                  4        FJ  w  ppW7,          V,          pV^ 8”  d   VV,          ^d,          M^ p\        R2VR3 R,VR- R&VR- R'VR. R/2	4       KL  	  K†  	  \        R>4       VP                  R4R54      p\        VR64      ;_uu_ 4       p\        V	V^R77       R8R8R84       \        VR6R9R:R;7      ;_uu_ 4       pV '       dO   \        P                  ! VV ^ ,          P!                  4       R<7      pVP#                  4        VP%                  V 4       R8R8R84       R8#   + '       g   i     Lˆ; i  + '       g   i     R8# ; i)Búcoarse perceptionzimage scene and topiczimage style & qualityzimage emotionúfine-grained perceptionzobject countingÚrecognitionÚlocalizationúinstance reasoningzsingle-instance reasoningz"cross-instance attribute reasoningz!cross-instance relation reasoningúlogical reasoningzcode & sequence reasoningzdiagram reasoningzcommon reasoningúscience & technologyzbiology & chemistry & physicsz&electronics & energy & mechanical eng.z'geography & earth science & agricultureÚmathÚgeometryz#numeric commonsense and calculationzstatistical reasoningÚ
Evaluating©ÚdescÚ
predictionr$   ÚcategoryÚl2_categoryÚscorezfinal scoreÚ(Ú)g        r
   zMMStar Evaluation Resultsz
Final Score: Ú/z = z.2fz%
zCategory Scores:Ú30sz: Ú4dz6.2fÚ%zSubcategory Scores:Ú:z  Ú55sz.csvz_score.jsonÚw)ÚindentNÚ úutf-8©ÚnewlineÚencoding©Ú
fieldnameszP================================================================================zQ
================================================================================zP--------------------------------------------------------------------------------)r:   r;   r>   r?   r@   rA   zQ
--------------------------------------------------------------------------------)r   r   r7   r2   ÚitemsÚsumÚvaluesÚfloatÚlenÚprintÚupperr   Úopenr   ÚcsvÚ
DictWriterÚkeysÚwriteheaderÚ	writerows)r4   r5   ÚMMStar_score_l2ÚMMStar_counterÚliner#   ÚanswersrG   rH   ÚMMStar_scoreÚtotal_correctÚkÚvÚ	cat_totalÚcat_correctÚl2_kÚl2_vÚcountÚl2_catrI   ÚpctÚ	score_pthÚfÚwriters   &&                      r   ÚMMStar_evalrz   I   sƒ  € àØ# QØ# QØ˜Qð
ð
 	"Ø˜qØ˜1Ø˜Að$
ð
 	Ø'¨Ø0°!Ø/°ð
ð
 	Ø'¨Ø Ø ð
ð
 	Ø+¨QØ4°aØ5°qð!
ð
 	Ø˜Ø1°1Ø# Qð
ð5€Oô@ ˜oÓ.€Nä�T ×-ˆÜ�d˜<Õ(Ó)ˆÜ�d˜8•nÓ%ˆÜ�t˜JÕ'Ó(ˆÜ˜$˜}Õ-Ó.ˆàÕ  ×-°Õ2Ó-ô ˜'×+Ò+ØÕ% k×2°aÕ7Ó2àˆD�‹MàˆD�‹Mñ .ð" €LØ"#€L�ÑØ€Mà×%Ñ%Ö'‰ˆÜ˜Õ)×0Ñ0Ó2Ó3ˆ	ØˆØŸ'™'ž)‰JˆD�$Ø"Õ% dÕ+ˆEØ�qŒyÜ/4°T«{¼UÀ5»\Õ/I�˜s ! D 6¨˜^Ò,à/2�˜s ! D 6¨˜^Ñ,Ø˜4ÕˆKØ˜TÕ!ŠMñ $ð =FÈ¼Mœ% Ó,¨yÖ8Èsˆ‰Ø�]×# {Õ2Õ#ñ (ô ˆ4ƒy�1„}Ü&+¨L¸Õ,GÓ&HÌ5Ü�‹IóL
õ '
ˆ�]Ñ#ô
 
ˆ/ÔÜ	Ð
%Ô&Ü	ˆ(„OÜ	Ø
˜-˜¨¬#¨d«)¨°C¸À]Õ8SÐTWÕ8WÐX[Ð7\Ð\_Ð`ôô 
ˆ(„OÜ	Ð
ÔÜ	ˆ(„Oóˆð Ö#Ü˜NÕ4×;Ñ;Ó=Ó>ˆIÜ˜oÕ7×>Ñ>Ó@ÓAˆKÜØ˜C�.  ;¨rÐ"2°!°I¸b°>ÀÀ\ÕE[Ð\_ÕE_Ð`dÐDeÐefÐgöñô 
ˆ/ÔÜ	Ð
Ô Ü	ˆ(„Oóˆô 	��8—>‘>Ó#Ð$ AÐ&Ô'Ø,Õ6×<Ñ<Ö>‰MˆF�EØ"Õ,¨VÕ4ˆEØ+0°1¬9�5˜5•= 3Ö&¸!ˆCÜ�B�v˜c�l " U¨2 J¨a°°b¨z¸¸SÀ¸JÀaÐHÖIó ?ñô 
ˆ/Ôð ×!Ñ! &¨-Ó8€IÜ	ˆi˜×	Ô	 Üˆ\˜1 QÕ'÷ 
ô 
ˆi˜ b°7×	;Õ	;¸qßÜ—^’^ A°$°qµ'·,±,³.ÔAˆFØ×ÑÔ Ø×Ñ˜TÔ"÷	 
<Ñ	;÷ 
×	ú÷ 
<×	;Ð	;ús   ÏQ!Ð AQ4Ñ!Q1	Ñ4R	c                  óÈ  € \         P                  ! R R7      p V P                  R\        RRR7       V P                  R\        RRR7       V P                  R	\        R
RR7       V P                  R\        RRR7       V P                  RRRR7       V P                  R\        RRR7       V P                  R\        RRR7       V P                  R\
        RRR7       V P                  R\        ^RRR7       V P                  RRRR7       V P                  R \        RR!R7       V P                  R"\        R#R$R7       V P                  R%\        ^*R&R7       V P                  4       # )'zMMStar Evaluation)Údescriptionz--modelz'mlx-community/Qwen2-VL-2B-Instruct-bf16z
Model path)ÚtypeÚdefaultÚhelpz--adapter-pathNzAdapter pathz	--datasetzLin-Chen/MMStarzDataset pathz--splitÚvalzSplit to use for evaluationz--streamingÚ
store_truezUse streaming dataset loading)Úactionr   z--max-samplesz5Maximum number of samples to evaluate (for debugging)z--max-tokensi¸  z$Maximum number of tokens to generatez--temperaturegffffffæ?zTemperature for samplingz--resize-shapezResize shape for the image)r}   Únargsr~   r   z	--verbosezVerbose outputz--prediction-filezPath to the prediction filez--output-dirzresults/mmstarz$Directory to save evaluation resultsz--seedzRandom seed)ÚargparseÚArgumentParserÚadd_argumentr7   Úintr^   Ú
parse_args)Úparsers    r   Úparse_argumentsrŠ   Í   s¨  € Ü×$Ò$Ð1DÔE€FØ
×ÑØÜØ9Øð	 ô ð ×ÑÐ(¬s¸DÀ~ÐÔVØ
×ÑØœ#Ð'8¸~ð ô ð ×ÑØœ UÐ1Nð ô ð ×ÑØ˜lÐ1Pð ô ð ×ÑØÜØØDð	 ô ð ×ÑØÜØØ3ð	 ô ð ×ÑØœe¨SÐ7Qð ô ð ×ÑØÜØØØ)ð ô ð ×Ñ˜¨LÐ?OÐÔPØ
×ÑØ¤#¨tÐ:Wð ô ð ×ÑØÜØ Ø3ð	 ô ð ×Ñ˜¤s°B¸]ÐÔKà×ÑÓÐr   c                  ó6	  € \        4       p \        P                  ! V P                  4       \        P                  ! V P
                  '       d   \        P                  M\        P                  R R7       \        P                  ! R4       V P                  '       d£   \        P                  ! RV P                   R24       . p\        V P                  RRR7      ;_uu_ 4       p\        P                  ! V4      pV Uu. uF  qDNK  	  ppRRR4       \        WP                  4       \        P                  ! R	4       R# \        P                  ! R
V P                   R24       \        V P                  V P                   V P"                  R7      pV P$                  '       d   VP'                  V P$                  4      p\        P                  ! RV P(                   R24       \+        V P(                  V P,                  RR7      w  rgVP.                  p\        P                  ! RV R24       V P0                   RV P(                  P!                  R4      R+,           RV P                  P!                  R4      R+,           RV P                    R2p	\2        P4                  ! V P0                  RR7       . p\7        VRR7       F—  p
V
R,          pV
R,          P9                  R4      p\;        VVVVV P<                  V P>                  V P@                  V P
                  4      pVPC                  RVRV
R,          RV
R,          RV
R,          RV
R,          RV/4       K™  	  \E        R4       \G        VR ,          4       FF  w  rï\E        V^,            R!VR,          R",           R#VR,           R$VR,          R",           R%24       KH  	  \        V	R&R'RR(7      ;_uu_ 4       pV'       dN   \        PH                  ! W!^ ,          PK                  4       R)7      pVPM                  4        VPO                  V4       RRR4       \        W4       \        P                  ! R*V	 R24       \        P                  ! R	4       R# u upi   + '       g   i     ELu; i  + '       g   i     Lf; i),z)%(asctime)s - %(levelname)s - %(message)s)Úlevelr   z#[32mStarting MMStar Evaluation[0mz[32mLoading predictions from z for evaluation[0mÚrrU   )rX   Nz[32mEvaluation complete[0mz[32mLoading dataset from z[0m)ÚsplitÚ	streamingz[32mLoading model from T)Úadapter_pathÚtrust_remote_codez[32mConfig: rL   Ú_z_predictions.csv)Úexist_okzRunning inferencerD   ÚquestionÚimageÚRGBr$   rG   rH   Ú	meta_inforF   z
First 5 results::Né   Nz. Question: :Né2   Nz... | Answer: z | Prediction: z...rR   rT   rV   rY   z[32mSaving results to éÿÿÿÿ)(rŠ   ÚrandomÚseedÚloggingÚbasicConfigÚverboseÚINFOÚWARNINGÚinfoÚprediction_filerb   rc   Ú
DictReaderrz   Údatasetr   rŽ   r�   Úmax_samplesÚtakeÚmodelr   r�   ÚconfigÚ
output_dirÚosÚmakedirsr   Úconvertr   Ú
max_tokensÚtemperatureÚresize_shaper    r`   Ú	enumeraterd   re   rf   rg   )ÚargsÚresultsrx   ÚreaderÚrowr¥   r¨   Ú	processorr©   Úresult_fileÚexampler”   r•   rF   ÚiÚresultry   s                    r   Úmainr»     sÀ  € ÜÓ€Dä
‡K‚K�—	‘	Ôô ×ÒØ"ŸlŸl˜lŒg�lŠl´·±Ø:õô
 ‡L‚LÐ<Ô=Ø××ÐÜ�ŠØ/°×0DÑ0DÐ/EÐE[Ð\ô	
ð ˆÜ�$×&Ñ&¨°g×>Õ>À!Ü—^’^ AÓ&ˆFÙ&,Ó-¡f˜s’s¡fˆGÐ-÷ ?ô 	�G×1Ñ1Ô2Ü�ŠÐ9Ô;ÙÜ‡L‚LÐ0°·±°¸gÐFÔGÜ˜4Ÿ<™<¨t¯z©zÀTÇ^Á^ÔT€GØ××ÐØ—,‘,˜t×/Ñ/Ó0ˆä‡L‚LÐ.¨t¯z©z¨l¸'ÐBÔCÜØ�
‰
 ×!2Ñ!2ÀdôÑ€Eð �\‰\€FÜ‡L‚LÐ# F 8¨7Ð3Ô4à—_‘_Ð% Q t§z¡z×'7Ñ'7¸Ó'<¸RÕ'@Ð&AÀÀ4Ç<Á<×CUÑCUÐVYÓCZÐ[]ÕC^ÐB_Ð_`Ðae×akÑakÐ`lÐl|Ð}€KÜ‡K‚K�—‘¨$Õ/à€GÜ˜Ð&9×:ˆØ˜:Õ&ˆØ˜Õ ×(Ñ(¨Ó/ˆÜØØØØØ�O‰OØ×ÑØ×ÑØ�L‰Ló	
ˆ
ð 	�‰à˜HØ˜' (Õ+Ø˜G JÕ/Ø˜w }Õ5Ø˜W [Õ1Ø˜jðö		
ñ ;ô2 
Ð
ÔÜ˜w r�{Ö+‰	ˆÜØ��sˆe�<  zÕ 2°3Õ 7Ð8¸ÀvÈhÕGWÐFXÐXgÐhnÐo{Õh|ð  ~Aõ  iBð  hCð  CFð  Gö	
ñ ,ô
 
ˆk˜3¨°W×	=Õ	=ÀßÜ—^’^ A¸!µ*·/±/Ó2CÔDˆFØ×ÑÔ Ø×Ñ˜WÔ%÷	 
>ô �Ô%ä‡L‚LÐ-¨k¨]¸'ÐBÔCÜ‡L‚LÐ5Ö7ùòy .÷ ?×>Ð>ú÷j 
>×	=ús+   Ã!Q4Ã<
Q/ÄQ4ÏARÑ/Q4Ñ4R	ÒR	Ú__main__)r„   rc   r�   r«   r›   r   Úcopyr   Újsonr   Údatasetsr   r   Úmlx_vlmr   Úmlx_vlm.evals.utilsr   r2   rz   rŠ   r»   Ú__name__r   r   r   Ú<module>rÃ      sW   ðÛ Û 
Û Û 	Û Û 	Ý Ý å !Ý å Ý )ò5&õpA#òH4ònO8ðd ˆzÔÙ†Fñ r   