(tiW^ ddlZddlZddlmZddlZddlZddlmZddlm Z m Z e rddl Z ddl m Z ddl mZmZej ddj#d vxr<ej$j'xr ej$j)dd k\Zere rdd lmZed ZndZej2j4ej2j6ej2j8hZej2j<ej2j>ej2j@ej2jBej2jDej2jFhZ$ej2jJej2jLej2jNej2jPej2jRhZ*ej2jVej2jXej2jZej2j\ej2j^ej2j`ej2jbej2jdej2jfh Z4e$e*ze4zZ5e$e*ze4zZ6e$e*zZ7d ejpdejpde9dejpfdZ:dZ;dgfdZdZ?dZ@dZAdZBd.dZCd.dZDd.dZEd.dZFd.dZGd.d ZHd.d!ZId.d"ZJd.d#ZKd.d$ZLd.d%ZMd.d&ZNd.d'ZOejZPej2jfeNej2jdeOej2j8eMej2jDeCej2jBeDej2j@eEej2j>eFej2j<eGej2jReHej2jPeIej2jNeJej2jLeKej2jJeLi ZQeReQjZTd(ZUd)ZVGd*d+ejjZXGd,d-ejZZy)/N) nullcontext)is_accelerate_availableis_kernels_available)init_empty_weights)add_hook_to_moduleremove_hook_from_moduleDIFFUSERS_GGUF_CUDA_KERNELSfalse)1trueyes) get_kernelz Isotr0py/ggmlxqweight qweight_typereturnc|tvrt|}||jzS|tvrtj |\}}|j d|j d|z|zf}tj||g|}||j|jjz}|jSt j|}td|)Nrz$Unsupported GGUF quantization type: )UNQUANTIZED_TYPESdequantize_gguf_tensorT DEQUANT_TYPESggufGGML_QUANT_SIZESshapeopsggml_dequantizetodtypeGGMLQuantizationTypeNotImplementedError as_tensor)rrrweight block_size type_sizerys n/Volumes/fast/ai/experiments/MLX_z-image/.venv/lib/python3.12/site-packages/diffusers/quantizers/gguf/utils.py_fused_mul_mat_ggufr*Os(('0688|}$ $ 5 5l C Iq!7==#3y#@:#MN$$WlCUC  !''"$$ $ ;;=00> !$H"WXXc$ttj|jj}|j }i}t j|j}|jD]}||jvs||||<|di|}|S)a Creates a new hook based on the old hook. Use it only if you know what you are doing ! This method is a copy of: https://github.com/huggingface/peft/blob/748f7968f3a31ec06a1c2b0328993319ad9a150a/src/peft/utils/other.py#L245 with some changes ) getattr acceleratehooks __class____name____dict__inspect signature__init__keys parameters)old_hook old_hook_cls old_hook_attrfiltered_old_hook_attrold_hook_init_signatureknew_hooks r)_create_accelerate_new_hookr@ns :++X-?-?-H-HIL%%M%// 0E0EF    ! '22 2(5a(8 "1 %"545H Or+cHd}t|j}|sy|jD]\}}||zdz} t|||| |t |t j s8||| sB||vsGtrtnt} | 5t|j|j|jdu||j|<dddt||j|_|j|j#d|S#1swYHxYw)Nc>|dz}||vxrt||tS)Nr%) isinstance GGUFParameter) state_dictprefix weight_keys r)_should_convert_to_ggufz:_replace_with_gguf_linear.._should_convert_to_ggufs(h& Z']Jz*7M},]]r+.) compute_dtypeF)listchildrennamed_children_replace_with_gguf_linearrDnnLinearrrr GGUFLinear in_features out_featuresbias_modulestype source_clsrequires_grad_) modelrKrFrGmodules_to_not_convertrI has_childrennamemodule module_prefixctxs r)rOrOs^()L ,,. f + !&-]Tjk vryy )' MB22(?(A${C'1&&''KKt+"/ (t$/36lENN4 + NN4 / / 6'/* Ls =DD! c|jD]W\}}t|tr||vr|jj}t |dd}t rtnt}|5tj|j|j|jdu|}dddtjt|j_|||_ t!|dr.|j"}t%|} t'|t)|| |j+|||j,|<t/|j1} | sLt3||Z|S#1swYxYw)NrU)device_hf_hook)rNrDrRr%rbr.rrrrPrQrSrTrU Parameterrhasattrrcr@r rr rVrLrM#_dequantize_gguf_and_restore_linear) rZr[r]r^rbrUr` new_moduler9r?r\s r)rfrfs2,,. f fj )d:P.P]]))F6640D(?(A${CYY&&''KKt+!  !# -CFMM-R SJ "& vz*!??6x@'/":x8 MM& !#-ENN4 FOO-.  /8N O=/@ L5s .:E..E7  c|jtjjtj}|dddf|dddfdzz|dddfdzz|dddfdzzj dS)Nrrr)viewtorchuint8r int32 unsqueeze)rs r) to_uint32rtso u{{u{{+A adGa1gl "Qq!tW] 2Qq!tW] B M Ma PPr+c|jd}t||t|z gz}tj||dS)Nrdim)rrLsumrpsplit)blocksargsn_maxdimss r)split_block_dimsr~s< LLOE :T*+ +D ;;vt ++r+c|jd}|jtj}|j |ddf}tj ||jddzd\}}}tj |dz|dz|dz d zzgd }tj |dz|dz |dz d zzgd }|j |d f|j |d ffS) Nrrrv?rl0rk)rrorprqreshaperycat)scalesn_blocksdmm_dscmins r) get_scale_minrs||AH [[ %F ^^Xq!, -F FFLL$4$9rBIAq# AHsTzqAvo>?R HB ))QXqa1f_=>B GC JJ!} %s{{Ha='A BBr+ct|d\}}|jtjj |}|jtj }||zS)Nrl)r~rorpfloat16r int8)rzr&r'r!rrs r)dequantize_blocks_Q8_0rsJ FA &DAq u}}  'A uzzA q5Lr+c|jd}t|ddd\}}}}|jtjj |}|jtjj |}t |}|j|dftjd|jtjjddz }|j|dd|dzftjddg|jtjjddddz } |dzj tj}| dzj|df} | |dzz}||z|zS) Nrrlrr rbr!rr) rr~rorprr rtrarangerbrrtensorrq) rzr&r'r!rrrqhqsqls r)dequantize_blocks_Q5_1rs>||AH#FAq!4LAq"b u}}  'A u}}  'A 2B XqM "ell2ahhekk&Z&b&bcdfh&i iB Xr1jAo6 75<< Aqxxu{{< gaAq B q&U[[ !B t)  h^ ,B rQwB Fa<r+c|jd}t|dd\}}}|jtjj |}t |}|j|dtjd|jtjjddz }|j|dd|dztjddg|jtjjddddz }|dzj tj}|dzj|d}||dzzj tjd z }||zS) Nrrlrrrrrrrm)rr~rorprr rtrrrbrrrrqr) rzr&r'r!rrrrrs r)dequantize_blocks_Q5_0rs%||AH A.IAr2 u}}  'A 2B Ha ELLAHHEKK$X$`$`abdf$g gB Hb!Z1_ 5 Aqxxu{{: gaAq B q&U[[ !B t)  Xr *B a.  UZZ (2 -B r6Mr+c|jd}t|dd\}}}|jtjj |}|jtjj |}|j |dd|dzftjddg|jtjj ddddz }|dzj |d}||z|zS)Nrrlrrrrr) rr~rorprr rrrbrq)rzr&r'r!rrrrs r)dequantize_blocks_Q4_1rs||AH1-HAq" u}}  'A u}}  'A Xr1jAo6 75<< Aqxxu{{< gaAq B t)  Xr *B Fa<r+c|jd}t|d\}}|jtjj |}|j |dd|dzftjddg|jtjj dz }|dzj |dfj tjd z }||zS) Nrrlrrrrrrrlrrrk) rr~rorprr rrrbrqr)rzr&r'r!rrrs r)dequantize_blocks_Q4_0r!s||AH VQ 'EAr u}}  'A Xr1jAo6 75<< Aqxxu{{< gl B t)  h^ , / / ;a ?B r6Mr+c|jd}t|tdztdztdz\}}}}|jtj j |}|jtjj |}||zj|tdzdf}|j|dddft jddg|jtjjd z }|d zj|dd f}|j|ddd ft jgd |jtjjd z }|dzj|dd f}||dzzj tj d z } | j|tdzdf} || zj|tfS)Nrrlrrmrr@rrrrrrlrrrrrr) rr~QK_Krorprr rrrrbrq) rzr&r'r!rrrrrqs r)dequantize_blocks_Q6_Kr.s||AH DAItrzB    [[ $ ' ' .F u}}  'A Vh A67A Xr1b) *ellAq6!((Z_ZeZe.f.n.n/ B t)  hB/ 0B Xr1b) *ellllRmm}}G ||AH VQ 'EAr u}}  'A Xr1jAo6 75<< Av}}EKK< gl B r'  Hb> * - -ekk :Bll1a$G b B gnnRXXa["((1+rBAr JB B  5 !B r6Mr+c tjgdtj|j}|jd}t |ddt dz\}}}} |jtjj|}|jtj}|j|ddftjddg|jtj jd z }|j|ddftjtt d zD cgc]} d| z c} |jtj jd z }|j|dfd z}|j|dfdz}||dzzd z } || j|zj|ddf} tjddg|jtj jdddd} | j|dddf| z } | d zj|dd fjtj} |jdddd}| jd} tj |j#| jd| jd| jddd| } | j%dj|} | | zj|dScc} w)Nrrrrlrrrrr)rrrlr)rrrrrrm)rprrrbrr~rrorr rrrqrangerrsrrr)rzr&r'r!rrrscales_hscales_lrirrshifts_qs r)dequantize_blocks_IQ4_XSrsllRmm}}G ||AH 0Atrz JAx2 u}}  'A}}U[[)H2q 12ell Av}}EKK7 giH1b 12elldbj)*)1Q)*6== 7 gjH2/$6H2/$6H(a-(B .F fii  ' '2q(9 :B||QF6== LTTUVXY[\^_`H Xr1b) *h 6B r'  Hb"- . 1 1%++ >Bll1aB'G b B gnnRXXa["((1+rxx{BOQRTV WB B  5 !B G  Xr ** +s5 L c(g|dd|d|z|zS)Nrr-)rr'r&s r)_quant_shape_from_byte_shapers& =U3BZ =ri/*< ==r+crt|ds|S|j}t|}t|\}}|j }|j t j}t|j||}|j|z}|j||f}||||}|j|}|S)N quant_type) rerdequantize_functionsrr$rorprqrrnumelr) rr dequant_fnr&r'rrrzdequants r)rrs 6< ( ""J%j1J,Z8J    F [[ %F (y* ME||~*H ^^Xy1 2FY7Gooe$G Nr+cHeZdZddZdZedZedfd ZxZ S)rEc||ntjd}tjj|||}||_t |\}}t |j|||_|SNr) rpemptyTensor_make_subclassrrrr quant_shape)clsdata requires_gradrselfr&r's r)__new__zGGUFParameter.__new__s^'tU[[^||**3mD$ 0 < I7 IzZ r+cttjjtj||jSN)rprrr)rs r)r$zGGUFParameter.as_tensor s%||**5<<t?Q?QRRr+c|D]T}t|tr$t|dtr|djcSt|tsH|jcSyr)rDrLrEr)r{args r)_extract_quant_typez!GGUFParameter._extract_quant_type#sN C#t$CFM)J1v(((#}-~~%  r+c|i}t |||||}t|tjr|j |}|||St |ttfvrU|j |}|Dcgc](}t|tjr |||n|*}}t ||S|Scc}w)N)r) super__torch_function__rDrprrrWrLtuple) rfunctypesr{kwargsresultrrwrappedr1s r)rz GGUFParameter.__torch_function__/s >F+D%vF fell +006Jv*5 5 &\dE] *006Jdjkdj_` 1ell8Ss14YZZdjGk4<( (Mls9-B;)FN)r-N) r2 __module__ __qualname__rr$ staticmethodr classmethodr __classcell__r1s@r)rErEs6S  r+rEceZdZ d dfd Zdej fdZdej fdZdej fdZxZ S)rRcFt|||||||_||_yr)rr6rKrb)rrSrTrUrKrbr1s r)r6zGGUFLinear.__init__Ds' lD&A* r+inputsct3|jjr|jr|j|S|j |Sr)rr%is_cuda forward_cudaforward_native)rrs r)forwardzGGUFLinear.forwardPs; ?t{{22v~~$$V, ,""6**r+c"t|j}|j|j}|j%|jj|jnd}t j jj|||}|Sr) rr%r rKrUrprP functionallinear)rrr%rUoutputs r)rzGGUFLinear.forward_nativeUsj' 44--.37993Htyy||D../d$$++FFDA r+c|jj}t|j|j|j|}|j (||j j|jz }|Sr)r%rr*r rKrU)rrrr s r)rzGGUFLinear.forward_cuda]s_[[++ $VYYt/A/A%BDKKQ[\ 99 diill4#5#56 6F r+)FNN)rN) r2rrr6rprrrrrrs@r)rRrRCsM    +ell+ U\\5<<r+rRr)[r4os contextlibrrrptorch.nnrPutilsrrr/raccelerate.hooksrr getenvlowercuda is_availableget_device_capabilitycan_use_cuda_kernelskernelsrrr"F32F16BF16rQ4_0Q4_1Q5_0Q5_1Q8_0Q8_1STANDARD_QUANT_TYPESQ2_KQ3_KQ4_KQ5_KQ6_K KQUANT_TYPESIQ1_MIQ1_SIQ2_XXSIQ2_XSIQ2_SIQ3_XXSIQ3_SIQ4_XSIQ4_NLIMATRIX_QUANT_TYPESrMMVQ_QUANT_TYPESMMQ_QUANT_TYPESrintr*r@rOrfrrrtr~rrrrrrrrrrrrrrrrrLr7SUPPORTED_GGUF_QUANT_TYPESrrrdrErQrRr-r+r)r8sE " B-LBII+W5;;=AUU3 !3 ((*1-2 02" _ %C C..22D4M4M4Q4QSWSlSlSqSqr""""""""""""  """"""""""   ####%%$$##%%##$$$$ %|36II ',69LL&55<<%,,cV[VbVb>"HJbdBGI!R  Q , C&&  -8243(.B&(P(+<(($$&>$$&>""$:""$:""$:""$:""$:""$:""$:""$:""$:""$:""$:""6";";"=>>0*EHH&&*Zr+