iSddlmZddlZddlZddlZddlmZddlmZm Z m Z ddl Z ddl m Z mZejeZerddlmZmZ ddlmZ ddlmZmZ d dd Z d dd Z d dd Z d dd Zy#e$rYuwxYw#e$rYwwxYw)) annotationsN)Path) TYPE_CHECKINGCallableLiteral)disable_datasets_cachingis_datasets_available CrossEncoderSentenceTransformer)OVQuantizationConfig)OptimizationConfigQuantizationConfigc Fddlm}m} ddlm}m} m} ddlm} t||xr#t|xrt|j|} t||xrt|j| } | s | s td| r |j}n |j}| j|tt r0| j"vr tdxst%| d t'fd d |||d | y#t$r tdwxYw)a Export an optimized ONNX model from a SentenceTransformer or CrossEncoder model. The O1-O4 optimization levels are defined by Optimum and are documented here: https://huggingface.co/docs/optimum/main/en/onnxruntime/usage_guides/optimization The optimization levels are: - O1: basic general optimizations. - O2: basic and extended general optimizations, transformers-specific fusions. - O3: same as O2 with GELU approximation. - O4: same as O3 with mixed precision (fp16, GPU-only) See the following pages for more information & benchmarks: - `Sentence Transformer > Usage > Speeding up Inference `_ - `Cross Encoder > Usage > Speeding up Inference `_ Args: model (SentenceTransformer | CrossEncoder): The SentenceTransformer or CrossEncoder model to be optimized. Must be loaded with `backend="onnx"`. optimization_config (OptimizationConfig | Literal["O1", "O2", "O3", "O4"]): The optimization configuration or level. model_name_or_path (str): The path or Hugging Face Hub repository name where the optimized model will be saved. push_to_hub (bool, optional): Whether to push the optimized model to the Hugging Face Hub. Defaults to False. create_pr (bool, optional): Whether to create a pull request when pushing to the Hugging Face Hub. Defaults to False. file_suffix (str | None, optional): The suffix to add to the optimized model file name. Defaults to None. Raises: ImportError: If the required packages `optimum` and `onnxruntime` are not installed. ValueError: If the provided model is not a valid SentenceTransformer or CrossEncoder model loaded with `backend="onnx"`. ValueError: If the provided optimization_config is not valid. Returns: None rr )ORTModelForFeatureExtraction!ORTModelForSequenceClassification ORTOptimizer)AutoOptimizationConfigPlease install Optimum and ONNX Runtime to use this function. You can install them with pip: `pip install optimum[onnxruntime]` or `pip install optimum[onnxruntime-gpu]`mThe model must be a Transformer-based SentenceTransformer or CrossEncoder model loaded with `backend="onnx"`.z\optimization_config must be an OptimizationConfig instance or one of 'O1', 'O2', 'O3', 'O4'.N optimizedc,j|SN) file_suffix)optimize)save_dirroptimization_config optimizers [/var/www/html/eduruby.in/venv/lib/python3.12/site-packages/sentence_transformers/backend.pyz-export_optimized_onnx_model..q););QRT! s:"- I   8   s D D c ddlm}m} ddlm}m} m} ddlm} t||xr#t|xrt|j|} t||xrt|j| } | s | s td| r |j}n |j}| j|tt rTdvr tdd d }t#| d xs)j$j&j)d |'j$j&j)d t+fdd|||d| y #t$r tdwxYw)a Export a quantized ONNX model from a SentenceTransformer or CrossEncoder model. This function applies dynamic quantization, i.e. without a calibration dataset. Each of the default quantization configurations quantize the model to int8, allowing for faster inference on CPUs, but are likely slower on GPUs. See the following pages for more information & benchmarks: - `Sentence Transformer > Usage > Speeding up Inference `_ - `Cross Encoder > Usage > Speeding up Inference `_ Args: model (SentenceTransformer | CrossEncoder): The SentenceTransformer or CrossEncoder model to be quantized. Must be loaded with `backend="onnx"`. quantization_config (QuantizationConfig): The quantization configuration. model_name_or_path (str): The path or Hugging Face Hub repository name where the quantized model will be saved. push_to_hub (bool, optional): Whether to push the quantized model to the Hugging Face Hub. Defaults to False. create_pr (bool, optional): Whether to create a pull request when pushing to the Hugging Face Hub. Defaults to False. file_suffix (str | None, optional): The suffix to add to the quantized model file name. Defaults to None. Raises: ImportError: If the required packages `optimum` and `onnxruntime` are not installed. ValueError: If the provided model is not a valid SentenceTransformer or CrossEncoder model loaded with `backend="onnx"`. ValueError: If the provided quantization_config is not valid. Returns: None rr )rr ORTQuantizer)AutoQuantizationConfigrr)arm64avx2avx512 avx512_vnnizqquantization_config must be an QuantizationConfig instance or one of 'arm64', 'avx2', 'avx512', or 'avx512_vnni'.NF) is_static_ _quantizedc,j|Srquantize)rrquantization_config quantizers rr z5export_dynamic_quantized_onnx_model..r!r"#export_dynamic_quantized_onnx_modelr$r%)r.r r r/rrr?r0r@r1r2r3r4r-r5r6r7r9 weights_dtypenamelowerr:)r-rKr)r*r+rr r rrr?r@r;r<r=quantization_config_namerLs ` ` @rrMrM}sJH uuL 5-. O J O u//1M N ! 5t*U[[Rs:tO  {  272J2J 7<{{ ,,Y7I%s+ &P PD $7q#9 Rg&<>QR]bc!s(;(I(I(N(N(T(T(V'WWXYqXr%s ,::??EEGH S sB"- K   8   s EE)c Dddlm} m}  ddlm} m} m}m}m}ts tdt| xr#txrtj| }t| xrtj|}|s |s td||}|r j}n j}| ||j!|t#d |||fDr!t%d |||fDs td fd ||nd }||nd}||nd}ndt'5j)||fd| |j*nd|dddt-fdd||||| d y#t$r tdwxYw#1swY;xYw)a+ Export a quantized OpenVINO model from a SentenceTransformer or CrossEncoder model. This function applies Post-Training Static Quantization (PTQ) using a calibration dataset, which calibrates quantization constants without requiring model retraining. Each default quantization configuration converts the model to int8 precision, enabling faster inference while maintaining accuracy. See the following pages for more information & benchmarks: - `Sentence Transformer > Usage > Speeding up Inference `_ - `Cross Encoder > Usage > Speeding up Inference `_ Args: model (SentenceTransformer | CrossEncoder): The SentenceTransformer or CrossEncoder model to be quantized. Must be loaded with `backend="openvino"`. quantization_config (OVQuantizationConfig | dict | None): The quantization configuration. If None, default values are used. model_name_or_path (str): The path or Hugging Face Hub repository name where the quantized model will be saved. dataset_name(str, optional): The name of the dataset to load for calibration. If not specified, the `sst2` subset of the `glue` dataset will be used by default. dataset_config_name (str, optional): The specific configuration of the dataset to load. dataset_split (str, optional): The split of the dataset to load (e.g., 'train', 'test'). Defaults to None. column_name (str, optional): The column name in the dataset to use for calibration. Defaults to None. push_to_hub (bool, optional): Whether to push the quantized model to the Hugging Face Hub. Defaults to False. create_pr (bool, optional): Whether to create a pull request when pushing to the Hugging Face Hub. Defaults to False. file_suffix (str, optional): The suffix to add to the quantized model file name. Defaults to `qint8_quantized`. Raises: ImportError: If the required packages `optimum` and `openvino` are not installed. ValueError: If the provided model is not a valid SentenceTransformer or CrossEncoder model loaded with `backend="openvino"`. ValueError: If the provided quantization_config is not valid. Returns: None rr )OVConfigOVModelForFeatureExtraction OVModelForSequenceClassificationr OVQuantizerzPlease install datasets, optimum-intel and openvino to use this function. You can install them with pip: `pip install datasets optimum[openvino]`zaPlease install datasets to use this function. You can install it with pip: `pip install datasets`zqThe model must be a Transformer-based SentenceTransformer or CrossEncoder model loaded with `backend="openvino"`.N)rKc3$K|]}|du ywN.0params r z9export_static_quantized_openvino_model..0s j5  jc3$K|]}|du ywrXrYrZs rr]z9export_static_quantized_openvino_model..0sw#Twr^zEither specify all of `dataset_name`, `dataset_config_name`, `dataset_split`, and `column_name`, or leave them all unspecified.c.j|dddS)N max_lengthiT)paddingra truncation) tokenizer)examplesr-s rpreprocess_functionzCexport_static_quantized_openvino_model..preprocess_function7sx#Z^__r"gluesst2trainsentencec|SrXrY)re column_namerfs rr z8export_static_quantized_openvino_model..Bs1DXkEZ1[r"i,) dataset_namedataset_config_namerf num_samples dataset_splitc,j|S)N)save_directory ov_configrI)rcalibration_datasetrsrLs rr z8export_static_quantized_openvino_model..Hs);); I*<* r"&export_static_quantized_openvino_modelopenvinor%)r.r r optimum.intelrSrTrUr rVr1r r2r3r4r-r5r6anyallrget_calibration_datasetror:)r-rKr)rmrnrprlr*r+rr r rSrTrUr rVr;r<ov_modelrtrsrfrLs` ` @@@@rrurus \H     ! " o  5-. N J N u//1L M ! 5s*U[[Rr:sO    "24050H0H5:[[-@AI++H5I j<9Lm]h*i jjsvw(46I=Ze'fwt N  `$0#;<L1D1P-V\%2%>MGM!,!8+jK ! # '??% 3 [;N;Z+77`c' @   F"- m   V   Z  sE>3)F>FFc ddlm} m} |dk(rd|d} n |dk(rd|d} tj5} || |dk(ret | |z } t j| d z |  z t j| d z | | z jd | j} |dk(rZt | |z } | jd d t |  z }| | z }t j||| j} |rd}|rt|jddjddjdd}| t|| rd|d|d|d|d|d d }n!t|| rd|d|d|d|d|d d }tj| ||dd|d  ||!nt ||z } | jd d t |  z }| | z }t j |||dk(rPt | | z jd }t | | z jd }t j ||dddy#1swYyxYw)"Nrr r$model_z.onnxrvopenvino_model_z.xmlzopenvino_model.xmlzopenvino_model.binz.binT)parentsexist_ok(z( z, z, )z )zGHello! *This pull request has been automatically generated from the [`zT`](https://sbert.net/docs/package_reference/util.html#sentence_transformers.backend.zI) function from the Sentence Transformers library.* ## Config ```python a ``` ## Tip: Consider testing this pull request before merging by loading the model from this PR with the `revision` argument: ```python from sentence_transformers import SentenceTransformer # TODO: Fill in the PR number pr_number = 2 model = SentenceTransformer( "z5", revision=f"refs/pr/{pr_number}", backend="z#", model_kwargs={"file_name": "a "}, ) # Verify that everything works as expected embeddings = model.encode(["The weather is lovely today.", "It's so sunny outside!", "He drove to the stadium."]) print(embeddings.shape) similarities = model.similarity(embeddings, embeddings) print(similarities) ``` a ``` ## Tip: Consider testing this pull request before merging by loading the model from this PR with the `revision` argument: ```python from sentence_transformers import CrossEncoder # TODO: Fill in the PR number pr_number = 2 model = CrossEncoder( "a "}, ) # Verify that everything works as expected query = "Which planet is known as the Red Planet?" passages = [ "Venus is often called Earth's twin because of its similar size and proximity.", "Mars, known for its reddish appearance, is often referred to as the Red Planet.", "Jupiter, the largest planet in our solar system, has a prominent red spot.", "Saturn, famous for its rings, is sometimes mistaken for the Red Planet." ] scores = model.predict([(query, passage) for passage in passages]) print(scores) ``` r-z Add exported z model ) folder_path path_in_reporepo_id repo_typecommit_messagecommit_descriptionr+)r.r r tempfileTemporaryDirectoryrshutilmove with_suffixas_posixmkdirreprreplacer2huggingface_hub upload_foldercopy)r&r'r(r)r*r+rr,r-r r file_namerdst_dirsource destinationropt_config_string bin_sourcebin_destinations rr:r:VsH&[M/ J %k]$7  $ $ &x9(! j H~/H KK#77I9M N KK#77(Y:N9[9[\b9c d((*H f 8nw.G MM$M 6(^i/F!I-K KK ,'')H !# $(L$8$8g$F$N$NtU\$]$e$efikp$q!=Ju6I$J.@AU?UVjkj@@ Y""+ -+*&@ |4.@AU?UVjkj@@ Y""+-+$*&L  ) )$$*!!.wiwymL#5# -.8G MM$M 6(^i/F!I-K KK ,*$"8ny8EEfM #'=9#<"I"I&"Q J8qx9x9x9s HII )FFN)r-"SentenceTransformer | CrossEncoderrz4OptimizationConfig | Literal['O1', 'O2', 'O3', 'O4']r)r7r*boolr+rr str | NonereturnNone)r-rrKzFQuantizationConfig | Literal['arm64', 'avx2', 'avx512', 'avx512_vnni']r)r7r*rr+rrrrr)NNNNFFqint8_quantized)r-rrKz"OVQuantizationConfig | dict | Noner)r7rmrrnrrprrlrr*rr+rrr7rr)FFNr$N)r&rr'r7r)r7r*rr+rrrr,r7r-z)SentenceTransformer | CrossEncoder | None) __future__rloggingrrpathlibrtypingrrrrsentence_transformers.utilrr getLogger__name__loggerr.r r rwr r1r0rrr#rMrur:rYr"rrs$" 33V   8 $G 6 \" ^ -^M^^ ^  ^  ^ ^J" Y -Y_YY Y  Y  Y Y@ $&* $"(z -z;zz z $ z  zzzzz zD"7;J9J9J9 J9  J9  J9J9J9 5J9E        s$ CC CC CC