
    i)                        d dl mZ d dlmZ d dlmZmZ d dlZd dlmZm	Z	 d dl
mZ d dlmZ d dlmZ  G d	 d
e	j                         Zy)    )annotations)Iterable)AnyLiteralN)Tensornn)PreTrainedTokenizerBase)StaticEmbedding)SentenceTransformerc                  h     e Zd Z	 	 	 d	 	 	 	 	 	 	 	 	 	 	 d fdZddZd	dZd
dZedd       Z xZ	S )GISTEmbedLossc                   t         |           || _        || _        || _        t        j                  d      | _        t        |d      rt        |d      st        d      t        |j                  t              rt        |j                  t              st        d      |j                  j                         |j                  j                         k7  xs |j                  |j                  k  | _        | j                  rC| j                  j                  | _        t        | j                  d   t               rt        d      |dvrt        d	      || _        || _        y
)a|  
        This loss is used to train a SentenceTransformer model using the GISTEmbed algorithm.
        It takes a model and a guide model as input, and uses the guide model to guide the
        in-batch negative sample selection. The cosine similarity is used to compute the loss
        and the temperature parameter is used to scale the cosine similarities.

        You can apply different false-negative filtering strategies to discard hard negatives that are too similar to
        the positive. Two strategies are supported:

            - "absolute": Discards negatives whose similarity score is greater than or equal to ``positive_score - margin``.
            - "relative": Discards negatives whose similarity score is greater than or equal to ``positive_score * (1 - margin)``.

        Args:
            model: SentenceTransformer model based on a `transformers` model.
            guide: SentenceTransformer model to guide the in-batch negative sample selection.
            temperature: Temperature parameter to scale the cosine similarities.
            margin_strategy: Strategy used for false negative filtering. One of {"absolute", "relative"}.
            margin: The margin value for filtering negatives. Defaults to 0.0, together with the "absolute" strategy,
                this only removes negatives that are more similar to the query than the positive is to the query.

        References:
            - For further details, see: https://arxiv.org/abs/2402.16829

        Requirements:
            1. (anchor, positive, negative) triplets
            2. (anchor, positive) pairs

        Inputs:
            +---------------------------------------+--------+
            | Texts                                 | Labels |
            +=======================================+========+
            | (anchor, positive, negative) triplets | none   |
            +---------------------------------------+--------+
            | (anchor, positive) pairs              | none   |
            +---------------------------------------+--------+

        Recommendations:
            - Use ``BatchSamplers.NO_DUPLICATES`` (:class:`docs <sentence_transformers.training_args.BatchSamplers>`) to
              ensure that no in-batch negatives are duplicates of the anchor or positive samples.

        Relations:
            - :class:`MultipleNegativesRankingLoss` is similar to this loss, but it does not use
              a guide model to guide the in-batch negative sample selection. `GISTEmbedLoss` yields
              a stronger training signal at the cost of some training overhead.

        Example:
            ::

                from sentence_transformers import SentenceTransformer, SentenceTransformerTrainer, losses
                from datasets import Dataset

                model = SentenceTransformer("microsoft/mpnet-base")
                guide = SentenceTransformer("all-MiniLM-L6-v2")
                train_dataset = Dataset.from_dict({
                    "anchor": ["It's nice weather outside today.", "He drove to work."],
                    "positive": ["It's so sunny.", "He took the car to the office."],
                })
                loss = losses.GISTEmbedLoss(model, guide)

                trainer = SentenceTransformerTrainer(
                    model=model,
                    train_dataset=train_dataset,
                    loss=loss,
                )
                trainer.train()
        dim	tokenizerzNBoth the training model and the guiding model must have a tokenizer attribute.z_Both the training model and the guiding model must use a PreTrainedTokenizer from transformers.r   zIf we must retokenize because the guide model has a different tokenizer, then the Sentence Transformer model must not be based on a StaticEmbedding.)absoluterelativez1margin_strategy must be 'absolute' or 'relative'.N)super__init__modelguidetemperaturer   CosineSimilaritysimilarity_fcthasattr
ValueError
isinstancer   r	   	get_vocabmax_seq_lengthmust_retokenizer
   margin_strategymargin)selfr   r   r   r"   r#   	__class__s         h/var/www/html/eduruby.in/venv/lib/python3.12/site-packages/sentence_transformers/losses/GISTEmbedLoss.pyr   zGISTEmbedLoss.__init__   s6   T 	

& 11b9uk*'%2Mmnn%//+BC:OO4L
 q  OO%%'5??+D+D+FFu%J^J^afauauJu 	 !ZZ11DN$**Q-9 b 
 "::PQQ.    c                b    | j                  |j                  d      |j                  d            S )N   r   )r   	unsqueeze)r$   embed1embed2s      r&   
sim_matrixzGISTEmbedLoss.sim_matrixw   s+    ""6#3#3A#68H8H8KLLr'   c                    |D cg c]  } j                  |      d    }}t        j                         5   j                  r|D cg c]"  } j                  j                  |d   d      $ }}|D cg c]  } j                  j                  |       }}|D cg c]I  }|j                         D ci c]+  \  }}||j                   j                  j                        - c}}K }}}}|D cg c]  } j                  |      d    }	}d d d        d }
d }t        |      dk(  r|\  }}	\  }}n2t        |      dk(  r|\  }}}
	\  }}}nt        dt        |              j                  ||      } j                  ||      } j                  ||      } j                  ||      } j                  ||      } j                  ||      }|j                         j                  dd	      dd fd
}t        j                   |j"                  t        j$                  |j                  d} ||||      } |||      } |||      }|||g}|
> j                  ||
      } j                  ||      } |||      }|j'                  |       t        j(                  |d	       j*                  z  }t        j,                  |j/                  d            j1                         j                  |j                        } t3        j4                         ||      S c c}w c c}w c c}w c c}}w c c}}}w c c}w # 1 sw Y   2xY w)Nsentence_embedding	input_idsT)skip_special_tokens      z Expected 2 or 3 embeddings, got r   r)   c                    j                   dk(  r| j                  z
  kD  }n$j                   dk(  r| dj                  z
  z  kD  }|| z  }t        j                   |<   |S )Nr   r   r)   )r"   r#   torchinf)guided_sim_matsim_matpositive_maskmask
guided_simr$   s       r&   mask_false_negativesz3GISTEmbedLoss.forward.<locals>.mask_false_negatives   so    ##z1%dkk)AB%%3%q4;;)GH(}n,"YYJGDMNr'   )dtypedevice)r9   r   r   )N)r9   zTensor | None)r   r5   no_gradr!   r   batch_decoder   tokenizeitemstor>   lenr   r-   diagonalvieweyeshapeboolappendcatr   arangesizelongr   CrossEntropyLoss)r$   sentence_featureslabelssentence_feature
embeddingsdecoded	sentenceskeyvalueguide_embeddingsnegativenegative_guideanchorpositiveanchor_guidepositive_guideap_simaa_simpp_simguided_ap_simguided_aa_simguided_pp_simr<   r9   scoresan_simguided_an_simr;   s   `                          @r&   forwardzGISTEmbedLoss.forwardz   s>   arsM]djj!123GHs
s]]_ 	## ->( NN//0@0Mcg/h  V]$]	TZZ%8%8%C$]!$] ->% %( IYH^H^H`a*#uS%((4::#4#455a%! % \m GW

+,-AB   	  z?a)FH+;(L._!)3&FHh;K8L..?J?PQRR 2084 nElCG #++-222q9
	 		=#6#6ejjQ^QeQef &mV=Y%mV<%mV<&&) __VX6F OOL.IM)-@FMM&!6q)D,<,<< fkk!n-22477F$r""$VV44c t %^a%
 	 	s^   L"M	'L'0M6"L,M L7
80L1(L7
.M6L>M'
M1L7
7MMc                `    | j                   | j                  | j                  | j                  dS )Nr   r   r"   r#   rj   r$   s    r&   get_config_dictzGISTEmbedLoss.get_config_dict   s,    ZZ++#33kk	
 	
r'   c                     y)Na  
@misc{solatorio2024gistembed,
    title={GISTEmbed: Guided In-sample Selection of Training Negatives for Text Embedding Fine-tuning},
    author={Aivin V. Solatorio},
    year={2024},
    eprint={2402.16829},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}
 rk   s    r&   citationzGISTEmbedLoss.citation   s    	r'   )g{Gz?r   g        )r   r   r   r   r   floatr"   zLiteral['absolute', 'relative']r#   rp   returnNone)r+   r   r,   r   rq   r   )rP   zIterable[dict[str, Tensor]]rQ   r   rq   r   )rq   zdict[str, Any])rq   str)
__name__
__module____qualname__r   r-   rh   rl   propertyro   __classcell__)r%   s   @r&   r   r      s{    
 ";Ef"f #f 	f
 9f f 
fPMR5h
 
 
r'   r   )
__future__r   collections.abcr   typingr   r   r5   r   r   transformersr	   sentence_transformers.modelsr
   )sentence_transformers.SentenceTransformerr   Moduler   rn   r'   r&   <module>r      s/    " $    0 8 ISBII Sr'   