U
    9%e`                     @   s  d dl mZmZmZ d dlmZmZ dd Zdd Zdd Z	d	d
 Z
dd Zdd Zdd Zdd Zdd Zdd Zdd ZdddZdd Zdd Zeee	ee
eeeeeeeeed ZG d!d" d"Zed#d$d%gd&gd'd(ed)d*d+gd,gd-d.gd(ed/d0gd1d2gd3d(ed4d5d6gd7gd8d(ed9d:d;gd&d,gd<d(ed=d>gd7gd?d(ed@gd1gdAd(edBdCdDgd,d2gdEd(edFdGdHgd&gdId(edJdKdLgd2dMidNd(edOgd7gdPd(gZedQdQgd&gdRd(edSdTgg dUd(gedVdWdXgd,gdYdZgd(ed[d\d]gg d^d^gd(ged@d_gd1gd`d(edadbgd2gdEd(gedcddgd7gded(edfdggg dhd(gedidjgd&gdkd(edldmgd,gdnd(gedodpgd7gdqd(edrdrgg dsd(edtdugg dvd(ged@gd1gdAd(gedBdCdDgd,d2gdEd(gedwdxgd&gdsd(edyd]gg dzd(ed{d|gg d}d(ed~dgg dd(gedJdKdLgd2dMidNd(gedcddgd7gded(eddgg dd(ggZdd ZdddZdedddZdddZdddZdddZdS )   )BASE_PYTHON_TOOLSclean_code_for_chatclean_code_for_run)InterpretorErrorevaluatec                 C   s   d|  d| dS )NzThis is the classification of z along . )textlabelsr   r   `/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/transformers/tools/evaluate_agent.py
classifier   s    r   c                 C   s   d|  d| d| dS )NzThis is the translation of  from z to r   r   )r	   Zsrc_langZtgt_langr   r   r   
translator   s    r   c                 C   s   d|  dS )Nz!This is actually a sound reading r   r   r	   r   r   r   speaker   s    r   c                 C   s$   d| krt d|  dd|  dS )Nsoundz	`audio` (z) is not a sound.z"This is the transcribed text from r   
ValueError)Zaudior   r   r   transcriber"   s    r   c                 C   s   d|  dS )Nz'This is actually an image representing r   r   )promptr   r   r   image_generator(   s    r   c                 C   s$   d| krt d|  dd|  dS )Nimage	`image` () is not an image.zThis is a description of r   r   )r   r   r   r   image_captioner,   s    r   c                 C   s*   d| krt d|  dd|  d| dS )Nr   r   r   zThis is a transformation of z according to r   r   r   r   r   r   r   image_transformer2   s    r   c                 C   s   d| d|  dS )NThis is the answer to r   r   r   )r	   questionr   r   r   question_answerer8   s    r   c                 C   s*   d| krt d|  dd| d|  dS )Nr   r   r   r   r   r   r   r   r   r   r   r   image_qa<   s    r!   c                 C   s   d|  dS )NzThis is the content of r   r   )urlr   r   r   text_downloaderB   s    r#   c                 C   s   d|  dS )NzThis is a summary of r   r   r   r   r   r   
summarizerF   s    r$      c                 C   s
   d|  S )NzA video of r   )r   secondsr   r   r   video_generatorJ   s    r'   c                 C   s   d| d|  dS )Nr   z from the document r   r   r    r   r   r   document_qaN   s    r(   c                 C   s   d| d|  S )NzThis is the mask of z in r   r   r   r   r   image_segmenterR   s    r)   )Ztext_classifierr   Ztext_readerr$   r   r   r   r   Ztext_qar#   r!   r'   r(   r)   c                   @   s   e Zd ZdZdd ZdS )Problema#  
    A class regrouping all the information to solve a problem on which we will evaluate agents.

    Args:
        task (`str` ou `list[str]`):
            One or several descriptions of the task to perform. If a list, it should contain variations on the
            phrasing, but for the same task.
        inputs (`list[str]` or `dict[str, str]`):
            The inputs that will be fed to the tools. For this testing environment, only strings are accepted as
            values. Pass along a dictionary when you want to specify the values of each inputs, or just the list of
            inputs expected (the value used will be `<<input_name>>` in this case).
        answer (`str` or `list[str`]):
            The theoretical answer (or list of possible valid answers) to the problem, as code.
    c                 C   s   || _ || _|| _d S Ntaskinputsanswer)selfr-   r.   r/   r   r   r   __init__x   s    zProblem.__init__N)__name__
__module____qualname____doc__r1   r   r   r   r   r*   h   s   r*   z:Is the following `text` (in Spanish) positive or negative?zEIs the text in the variable `text` (in Spanish) positive or negative?z`Translate the following `text` from Spanish to English then tell me if its positive or negative.r	   zjtext_classifier(translator(text, src_lang="Spanish", tgt_lang="English"), labels=["positive", "negative"])r,   z+Tell me out loud what the `image` contains.z(Describe the following `image` out loud.zDFind what is in the picture stored in `image` then read it out loud.r   z#text_reader(image_captioner(image))z>text_reader(image_qa(image, question='What is in the image?'))zkGenerate an image from the text given in `text_input`. Then transform it according to the text in `prompt`.zeUse the following `text_input` to generate an image, then transform it by using the text in `prompt`.Z
text_inputr   z6image_transformer(image_generator(text_input), prompt)zTDownload the content of `url`, summarize it then generate an image from its content.z<Use a summary of the web page at `url` to generate an image.zXSummarize the content of the web page at `url`, and use the result to generate an image.r"   z1image_generator(summarizer(text_downloader(url)))zUTransform the following `image` using the prompt in `text`. The prompt is in Spanish.zNUse the text prompt in `text` (in Spanish) to transform the following `image`.z]Translate the `text` from Spanish to English then use it to transform the picture in `image`.zRimage_transformer(image, translator(text, src_lang='Spanish', tgt_lang='English'))zHDownload the content of `url`, summarize it then read it out loud to me.z+Read me a summary of the web page at `url`.z-text_reader(summarizer(text_downloader(url)))z6Generate an image from the text given in `text_input`.zimage_generator(text_input)z2Replace the beaver in the `image` by the `prompt`.z7Transform the `image` so that it contains the `prompt`.z'Use `prompt` to transform this `image`.z image_transformer(image, prompt)zmProvide me the summary of the `text`, then read it to me before transcribing it and translating it in French.zXSummarize `text`, read it out loud then transcribe the audio and translate it in French.zYRead me a summary of the the `text` out loud. Transcribe this and translate it in French.z]translator(transcriber(text_reader(summarizer(text))), src_lang='English', tgt_lang='French')z Generate a video of the `prompt`zAnimate a `prompt`z%Make me a short video using `prompt`.zA lobster swimmingz%video_generator('A lobster swimming')zDownload the following file `url`, summarize it in a few words and generate a video from it.Fetch the file at this `url`, summarize it, and create an animation out of it.z1video_generator(summarizer(text_downloader(url)))z7Translate the following `text` from Spanish to English.zHtranslated_text=translator(text, src_lang='Spanish', tgt_lang='English')zIs it positive or negative?z$Tell me if its positive or negative.zAtext_classifier(translated_text, labels=['positive', 'negative'])zWhat does this `image` contain?zDescribe the following `image`.z-Find what is in the picture stored in `image`z"description=image_captioner(image)z=description=image_qa(image, question='What is in the image?')z#Now, read the description out loud.z Great! Can you read it out loud?zRead it out loud.zaudio=text_reader(description)z3Use the following `text_input` to generate an imagez#image = image_generator(text_input)z/Transform it according to the text in `prompt`.z+Transform it by using the text in `prompt`.z/Download the content of `url` and summarize it.z/Summarize the content of the web page at `url`.z*summary = summarizer(text_downloader(url))z#Generate an image from its content.z-Use the previous result to generate an image.zimage_generator(summary)z)Translate this Spanish `text` in English.z-Translate the `text` from Spanish to English.zJtranslated_text = translator(text, src_lang='Spanish', tgt_lang='English')z<Transform the following `image` using the translated `text`.z;Use the previous result to transform the following `image`.z)image_transformer(image, translated_text)zDownload the content of `url`.z&Get me the text on the weg page `url`.ztext = text_downloader(url)zSummarize this text.zsummary = summarizer(text)zRead it out loud to me.zRead me the previous result.ztext_reader(summary)z%Provide me the summary of the `text`.zSummarize `text`.zRead this summary to me.z%audio = text_reader(summarizer(text))z.Transcribing the previous result back in text.zTranscribe the audio.ztext = transcriber(audio)z&Translating the last result in French.zTranslate this in French.z7translator(text, src_lang='English', tgt_lang='French')zgenerate a video from it.z)Create an animation from the last result.zvideo_generator(summary)c                    s   t |ts fddtD S t | tr\t| D ]*\}||  kr0fddtD   S q0t| D ]&\}| |krffddtD   S qf fddtD S )Nc                    s   h | ]}| kr|qS r   r   .0namecode_answerr   r   	<setcomp>  s      z(get_theoretical_tools.<locals>.<setcomp>c                    s   h | ]}| kr|qS r   r   r6   one_coder   r   r;     s      c                    s   h | ]}| kr|qS r   r   r6   r<   r   r   r;     s      c                    s   h | ]}| d  kr|qS )    r   r6   r9   r   r   r;     s      )
isinstancelist
TEST_TOOLSdictzipvalues)agent_answertheoretical_answerr:   Z
one_answerr   )r:   r=   r   get_theoretical_tools  s    

rG   NFc           	   
   C   s   t  }t D ]\}}|| kr"q|||< qt|tr@| }n|d k	rVdd |D }|d k	rj|| n|}zt| ||W S  tk
r } zt	| W Y S d }~X Y n4 t
k
r } z|rt| W Y d S d }~X Y nX d S )Nc                 S   s   i | ]}|d | dqS )z<<z>>r   )r7   Zinpr   r   r   
<dictcomp>  s      z!evaluate_code.<locals>.<dictcomp>)r   copyrA   itemsr?   rB   updater   r   str	Exceptionprint)	coder.   stateverboseZreturn_interpretor_errorZtoolsr8   Ztooler   r   r   evaluate_code  s(    


rS   rQ   c                    s   |rt |   t tr n g |  kr:|r6t d dS t| trnt fdd|  D rn|rjt d dS |rzt d dS d S )	NzPerfect!r   c                 3   s   | ]}| kV  qd S r+   r   )r7   vrF   r   r   	<genexpr>  s     zscore_code.<locals>.<genexpr>z Almsot perfect, result in state!g      ?z.Result is not the right one but code executed.g333333?)rN   r?   r@   rB   anyrD   )rE   rF   rQ   r   rV   r   
score_code  s    
$rY   c                    s  fddt D }t|||}||kr0d}d }	n<t|| }
t|| }tddd|
  d|  }||d}	 fddt D }||krd}d }n<t|| }
t|| }tddd|
  d|  }||d}t|||d}|dk r ||d	}nd }|||f|	||ffS )
Nc                    s    h | ]}d | d  kr|qS )`r   r6   )explanationr   r   r;     s      z&evaluate_one_result.<locals>.<setcomp>g      ?r>   g      ?)Zselected_toolstheoretical_toolsc                    s   h | ]}| kr|qS r   r   r6   )rO   r   r   r;     s      rT   )Zcode_producedZ
evaluationrF   )rA   rG   lenmaxrY   )r[   rO   rE   rF   r/   rQ   Ztools_in_explanationr\   tool_selection_scoretool_selection_errorsmissing_toolsunexpected_toolsZtools_in_codetool_used_scoretool_used_errorsZscorecode_errorsr   )rO   r[   r   evaluate_one_result  s:    rf      c              
      s  t  j }|t tkrHt t| }t |t }td| d| dg }g }ttD ]N\}	}
t|
jt	r|
|
j |
|	gt|
j  qX||
j ||	 qXd}d}d}|ri }i }i }tdt||D ]z}t|| t|}||| } fdd|D } j|dgd}t|D ]*\}	}t|||	   |r`td	||	  d
||	  d t|\}}t|j|d}tjt	rfddjD }ntjj}t||||j|d\}}||d 7 }||d 7 }||d 7 }|r"|d dk	r|d |||	 < |d dk	r.|d |||	 < |d dk	r"|d |||	 < q"qd|t|  d|t|  d|t|  d}|r||||fS |S dS )z
    Evaluates a new agent on all `EVALUATION_TASKS`.

    Example:

    ```py
    agent = NewOpenAiAgent(model="text-davinci-003", api_key=your_api_key)
    bads = new_evaluate_agent(agent)
    for bad in bads:
        print(bad)
    ```
    ?Fix the test tools in the evaluate_agent module. Tools mising: . Extra tools: r   r>   c                    s   g | ]}  |qS r   )format_prompt)r7   r-   )agentr   r   
<listcomp>&  s     z"evaluate_agent.<locals>.<listcomp>zTask:stopz	====Task z====

rT   c                    s   g | ]}t | jqS r   rS   r.   )r7   r/   )problemr   r   rl   2  s     r   r%   Nd   ztool selection scoreztool used scorez
code score)settoolboxkeysrA   r   	enumerateEVALUATION_TASKSr?   r-   r@   extendr]   appendrangeminZgenerate_manyrN   r   rS   r.   r/   rf   )rk   Z
batch_sizerQ   return_errorsagent_toolsra   rb   Z
eval_tasksZeval_idxidxpbr_   rc   
code_scorer`   rd   re   Z	start_idxZend_idxZbatch_tasksZpromptsresultsresultr[   rO   rE   rF   scoreserrorsr   )rk   rq   r   evaluate_agent  sz         
r   c                    s  t | j }|t tkrHt t| }|t t }td| d| dd}d}d}d}	|rhi }
i }i }tD ] t d jtr g}n  fddt	t
 d jD }|D ]}|   i }t|d jtrdd t	t
|d jD ni }t|D ]^\}|rtj |	d7 }	| jjd	d
}| j|ddgd}|| d | _t|\}}|rxtd|  td|  t|j||d}j}t|trfddt||D }nt|j|d}t||||||d\}}||d 7 }||d 7 }||d 7 }|r|d dk	r"|d |
j< |d dk	r>|d |j< |d dk	r|d |j< qqqld||	  d||	  d||	  d}|r||
||fS |S dS )z
    Evaluates a new agent on all `EVALUATION_CHATS`.

    Example:

    ```py
    agent = NewOpenAiAgent(model="text-davinci-003", api_key=your_api_key)
    bads = new_evaluate_agent(agent)
    for bad in bads:
        print(bad)
    ```
    rh   ri   r   r>   c                    s   g | ]  fd dD qS )c                    s$   g | ]}t |j  |j|jd qS )r,   )r*   r-   r.   r/   )r7   r   ir   r   rl   w  s     z2evaluate_chat_agent.<locals>.<listcomp>.<listcomp>r   )r7   )chat_problemr   r   rl   v  s   z'evaluate_chat_agent.<locals>.<listcomp>c                 S   s   g | ]}i qS r   r   )r7   _r   r   r   rl   ~  s     r   T)Z	chat_modezHuman:z=====rm   ro   z==Explanation from the agent==
z!
==Code generated by the agent==
)rP   rQ   c                    s    g | ]\}}t | j|d qS )rP   rp   )r7   arP   )step_problemr   r   rl     s   r   rT   r%   Nrr   rs   )rt   ru   rv   rA   r   EVALUATION_CHATSr?   r-   rL   r{   r]   Zprepare_for_new_chatr/   r@   rw   rN   rj   Zgenerate_oneZchat_historyr   rS   r.   rC   rf   )rk   rQ   r}   r~   ra   rb   r_   rc   r   Ztotal_stepsr`   rd   re   Zresolved_problemsrq   Zagent_stateZtheoretical_statestepr   r   r[   rO   rE   r/   rF   r   r   r   )r   r   r   evaluate_chat_agentR  s    


.

     



r   )r%   )NNFF)F)F)rg   FF)FF) Zagentsr   r   r   Zpython_interpreterr   r   r   r   r   r   r   r   r   r   r!   r#   r$   r'   r(   r)   rA   r*   rx   r   rG   rS   boolrY   rf   r   r   r   r   r   r   <module>   s  
					c
  1

+
Y