o
    áiÑ  ã                   @   s„   d dl mZmZmZ d dlmZ 	 d dlmZ d dlm	Z
mZ e
e_	ee_dgZG dd„ deƒZG dd„ deƒZG d	d
„ d
eƒZdS )é    )Úabsolute_importÚdivisionÚunicode_literals©Ústr)Úurllib)ÚparseÚrequestÚRobotFileParserc                   @   sZ   e Zd ZdZddd„Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ ZdS )r
   zs This class provides a set of methods to read, parse and answer
    questions about a single robots.txt file.

    Ú c                 C   s,   g | _ d | _d| _d| _|  |¡ d| _d S )NFr   )ÚentriesÚdefault_entryÚdisallow_allÚ	allow_allÚset_urlÚlast_checked©ÚselfÚurl© r   úr/var/www/html/eduruby.in/lip-sync/lip-sync-env/lib/python3.10/site-packages/future/backports/urllib/robotparser.pyÚ__init__   s   

zRobotFileParser.__init__c                 C   s   | j S )z·Returns the time the robots.txt file was last fetched.

        This is useful for long-running web spiders that need to
        check for new robots.txt files periodically.

        )r   ©r   r   r   r   Úmtime&   s   zRobotFileParser.mtimec                 C   s   ddl }|  ¡ | _dS )zYSets the time the robots.txt file was last fetched to the
        current time.

        r   N)Útimer   )r   r   r   r   r   Úmodified/   s   zRobotFileParser.modifiedc                 C   s&   || _ tj |¡dd… \| _| _dS )z,Sets the URL referring to a robots.txt file.é   é   N)r   r   r   ÚurlparseÚhostÚpathr   r   r   r   r   7   s    zRobotFileParser.set_urlc              
   C   sœ   z	t j | j¡}W n4 t jjy= } z&|jdv rd| _n|jdkr+d| _W Y d}~dS W Y d}~dS W Y d}~dS d}~ww | 	¡ }|  
| d¡ ¡ ¡ dS )z4Reads the robots.txt URL and feeds it to the parser.)i‘  i“  Ti  Nzutf-8)r   r	   Úurlopenr   ÚerrorÚ	HTTPErrorÚcoder   r   Úreadr   ÚdecodeÚ
splitlines)r   ÚfÚerrÚrawr   r   r   r%   <   s   

ÿÿ€þzRobotFileParser.readc                 C   s2   d|j v r| jd u r|| _d S d S | j |¡ d S ©NÚ*)Ú
useragentsr   r   Úappend)r   Úentryr   r   r   Ú
_add_entryI   s
   


þzRobotFileParser._add_entryc                 C   sj  d}t ƒ }|D ] }|s#|dkrt ƒ }d}n|dkr#|  |¡ t ƒ }d}| d¡}|dkr2|d|… }| ¡ }|s9q| dd¡}t|ƒdkr§|d  ¡  ¡ |d< tj 	|d  ¡ ¡|d< |d dkrx|dkrm|  |¡ t ƒ }|j
 |d ¡ d}q|d dkr|dkr|j t|d d	ƒ¡ d}q|d d
kr§|dkr§|j t|d dƒ¡ d}q|dkr³|  |¡ dS dS )z”Parse the input lines from a robots.txt file.

        We allow that a user-agent: line is not preceded by
        one or more blank lines.
        r   r   é   ú#Nú:z
user-agentZdisallowFZallowT)ÚEntryr0   ÚfindÚstripÚsplitÚlenÚlowerr   r   Úunquoter-   r.   Ú	rulelinesÚRuleLine)r   ÚlinesÚstater/   ÚlineÚir   r   r   r   R   sP   



€€ÿzRobotFileParser.parsec                 C   s–   | j rdS | jr
dS tj tj |¡¡}tj dd|j|j|j	|j
f¡}tj |¡}|s.d}| jD ]}| |¡r?| |¡  S q1| jrI| j |¡S dS )z=using the parsed robots.txt decide if useragent can fetch urlFTr   ú/)r   r   r   r   r   r:   Ú
urlunparser    ÚparamsÚqueryÚfragmentÚquoter   Ú
applies_toÚ	allowancer   )r   Ú	useragentr   Ú
parsed_urlr/   r   r   r   Ú	can_fetch…   s$   ÿ

ÿzRobotFileParser.can_fetchc                 C   s   d  dd„ | jD ƒ¡S )Nr   c                 S   s   g | ]}t |ƒd  ‘qS )Ú
r   )Ú.0r/   r   r   r   Ú
<listcomp>   s    z+RobotFileParser.__str__.<locals>.<listcomp>)Újoinr   r   r   r   r   Ú__str__œ   ó   zRobotFileParser.__str__N)r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r%   r0   r   rK   rP   r   r   r   r   r
      s    
		3c                   @   s(   e Zd ZdZdd„ Zdd„ Zdd„ ZdS )	r<   zoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
       (allowance==False) followed by a path.c                 C   s(   |dkr|sd}t j |¡| _|| _d S )Nr   T)r   r   rF   r    rH   )r   r    rH   r   r   r   r   £   s   
zRuleLine.__init__c                 C   s   | j dkp
| | j ¡S r+   )r    Ú
startswith)r   Úfilenamer   r   r   rG   ª   rQ   zRuleLine.applies_toc                 C   s   | j rdpdd | j S )NZAllowZDisallowz: )rH   r    r   r   r   r   rP   ­   s   zRuleLine.__str__N)rR   rS   rT   rU   r   rG   rP   r   r   r   r   r<       s
    r<   c                   @   s0   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
S )r4   z?An entry has one or more user-agents and zero or more rulelinesc                 C   s   g | _ g | _d S )N)r-   r;   r   r   r   r   r   ³   s   
zEntry.__init__c                 C   sH   g }| j D ]
}| d|dg¡ q| jD ]}| t|ƒdg¡ qd |¡S )NzUser-agent: rL   r   )r-   Úextendr;   r   rO   )r   ÚretÚagentr?   r   r   r   rP   ·   s   


zEntry.__str__c                 C   sF   |  d¡d  ¡ }| jD ]}|dkr dS | ¡ }||v r  dS qdS )z2check if this entry applies to the specified agentrA   r   r,   TF)r7   r9   r-   )r   rI   rZ   r   r   r   rG   ¿   s   
ÿzEntry.applies_toc                 C   s$   | j D ]}| |¡r|j  S qdS )zZPreconditions:
        - our agent applies to this entry
        - filename is URL decodedT)r;   rG   rH   )r   rW   r?   r   r   r   rH   Ì   s
   


ÿzEntry.allowanceN)rR   rS   rT   rU   r   rP   rG   rH   r   r   r   r   r4   ±   s    r4   N)Ú
__future__r   r   r   Zfuture.builtinsr   Zfuture.backportsr   Zfuture.backports.urllibr   Ú_parser	   Ú_requestÚ__all__Úobjectr
   r<   r4   r   r   r   r   Ú<module>   s     	