o i?@sRdZddlmZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl mZmZmZddlmZddlmZddlmZddlmZmZmZdd lmZdd lmZdd lm Z m!Z!dd l"m#Z#dd l$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5m6Z6m7Z7e 8e9Z:ee;e;fZZ?dUd"d#Z@Gd$d%d%e>ZAdVd(d)ZBdWd*d+ZCdXd.d/ZDGd0d1d1ZEGd2d3d3eZFdYd5d6ZGeGdZd:d;ZHedd8d8ZIGd?d@d@eZJ d[d\dGdHZK httpshttpT)allow_redirectsN)urllibparseurlsplitrBheadrrA)rrCr&netlocpathqueryfragmentrespr'r'r(_ensure_api_responseZs  rPcCsxtt|jr t||dtdt||j|dgdddd}t |t |tdt||j d d |S) aYAccess an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. rCzGetting page %sz, )r;z*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z max-age=0)Acceptz Cache-Control)r<zFetched page %s as %sr9r:) rrfilenamerPloggerdebugrr=joinrrAr<)rrCrOr'r'r(_get_simple_responsels&  rWr<ResponseHeaderscCs<|rd|vrtj}|d|d<|d}|rt|SdS)z=Determine if we have any encoding information in our headers.r9z content-typecharsetN)emailmessageMessage get_paramr)r<mrYr'r'r(_get_encoding_from_headerss    r_c@s*eZdZdddZdd d Zdd dZdS)CacheablePageContentpage IndexContentr r-cCs|jsJ||_dSr.)cache_link_parsingrar1rar'r'r(r0s  zCacheablePageContent.__init__otherobjectboolcCst|t|o|jj|jjkSr.) isinstancetyperar)r1rer'r'r(__eq__szCacheablePageContent.__eq__intcCs t|jjSr.)hashrarr1r'r'r(__hash__s zCacheablePageContent.__hash__N)rarbr r-)rerfr rg)r rk)r4r5r6r0rjrnr'r'r'r(r`s  r`c@seZdZdddZdS) ParseLinksrarbr Iterable[Link]cCsdSr.r'rdr'r'r(__call__szParseLinks.__call__Nrarbr rp)r4r5r6rqr'r'r'r(rosrofncs2tjd fdd td fd d }|S)z Given a function that parses an Iterable[Link] from an IndexContent, cache the function's result (keyed by CacheablePageContent), unless the IndexContent `page` has `page.cache_link_parsing == False`. cacheable_pager`r list[Link]cst|jSr.)listra)rt)rsr'r(wrappersz*with_cached_index_content..wrapperrarbcs|jr t|St|Sr.)rcr`rv)rarsrwr'r(wrapper_wrappers  z2with_cached_index_content..wrapper_wrapperN)rtr`r ru)rarbr ru) functoolscachewraps)rsryr'rxr(with_cached_index_contents r}rarbrpc cs|j}|dr+t|j}|dgD]}t||j }|dur%q|VqdSt |j }|j p4d}| |j ||j }|jpE|}|jD]} tj| ||d}|durXqI|VqIdS)z\ Parse a Simple API's Index Content, and yield its anchor elements as Link objects. r;filesNzutf-8)page_urlbase_url)r+r#r$jsonloadscontentr=r from_jsonrHTMLLinkParserencodingfeeddecoderanchors from_element) rar@datafilelinkparserrrranchorr'r'r( parse_linkss*       rT)frozenc@sHeZdZUdZded<ded<ded<ded<d Zd ed <dd dZdS)rbaRepresents one response (or page), along with its URL. :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. bytesrrr+r!rrTrgrcr cCs t|jSr.)rrrmr'r'r(__str__s zIndexContent.__str__N)r r)r4r5r6__doc____annotations__rcrr'r'r'r(rbs   cs6eZdZdZdfdd Zdd d ZdddZZS)rzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. rrr r-cs$tjdd||_d|_g|_dS)NT)convert_charrefs)r/r0rrr)r1rr2r'r(r0s zHTMLLinkParser.__init__tagattrslist[tuple[str, str | None]]cCsR|dkr|jdur||}|dur||_dSdS|dkr'|jt|dSdS)Nbasea)rget_hrefrappenddict)r1rrhrefr'r'r(handle_starttags  zHTMLLinkParser.handle_starttagr!cCs"|D] \}}|dkr|SqdS)Nrr')r1rnamevaluer'r'r(r!s zHTMLLinkParser.get_href)rrr r-)rrrrr r-)rrr r!)r4r5r6rr0rrr7r'r'r2r(r s  rrrreasonstr | ExceptionmethCallable[..., None] | NonecCs|durtj}|d||dS)Nz%Could not fetch URL %s: %s - skipping)rTrU)rrrr'r'r(_handle_get_simple_fail(srrcrgcCs&t|j}t|j|jd||j|dS)Nr9)rrrc)r_r<rbrr)r8rcrr'r'r(_make_index_content2s rIndexContent | Nonec Cs|jddd}t|}|rtd||dS|dr;tjt |r;| ds.|d7}t j |d}td|zt||d }WntyStd |YdStyo}ztd ||j|jWYd}~dSd}~wty}z t||WYd}~dSd}~wty}z t||WYd}~dSd}~wty}zd }|t|7}t||tjd WYd}~dSd}~wtjy}zt|d|WYd}~dSd}~wtjyt|dYdSwt||jdS)N#rrzICannot look at %s URL %s because it does not support lookup as web pages.zfile:/z index.htmlz# file: URL is directory, getting %srQz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: z timed out)rc)rsplitr)rTwarningr$osrLisdirrendswithrGrHurljoinrUrWrBr*r,r+rrrrrinfor ConnectionErrorTimeoutrrc)rrCr vcs_schemerOexcrr'r'r(_get_index_content?sl       rc@seZdZUded<ded<dS)CollectedSourceszSequence[LinkSource | None] find_links index_urlsN)r4r5r6rr'r'r'r(r{s  rc@sNeZdZdZd!dd Ze d"d#ddZed$ddZd%ddZ d&ddZ d S)' LinkCollectorz Responsible for collecting Link objects from all configured locations, making network requests as needed. The class's main method is its collect_sources() method. rCr search_scoperr r-cCs||_||_dSr.)rrC)r1rCrr'r'r(r0s zLinkCollector.__init__Foptionsrsuppress_no_indexrgcCsd|jg|j}|jr|stdddd|Dg}|jp g}tj|||jd}t ||d}|S)z :param session: The Session to use to make requests. :param suppress_no_index: Whether to ignore the --no-index option when constructing the SearchScope object. zIgnoring indexes: %s,css|]}t|VqdSr.r).0rr'r'r( sz'LinkCollector.create..)rrno_index)rCr) index_urlextra_index_urlsrrTrUrVrrcreater)clsrCrrrrrlink_collectorr'r'r(rs$  zLinkCollector.create list[str]cCs|jjSr.)rrrmr'r'r(rszLinkCollector.find_linkslocationrrcCst||jdS)z> Fetch an HTML page containing package links. rQ)rrC)r1rr'r'r(fetch_responseszLinkCollector.fetch_response project_namercandidates_from_pagerrcstfddjD}tfddjD}ttj rKddt ||D}t |ddg|}t d|tt|t|d S) Nc 3(|]}t|jjdddVqdS)Frpage_validator expand_dirrcrNrrCis_secure_originrlocrrr1r'r(r  z0LinkCollector.collect_sources..c 3r)TrNrrrr'r(rrcSs*g|]}|dur|jdurd|jqS)Nz* )r)rsr'r'r( s  z1LinkCollector.collect_sources..z' location(s) to search for versions of : )rr) collections OrderedDictrget_index_urls_locationsvaluesrrT isEnabledForloggingDEBUG itertoolschainr%rUrVrrv)r1rrindex_url_sourcesfind_links_sourceslinesr'rr(collect_sourcess2        zLinkCollector.collect_sourcesN)rCrrrr r-)F)rCrrrrrgr r)r r)rrr r)rrrrr r) r4r5r6rr0 classmethodrpropertyrrrr'r'r'r(rs !  r)rrr r!)r8r r r-)rrrCrr r-)rrrCrr r )r<rXr r!)rsror rorrr.)rrrrrrr r-)T)r8r rcrgr rb)rrrCrr r)Pr __future__rr email.messagerZrzrrrr urllib.parserGcollections.abcrrr dataclassesr html.parserroptparsertypingr r r pip._vendorr Zpip._vendor.requestsr Zpip._vendor.requests.exceptionsrrpip._internal.exceptionsrpip._internal.models.linkr!pip._internal.models.search_scoperpip._internal.network.sessionrpip._internal.network.utilsrpip._internal.utils.filetypesrpip._internal.utils.miscrpip._internal.utils.urlsrpip._internal.vcsrsourcesrrr getLoggerr4rTrrXr) Exceptionr*rArBrPrWr_r`ror}rrbrrrrrrr'r'r'r(sh                      ?     <