U /j2@sddlmZddlmZddlmZddlmZddlm Z m Z m Z m Z ddl mZmZddlmZmZmZGd d d ZGd d d Ze eefZe eZGd ddZdS)) annotations)aliases)dumps)sub)AnyIteratorListTuple)RE_POSSIBLE_ENCODING_INDICATIONTOO_BIG_SEQUENCE) iana_nameis_multi_byte_encoding unicode_rangec @seZdZdCddddddddd d Zd dd d dZd dd ddZeddddZddddZddddZ ddd ddZ eddddZ eddddZ eddd d!Z eddd"d#Zeddd$d%Zeddd&d'Zeddd(d)Zeddd*d+Zeddd,d-Zeddd.d/Zeddd0d1Zed2dd3d4Zeddd5d6Zeddd7d8Zeddd9d:ZdDddd?Zed@ddAdBZdS)E CharsetMatchNzbytes | bytearraystrfloatboolCoherenceMatches str | None)payloadguessed_encodingmean_mess_ratiohas_sig_or_bom languagesdecoded_payloadpreemptive_declarationcCsL||_||_||_||_||_d|_g|_d|_d|_d|_ ||_ ||_ dS)N) _payload _encoding_mean_mess_ratio _languages_has_sig_or_bom_unicode_ranges_leavesZ_mean_coherence_ratio_output_payload_output_encoding_string_preemptive_declaration)selfrrrrrrrr*=/tmp/pip-unpacked-wheel-s9saxz3h/charset_normalizer/models.py__init__ s zCharsetMatch.__init__object)otherreturncCs>t|ts&t|tr"t||jkSdS|j|jko<|j|jkS)NF) isinstancerrr encoding fingerprintr)r.r*r*r+__eq__)s   zCharsetMatch.__eq__cCst|tstt|j|j}t|j|j}|dkrJ|dkrJ|j|jkS|dkr|dkrt|jtkrt|j|jkS|j |j kS|j|jkS)zQ Implemented to make sorted available upon CharsetMatches items. g{Gzt?g{Gz?) r0r ValueErrorabschaos coherencelenrr multi_byte_usage)r)r.Zchaos_differenceZcoherence_differencer*r*r+__lt__0s    zCharsetMatch.__lt__r/cCsdtt|t|jS)Ng?)r9rrawr)r*r*r+r:FszCharsetMatch.multi_byte_usagecCsV|jdkrPt|j|jd|_|jrP|jdkrP|jrP|jddkrP|jdd|_|jS)Nstrictutf_7rur )r'rrrr"r>r*r*r+__str__Js  zCharsetMatch.__str__cCsd|jd|jdS)Nz)r1r2r>r*r*r+__repr__ZszCharsetMatch.__repr__NonecCs8t|tr||kr"td|jd|_|j|dS)Nz;Unable to add instance <{}> as a submatch of a CharsetMatch)r0rr5format __class__r'r$appendr3r*r*r+ add_submatch]szCharsetMatch.add_submatchcCs|jSN)rr>r*r*r+r1hszCharsetMatch.encoding list[str]cCsDg}tD]2\}}|j|kr*||q |j|kr ||q |S)z Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855. )ritemsr1rF)r)Z also_known_asupr*r*r+encoding_aliasesls    zCharsetMatch.encoding_aliasescCs|jSrHr"r>r*r*r+bomyszCharsetMatch.bomcCs|jSrHrNr>r*r*r+byte_order_mark}szCharsetMatch.byte_order_markcCsdd|jDS)z Return the complete list of possible languages found in decoded sequence. Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'. cSsg|] }|dqS)rr*).0er*r*r+ sz*CharsetMatch.languages..r!r>r*r*r+rszCharsetMatch.languagescCsp|jsbd|jkrdSddlm}m}t|jr8||jn||j}t|dksVd|krZdS|dS|jddS)z Most probable language found in decoded sequence. If none were detected or inferred, the property will return "Unknown". asciiZEnglishr)encoding_languagesmb_encoding_languagesz Latin BasedUnknown)r!could_be_from_charsetZcharset_normalizer.cdrVrWrr1r9)r)rVrWrr*r*r+languages  zCharsetMatch.languagecCs|jSrH)r r>r*r*r+r7szCharsetMatch.chaoscCs|js dS|jddS)Nrrr rTr>r*r*r+r8szCharsetMatch.coherencecCst|jdddSNd)ndigits)roundr7r>r*r*r+ percent_chaosszCharsetMatch.percent_chaoscCst|jdddSr[)r_r8r>r*r*r+percent_coherenceszCharsetMatch.percent_coherencecCs|jS)z+ Original untouched bytes. )rr>r*r*r+r=szCharsetMatch.rawzlist[CharsetMatch]cCs|jSrH)r$r>r*r*r+submatchszCharsetMatch.submatchcCst|jdkSNr)r9r$r>r*r*r+ has_submatchszCharsetMatch.has_submatchcCs@|jdk r|jSddt|D}ttdd|D|_|jS)NcSsg|] }t|qSr*)r)rQcharr*r*r+rSsz*CharsetMatch.alphabets..cSsh|] }|r|qSr*r*)rQrr*r*r+ sz)CharsetMatch.alphabets..)r#rsortedlist)r)Zdetected_rangesr*r*r+ alphabetss  zCharsetMatch.alphabetscCs|jgdd|jDS)z The complete list of encoding that output the exact SAME str result and therefore could be the originating encoding. This list does include the encoding available in property 'encoding'. cSsg|] }|jqSr*)r1)rQmr*r*r+rSsz6CharsetMatch.could_be_from_charset..)rr$r>r*r*r+rYsz"CharsetMatch.could_be_from_charsetutf_8bytes)r1r/cs~jdksj|krx|_t}jdk rjjdkrjttfdd|dddd}||dd}||d_jS) z Method to get re-encoded bytes payload using given target encoding. Default to UTF-8. Any errors will be simply ignored by the encoder NOT replaced. N)zutf-8utf8rlcs<|j|d|d|dtjddS)Nrr _-)stringspanreplacegroupsr r&)rkr>r*r+s z%CharsetMatch.output..i r )countrs)r&rr(lowerrr encoder%)r)r1decoded_stringZpatched_headerr*r>r+outputs$   zCharsetMatch.outputintcCs tt|S)z] Retrieve a hash fingerprint of the decoded payload, used for deduplication. )hashrr>r*r*r+r2szCharsetMatch.fingerprint)NN)rl)__name__ __module__ __qualname__r,r4r;propertyr:rArBrGr1rMrOrPrrZr7r8r`rar=rbrdrjrYrzr2r*r*r*r+r sV   rc@seZdZdZdddddZddd d Zd d d ddZddddZddddZd dd ddZ ddddZ ddddZ dS)CharsetMatchesz Container with every CharsetMatch items ordered by default from most probable to the less one. Act like a list(iterable) but does not implements all related methods. Nzlist[CharsetMatch] | None)resultscCs|r t|ng|_dSrH)rh_results)r)rr*r*r+r,szCharsetMatches.__init__zIterator[CharsetMatch]r<ccs|jEdHdSrHrr>r*r*r+__iter__szCharsetMatches.__iter__z int | strr)itemr/cCsNt|tr|j|St|trFt|d}|jD]}||jkr.|Sq.tdS)z Retrieve a single item either by its position or encoding name (alias may be used here). Raise KeyError upon invalid index or encoding not present in results. FN)r0r{rrr rYKeyError)r)rresultr*r*r+ __getitem__ s       zCharsetMatches.__getitem__r{cCs t|jSrHr9rr>r*r*r+__len__szCharsetMatches.__len__rcCst|jdkSrcrr>r*r*r+__bool__szCharsetMatches.__bool__rCcCs|t|tstdt|jt|jtkr`|j D],}|j |j kr2|j |j kr2| |dSq2|j |t|j |_ dS)z~ Insert a single match. Will be inserted accordingly to preserve sort. Can be inserted as a submatch. z-Cannot append instance '{}' to CharsetMatchesN)r0rr5rDrrEr9r=r rr2r7rGrFrh)r)rmatchr*r*r+rFs    zCharsetMatches.appendzCharsetMatch | NonecCs|js dS|jdS)zQ Simply return the first match. Strict equivalent to matches[0]. Nrrr>r*r*r+best2szCharsetMatches.bestcCs|S)zP Redundant method, call the method best(). Kept for BC reasons. )rr>r*r*r+first:szCharsetMatches.first)N) r}r~r__doc__r,rrrrrFrrr*r*r*r+rsrc @sNeZdZdddddddddddd ddZed d d d Zdd d dZdS)CliDetectionResultrrrIrr pathr1rMalternative_encodingsrZrjrr7r8 unicode_path is_preferredc CsF||_| |_||_||_||_||_||_||_||_| |_ | |_ dSrH) rrr1rMrrZrjrr7r8r) r)rr1rMrrZrjrr7r8rrr*r*r+r,FszCliDetectionResult.__init__zdict[str, Any]r<c Cs2|j|j|j|j|j|j|j|j|j|j |j d S)Nrrr>r*r*r+__dict__`szCliDetectionResult.__dict__cCst|jdddS)NT) ensure_asciiindent)rrr>r*r*r+to_jsonpszCliDetectionResult.to_jsonN)r}r~rr,rrrr*r*r*r+rEs"rN) __future__rZencodings.aliasesrjsonrrertypingrrrr Zconstantr r utilsr rrrrrrZCoherenceMatchrrr*r*r*r+s    sC