o i%@s8ddlZddlZGdddZGdddZddgZdS)Nc@s"eZdZdZddZdddZdS) Perlunipropsa3 This class is used to read lists of characters from the Perl Unicode Properties (see http://perldoc.perl.org/perluniprops.html). The files in the perluniprop.zip are extracted using the Unicode::Tussle module from http://search.cpan.org/~bdfoy/Unicode-Tussle-1.11/lib/Unicode/Tussle.pm cCs(tjtjtd|_gd|_dS)Nz/data/perluniprops/)Close_PunctuationCurrency_SymbolIsAlnumIsAlphaIsLowerIsNIsScIsSoIsUpperLine_SeparatorNumberOpen_Punctuation Punctuation SeparatorSymbolLowercase_LetterTitlecase_LetterUppercase_LetterIsPfIsPi CJKSymbolsCJK)ospathdirnameabspath__file__datadiravailable_categoriesselfr"L/var/www/edux/Edux_v2/venv/lib/python3.10/site-packages/sacremoses/corpus.py__init__szPerluniprops.__init__Nccs<tjdd|d}td|}|dD]}|VqdS)u This module returns a list of characters from the Perl Unicode Properties. They are very useful when porting Perl tokenizers to Python. >>> from sacremoses.corpus import Perluniprops >>> pup = Perluniprops() >>> list(pup.chars('Open_Punctuation'))[:5] == [u'(', u'[', u'{', u'༺', u'༼'] True >>> list(pup.chars('Currency_Symbol'))[:5] == [u'$', u'¢', u'£', u'¤', u'¥'] True >>> pup.available_categories[:5] ['Close_Punctuation', 'Currency_Symbol', 'IsAlnum', 'IsAlpha', 'IsLower'] :return: a generator of characters given the specific unicode character category data perlunipropsz.txt sacremosesutf-8N)rrjoinpkgutilget_datadecode)r!category relative_path binary_datachr"r"r#chars.s  zPerluniprops.chars)N)__name__ __module__ __qualname____doc__r$r1r"r"r"r#rsrc@s"eZdZdZddZdddZdS) NonbreakingPrefixesz This is a class to read the nonbreaking prefixes textfiles from the Moses Machine Translation toolkit. These lists are used in the Python port of the Moses' word tokenizer. cCstjtjtd|_idddddddd d d d d ddddddddddddddddddd d!d"d#id$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;dd?d@dAdBdCdDdEdFdGdHdIdJ|_|jdKdL|jDdS)MNz/data/nonbreaking_prefixes/assameseasbengalibncatalancaczechcsgermandegreekelenglishenspanishesestonianetfinnishfifrenchfririshgagujaratiguhindihi hungarianhu icelandicisitalianitkannadakn lithuanianltlatvianlv malayalammlmanipurimnimarathimrdutchnloriyaorpunjabipapolishpl portugueseptromanianrorussianruslovaksk slovenianslswedishsvtamiltatetdtyuezh)telugutetum cantonesechinesecSsi|]}||qSr"r".0vr"r"r# xsz0NonbreakingPrefixes.__init__..) rrrrrravailable_langsupdatevaluesr r"r"r#r$Ks      !"# )zNonbreakingPrefixes.__init__N#ccs||jvrd|j|g}n|dkr ddt|jD}ndg}|D](}tjdd|}td|}|d  D]}| }|rL| |sL|Vq>> from sacremoses.corpus import NonbreakingPrefixes >>> nbp = NonbreakingPrefixes() >>> list(nbp.words('en'))[:10] == [u'A', u'B', u'C', u'D', u'E', u'F', u'G', u'H', u'I', u'J'] True >>> list(nbp.words('ta'))[:5] == ['ர', 'ூ', 'திரு', 'ஏ', 'பீ'] True :return: a generator words for the specified language(s). nonbreaking_prefix.NcSsg|]}d|qS)rr"rr"r"r# sz-NonbreakingPrefixes.words..znonbreaking_prefix.enr%nonbreaking_prefixesr'r() rsetrrrr)r*r+r, splitlinesstrip startswith)r!langignore_lines_startswith filenamesfilenamer.r/liner"r"r#wordszs$   zNonbreakingPrefixes.words)Nr)r2r3r4r5r$rr"r"r"r#r6Ds/r6)rr*rr6__all__r"r"r"r#s < X