ž ¬ÿfŠmc@sêdZddlmZdZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z mZmZddlmZdd lmZmZdd lmZdd lmZed d „ZGdd„deƒZdd„Zdd„Zdd„Zdd„Z ej!ddfkrpddl"Z"e"j#Z$dd„Z%dd„Z&ne#Z$e Z%e Z&dd„Z'Gdd „d eƒZ(Gd!d"„d"e)ƒZ*Gd#d$„d$eƒZ+Gd%d&„d&e*ƒZ,dS('uÛRefactoring framework. Used as a main program, this can refactor any number of files and/or recursively descend down directories. Imported as a module, this provides infrastructure to write your own refactoring tool. i(uwith_statementu#Guido van Rossum N(uchaini(udriverutokenizeutoken(u find_root(upytreeupygram(u btm_utils(u btm_matchercCs¨t|ggdgƒ}tjj|jƒ}g}xnttj|ƒƒD]W}|jdƒrI|jdƒrI|r†|dd…}n|j |dd…ƒqIqIW|S(uEReturn a sorted list of all available fix names in the given package.u*ufix_u.pyiNiiýÿÿÿ( u __import__uosupathudirnameu__file__usortedulistdiru startswithuendswithuappend(u fixer_pkgu remove_prefixupkgu fixer_diru fix_namesuname((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyuget_all_fix_names!suget_all_fix_namescBs|EeZdZdS(u _EveryNodeN(u__name__u __module__u __qualname__(u __locals__((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu _EveryNode.su _EveryNodecCsßt|tjtjfƒrC|jdkr3t‚nt|jgƒSt|tjƒrt|j rkt |j ƒSt‚nt|tj ƒrËtƒ}x5|j D]*}x!|D]}|j t |ƒƒq¦Wq™W|St d|ƒ‚dS(uf Accepts a pytree Pattern Node and returns a set of the pattern types which will match first. u$Oh no! I don't understand pattern %sN(u isinstanceupytreeu NodePatternu LeafPatternutypeuNoneu _EveryNodeusetuNegatedPatternucontentu_get_head_typesuWildcardPatternuupdateu Exception(upaturupux((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu_get_head_types2s      u_get_head_typesc Cstjtƒ}g}x¤|D]œ}|jr…yt|jƒ}Wntk r_|j|ƒYq¸XxU|D]}||j|ƒqgWq|jdk r«||jj|ƒq|j|ƒqWx:t t j j j ƒt j jƒD]}||j|ƒqÞWt|ƒS(u^ Accepts a list of fixers and returns a dictionary of head node type --> fixer list. N(u collectionsu defaultdictulistupatternu_get_head_typesu _EveryNodeuappendu _accept_typeuNoneuchainupygramupython_grammaru symbol2numberuvaluesutokensuextendudict(u fixer_listu head_nodesueveryufixeruheadsu node_type((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu_get_headnode_dictNs"    u_get_headnode_dictcs ‡fdd†tˆdƒDƒS(uN Return the fully qualified names for fixers in the package pkg_name. csg|]}ˆd|‘qS(u.((u.0ufix_name(upkg_name(u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu ks u+get_fixers_from_package..F(uget_all_fix_namesuFalse(upkg_name((upkg_nameu5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyuget_fixers_from_packagegsuget_fixers_from_packagecCs|S(N((uobj((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu _identitynsu _identityicCs|jddƒS(Nu u (ureplace(uinput((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu_from_system_newlinesusu_from_system_newlinescCs*tjdkr"|jdtjƒS|SdS(Nu (uosulinesepureplace(uinput((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu_to_system_newlineswsu_to_system_newlinesc sÝd}tjtj|ƒjƒ‰‡fdd†}ttjtj tj fƒ}t ƒ}ydx]|ƒ\}}||krq`q`|tj kr£|ršPnd }q`|tjkr¹|dkr¹|ƒ\}}|tjksè|dkrìPn|ƒ\}}|tjks|dkrPn|ƒ\}}|tjkrV|dkrV|ƒ\}}nxa|tjkrµ|j|ƒ|ƒ\}}|tjksŸ|dkr£Pn|ƒ\}}qYWq`Pq`Wntk rÒYnXt|ƒS( Ncstˆƒ}|d|dfS(Nii(unext(utok(ugen(u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyuadvance…s u(_detect_future_features..advanceufromu __future__uimportu(u,FT(uFalseutokenizeugenerate_tokensuiouStringIOureadlineu frozensetutokenuNEWLINEuNLuCOMMENTusetuSTRINGuTrueuNAMEuOPuaddu StopIteration(usourceuhave_docstringuadvanceuignoreufeaturesutpuvalue((ugenu5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu_detect_future_features‚sD     u_detect_future_featurescBs|EeZdZdZdS(u FixerErroruA fixer could not be loaded.N(u__name__u __module__u __qualname__u__doc__(u __locals__((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu FixerErrorªsu FixerErrorcBso|EeZdZid4d6d4d6ZdZdZd3d3dd„Zdd„Z d d „Z d d „Z d d„Z dd„Z d4d4dd„Zd4d4dd„Zdd„Zd4d4dd„Zdd„Zd4dd„Zdd„Zdd „Zd3d4d3d!d"„Zd3d#d$„Zd%Zd&Zd'd(„Zd)d*„Zd+d,„Zd-d.„Zd/d0„Zd1d2„Zd3S(5uRefactoringTooluprint_functionuwrite_unchanged_filesuFixufix_cCsÄ||_|pg|_|jjƒ|_|dk rI|jj|ƒn|jdretj|_ n tj |_ |jj dƒ|_ g|_ tjdƒ|_g|_d|_tj|j dtjd|jƒ|_|jƒ\|_|_g|_tjƒ|_g|_g|_ x}t!|j|jƒD]f}|j"rT|jj#|ƒq2||jkrv|jj$|ƒq2||jkr2|j j$|ƒq2q2Wt%|jƒ|_&t%|j ƒ|_'dS(uÒInitializer. Args: fixer_names: a list of fixers to import options: an dict with configuration. explicit: a list of fixers to run even if they are explicit. uprint_functionuwrite_unchanged_filesuRefactoringTooluconvertuloggerNF((ufixersuexplicitu_default_optionsucopyuoptionsuNoneuupdateupygramu!python_grammar_no_print_statementugrammarupython_grammarugetuwrite_unchanged_filesuerrorsuloggingu getLoggeruloggeru fixer_loguFalseuwroteudriveruDriverupytreeuconvertu get_fixersu pre_orderu post_orderufilesubmu BottomMatcheruBMu bmi_pre_orderubmi_post_orderuchainu BM_compatibleu add_fixeruappendu_get_headnode_dictubmi_pre_order_headsubmi_post_order_heads(uselfu fixer_namesuoptionsuexplicitufixer((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu__init__¶s<            uRefactoringTool.__init__c CsØg}g}x|jD]…}t|iidgƒ}|jddƒd}|j|jƒrx|t|jƒd…}n|jdƒ}|jdjdd„|Dƒƒ}yt ||ƒ}Wn(t k rçt d ||fƒ‚YnX||j |j ƒ} | jr:|jdk r:||jkr:|jd |ƒqn|jd |ƒ| jd kri|j| ƒq| jd krˆ|j| ƒqt d| jƒ‚qWtjdƒ} |jd| ƒ|jd| ƒ||fS(uInspects the options to load the requested patterns and handlers. Returns: (pre_order, post_order), where pre_order is the list of fixers that want a pre-order AST traversal, and post_order is the list that want post-order traversal. u*u.iNu_ucSsg|]}|jƒ‘qS((utitle(u.0up((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu ús u.RefactoringTool.get_fixers..uCan't find %s.%suSkipping implicit fixer: %suAdding transformation: %supreupostuIllegal fixer order: %ru run_orderukeyiÿÿÿÿT(ufixersu __import__ursplitu startswithu FILE_PREFIXulenusplitu CLASS_PREFIXujoinugetattruAttributeErroru FixerErroruoptionsu fixer_loguexplicituTrueu log_messageu log_debuguorderuappenduoperatoru attrgetterusort( uselfupre_order_fixersupost_order_fixersu fix_mod_pathumodufix_nameupartsu class_nameu fix_classufixerukey_func((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu get_fixersês8# uRefactoringTool.get_fixerscOs‚dS(uCalled when an error occurs.N((uselfumsguargsukwds((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu log_errorsuRefactoringTool.log_errorcGs'|r||}n|jj|ƒdS(uHook to log a message.N(uloggeruinfo(uselfumsguargs((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu log_messages uRefactoringTool.log_messagecGs'|r||}n|jj|ƒdS(N(uloggerudebug(uselfumsguargs((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu log_debugs uRefactoringTool.log_debugcCsdS(uTCalled with the old version, new version, and filename of a refactored file.N((uselfuold_textunew_textufilenameuequal((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu print_output!suRefactoringTool.print_outputcCsPxI|D]A}tjj|ƒr5|j|||ƒq|j|||ƒqWdS(u)Refactor a list of files and directories.N(uosupathuisdiru refactor_diru refactor_file(uselfuitemsuwriteu doctests_onlyu dir_or_file((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyurefactor&s uRefactoringTool.refactorc CsÝtjd}xÉtj|ƒD]¸\}}}|jd|ƒ|jƒ|jƒxe|D]]}|jdƒ rWtjj|ƒd|krWtjj||ƒ} |j | ||ƒqWqWWdd„|Dƒ|dd…As u0RefactoringTool.refactor_dir..N( uosuextsepuwalku log_debugusortu startswithupathusplitextujoinu refactor_file( uselfudir_nameuwriteu doctests_onlyupy_extudirpathudirnamesu filenamesunameufullname((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu refactor_dir/s    uRefactoringTool.refactor_dircCs²yt|dƒ}Wn<tk rQ}z|jd||ƒdSWYdd}~XnXztj|jƒd}Wd|jƒXt|dd|ƒ}t |j ƒƒ|fSWdQXdS(uG Do our best to decode a Python source file correctly. urbuCan't open %s: %sNiuruencoding(NN( uopenuIOErroru log_erroruNoneutokenizeudetect_encodingureadlineucloseu_open_with_encodingu_from_system_newlinesuread(uselfufilenameufuerruencoding((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu_read_python_sourceCs u#RefactoringTool._read_python_sourcec Cs|j|ƒ\}}|dkr%dS|d7}|r›|jd|ƒ|j||ƒ}|jsl||krˆ|j|||||ƒq|jd|ƒni|j||ƒ}|jsÅ|rô|jrô|jt|ƒdd …|d|d|ƒn|jd|ƒdS( uRefactors a file.Nu uRefactoring doctests in %suNo doctest changes in %siuwriteuencodinguNo changes in %siÿÿÿÿ( u_read_python_sourceuNoneu log_debugurefactor_docstringuwrite_unchanged_filesuprocessed_fileurefactor_stringu was_changedustr(uselfufilenameuwriteu doctests_onlyuinputuencodinguoutpututree((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu refactor_fileSs  uRefactoringTool.refactor_filecCsÌt|ƒ}d|kr*tj|j_nzby|jj|ƒ}WnEtk rŠ}z%|jd||jj |ƒdSWYdd}~XnXWd|j|j_X||_ |j d|ƒ|j ||ƒ|S(uFRefactor a given input string. Args: data: a string holding the code to be refactored. name: a human-readable name for use in error/log messages. Returns: An AST corresponding to the refactored input stream; None if there were errors during the parse. uprint_functionuCan't parse %s: %s: %sNuRefactoring %s( u_detect_future_featuresupygramu!python_grammar_no_print_statementudriverugrammaru parse_stringu Exceptionu log_erroru __class__u__name__ufuture_featuresu log_debugu refactor_tree(uselfudataunameufeaturesutreeuerr((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyurefactor_stringjs    uRefactoringTool.refactor_stringcCsÆtjjƒ}|ro|jdƒ|j|dƒ}|jsI||kr_|j|d|ƒqÂ|jdƒnS|j|dƒ}|js™|rµ|jrµ|jt |ƒd|ƒn |jdƒdS(NuRefactoring doctests in stdinuuNo doctest changes in stdinuNo changes in stdin( usysustdinureadu log_debugurefactor_docstringuwrite_unchanged_filesuprocessed_fileurefactor_stringu was_changedustr(uselfu doctests_onlyuinputuoutpututree((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyurefactor_stdin…s uRefactoringTool.refactor_stdinc Csœx-t|j|jƒD]}|j||ƒqW|j|j|jƒƒ|j|j|jƒƒ|jj|j ƒƒ}xèt |j ƒƒrdxÏ|jj D]Á}||krœ||rœ||j dtjjddƒ|jrþ||j dtjjƒnx\t||ƒD]G}|||kr9||j|ƒnyt|ƒWntk r^wYnX|jr}||jkr}qn|j|ƒ}|r|j||ƒ}|dk rV|j|ƒx9|jƒD]+}|jsåg|_n|jj|ƒqÊW|jj|j ƒƒ}x?|D]4} | |kr7g|| >> u... c Csg}d}d}d}d}x4|jddƒD] }|d7}|jƒj|jƒr»|dk r‡|j|j||||ƒƒn|}|g}|j|jƒ} |d| …}q1|dk r |j||j ƒsú|||j j ƒdkr |j |ƒq1|dk r8|j|j||||ƒƒnd}d}|j |ƒq1W|dk rƒ|j|j||||ƒƒndj |ƒS(uËRefactors a docstring, looking for doctests. This returns a modified version of the input string. It looks for doctests, which start with a ">>>" prompt, and may be continued with "..." prompts, as long as the "..." is indented the same as the ">>>". (Unfortunately we can't use the doctest module's parser, since, like most parsers, it is not geared towards preserving the original source.) iukeependsiNu uT( uNoneu splitlinesuTrueulstripu startswithuPS1uextendurefactor_doctestufinduPS2urstripuappendujoin( uselfuinputufilenameuresultublocku block_linenouindentulinenoulineui((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyurefactor_docstring(s:       u"RefactoringTool.refactor_docstringc syˆj||ˆƒ}WnŠtk r¥}zjˆjjtjƒrpx*|D]}ˆjd|jdƒƒqJWnˆjd|||j j |ƒ|SWYdd}~XnXˆj ||ƒr‰t |ƒj dd ƒ}|d|d…||dd…} }| dg|dkst| ƒ‚|d jdƒs@|d d7>>" (possibly indented), while the remaining lines start with "..." (identically indented). u Source: %su u+Can't parse docstring in %s line %s: %s: %sNukeependsiics!g|]}ˆˆj|‘qS((uPS2(u.0uline(uindentuself(u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu ms u4RefactoringTool.refactor_doctest..Tiÿÿÿÿiÿÿÿÿ(u parse_blocku Exceptionuloggeru isEnabledForulogginguDEBUGu log_debugurstripu log_erroru __class__u__name__u refactor_treeustru splitlinesuTrueuAssertionErroruendswithuPS1upop( uselfublockulinenouindentufilenameutreeuerrulineunewuclipped((uindentuselfu5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyurefactor_doctestSs&   )#&u RefactoringTool.refactor_doctestcCs|jrd}nd}|js4|jd|ƒn1|jd|ƒx|jD]}|j|ƒqNW|jrŸ|jdƒx!|jD]}|j|ƒq…Wn|jrt|jƒdkrÍ|jdƒn|jdt|jƒƒx0|jD]"\}}}|j|||ŽqðWndS( Nuwereu need to beuNo files %s modified.uFiles that %s modified:u$Warnings/messages while refactoring:iuThere was 1 error:uThere were %d errors:(uwroteufilesu log_messageu fixer_loguerrorsulen(uselfuwereufileumessageumsguargsukwds((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu summarizeps$      uRefactoringTool.summarizecCs1|jj|j|||ƒƒ}tƒ|_|S(u³Parses a block into a tree. This is necessary to get correct line number / offset information in the parser diagnostics and embedded into the parse tree. (udriveru parse_tokensu wrap_toksu frozensetufuture_features(uselfublockulinenouindentutree((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu parse_block‡s! uRefactoringTool.parse_blockc csŠtj|j||ƒjƒ}xe|D]]\}}\}}\} } } ||d7}| |d7} ||||f| | f| fVq%WdS(u;Wraps a tokenize stream to systematically modify start/end.iN(utokenizeugenerate_tokensu gen_linesu__next__( uselfublockulinenouindentutokensutypeuvalueuline0ucol0uline1ucol1u line_text((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu wrap_toks‘s (uRefactoringTool.wrap_toksccs¡||j}||j}|}xo|D]g}|j|ƒrT|t|ƒd…Vn4||jƒdkrrdVntd||fƒ‚|}q'WxdVq•dS(u–Generates lines as expected by tokenize from a list of lines. This strips the first len(indent + self.PS1) characters off each line. Nu uline=%r, prefix=%ru(uPS1uPS2u startswithulenurstripuAssertionError(uselfublockuindentuprefix1uprefix2uprefixuline((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu gen_linesŸs    uRefactoringTool.gen_linesNF( u__name__u __module__u __qualname__uFalseu_default_optionsu CLASS_PREFIXu FILE_PREFIXuNoneu__init__u get_fixersu log_erroru log_messageu log_debugu print_outputurefactoru refactor_diru_read_python_sourceu refactor_fileurefactor_stringurefactor_stdinu refactor_treeu traverse_byuprocessed_fileu write_fileuPS1uPS2urefactor_docstringurefactor_doctestu summarizeu parse_blocku wrap_toksu gen_lines(u __locals__((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyuRefactoringTool®s:  4 (        O  +   uRefactoringToolcBs|EeZdZdS(uMultiprocessingUnsupportedN(u__name__u __module__u __qualname__(u __locals__((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyuMultiprocessingUnsupported³suMultiprocessingUnsupportedcse|EeZdZ‡fdd†Zd d d‡fdd†Z‡fdd†Z‡fdd †Z‡S( uMultiprocessRefactoringToolcs/tt|ƒj||Žd|_d|_dS(N(usuperuMultiprocessRefactoringToolu__init__uNoneuqueueu output_lock(uselfuargsukwargs(u __class__(u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu__init__¹s u$MultiprocessRefactoringTool.__init__ics[|dkr(ttˆƒj|||ƒSyddl‰Wntk rRt‚YnXˆjdk rqtdƒ‚nˆj ƒˆ_ˆj ƒˆ_ ‡‡fdd†t |ƒDƒ}z;x|D]}|j ƒq»Wttˆƒj|||ƒWdˆjjƒx$t |ƒD]}ˆjjdƒq Wx'|D]}|jƒr*|jƒq*q*Wdˆ_XdS(Niiu already doing multiple processescs%g|]}ˆjdˆjƒ‘qS(utarget(uProcessu_child(u.0ui(umultiprocessinguself(u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu Ës u8MultiprocessRefactoringTool.refactor..(usuperuMultiprocessRefactoringToolurefactorumultiprocessingu ImportErroruMultiprocessingUnsupporteduqueueuNoneu RuntimeErroru JoinableQueueuLocku output_lockurangeustartujoinuputuis_alive(uselfuitemsuwriteu doctests_onlyu num_processesu processesupui(u __class__(umultiprocessinguselfu5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyurefactor¾s2         u$MultiprocessRefactoringTool.refactorc so|jjƒ}xY|dk rj|\}}ztt|ƒj||ŽWd|jjƒX|jjƒ}qWdS(N(uqueueugetuNoneusuperuMultiprocessRefactoringToolu refactor_fileu task_done(uselfutaskuargsukwargs(u __class__(u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu_childÛs u"MultiprocessRefactoringTool._childcsE|jdk r(|jj||fƒntt|ƒj||ŽSdS(N(uqueueuNoneuputusuperuMultiprocessRefactoringToolu refactor_file(uselfuargsukwargs(u __class__(u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu refactor_fileæsu)MultiprocessRefactoringTool.refactor_fileF(u__name__u __module__u __qualname__u__init__uFalseurefactoru_childu refactor_file(u __locals__((u __class__u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyuMultiprocessRefactoringTool·s  uMultiprocessRefactoringTool(-u__doc__u __future__uwith_statementu __author__uosusysulogginguoperatoru collectionsuiou itertoolsuchainupgen2udriverutokenizeutokenu fixer_utilu find_rootuupytreeupygramu btm_utilsubuu btm_matcherubmuTrueuget_all_fix_namesu Exceptionu _EveryNodeu_get_head_typesu_get_headnode_dictuget_fixers_from_packageu _identityu version_infoucodecsuopenu_open_with_encodingu_from_system_newlinesu_to_system_newlinesu_detect_future_featuresu FixerErroruobjectuRefactoringTooluMultiprocessingUnsupporteduMultiprocessRefactoringTool(((u5/opt/alt/python33/lib64/python3.3/lib2to3/refactor.pyu sF               (ÿÿ