a 5êdg½üã@sNddlmZddlmZmZmZmZmZmZmZm Z m Z m Z m Z m ZmZmZmZddlmZmZddlZddlmZmZddlmZmZddlZddlZdd„ZGd d „d eƒZGd d „d eƒZd d„Z Gdd„deƒZ!Gdd„deƒZ"Gdd„deƒZ#Gdd„deƒZ$Gdd„deƒZ%Gdd„deƒZ&Gdd„deƒZ'e(dkrJe )¡dS)é)Úsupport)ÚtokenizeÚ _tokenizeÚ untokenizeÚNUMBERÚNAMEÚOPÚSTRINGÚ ENDMARKERÚENCODINGÚtok_nameÚdetect_encodingÚopenÚ UntokenizerÚgenerate_tokensÚNEWLINE)ÚBytesIOÚStringION)ÚTestCaseÚmock)ÚVALID_UNDERSCORE_LITERALSÚINVALID_UNDERSCORE_LITERALSc Csˆg}t| ¡ƒ}|ddv}|D]b\}}}}} |tkr:q„|rT|tkrT|d|krTq t|}| d|d›d|d›d|›d|›¡q |S)Néÿÿÿÿz rú Z10ú Z13)ÚlenÚ splitlinesr rr Úappend) Ztoken_generatorZ source_stringÚresultZ num_linesZmissing_trailing_nlÚtypeÚtokenÚstartÚendÚline©r$ú./usr/local/lib/python3.9/test/test_tokenize.pyÚstringify_tokens_from_sources  (r&c@s¬eZdZdd„Zdd„Zdd„Zdd„Zd d „Zd d „Zd d„Z dd„Z dd„Z dd„Z dd„Z dd„Zdd„Zdd„Zdd„Zdd „Zd!d"„Zd#d$„Zd%d&„Zd'd(„Zd)S)*Ú TokenizeTestcCs<t| d¡ƒ}tt|jƒ|ƒ}| |dg| ¡ ¡¡dS)Núutf-8z* ENCODING 'utf-8' (0, 0) (0, 0))rÚencoder&rÚreadlineÚ assertEqualÚrstripr©ÚselfÚsÚexpectedÚfrr$r$r%Úcheck_tokenize*s ÿÿzTokenizeTest.check_tokenizecCsDtd d¡ƒ}tt|jƒƒ}| |djt¡| |djt¡dS)NÚxr(éþÿÿÿr) rr)Úlistrr*r+rrr )r.r1Útokensr$r$r%Útest_implicit_newline4sz"TokenizeTest.test_implicit_newlinecCsd| dd¡| dd¡d}t|ƒj}| td¡t|ƒD]}q ~0x15z´ NUMBER '1234567' (1, 0) (1, 7) OP '>' (1, 8) (1, 9) OP '~' (1, 10) (1, 11) NUMBER '0x15' (1, 11) (1, 15) z2134568 != 1231515zˆ NUMBER '2134568' (1, 0) (1, 7) OP '!=' (1, 8) (1, 10) NUMBER '1231515' (1, 11) (1, 18) z(-124561-1) & 200000000ac OP '(' (1, 0) (1, 1) OP '-' (1, 1) (1, 2) NUMBER '124561' (1, 2) (1, 8) OP '-' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP ')' (1, 10) (1, 11) OP '&' (1, 12) (1, 13) NUMBER '200000000' (1, 14) (1, 23) z0xdeadbeef != -1z· NUMBER '0xdeadbeef' (1, 0) (1, 10) OP '!=' (1, 11) (1, 13) OP '-' (1, 14) (1, 15) NUMBER '1' (1, 15) (1, 16) z0xdeadc0de & 12345zŠ NUMBER '0xdeadc0de' (1, 0) (1, 10) OP '&' (1, 11) (1, 12) NUMBER '12345' (1, 13) (1, 18) z0xFF & 0x15 | 1234zà NUMBER '0xFF' (1, 0) (1, 4) OP '&' (1, 5) (1, 6) NUMBER '0x15' (1, 7) (1, 11) OP '|' (1, 12) (1, 13) NUMBER '1234' (1, 14) (1, 18) ©r2©r.r$r$r%Útest_intas       zTokenizeTest.test_intcCs4| dd¡| dd¡| dd¡| dd¡dS) Nzx = 0z… NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0' (1, 4) (1, 5) zx = 0xfffffffffffzˆ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '0xfffffffffff' (1, 4) (1, 17) zx = 123141242151251616110z NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '123141242151251616110' (1, 4) (1, 25) zx = -15921590215012591z· NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '-' (1, 4) (1, 5) NUMBER '15921590215012591' (1, 5) (1, 22) r<r=r$r$r%Ú test_longšs   zTokenizeTest.test_longcCsX| dd¡| dd¡| dd¡| dd¡| d d ¡| d d ¡| d d¡dS)Nz x = 3.14159z† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14159' (1, 4) (1, 11) z x = 314159.z† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '314159.' (1, 4) (1, 11) z x = .314159z† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '.314159' (1, 4) (1, 11) z x = 3e14159z† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3e14159' (1, 4) (1, 11) z x = 3E123z… NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3E123' (1, 4) (1, 9) z x+y = 3e-1230zÜ NAME 'x' (1, 0) (1, 1) OP '+' (1, 1) (1, 2) NAME 'y' (1, 2) (1, 3) OP '=' (1, 4) (1, 5) NUMBER '3e-1230' (1, 6) (1, 13) z x = 3.14e159z† NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '3.14e159' (1, 4) (1, 12) r<r=r$r$r%Ú test_float²s      zTokenizeTest.test_floatcCsJdd„}tD]}d|vrq | ||ƒ|¡q tD]}| ||ƒ|¡q0dS)NcSs<t| d¡ƒ}t|jƒD]\}}}}}|tkr|SqdS)Nr(z invalid token)rr)rr*r)r/r1Ztoktyper r!r"r#r$r$r%Ú number_tokenÛs  z;TokenizeTest.test_underscore_literals..number_tokenú()rr+rZassertNotEqual)r.rAZlitr$r$r%Útest_underscore_literalsÚsz%TokenizeTest.test_underscore_literalscCs<| dd¡| dd¡| dd¡| dd¡| d d ¡| d d ¡| d d¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd ¡| d!d"¡| d#d$¡| d%d&¡| d'd(¡| d)d*¡| d+d,¡| d-d.¡| d/d0¡| d1d2¡| d3d4¡dS)5Nzx = ''; y = ""a5 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "''" (1, 4) (1, 6) OP ';' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '=' (1, 10) (1, 11) STRING '""' (1, 12) (1, 14) zx = '"'; y = "'"a6 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '\'"\'' (1, 4) (1, 7) OP ';' (1, 7) (1, 8) NAME 'y' (1, 9) (1, 10) OP '=' (1, 11) (1, 12) STRING '"\'"' (1, 13) (1, 16) zx = "doesn't "shrink", does it"zà NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '"doesn\'t "' (1, 4) (1, 14) NAME 'shrink' (1, 14) (1, 20) STRING '", does it"' (1, 20) (1, 31) zx = 'abc' + 'ABC'zß NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "'abc'" (1, 4) (1, 9) OP '+' (1, 10) (1, 11) STRING "'ABC'" (1, 12) (1, 17) zy = "ABC" + "ABC"zß NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING '"ABC"' (1, 4) (1, 9) OP '+' (1, 10) (1, 11) STRING '"ABC"' (1, 12) (1, 17) z%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a” NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING "r'abc'" (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING "r'ABC'" (1, 13) (1, 19) OP '+' (1, 20) (1, 21) STRING "R'ABC'" (1, 22) (1, 28) OP '+' (1, 29) (1, 30) STRING "R'ABC'" (1, 31) (1, 37) z%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a” NAME 'y' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) STRING 'r"abc"' (1, 4) (1, 10) OP '+' (1, 11) (1, 12) STRING 'r"ABC"' (1, 13) (1, 19) OP '+' (1, 20) (1, 21) STRING 'R"ABC"' (1, 22) (1, 28) OP '+' (1, 29) (1, 30) STRING 'R"ABC"' (1, 31) (1, 37) zu'abc' + U'abc'z† STRING "u'abc'" (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING "U'abc'" (1, 9) (1, 15) zu"abc" + U"abc"z† STRING 'u"abc"' (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING 'U"abc"' (1, 9) (1, 15) zb'abc' + B'abc'z† STRING "b'abc'" (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING "B'abc'" (1, 9) (1, 15) zb"abc" + B"abc"z† STRING 'b"abc"' (1, 0) (1, 6) OP '+' (1, 7) (1, 8) STRING 'B"abc"' (1, 9) (1, 15) z%br'abc' + bR'abc' + Br'abc' + BR'abc'a; STRING "br'abc'" (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING "bR'abc'" (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING "Br'abc'" (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING "BR'abc'" (1, 30) (1, 37) z%br"abc" + bR"abc" + Br"abc" + BR"abc"a; STRING 'br"abc"' (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING 'bR"abc"' (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING 'Br"abc"' (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING 'BR"abc"' (1, 30) (1, 37) z%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a; STRING "rb'abc'" (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING "rB'abc'" (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING "Rb'abc'" (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING "RB'abc'" (1, 30) (1, 37) z%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a; STRING 'rb"abc"' (1, 0) (1, 7) OP '+' (1, 8) (1, 9) STRING 'rB"abc"' (1, 10) (1, 17) OP '+' (1, 18) (1, 19) STRING 'Rb"abc"' (1, 20) (1, 27) OP '+' (1, 28) (1, 29) STRING 'RB"abc"' (1, 30) (1, 37) z "a\ de\ fg"z3 STRING '"a\\\nde\\\nfg"' (1, 0) (3, 3) zu"a\ de"z/ STRING 'u"a\\\nde"' (1, 0) (2, 3) zrb"a\ d"z/ STRING 'rb"a\\\nd"' (1, 0) (2, 2) z """a\ b"""z0 STRING '"""a\\\nb"""' (1, 0) (2, 4) z u"""a\ b"""z1 STRING 'u"""a\\\nb"""' (1, 0) (2, 4) zrb"""a\ b\ c"""z7 STRING 'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4) zf"abc"z/ STRING 'f"abc"' (1, 0) (1, 6) z fR"a{b}c"z/ STRING 'fR"a{b}c"' (1, 0) (1, 9) z f"""abc"""z0 STRING 'f"""abc"""' (1, 0) (1, 10) z f"abc\ def"z1 STRING 'f"abc\\\ndef"' (1, 0) (2, 4) z Rf"abc\ def"z2 STRING 'Rf"abc\\\ndef"' (1, 0) (2, 4) r<r=r$r$r%Ú test_stringésT        þÿÿÿÿþ   ÿÿzTokenizeTest.test_stringcCs(| dd¡| dd¡| dd¡dS)Nz!def d22(a, b, c=2, d=2, *k): passa NAME 'def' (1, 0) (1, 3) NAME 'd22' (1, 4) (1, 7) OP '(' (1, 7) (1, 8) NAME 'a' (1, 8) (1, 9) OP ',' (1, 9) (1, 10) NAME 'b' (1, 11) (1, 12) OP ',' (1, 12) (1, 13) NAME 'c' (1, 14) (1, 15) OP '=' (1, 15) (1, 16) NUMBER '2' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) NAME 'd' (1, 19) (1, 20) OP '=' (1, 20) (1, 21) NUMBER '2' (1, 21) (1, 22) OP ',' (1, 22) (1, 23) OP '*' (1, 24) (1, 25) NAME 'k' (1, 25) (1, 26) OP ')' (1, 26) (1, 27) OP ':' (1, 27) (1, 28) NAME 'pass' (1, 29) (1, 33) zdef d01v_(a=1, *k, **w): passa¢ NAME 'def' (1, 0) (1, 3) NAME 'd01v_' (1, 4) (1, 9) OP '(' (1, 9) (1, 10) NAME 'a' (1, 10) (1, 11) OP '=' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP ',' (1, 13) (1, 14) OP '*' (1, 15) (1, 16) NAME 'k' (1, 16) (1, 17) OP ',' (1, 17) (1, 18) OP '**' (1, 19) (1, 21) NAME 'w' (1, 21) (1, 22) OP ')' (1, 22) (1, 23) OP ':' (1, 23) (1, 24) NAME 'pass' (1, 25) (1, 29) z&def d23(a: str, b: int=3) -> int: passaø NAME 'def' (1, 0) (1, 3) NAME 'd23' (1, 4) (1, 7) OP '(' (1, 7) (1, 8) NAME 'a' (1, 8) (1, 9) OP ':' (1, 9) (1, 10) NAME 'str' (1, 11) (1, 14) OP ',' (1, 14) (1, 15) NAME 'b' (1, 16) (1, 17) OP ':' (1, 17) (1, 18) NAME 'int' (1, 19) (1, 22) OP '=' (1, 22) (1, 23) NUMBER '3' (1, 23) (1, 24) OP ')' (1, 24) (1, 25) OP '->' (1, 26) (1, 28) NAME 'int' (1, 29) (1, 32) OP ':' (1, 32) (1, 33) NAME 'pass' (1, 34) (1, 38) r<r=r$r$r%Ú test_functions  zTokenizeTest.test_functioncCs| dd¡dS)NzXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa› NAME 'if' (1, 0) (1, 2) NUMBER '1' (1, 3) (1, 4) OP '<' (1, 5) (1, 6) NUMBER '1' (1, 7) (1, 8) OP '>' (1, 9) (1, 10) NUMBER '1' (1, 11) (1, 12) OP '==' (1, 13) (1, 15) NUMBER '1' (1, 16) (1, 17) OP '>=' (1, 18) (1, 20) NUMBER '5' (1, 21) (1, 22) OP '<=' (1, 23) (1, 25) NUMBER '0x15' (1, 26) (1, 30) OP '<=' (1, 31) (1, 33) NUMBER '0x12' (1, 34) (1, 38) OP '!=' (1, 39) (1, 41) NUMBER '1' (1, 42) (1, 43) NAME 'and' (1, 44) (1, 47) NUMBER '5' (1, 48) (1, 49) NAME 'in' (1, 50) (1, 52) NUMBER '1' (1, 53) (1, 54) NAME 'not' (1, 55) (1, 58) NAME 'in' (1, 59) (1, 61) NUMBER '1' (1, 62) (1, 63) NAME 'is' (1, 64) (1, 66) NUMBER '1' (1, 67) (1, 68) NAME 'or' (1, 69) (1, 71) NUMBER '5' (1, 72) (1, 73) NAME 'is' (1, 74) (1, 76) NAME 'not' (1, 77) (1, 80) NUMBER '1' (1, 81) (1, 82) OP ':' (1, 82) (1, 83) NAME 'pass' (1, 84) (1, 88) r<r=r$r$r%Útest_comparisonËsÿzTokenizeTest.test_comparisoncCs| dd¡dS)Nzx = 1 << 1 >> 5a6 NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '<<' (1, 6) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '>>' (1, 11) (1, 13) NUMBER '5' (1, 14) (1, 15) r<r=r$r$r%Ú test_shiftñszTokenizeTest.test_shiftcCs| dd¡dS)Nz%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '-' (1, 6) (1, 7) NAME 'y' (1, 8) (1, 9) OP '+' (1, 10) (1, 11) NUMBER '15' (1, 12) (1, 14) OP '-' (1, 15) (1, 16) NUMBER '1' (1, 17) (1, 18) OP '+' (1, 19) (1, 20) NUMBER '0x124' (1, 21) (1, 26) OP '+' (1, 27) (1, 28) NAME 'z' (1, 29) (1, 30) OP '+' (1, 31) (1, 32) NAME 'a' (1, 33) (1, 34) OP '[' (1, 34) (1, 35) NUMBER '5' (1, 35) (1, 36) OP ']' (1, 36) (1, 37) r<r=r$r$r%Ú test_additiveýszTokenizeTest.test_additivecCs| dd¡dS)Nzx = 1//1*1/5*12%0x12@42aš NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) NUMBER '1' (1, 4) (1, 5) OP '//' (1, 5) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '/' (1, 10) (1, 11) NUMBER '5' (1, 11) (1, 12) OP '*' (1, 12) (1, 13) NUMBER '12' (1, 13) (1, 15) OP '%' (1, 15) (1, 16) NUMBER '0x12' (1, 16) (1, 20) OP '@' (1, 20) (1, 21) NUMBER '42' (1, 21) (1, 23) r<r=r$r$r%Útest_multiplicativesz TokenizeTest.test_multiplicativecCs| dd¡| dd¡dS)Nz~1 ^ 1 & 1 |1 ^ -1aè OP '~' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '^' (1, 3) (1, 4) NUMBER '1' (1, 5) (1, 6) OP '&' (1, 7) (1, 8) NUMBER '1' (1, 9) (1, 10) OP '|' (1, 11) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '^' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) NUMBER '1' (1, 17) (1, 18) z-1*1/1+1*1//1 - ---1**1aH OP '-' (1, 0) (1, 1) NUMBER '1' (1, 1) (1, 2) OP '*' (1, 2) (1, 3) NUMBER '1' (1, 3) (1, 4) OP '/' (1, 4) (1, 5) NUMBER '1' (1, 5) (1, 6) OP '+' (1, 6) (1, 7) NUMBER '1' (1, 7) (1, 8) OP '*' (1, 8) (1, 9) NUMBER '1' (1, 9) (1, 10) OP '//' (1, 10) (1, 12) NUMBER '1' (1, 12) (1, 13) OP '-' (1, 14) (1, 15) OP '-' (1, 16) (1, 17) OP '-' (1, 17) (1, 18) OP '-' (1, 18) (1, 19) NUMBER '1' (1, 19) (1, 20) OP '**' (1, 20) (1, 22) NUMBER '1' (1, 22) (1, 23) r<r=r$r$r%Ú test_unary(s zTokenizeTest.test_unarycCs| dd¡dS)Nz/import sys, time x = sys.modules['time'].time()aõ NAME 'import' (1, 0) (1, 6) NAME 'sys' (1, 7) (1, 10) OP ',' (1, 10) (1, 11) NAME 'time' (1, 12) (1, 16) NEWLINE '\n' (1, 16) (1, 17) NAME 'x' (2, 0) (2, 1) OP '=' (2, 2) (2, 3) NAME 'sys' (2, 4) (2, 7) OP '.' (2, 7) (2, 8) NAME 'modules' (2, 8) (2, 15) OP '[' (2, 15) (2, 16) STRING "'time'" (2, 16) (2, 22) OP ']' (2, 22) (2, 23) OP '.' (2, 23) (2, 24) NAME 'time' (2, 24) (2, 28) OP '(' (2, 28) (2, 29) OP ')' (2, 29) (2, 30) r<r=r$r$r%Ú test_selectorMszTokenizeTest.test_selectorcCs| dd¡dS)Nz @staticmethod def foo(x,y): passa OP '@' (1, 0) (1, 1) NAME 'staticmethod' (1, 1) (1, 13) NEWLINE '\n' (1, 13) (1, 14) NAME 'def' (2, 0) (2, 3) NAME 'foo' (2, 4) (2, 7) OP '(' (2, 7) (2, 8) NAME 'x' (2, 8) (2, 9) OP ',' (2, 9) (2, 10) NAME 'y' (2, 10) (2, 11) OP ')' (2, 11) (2, 12) OP ':' (2, 12) (2, 13) NAME 'pass' (2, 14) (2, 18) r<r=r$r$r%Ú test_methodcszTokenizeTest.test_methodcCs| dd¡dS)Nzdef f(): if x passa_ NAME 'def' (1, 0) (1, 3) NAME 'f' (1, 4) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) OP ':' (1, 7) (1, 8) NEWLINE '\n' (1, 8) (1, 9) INDENT '\t' (2, 0) (2, 1) NAME 'if' (2, 1) (2, 3) NAME 'x' (2, 4) (2, 5) NEWLINE '\n' (2, 5) (2, 6) INDENT ' \t' (3, 0) (3, 9) NAME 'pass' (3, 9) (3, 13) DEDENT '' (4, 0) (4, 0) DEDENT '' (4, 0) (4, 0) r<r=r$r$r%Ú test_tabstsþzTokenizeTest.test_tabscCs| dd¡dS)Nu!Örter = 'places' grün = 'green'u7 NAME 'Örter' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) STRING "'places'" (1, 8) (1, 16) NEWLINE '\n' (1, 16) (1, 17) NAME 'grün' (2, 0) (2, 4) OP '=' (2, 5) (2, 6) STRING "'green'" (2, 7) (2, 14) r<r=r$r$r%Útest_non_ascii_identifiers‰sz'TokenizeTest.test_non_ascii_identifierscCs| dd¡dS)Nu#Örter = u'places' grün = U'green'u7 NAME 'Örter' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) STRING "u'places'" (1, 8) (1, 17) NEWLINE '\n' (1, 17) (1, 18) NAME 'grün' (2, 0) (2, 4) OP '=' (2, 5) (2, 6) STRING "U'green'" (2, 7) (2, 15) r<r=r$r$r%Ú test_unicode•szTokenizeTest.test_unicodecCs| dd¡| dd¡| dd¡| dd¡| d d ¡| d d ¡| d d¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd¡| dd ¡| d!d"¡| d#d$¡| d%d&¡| d'd(¡| d)d*¡dS)+Nz async = 1z… NAME 'async' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) NUMBER '1' (1, 8) (1, 9) za = (async = 1)a8 NAME 'a' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '(' (1, 4) (1, 5) NAME 'async' (1, 5) (1, 10) OP '=' (1, 11) (1, 12) NUMBER '1' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) zasync()z… NAME 'async' (1, 0) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) zclass async(Bar):passa< NAME 'class' (1, 0) (1, 5) NAME 'async' (1, 6) (1, 11) OP '(' (1, 11) (1, 12) NAME 'Bar' (1, 12) (1, 15) OP ')' (1, 15) (1, 16) OP ':' (1, 16) (1, 17) NAME 'pass' (1, 17) (1, 21) zclass async:passzµ NAME 'class' (1, 0) (1, 5) NAME 'async' (1, 6) (1, 11) OP ':' (1, 11) (1, 12) NAME 'pass' (1, 12) (1, 16) z await = 1z… NAME 'await' (1, 0) (1, 5) OP '=' (1, 6) (1, 7) NUMBER '1' (1, 8) (1, 9) z foo.asyncz… NAME 'foo' (1, 0) (1, 3) OP '.' (1, 3) (1, 4) NAME 'async' (1, 4) (1, 9) zasync for a in b: passa; NAME 'async' (1, 0) (1, 5) NAME 'for' (1, 6) (1, 9) NAME 'a' (1, 10) (1, 11) NAME 'in' (1, 12) (1, 14) NAME 'b' (1, 15) (1, 16) OP ':' (1, 16) (1, 17) NAME 'pass' (1, 18) (1, 22) zasync with a as b: passa< NAME 'async' (1, 0) (1, 5) NAME 'with' (1, 6) (1, 10) NAME 'a' (1, 11) (1, 12) NAME 'as' (1, 13) (1, 15) NAME 'b' (1, 16) (1, 17) OP ':' (1, 17) (1, 18) NAME 'pass' (1, 19) (1, 23) z async.fooz… NAME 'async' (1, 0) (1, 5) OP '.' (1, 5) (1, 6) NAME 'foo' (1, 6) (1, 9) Úasyncz/ NAME 'async' (1, 0) (1, 5) zasync #comment awaitzÛ NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) COMMENT '#comment' (2, 0) (2, 8) NL '\n' (2, 8) (2, 9) NAME 'await' (3, 0) (3, 5) zasync ... awaitzÛ NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) OP '...' (2, 0) (2, 3) NEWLINE '\n' (2, 3) (2, 4) NAME 'await' (3, 0) (3, 5) z async awaitz… NAME 'async' (1, 0) (1, 5) NEWLINE '\n' (1, 5) (1, 6) NAME 'await' (2, 0) (2, 5) z foo.async + 1zß NAME 'foo' (1, 0) (1, 3) OP '.' (1, 3) (1, 4) NAME 'async' (1, 4) (1, 9) OP '+' (1, 10) (1, 11) NUMBER '1' (1, 12) (1, 13) zasync def foo(): passa; NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NAME 'pass' (1, 17) (1, 21) zNasync def foo(): def foo(await): await = 1 if 1: await async += 1 a× NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NEWLINE '\n' (1, 16) (1, 17) INDENT ' ' (2, 0) (2, 2) NAME 'def' (2, 2) (2, 5) NAME 'foo' (2, 6) (2, 9) OP '(' (2, 9) (2, 10) NAME 'await' (2, 10) (2, 15) OP ')' (2, 15) (2, 16) OP ':' (2, 16) (2, 17) NEWLINE '\n' (2, 17) (2, 18) INDENT ' ' (3, 0) (3, 4) NAME 'await' (3, 4) (3, 9) OP '=' (3, 10) (3, 11) NUMBER '1' (3, 12) (3, 13) NEWLINE '\n' (3, 13) (3, 14) DEDENT '' (4, 2) (4, 2) NAME 'if' (4, 2) (4, 4) NUMBER '1' (4, 5) (4, 6) OP ':' (4, 6) (4, 7) NEWLINE '\n' (4, 7) (4, 8) INDENT ' ' (5, 0) (5, 4) NAME 'await' (5, 4) (5, 9) NEWLINE '\n' (5, 9) (5, 10) DEDENT '' (6, 0) (6, 0) DEDENT '' (6, 0) (6, 0) NAME 'async' (6, 0) (6, 5) OP '+=' (6, 6) (6, 8) NUMBER '1' (6, 9) (6, 10) NEWLINE '\n' (6, 10) (6, 11) z)async def foo(): async for i in 1: passaÉ NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) OP ')' (1, 14) (1, 15) OP ':' (1, 15) (1, 16) NEWLINE '\n' (1, 16) (1, 17) INDENT ' ' (2, 0) (2, 2) NAME 'async' (2, 2) (2, 7) NAME 'for' (2, 8) (2, 11) NAME 'i' (2, 12) (2, 13) NAME 'in' (2, 14) (2, 16) NUMBER '1' (2, 17) (2, 18) OP ':' (2, 18) (2, 19) NAME 'pass' (2, 20) (2, 24) DEDENT '' (3, 0) (3, 0) zasync def foo(async): awaitah NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'foo' (1, 10) (1, 13) OP '(' (1, 13) (1, 14) NAME 'async' (1, 14) (1, 19) OP ')' (1, 19) (1, 20) OP ':' (1, 20) (1, 21) NAME 'await' (1, 22) (1, 27) z@def f(): def baz(): pass async def bar(): pass await = 2aÐ NAME 'def' (1, 0) (1, 3) NAME 'f' (1, 4) (1, 5) OP '(' (1, 5) (1, 6) OP ')' (1, 6) (1, 7) OP ':' (1, 7) (1, 8) NEWLINE '\n' (1, 8) (1, 9) NL '\n' (2, 0) (2, 1) INDENT ' ' (3, 0) (3, 2) NAME 'def' (3, 2) (3, 5) NAME 'baz' (3, 6) (3, 9) OP '(' (3, 9) (3, 10) OP ')' (3, 10) (3, 11) OP ':' (3, 11) (3, 12) NAME 'pass' (3, 13) (3, 17) NEWLINE '\n' (3, 17) (3, 18) NAME 'async' (4, 2) (4, 7) NAME 'def' (4, 8) (4, 11) NAME 'bar' (4, 12) (4, 15) OP '(' (4, 15) (4, 16) OP ')' (4, 16) (4, 17) OP ':' (4, 17) (4, 18) NAME 'pass' (4, 19) (4, 23) NEWLINE '\n' (4, 23) (4, 24) NL '\n' (5, 0) (5, 1) NAME 'await' (6, 2) (6, 7) OP '=' (6, 8) (6, 9) NUMBER '2' (6, 10) (6, 11) DEDENT '' (7, 0) (7, 0) zFasync def f(): def baz(): pass async def bar(): pass await = 2a NAME 'async' (1, 0) (1, 5) NAME 'def' (1, 6) (1, 9) NAME 'f' (1, 10) (1, 11) OP '(' (1, 11) (1, 12) OP ')' (1, 12) (1, 13) OP ':' (1, 13) (1, 14) NEWLINE '\n' (1, 14) (1, 15) NL '\n' (2, 0) (2, 1) INDENT ' ' (3, 0) (3, 2) NAME 'def' (3, 2) (3, 5) NAME 'baz' (3, 6) (3, 9) OP '(' (3, 9) (3, 10) OP ')' (3, 10) (3, 11) OP ':' (3, 11) (3, 12) NAME 'pass' (3, 13) (3, 17) NEWLINE '\n' (3, 17) (3, 18) NAME 'async' (4, 2) (4, 7) NAME 'def' (4, 8) (4, 11) NAME 'bar' (4, 12) (4, 15) OP '(' (4, 15) (4, 16) OP ')' (4, 16) (4, 17) OP ':' (4, 17) (4, 18) NAME 'pass' (4, 19) (4, 23) NEWLINE '\n' (4, 23) (4, 24) NL '\n' (5, 0) (5, 1) NAME 'await' (6, 2) (6, 7) OP '=' (6, 8) (6, 9) NUMBER '2' (6, 10) (6, 11) DEDENT '' (7, 0) (7, 0) r<r=r$r$r%Ú test_async¡s:            ù,þ ú%úzTokenizeTest.test_asyncN)Ú__name__Ú __module__Ú __qualname__r2r7r;r>r?r@rCrDrErFrGrHrIrJrKrLrMrNrOrQr$r$r$r%r'#s* %9('<& %  r'c@seZdZdd„ZdS)ÚGenerateTokensTestcCs0t|ƒ}tt|jƒ|ƒ}| || ¡ ¡¡dS©N)rr&rr*r+r,rr-r$r$r%r2´sz!GenerateTokensTest.check_tokenizeN)rRrSrTr2r$r$r$r%rU³srUcCs€g}tt| d¡ƒjƒ}|D]T\}}}}}|tkrbd|vrb| tdftdftt |ƒftdfg¡q|  ||f¡qt |ƒ  d¡S)Nr(Ú.ÚDecimalrBú)) rrr)r*rÚextendrrr ÚreprrrÚdecode)r/rÚgÚtoknumÚtokvalÚ_r$r$r%Údecistmt¼s ürac@seZdZdd„ZdS)ÚTestMisccCsPddlm}d}| t|ƒd¡| tt|ƒƒd¡| tt|ƒƒ|dƒ¡dS)Nr)rXz+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)ÚdecimalrXr+raZ assertRegexr[Úeval)r.rXr/r$r$r%Ú test_decistmtÍs  ÿÿzTestMisc.test_decistmtN)rRrSrTrer$r$r$r%rbËsrbc@s@eZdZdZdd„Zdd„Zdd„Zdd „Zd d „Zd d „Z dS)ÚTestTokenizerAdheresToPep0263zU Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263. cCs,tj tj t¡|¡}t |t|dƒ¡dS)NÚrb)ÚosÚpathÚjoinÚdirnameÚ__file__Ú TestRoundtripÚcheck_roundtripr)r.Úfilenamerir$r$r%Ú _testFileçsz'TestTokenizerAdheresToPep0263._testFilecCsd}| |¡dS)Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt©rp©r.r1r$r$r%Ú'test_utf8_coding_cookie_and_no_utf8_bomëszETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bomcCsd}| t|j|¡dS)a† As per PEP 0263, if a file starts with a utf-8 BOM signature, the only allowed encoding for the comment is 'utf-8'. The text file used in this test starts with a BOM signature, but specifies latin1 as the coding, so verify that a SyntaxError is raised, which matches the behaviour of the interpreter when it encounters a similar condition. z8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtN©Ú assertRaisesÚ SyntaxErrorrprrr$r$r%Ú&test_latin1_coding_cookie_and_utf8_bomïszDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bomcCsd}| |¡dS)Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtrqrrr$r$r%Ú"test_no_coding_cookie_and_utf8_bomúsz@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bomcCsd}| |¡dS)Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtrqrrr$r$r%Ú$test_utf8_coding_cookie_and_utf8_bomþszBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bomcCs$| t|jd¡| t|jd¡dS)Nz bad_coding.pyzbad_coding2.pyrtr=r$r$r%Útest_bad_coding_cookiesz4TestTokenizerAdheresToPep0263.test_bad_coding_cookieN) rRrSrTÚ__doc__rprsrwrxryrzr$r$r$r%rfâs rfc@seZdZdd„Zdd„ZdS)Ú Test_TokenizecsPd}| d¡‰d‰‡‡fdd„}tt|ddƒdd…}d g}| ||d ¡dS) Nõ "ЉЊЈÐЂ"r(Fcsˆs d‰ˆSdSdS©NTór$r$©Úfirstr#r$r%r* szNTest_Tokenize.test__tokenize_decodes_with_specified_encoding..readline©Úencodingér4©ér})r„r)r„ér}zbytes not decoded with encoding)r)r5rr+)r.Úliteralr*r6Úexpected_tokensr$r€r%Ú.test__tokenize_decodes_with_specified_encoding s  ÿz.readliner‚r4r…z*string not tokenized when encoding is None)r5rr+)r.r*r6r‰r$r‹r%Ú1test__tokenize_does_not_decode_with_encoding_nones ÿz?Test_Tokenize.test__tokenize_does_not_decode_with_encoding_noneN)rRrSrTrŠrŒr$r$r$r%r|sr|c@s¬eZdZdd„Zdd„Zdd„Zdd„Zd d „Zd d „Zd d„Z dd„Z dd„Z dd„Z dd„Z dd„Zdd„Zdd„Zdd„Zdd „Zd!d"„Zd#d$„Zd%d&„Zd'd(„Zd)S)*ÚTestDetectEncodingcsd‰‡‡fdd„}|S)Nrcs$ˆtˆƒkrt‚ˆˆ}ˆd7‰|S©Nr„)rÚ StopIteration)r#©ÚindexÚlinesr$r%r*1s  z1TestDetectEncoding.get_readline..readliner$©r.r’r*r$rr%Ú get_readline/szTestDetectEncoding.get_readlinecCs>d}t| |¡ƒ\}}| |d¡| |t|dd…ƒ¡dS)N)ó # something óprint(something) ódo_something(else) r(é)r r”r+r5©r.r’rƒÚconsumed_linesr$r$r%Útest_no_bom_no_encoding_cookie:s z1TestDetectEncoding.test_no_bom_no_encoding_cookiecCs6d}t| |¡ƒ\}}| |d¡| |ddg¡dS)N)s# something r–r—ú utf-8-sigr•r–©r r”r+r™r$r$r%Útest_bom_no_cookieDs  ÿz%TestDetectEncoding.test_bom_no_cookiecCs4d}t| |¡ƒ\}}| |d¡| |dg¡dS)N)ó# -*- coding: latin-1 -*- r–r—ú iso-8859-1rŸrr™r$r$r%Útest_cookie_first_line_no_bomOs z0TestDetectEncoding.test_cookie_first_line_no_bomcCs4d}t| |¡ƒ\}}| |d¡| |dg¡dS)N)s# coding=utf-8 r–r—rœs# coding=utf-8 rr™r$r$r%Ú&test_matched_bom_and_cookie_first_lineYs z9TestDetectEncoding.test_matched_bom_and_cookie_first_linecCs d}| |¡}| tt|¡dS)N)s## vim: set fileencoding=ascii : r–r—©r”rurvr r“r$r$r%Ú.BunkcSs||_||_d|_dS)Nr)ÚnameZ_linesÚ_index)r.r’rir$r$r%Ú__init__szDTestDetectEncoding.test_filename_in_exception..Bunk.__init__cs.|jtˆƒkrt‚ˆ|j}|jd7_|SrŽ)rÔrr)r.r#©r’r$r%r*s  zDTestDetectEncoding.test_filename_in_exception..Bunk.readlineN)rRrSrTrÕr*r$rÖr$r%ÚBunk sr×z.*{})rurvrÓr r*r8Úformat)r.rir×Zinsr$rÖr%Útest_filename_in_exceptions  ( z-TestDetectEncoding.test_filename_in_exceptioncCsTtdƒ}tjd|d| ttd¡Wdƒn1s:0Y| |j¡dS)Ns #coding:xxxztokenize._builtin_open)Z return_valueZfoobar)rrZpatchrurvrÏZ assertTrueÚclosed)r.Úmr$r$r%Útest_open_error#s,z"TestDetectEncoding.test_open_errorN)rRrSrTr”r›ržr¡r¢r¤r©r¬r­r±r´r¶rÁrÃrÄrÆrÈrÒrÙrÜr$r$r$r%r-s(              rc@s4eZdZdd„Zdd„Zdd„Zdd„Zd d „Zd S) Ú TestTokenizecsžddl}tƒ‰d‰‡fdd„}‡fdd„}d‰‡fdd„}|j}|j}||_||_z,t|ƒ}| t|ƒgd¢¡W||_||_n||_||_0| ˆˆ¡dS) Nrcs ˆddgfS)Nófirstósecondr$)r*r‚r$r%Úmock_detect_encoding1sz8TestTokenize.test_tokenize..mock_detect_encodingcs"|‰g}|ƒ}|r| |¡q|SrV)r)r*rƒÚoutZ next_line)Ú encoding_usedr$r%Úmock__tokenize4s z2TestTokenize.test_tokenize..mock__tokenizecs ˆd7‰ˆdkrdStˆƒ ¡S)Nr„ér)Ústrr)r$)Úcounterr$r%Ú mock_readline@sz1TestTokenize.test_tokenize..mock_readline)rÞrßó1ó2ó3ó4)rÚobjectr rr+r5)r.Ztokenize_moduleràrãrçZorig_detect_encodingZorig__tokenizeÚresultsr$)rærƒrâr%Ú test_tokenize-s*    ÿÿzTestTokenize.test_tokenizecCsfg}tdƒD]}| dj|d¡q | d¡d |¡}ttt| d¡ƒjƒƒ}|  |dj d¡dS)Niôzdef i{i}(): return {i})ÚiZOKÚ r(éýÿÿÿ) ÚrangerrØrjr5rrr)r*r+Ústring)r.ÚbufrïZtoksr$r$r%Útest_oneline_defsUs   zTestTokenize.test_oneline_defscGsÂttt| d¡ƒjƒƒ}t|ƒ}| t|ƒd|¡| t|djtt ¡t |ƒD]&}| t||djt||¡qV| t|d|jtt j ¡| t|d|jtt j ¡dS)Nr(r†rr„r˜)r5rrr)r*rr+r Ú exact_typer ròr rr )r.ZopstrZoptypesr6Z num_optypesrïr$r$r%ÚassertExactTypeEqualas ÿ  ÿÿÿz!TestTokenize.assertExactTypeEqualcCsø| dtjtj¡| dtjtj¡| dtj¡| dtj¡| dtj¡| dtj ¡| dtj ¡| dtj ¡| d tj ¡| d tj ¡| d tj¡| d tj¡| d tj¡| dtj¡| dtj¡| dtj¡| dtjtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj¡| dtj ¡| dtj!¡| dtj"¡| dtj#¡| d tj$¡| d!tj%¡| d"tj&¡| d"tj&¡| d#tj'¡| d$tj(¡| d%tj)¡| d&tj*¡| d'tj+¡| d(tj,¡| d)tj-¡| d*tj.¡| d+tj/¡| d,tj0¡| d-t1tjt2tj t1tjt2tjt1tjt2¡ | d.tjtj2tjtj2tjtj2tj¡| d/tjtjtj1tjtj2tj¡dS)0Nz()z[]ú:ú,ú;ú+r¸Ú*ú/ú|ú&ú<ú>ú=rWú%z{}z==z!=z<=z>=ú~ú^z<>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->ú@z@=za**2+b**2==c**2z {1, 2, 3}z ^(x & 0x1))3r÷r ÚLPARÚRPARÚLSQBÚRSQBÚCOLONÚCOMMAÚSEMIÚPLUSÚMINUSÚSTARÚSLASHÚVBARÚAMPERÚLESSÚGREATERÚEQUALÚDOTÚPERCENTÚLBRACEÚRBRACEÚEQEQUALÚNOTEQUALÚ LESSEQUALÚ GREATEREQUALÚTILDEÚ CIRCUMFLEXÚ LEFTSHIFTÚ RIGHTSHIFTÚ DOUBLESTARÚ PLUSEQUALÚMINEQUALÚ STAREQUALÚ SLASHEQUALÚ PERCENTEQUALÚ AMPEREQUALÚ VBAREQUALÚCIRCUMFLEXEQUALÚLEFTSHIFTEQUALÚRIGHTSHIFTEQUALÚDOUBLESTAREQUALÚ DOUBLESLASHÚDOUBLESLASHEQUALÚ COLONEQUALÚELLIPSISÚRARROWÚATÚATEQUALrrr=r$r$r%Útest_exact_typeos‚ûû üzTestTokenize.test_exact_typecCs| dtj¡dS)Nz @ )r÷r r4r=r$r$r%Ú%test_pathological_trailing_whitespace°sz2TestTokenize.test_pathological_trailing_whitespaceN)rRrSrTrîrõr÷r6r7r$r$r$r%rÝ+s ( ArÝc@s$eZdZdd„Zdd„Zdd„ZdS)ÚUntokenizeTestcCsntƒ}d|_d|_| t¡}| d¡Wdƒn1s<0Y| |jjdd¡| t|jd¡dS)Nr˜)r„r†rz'start (1,3) precedes previous end (2,2))r˜r„) rÚprev_rowÚprev_colruÚ ValueErrorÚadd_whitespacer+Ú exceptionÚargs)r.ÚuÚcmr$r$r%Útest_bad_input_order·s (ÿz#UntokenizeTest.test_bad_input_ordercCsdtƒ}d|_d|_g|_| d¡| |jdg¡d|_| d¡| |jgd¢¡t |d¡dS)Nr„)r˜rú\ r˜)érC)rBz\ \ rza b c \ c )rr9r:r6r<r+rmrn)r.r?r$r$r%Útest_backslash_continuationÃs  z*UntokenizeTest.test_backslash_continuationcCsštƒ}tdf}tdf|g}| |tgƒ¡| |jdg¡tƒ}| | t|gƒ¡d¡tƒ}| | t|ƒ¡d¡| |jd¡| tt|ƒƒd¡dS)NZHellor(zHello sHello ) rrr ÚcompatÚiterr+r6rrƒ)r.r?r r6r$r$r%Útest_iter_compatÐs zUntokenizeTest.test_iter_compatN)rRrSrTrArDrGr$r$r$r%r8µs  r8c@sLeZdZdd„Zdd„Zdd„Zdd„Zd d „Zd d „Zd d„Z dd„Z dS)rmc CsÆt|tƒr| d¡}n| ¡}| ¡t|jddƒj}tt |ƒƒ}dd„|Dƒ}t |ƒ}t|jddƒj}dd„t |ƒDƒ}|  ||¡t |ƒ} t| jddƒj} dd„t | ƒDƒ} |  | |¡dS) a) Test roundtrip for `untokenize`. `f` is an open file or a string. The source code in f is tokenized to both 5- and 2-tuples. Both sequences are converted back to source code via tokenize.untokenize(), and the latter tokenized again to 2-tuples. The test fails if the 3 pair tokenizations do not match. When untokenize bugs are fixed, untokenize with 5-tuples should reproduce code that does not contain a backslash continuation following spaces. A proper test should test this. r(T)ÚkeependscSsg|]}|dd…‘qS©Nr˜r$©Ú.0r:r$r$r%Ú ôrz1TestRoundtrip.check_roundtrip..cSsg|]}|dd…‘qSrIr$rJr$r$r%rLørcSsg|]}|dd…‘qSrIr$rJr$r$r%rLýrN) Ú isinstancerår)ÚreadÚcloserFrÚ__next__r5rrr+) r.r1Úcoder*Ztokens5Ztokens2Z bytes_from2Z readline2Z tokens2_from2Z bytes_from5Z readline5Z tokens2_from5r$r$r%rnàs    zTestRoundtrip.check_roundtripcCsˆ| d¡| d¡| d¡t d¡}t|dƒ}| |¡Wdƒn1sR0Y| d¡| d¡| d¡| d ¡dS) Nzif x == 1: print(x) z # This is a comment # This also zif x == 1 : print(x) útokenize_tests.txtrgzoif x == 1: # A comment by itself. print(x) # Comment here, too. # Another comment. after_if = True zPif (x # The comments need to go in the right place == 1): print('x==1') z¾class Test: # A comment here # A comment with weird indent after_com = 5 def x(m): return m*5 # a one liner def y(m): # A whitespace after the colon return y*4 # 3-space indent zttry: import somemodule except ImportError: # comment print('Can not import' # comment2 )else: print('Loaded') )rnrÚfindfiler)r.Úfnr1r$r$r%Útest_roundtrips     (   zTestRoundtrip.test_roundtripcCs| d¡dS)Nzna = (3,4, 5,6) y = [3, 4, 5] z = {'a': 5, 'b':15, 'c':True} x = len(y) + 5 - a[ 3] - a[2] + len(z) - z[ 'b'] ©rnr=r$r$r%Útest_continuation'szTestRoundtrip.test_continuationcCs| d¡| d¡dS)Nz)x=1+\ 1 # This is a comment\ # This also z# Comment \ x = 0rVr=r$r$r%rD4s z)TestRoundtrip.test_backslash_continuationcCs| d¡dS)Nz'' ''rVr=r$r$r%Útest_string_concatenation=sz'TestRoundtrip.test_string_concatenationc Csddl}ddl}t d¡}tj |¡p*tj}| tj |  |¡d¡¡}|  tj |d¡¡dD]}|  tj |d¡|¡q^t  d¡s’|  |d¡}|D]z}tj d kr®td |ƒt|d ƒF}|j|d | |¡Wdƒn1sæ0YWdƒq–1s0Yq–dS) NrrRztest*.pyztest_unicode_identifiers.py)ÚbufferZbuiltinZfileioÚinspectrhÚplatformÚsysz test_%s.pyZcpué r˜rrgrÊ)ÚglobÚrandomrrSrhrirkÚcurdirrjÚescapeÚremoveZis_resource_enabledÚsampleÚverboserÎrZsubTestrn)r.r^r_rTÚtempdirZ testfilesr1Ztestfiler$r$r%Útest_random_filesAs      zTestRoundtrip.test_random_filescCs,t|tƒr| d¡}ttt|ƒjƒƒ d¡S)Nr()rMrår)rrrr*r\)r.rQr$r$r%Ú roundtrip^s  zTestRoundtrip.roundtripcCs6d}| |¡ d¡}| |d|d¡| |¡dS)z– Ensure that although whitespace might be mutated in a roundtrip, the semantic meaning of the indentation remains consistent. zif False: x=3 x=3 rðr„r˜N)rgÚsplitr+rn)r.rQZ codelinesr$r$r%Ú#test_indentation_semantics_retainedcsz1TestRoundtrip.test_indentation_semantics_retainedN) rRrSrTrnrUrWrDrXrfrgrir$r$r$r%rmÞs '  rmÚ__main__)*Útestrrrrrrrr r r r r rrÏrrrÚiorrZunittestrrZtest.test_grammarrrrhr r&r'rUrarbrfr|rrÝr8rmrRÚmainr$r$r$r%Ús> D %& )