mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-05-22 21:32:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Improve Python tree-shaking

Browse source
This commit is contained in:
Justine Tunney 2021-09-06 19:24:10 -07:00
parent 5bb2275788
commit 4f41f2184d
169 changed files with 4182 additions and 2411 deletions
Download patch file Download diff file Expand all files Collapse all files

7
third_party/python/Lib/tokenize.py vendored
View file

@ -32,16 +32,11 @@ from itertools import chain
import itertools as _itertools
import re
import sys
from token import *
from token import AMPER, AMPEREQUAL, ASYNC, AT, ATEQUAL, AWAIT, CIRCUMFLEX, CIRCUMFLEXEQUAL, COLON, COMMA, DEDENT, DOT, DOUBLESLASH, DOUBLESLASHEQUAL, DOUBLESTAR, DOUBLESTAREQUAL, ELLIPSIS, ENDMARKER, EQEQUAL, EQUAL, ERRORTOKEN, GREATER, GREATEREQUAL, INDENT, ISEOF, ISNONTERMINAL, ISTERMINAL, LBRACE, LEFTSHIFT, LEFTSHIFTEQUAL, LESS, LESSEQUAL, LPAR, LSQB, MINEQUAL, MINUS, NAME, NEWLINE, NOTEQUAL, NT_OFFSET, NUMBER, N_TOKENS, OP, PERCENT, PERCENTEQUAL, PLUS, PLUSEQUAL, RARROW, RBRACE, RIGHTSHIFT, RIGHTSHIFTEQUAL, RPAR, RSQB, SEMI, SLASH, SLASHEQUAL, STAR, STAREQUAL, STRING, TILDE, VBAR, VBAREQUAL, tok_name
cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII)
blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII)
import token
__all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
"NL", "untokenize", "ENCODING", "TokenInfo"]
del token
COMMENT = N_TOKENS
tok_name[COMMENT] = 'COMMENT'
NL = N_TOKENS + 1