python-3.6.zip added from Github

README.cosmo contains the necessary links.

third_party/python/Lib/lib2to3/tests/__init__.py

@@ -0,0 +1,9 @@
+# Author: Collin Winter
+
+import os
+import unittest
+
+from test.support import load_package_tests
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)

third_party/python/Lib/lib2to3/tests/__main__.py

@@ -0,0 +1,4 @@
+from . import load_tests
+import unittest
+
+unittest.main()

third_party/python/Lib/lib2to3/tests/data/README

@@ -0,0 +1,6 @@
+In this directory:
+- py2_test_grammar.py -- test file that exercises most/all of Python 2.x's grammar.
+- py3_test_grammar.py -- test file that exercises most/all of Python 3.x's grammar.
+- infinite_recursion.py -- test file that causes lib2to3's faster recursive pattern matching
+  scheme to fail, but passes when lib2to3 falls back to iterative pattern matching.
+- fixes/ -- for use by test_refactor.py

third_party/python/Lib/lib2to3/tests/data/bom.py

@@ -0,0 +1,2 @@
+# coding: utf-8
+print "BOM BOOM!"

third_party/python/Lib/lib2to3/tests/data/crlf.py

@@ -0,0 +1,3 @@
+print "hi"
+
+print "Like bad Windows newlines?"

third_party/python/Lib/lib2to3/tests/data/different_encoding.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+print u'ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ'
+
+def f(x):
+    print '%s\t->  α(%2i):%s  β(%s)'

third_party/python/Lib/lib2to3/tests/data/false_encoding.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python
+print '#coding=0'

third_party/python/Lib/lib2to3/tests/data/fixers/bad_order.py

@@ -0,0 +1,5 @@
+from lib2to3.fixer_base import BaseFix
+
+class FixBadOrder(BaseFix):
+
+    order = "crazy"

third_party/python/Lib/lib2to3/tests/data/fixers/myfixes/fix_explicit.py

@@ -0,0 +1,6 @@
+from lib2to3.fixer_base import BaseFix
+
+class FixExplicit(BaseFix):
+    explicit = True
+
+    def match(self): return False

third_party/python/Lib/lib2to3/tests/data/fixers/myfixes/fix_first.py

@@ -0,0 +1,6 @@
+from lib2to3.fixer_base import BaseFix
+
+class FixFirst(BaseFix):
+    run_order = 1
+
+    def match(self, node): return False

third_party/python/Lib/lib2to3/tests/data/fixers/myfixes/fix_last.py

@@ -0,0 +1,7 @@
+from lib2to3.fixer_base import BaseFix
+
+class FixLast(BaseFix):
+
+    run_order = 10
+
+    def match(self, node): return False

third_party/python/Lib/lib2to3/tests/data/fixers/myfixes/fix_parrot.py

@@ -0,0 +1,13 @@
+from lib2to3.fixer_base import BaseFix
+from lib2to3.fixer_util import Name
+
+class FixParrot(BaseFix):
+    """
+    Change functions named 'parrot' to 'cheese'.
+    """
+
+    PATTERN = """funcdef < 'def' name='parrot' any* >"""
+
+    def transform(self, node, results):
+        name = results["name"]
+        name.replace(Name("cheese", name.prefix))

third_party/python/Lib/lib2to3/tests/data/fixers/myfixes/fix_preorder.py

@@ -0,0 +1,6 @@
+from lib2to3.fixer_base import BaseFix
+
+class FixPreorder(BaseFix):
+    order = "pre"
+
+    def match(self, node): return False

third_party/python/Lib/lib2to3/tests/data/fixers/no_fixer_cls.py

@@ -0,0 +1 @@
+# This is empty so trying to fetch the fixer class gives an AttributeError

third_party/python/Lib/lib2to3/tests/data/fixers/parrot_example.py

@@ -0,0 +1,2 @@
+def parrot():
+    pass

third_party/python/Lib/lib2to3/tests/data/py2_test_grammar.py

@@ -0,0 +1,974 @@
+# Python test set -- part 1, grammar.
+# This just tests whether the parser accepts them all.
+
+# NOTE: When you run this test as a script from the command line, you
+# get warnings about certain hex/oct constants.  Since those are
+# issued by the parser, you can't suppress them by adding a
+# filterwarnings() call to this module.  Therefore, to shut up the
+# regression test, the filterwarnings() call has been added to
+# regrtest.py.
+
+from test.test_support import run_unittest, check_syntax_error
+import unittest
+import sys
+# testing import *
+from sys import *
+
+class TokenTests(unittest.TestCase):
+
+    def testBackslash(self):
+        # Backslash means line continuation:
+        x = 1 \
+        + 1
+        self.assertEquals(x, 2, 'backslash for line continuation')
+
+        # Backslash does not means continuation in comments :\
+        x = 0
+        self.assertEquals(x, 0, 'backslash ending comment')
+
+    def testPlainIntegers(self):
+        self.assertEquals(0xff, 255)
+        self.assertEquals(0377, 255)
+        self.assertEquals(2147483647, 017777777777)
+        # "0x" is not a valid literal
+        self.assertRaises(SyntaxError, eval, "0x")
+        from sys import maxint
+        if maxint == 2147483647:
+            self.assertEquals(-2147483647-1, -020000000000)
+            # XXX -2147483648
+            self.assert_(037777777777 > 0)
+            self.assert_(0xffffffff > 0)
+            for s in '2147483648', '040000000000', '0x100000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        elif maxint == 9223372036854775807:
+            self.assertEquals(-9223372036854775807-1, -01000000000000000000000)
+            self.assert_(01777777777777777777777 > 0)
+            self.assert_(0xffffffffffffffff > 0)
+            for s in '9223372036854775808', '02000000000000000000000', \
+                     '0x10000000000000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        else:
+            self.fail('Weird maxint value %r' % maxint)
+
+    def testLongIntegers(self):
+        x = 0L
+        x = 0l
+        x = 0xffffffffffffffffL
+        x = 0xffffffffffffffffl
+        x = 077777777777777777L
+        x = 077777777777777777l
+        x = 123456789012345678901234567890L
+        x = 123456789012345678901234567890l
+
+    def testFloats(self):
+        x = 3.14
+        x = 314.
+        x = 0.314
+        # XXX x = 000.314
+        x = .314
+        x = 3e14
+        x = 3E14
+        x = 3e-14
+        x = 3e+14
+        x = 3.e14
+        x = .3e14
+        x = 3.1e4
+
+    def testStringLiterals(self):
+        x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
+        x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
+        x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
+        x = "doesn't \"shrink\" does it"
+        y = 'doesn\'t "shrink" does it'
+        self.assert_(len(x) == 24 and x == y)
+        x = "does \"shrink\" doesn't it"
+        y = 'does "shrink" doesn\'t it'
+        self.assert_(len(x) == 24 and x == y)
+        x = """
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+"""
+        y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
+        self.assertEquals(x, y)
+        y = '''
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+'''
+        self.assertEquals(x, y)
+        y = "\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the 'lazy' dog.\n\
+"
+        self.assertEquals(x, y)
+        y = '\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the \'lazy\' dog.\n\
+'
+        self.assertEquals(x, y)
+
+
+class GrammarTests(unittest.TestCase):
+
+    # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+    # XXX can't test in a script -- this rule is only used when interactive
+
+    # file_input: (NEWLINE | stmt)* ENDMARKER
+    # Being tested as this very moment this very module
+
+    # expr_input: testlist NEWLINE
+    # XXX Hard to test -- used only in calls to input()
+
+    def testEvalInput(self):
+        # testlist ENDMARKER
+        x = eval('1, 0 or 1')
+
+    def testFuncdef(self):
+        ### 'def' NAME parameters ':' suite
+        ### parameters: '(' [varargslist] ')'
+        ### varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' ('**'|'*' '*') NAME]
+        ###            | ('**'|'*' '*') NAME)
+        ###            | fpdef ['=' test] (',' fpdef ['=' test])* [',']
+        ### fpdef: NAME | '(' fplist ')'
+        ### fplist: fpdef (',' fpdef)* [',']
+        ### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test)
+        ### argument: [test '='] test   # Really [keyword '='] test
+        def f1(): pass
+        f1()
+        f1(*())
+        f1(*(), **{})
+        def f2(one_argument): pass
+        def f3(two, arguments): pass
+        def f4(two, (compound, (argument, list))): pass
+        def f5((compound, first), two): pass
+        self.assertEquals(f2.func_code.co_varnames, ('one_argument',))
+        self.assertEquals(f3.func_code.co_varnames, ('two', 'arguments'))
+        if sys.platform.startswith('java'):
+            self.assertEquals(f4.func_code.co_varnames,
+                   ('two', '(compound, (argument, list))', 'compound', 'argument',
+                                'list',))
+            self.assertEquals(f5.func_code.co_varnames,
+                   ('(compound, first)', 'two', 'compound', 'first'))
+        else:
+            self.assertEquals(f4.func_code.co_varnames,
+                  ('two', '.1', 'compound', 'argument',  'list'))
+            self.assertEquals(f5.func_code.co_varnames,
+                  ('.0', 'two', 'compound', 'first'))
+        def a1(one_arg,): pass
+        def a2(two, args,): pass
+        def v0(*rest): pass
+        def v1(a, *rest): pass
+        def v2(a, b, *rest): pass
+        def v3(a, (b, c), *rest): return a, b, c, rest
+
+        f1()
+        f2(1)
+        f2(1,)
+        f3(1, 2)
+        f3(1, 2,)
+        f4(1, (2, (3, 4)))
+        v0()
+        v0(1)
+        v0(1,)
+        v0(1,2)
+        v0(1,2,3,4,5,6,7,8,9,0)
+        v1(1)
+        v1(1,)
+        v1(1,2)
+        v1(1,2,3)
+        v1(1,2,3,4,5,6,7,8,9,0)
+        v2(1,2)
+        v2(1,2,3)
+        v2(1,2,3,4)
+        v2(1,2,3,4,5,6,7,8,9,0)
+        v3(1,(2,3))
+        v3(1,(2,3),4)
+        v3(1,(2,3),4,5,6,7,8,9,0)
+
+        # ceval unpacks the formal arguments into the first argcount names;
+        # thus, the names nested inside tuples must appear after these names.
+        if sys.platform.startswith('java'):
+            self.assertEquals(v3.func_code.co_varnames, ('a', '(b, c)', 'rest', 'b', 'c'))
+        else:
+            self.assertEquals(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c'))
+        self.assertEquals(v3(1, (2, 3), 4), (1, 2, 3, (4,)))
+        def d01(a=1): pass
+        d01()
+        d01(1)
+        d01(*(1,))
+        d01(**{'a':2})
+        def d11(a, b=1): pass
+        d11(1)
+        d11(1, 2)
+        d11(1, **{'b':2})
+        def d21(a, b, c=1): pass
+        d21(1, 2)
+        d21(1, 2, 3)
+        d21(*(1, 2, 3))
+        d21(1, *(2, 3))
+        d21(1, 2, *(3,))
+        d21(1, 2, **{'c':3})
+        def d02(a=1, b=2): pass
+        d02()
+        d02(1)
+        d02(1, 2)
+        d02(*(1, 2))
+        d02(1, *(2,))
+        d02(1, **{'b':2})
+        d02(**{'a': 1, 'b': 2})
+        def d12(a, b=1, c=2): pass
+        d12(1)
+        d12(1, 2)
+        d12(1, 2, 3)
+        def d22(a, b, c=1, d=2): pass
+        d22(1, 2)
+        d22(1, 2, 3)
+        d22(1, 2, 3, 4)
+        def d01v(a=1, *rest): pass
+        d01v()
+        d01v(1)
+        d01v(1, 2)
+        d01v(*(1, 2, 3, 4))
+        d01v(*(1,))
+        d01v(**{'a':2})
+        def d11v(a, b=1, *rest): pass
+        d11v(1)
+        d11v(1, 2)
+        d11v(1, 2, 3)
+        def d21v(a, b, c=1, *rest): pass
+        d21v(1, 2)
+        d21v(1, 2, 3)
+        d21v(1, 2, 3, 4)
+        d21v(*(1, 2, 3, 4))
+        d21v(1, 2, **{'c': 3})
+        def d02v(a=1, b=2, *rest): pass
+        d02v()
+        d02v(1)
+        d02v(1, 2)
+        d02v(1, 2, 3)
+        d02v(1, *(2, 3, 4))
+        d02v(**{'a': 1, 'b': 2})
+        def d12v(a, b=1, c=2, *rest): pass
+        d12v(1)
+        d12v(1, 2)
+        d12v(1, 2, 3)
+        d12v(1, 2, 3, 4)
+        d12v(*(1, 2, 3, 4))
+        d12v(1, 2, *(3, 4, 5))
+        d12v(1, *(2,), **{'c': 3})
+        def d22v(a, b, c=1, d=2, *rest): pass
+        d22v(1, 2)
+        d22v(1, 2, 3)
+        d22v(1, 2, 3, 4)
+        d22v(1, 2, 3, 4, 5)
+        d22v(*(1, 2, 3, 4))
+        d22v(1, 2, *(3, 4, 5))
+        d22v(1, *(2, 3), **{'d': 4})
+        def d31v((x)): pass
+        d31v(1)
+        def d32v((x,)): pass
+        d32v((1,))
+
+        # keyword arguments after *arglist
+        def f(*args, **kwargs):
+            return args, kwargs
+        self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
+                                                    {'x':2, 'y':5}))
+        self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
+        self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
+
+        # Check ast errors in *args and *kwargs
+        check_syntax_error(self, "f(*g(1=2))")
+        check_syntax_error(self, "f(**g(1=2))")
+
+    def testLambdef(self):
+        ### lambdef: 'lambda' [varargslist] ':' test
+        l1 = lambda : 0
+        self.assertEquals(l1(), 0)
+        l2 = lambda : a[d] # XXX just testing the expression
+        l3 = lambda : [2 < x for x in [-1, 3, 0L]]
+        self.assertEquals(l3(), [0, 1, 0])
+        l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
+        self.assertEquals(l4(), 1)
+        l5 = lambda x, y, z=2: x + y + z
+        self.assertEquals(l5(1, 2), 5)
+        self.assertEquals(l5(1, 2, 3), 6)
+        check_syntax_error(self, "lambda x: x = 2")
+        check_syntax_error(self, "lambda (None,): None")
+
+    ### stmt: simple_stmt | compound_stmt
+    # Tested below
+
+    def testSimpleStmt(self):
+        ### simple_stmt: small_stmt (';' small_stmt)* [';']
+        x = 1; pass; del x
+        def foo():
+            # verify statements that end with semi-colons
+            x = 1; pass; del x;
+        foo()
+
+    ### small_stmt: expr_stmt | print_stmt  | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt | exec_stmt
+    # Tested below
+
+    def testExprStmt(self):
+        # (exprlist '=')* exprlist
+        1
+        1, 2, 3
+        x = 1
+        x = 1, 2, 3
+        x = y = z = 1, 2, 3
+        x, y, z = 1, 2, 3
+        abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
+
+        check_syntax_error(self, "x + 1 = 1")
+        check_syntax_error(self, "a + 1 = b + 2")
+
+    def testPrintStmt(self):
+        # 'print' (test ',')* [test]
+        import StringIO
+
+        # Can't test printing to real stdout without comparing output
+        # which is not available in unittest.
+        save_stdout = sys.stdout
+        sys.stdout = StringIO.StringIO()
+
+        print 1, 2, 3
+        print 1, 2, 3,
+        print
+        print 0 or 1, 0 or 1,
+        print 0 or 1
+
+        # 'print' '>>' test ','
+        print >> sys.stdout, 1, 2, 3
+        print >> sys.stdout, 1, 2, 3,
+        print >> sys.stdout
+        print >> sys.stdout, 0 or 1, 0 or 1,
+        print >> sys.stdout, 0 or 1
+
+        # test printing to an instance
+        class Gulp:
+            def write(self, msg): pass
+
+        gulp = Gulp()
+        print >> gulp, 1, 2, 3
+        print >> gulp, 1, 2, 3,
+        print >> gulp
+        print >> gulp, 0 or 1, 0 or 1,
+        print >> gulp, 0 or 1
+
+        # test print >> None
+        def driver():
+            oldstdout = sys.stdout
+            sys.stdout = Gulp()
+            try:
+                tellme(Gulp())
+                tellme()
+            finally:
+                sys.stdout = oldstdout
+
+        # we should see this once
+        def tellme(file=sys.stdout):
+            print >> file, 'hello world'
+
+        driver()
+
+        # we should not see this at all
+        def tellme(file=None):
+            print >> file, 'goodbye universe'
+
+        driver()
+
+        self.assertEqual(sys.stdout.getvalue(), '''\
+1 2 3
+1 2 3
+1 1 1
+1 2 3
+1 2 3
+1 1 1
+hello world
+''')
+        sys.stdout = save_stdout
+
+        # syntax errors
+        check_syntax_error(self, 'print ,')
+        check_syntax_error(self, 'print >> x,')
+
+    def testDelStmt(self):
+        # 'del' exprlist
+        abc = [1,2,3]
+        x, y, z = abc
+        xyz = x, y, z
+
+        del abc
+        del x, y, (z, xyz)
+
+    def testPassStmt(self):
+        # 'pass'
+        pass
+
+    # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
+    # Tested below
+
+    def testBreakStmt(self):
+        # 'break'
+        while 1: break
+
+    def testContinueStmt(self):
+        # 'continue'
+        i = 1
+        while i: i = 0; continue
+
+        msg = ""
+        while not msg:
+            msg = "ok"
+            try:
+                continue
+                msg = "continue failed to continue inside try"
+            except:
+                msg = "continue inside try called except block"
+        if msg != "ok":
+            self.fail(msg)
+
+        msg = ""
+        while not msg:
+            msg = "finally block not called"
+            try:
+                continue
+            finally:
+                msg = "ok"
+        if msg != "ok":
+            self.fail(msg)
+
+    def test_break_continue_loop(self):
+        # This test warrants an explanation. It is a test specifically for SF bugs
+        # #463359 and #462937. The bug is that a 'break' statement executed or
+        # exception raised inside a try/except inside a loop, *after* a continue
+        # statement has been executed in that loop, will cause the wrong number of
+        # arguments to be popped off the stack and the instruction pointer reset to
+        # a very small number (usually 0.) Because of this, the following test
+        # *must* written as a function, and the tracking vars *must* be function
+        # arguments with default values. Otherwise, the test will loop and loop.
+
+        def test_inner(extra_burning_oil = 1, count=0):
+            big_hippo = 2
+            while big_hippo:
+                count += 1
+                try:
+                    if extra_burning_oil and big_hippo == 1:
+                        extra_burning_oil -= 1
+                        break
+                    big_hippo -= 1
+                    continue
+                except:
+                    raise
+            if count > 2 or big_hippo <> 1:
+                self.fail("continue then break in try/except in loop broken!")
+        test_inner()
+
+    def testReturn(self):
+        # 'return' [testlist]
+        def g1(): return
+        def g2(): return 1
+        g1()
+        x = g2()
+        check_syntax_error(self, "class foo:return 1")
+
+    def testYield(self):
+        check_syntax_error(self, "class foo:yield 1")
+
+    def testRaise(self):
+        # 'raise' test [',' test]
+        try: raise RuntimeError, 'just testing'
+        except RuntimeError: pass
+        try: raise KeyboardInterrupt
+        except KeyboardInterrupt: pass
+
+    def testImport(self):
+        # 'import' dotted_as_names
+        import sys
+        import time, sys
+        # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
+        from time import time
+        from time import (time)
+        # not testable inside a function, but already done at top of the module
+        # from sys import *
+        from sys import path, argv
+        from sys import (path, argv)
+        from sys import (path, argv,)
+
+    def testGlobal(self):
+        # 'global' NAME (',' NAME)*
+        global a
+        global a, b
+        global one, two, three, four, five, six, seven, eight, nine, ten
+
+    def testExec(self):
+        # 'exec' expr ['in' expr [',' expr]]
+        z = None
+        del z
+        exec 'z=1+1\n'
+        if z != 2: self.fail('exec \'z=1+1\'\\n')
+        del z
+        exec 'z=1+1'
+        if z != 2: self.fail('exec \'z=1+1\'')
+        z = None
+        del z
+        import types
+        if hasattr(types, "UnicodeType"):
+            exec r"""if 1:
+            exec u'z=1+1\n'
+            if z != 2: self.fail('exec u\'z=1+1\'\\n')
+            del z
+            exec u'z=1+1'
+            if z != 2: self.fail('exec u\'z=1+1\'')"""
+        g = {}
+        exec 'z = 1' in g
+        if g.has_key('__builtins__'): del g['__builtins__']
+        if g != {'z': 1}: self.fail('exec \'z = 1\' in g')
+        g = {}
+        l = {}
+
+        import warnings
+        warnings.filterwarnings("ignore", "global statement", module="<string>")
+        exec 'global a; a = 1; b = 2' in g, l
+        if g.has_key('__builtins__'): del g['__builtins__']
+        if l.has_key('__builtins__'): del l['__builtins__']
+        if (g, l) != ({'a':1}, {'b':2}):
+            self.fail('exec ... in g (%s), l (%s)' %(g,l))
+
+    def testAssert(self):
+        # assert_stmt: 'assert' test [',' test]
+        assert 1
+        assert 1, 1
+        assert lambda x:x
+        assert 1, lambda x:x+1
+        try:
+            assert 0, "msg"
+        except AssertionError, e:
+            self.assertEquals(e.args[0], "msg")
+        else:
+            if __debug__:
+                self.fail("AssertionError not raised by assert 0")
+
+    ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
+    # Tested below
+
+    def testIf(self):
+        # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+        if 1: pass
+        if 1: pass
+        else: pass
+        if 0: pass
+        elif 0: pass
+        if 0: pass
+        elif 0: pass
+        elif 0: pass
+        elif 0: pass
+        else: pass
+
+    def testWhile(self):
+        # 'while' test ':' suite ['else' ':' suite]
+        while 0: pass
+        while 0: pass
+        else: pass
+
+        # Issue1920: "while 0" is optimized away,
+        # ensure that the "else" clause is still present.
+        x = 0
+        while 0:
+            x = 1
+        else:
+            x = 2
+        self.assertEquals(x, 2)
+
+    def testFor(self):
+        # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
+        for i in 1, 2, 3: pass
+        for i, j, k in (): pass
+        else: pass
+        class Squares:
+            def __init__(self, max):
+                self.max = max
+                self.sofar = []
+            def __len__(self): return len(self.sofar)
+            def __getitem__(self, i):
+                if not 0 <= i < self.max: raise IndexError
+                n = len(self.sofar)
+                while n <= i:
+                    self.sofar.append(n*n)
+                    n = n+1
+                return self.sofar[i]
+        n = 0
+        for x in Squares(10): n = n+x
+        if n != 285:
+            self.fail('for over growing sequence')
+
+        result = []
+        for x, in [(1,), (2,), (3,)]:
+            result.append(x)
+        self.assertEqual(result, [1, 2, 3])
+
+    def testTry(self):
+        ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
+        ###         | 'try' ':' suite 'finally' ':' suite
+        ### except_clause: 'except' [expr [('as' | ',') expr]]
+        try:
+            1/0
+        except ZeroDivisionError:
+            pass
+        else:
+            pass
+        try: 1/0
+        except EOFError: pass
+        except TypeError as msg: pass
+        except RuntimeError, msg: pass
+        except: pass
+        else: pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError): pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError), msg: pass
+        try: pass
+        finally: pass
+
+    def testSuite(self):
+        # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
+        if 1: pass
+        if 1:
+            pass
+        if 1:
+            #
+            #
+            #
+            pass
+            pass
+            #
+            pass
+            #
+
+    def testTest(self):
+        ### and_test ('or' and_test)*
+        ### and_test: not_test ('and' not_test)*
+        ### not_test: 'not' not_test | comparison
+        if not 1: pass
+        if 1 and 1: pass
+        if 1 or 1: pass
+        if not not not 1: pass
+        if not 1 and 1 and 1: pass
+        if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
+
+    def testComparison(self):
+        ### comparison: expr (comp_op expr)*
+        ### comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+        if 1: pass
+        x = (1 == 1)
+        if 1 == 1: pass
+        if 1 != 1: pass
+        if 1 <> 1: pass
+        if 1 < 1: pass
+        if 1 > 1: pass
+        if 1 <= 1: pass
+        if 1 >= 1: pass
+        if 1 is 1: pass
+        if 1 is not 1: pass
+        if 1 in (): pass
+        if 1 not in (): pass
+        if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass
+
+    def testBinaryMaskOps(self):
+        x = 1 & 1
+        x = 1 ^ 1
+        x = 1 | 1
+
+    def testShiftOps(self):
+        x = 1 << 1
+        x = 1 >> 1
+        x = 1 << 1 >> 1
+
+    def testAdditiveOps(self):
+        x = 1
+        x = 1 + 1
+        x = 1 - 1 - 1
+        x = 1 - 1 + 1 - 1 + 1
+
+    def testMultiplicativeOps(self):
+        x = 1 * 1
+        x = 1 / 1
+        x = 1 % 1
+        x = 1 / 1 * 1 % 1
+
+    def testUnaryOps(self):
+        x = +1
+        x = -1
+        x = ~1
+        x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
+        x = -1*1/1 + 1*1 - ---1*1
+
+    def testSelectors(self):
+        ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
+        ### subscript: expr | [expr] ':' [expr]
+
+        import sys, time
+        c = sys.path[0]
+        x = time.time()
+        x = sys.modules['time'].time()
+        a = '01234'
+        c = a[0]
+        c = a[-1]
+        s = a[0:5]
+        s = a[:5]
+        s = a[0:]
+        s = a[:]
+        s = a[-5:]
+        s = a[:-1]
+        s = a[-4:-3]
+        # A rough test of SF bug 1333982.  http://python.org/sf/1333982
+        # The testing here is fairly incomplete.
+        # Test cases should include: commas with 1 and 2 colons
+        d = {}
+        d[1] = 1
+        d[1,] = 2
+        d[1,2] = 3
+        d[1,2,3] = 4
+        L = list(d)
+        L.sort()
+        self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
+
+    def testAtoms(self):
+        ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING
+        ### dictmaker: test ':' test (',' test ':' test)* [',']
+
+        x = (1)
+        x = (1 or 2 or 3)
+        x = (1 or 2 or 3, 2, 3)
+
+        x = []
+        x = [1]
+        x = [1 or 2 or 3]
+        x = [1 or 2 or 3, 2, 3]
+        x = []
+
+        x = {}
+        x = {'one': 1}
+        x = {'one': 1,}
+        x = {'one' or 'two': 1 or 2}
+        x = {'one': 1, 'two': 2}
+        x = {'one': 1, 'two': 2,}
+        x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
+
+        x = `x`
+        x = `1 or 2 or 3`
+        self.assertEqual(`1,2`, '(1, 2)')
+
+        x = x
+        x = 'x'
+        x = 123
+
+    ### exprlist: expr (',' expr)* [',']
+    ### testlist: test (',' test)* [',']
+    # These have been exercised enough above
+
+    def testClassdef(self):
+        # 'class' NAME ['(' [testlist] ')'] ':' suite
+        class B: pass
+        class B2(): pass
+        class C1(B): pass
+        class C2(B): pass
+        class D(C1, C2, B): pass
+        class C:
+            def meth1(self): pass
+            def meth2(self, arg): pass
+            def meth3(self, a1, a2): pass
+        # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        # decorators: decorator+
+        # decorated: decorators (classdef | funcdef)
+        def class_decorator(x):
+            x.decorated = True
+            return x
+        @class_decorator
+        class G:
+            pass
+        self.assertEqual(G.decorated, True)
+
+    def testListcomps(self):
+        # list comprehension tests
+        nums = [1, 2, 3, 4, 5]
+        strs = ["Apple", "Banana", "Coconut"]
+        spcs = ["  Apple", " Banana ", "Coco  nut  "]
+
+        self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco  nut'])
+        self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
+        self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
+        self.assertEqual([(i, s) for i in nums for s in strs],
+                         [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
+                          (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
+                          (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
+                         [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
+                         [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
+
+        def test_in_func(l):
+            return [None < x < 3 for x in l if x > 2]
+
+        self.assertEqual(test_in_func(nums), [False, False, False])
+
+        def test_nested_front():
+            self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
+                             [[1, 2], [3, 4], [5, 6]])
+
+        test_nested_front()
+
+        check_syntax_error(self, "[i, s for i in nums for s in strs]")
+        check_syntax_error(self, "[x if y]")
+
+        suppliers = [
+          (1, "Boeing"),
+          (2, "Ford"),
+          (3, "Macdonalds")
+        ]
+
+        parts = [
+          (10, "Airliner"),
+          (20, "Engine"),
+          (30, "Cheeseburger")
+        ]
+
+        suppart = [
+          (1, 10), (1, 20), (2, 20), (3, 30)
+        ]
+
+        x = [
+          (sname, pname)
+            for (sno, sname) in suppliers
+              for (pno, pname) in parts
+                for (sp_sno, sp_pno) in suppart
+                  if sno == sp_sno and pno == sp_pno
+        ]
+
+        self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
+                             ('Macdonalds', 'Cheeseburger')])
+
+    def testGenexps(self):
+        # generator expression tests
+        g = ([x for x in range(10)] for x in range(1))
+        self.assertEqual(g.next(), [x for x in range(10)])
+        try:
+            g.next()
+            self.fail('should produce StopIteration exception')
+        except StopIteration:
+            pass
+
+        a = 1
+        try:
+            g = (a for d in a)
+            g.next()
+            self.fail('should produce TypeError')
+        except TypeError:
+            pass
+
+        self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
+        self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
+
+        a = [x for x in range(10)]
+        b = (x for x in (y for y in a))
+        self.assertEqual(sum(b), sum([x for x in range(10)]))
+
+        self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
+        self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
+        self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
+        check_syntax_error(self, "foo(x for x in range(10), 100)")
+        check_syntax_error(self, "foo(100, x for x in range(10))")
+
+    def testComprehensionSpecials(self):
+        # test for outmost iterable precomputation
+        x = 10; g = (i for i in range(x)); x = 5
+        self.assertEqual(len(list(g)), 10)
+
+        # This should hold, since we're only precomputing outmost iterable.
+        x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
+        x = 5; t = True;
+        self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
+
+        # Grammar allows multiple adjacent 'if's in listcomps and genexps,
+        # even though it's silly. Make sure it works (ifelse broke this.)
+        self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
+        self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
+
+        # verify unpacking single element tuples in listcomp/genexp.
+        self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
+        self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
+
+    def test_with_statement(self):
+        class manager(object):
+            def __enter__(self):
+                return (1, 2)
+            def __exit__(self, *args):
+                pass
+
+        with manager():
+            pass
+        with manager() as x:
+            pass
+        with manager() as (x, y):
+            pass
+        with manager(), manager():
+            pass
+        with manager() as x, manager() as y:
+            pass
+        with manager() as x, manager():
+            pass
+
+    def testIfElseExpr(self):
+        # Test ifelse expressions in various cases
+        def _checkeval(msg, ret):
+            "helper to check that evaluation of expressions is done correctly"
+            print x
+            return ret
+
+        self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
+        self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
+        self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
+        self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
+        self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
+        self.assertEqual((5 and 6 if 0 else 1), 1)
+        self.assertEqual(((5 and 6) if 0 else 1), 1)
+        self.assertEqual((5 and (6 if 1 else 1)), 6)
+        self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
+        self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
+        self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
+        self.assertEqual((not 5 if 1 else 1), False)
+        self.assertEqual((not 5 if 0 else 1), 1)
+        self.assertEqual((6 + 1 if 1 else 2), 7)
+        self.assertEqual((6 - 1 if 1 else 2), 5)
+        self.assertEqual((6 * 2 if 1 else 4), 12)
+        self.assertEqual((6 / 2 if 1 else 3), 3)
+        self.assertEqual((6 < 4 if 0 else 2), 2)
+
+
+def test_main():
+    run_unittest(TokenTests, GrammarTests)
+
+if __name__ == '__main__':
+    test_main()

third_party/python/Lib/lib2to3/tests/data/py3_test_grammar.py

@@ -0,0 +1,945 @@
+# Python test set -- part 1, grammar.
+# This just tests whether the parser accepts them all.
+
+# NOTE: When you run this test as a script from the command line, you
+# get warnings about certain hex/oct constants.  Since those are
+# issued by the parser, you can't suppress them by adding a
+# filterwarnings() call to this module.  Therefore, to shut up the
+# regression test, the filterwarnings() call has been added to
+# regrtest.py.
+
+from test.support import run_unittest, check_syntax_error
+import unittest
+import sys
+# testing import *
+from sys import *
+
+class TokenTests(unittest.TestCase):
+
+    def testBackslash(self):
+        # Backslash means line continuation:
+        x = 1 \
+        + 1
+        self.assertEquals(x, 2, 'backslash for line continuation')
+
+        # Backslash does not means continuation in comments :\
+        x = 0
+        self.assertEquals(x, 0, 'backslash ending comment')
+
+    def testPlainIntegers(self):
+        self.assertEquals(type(000), type(0))
+        self.assertEquals(0xff, 255)
+        self.assertEquals(0o377, 255)
+        self.assertEquals(2147483647, 0o17777777777)
+        self.assertEquals(0b1001, 9)
+        # "0x" is not a valid literal
+        self.assertRaises(SyntaxError, eval, "0x")
+        from sys import maxsize
+        if maxsize == 2147483647:
+            self.assertEquals(-2147483647-1, -0o20000000000)
+            # XXX -2147483648
+            self.assert_(0o37777777777 > 0)
+            self.assert_(0xffffffff > 0)
+            self.assert_(0b1111111111111111111111111111111 > 0)
+            for s in ('2147483648', '0o40000000000', '0x100000000',
+                      '0b10000000000000000000000000000000'):
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        elif maxsize == 9223372036854775807:
+            self.assertEquals(-9223372036854775807-1, -0o1000000000000000000000)
+            self.assert_(0o1777777777777777777777 > 0)
+            self.assert_(0xffffffffffffffff > 0)
+            self.assert_(0b11111111111111111111111111111111111111111111111111111111111111 > 0)
+            for s in '9223372036854775808', '0o2000000000000000000000', \
+                     '0x10000000000000000', \
+                     '0b100000000000000000000000000000000000000000000000000000000000000':
+                try:
+                    x = eval(s)
+                except OverflowError:
+                    self.fail("OverflowError on huge integer literal %r" % s)
+        else:
+            self.fail('Weird maxsize value %r' % maxsize)
+
+    def testLongIntegers(self):
+        x = 0
+        x = 0xffffffffffffffff
+        x = 0Xffffffffffffffff
+        x = 0o77777777777777777
+        x = 0O77777777777777777
+        x = 123456789012345678901234567890
+        x = 0b100000000000000000000000000000000000000000000000000000000000000000000
+        x = 0B111111111111111111111111111111111111111111111111111111111111111111111
+
+    def testUnderscoresInNumbers(self):
+        # Integers
+        x = 1_0
+        x = 123_456_7_89
+        x = 0xabc_123_4_5
+        x = 0X_abc_123
+        x = 0B11_01
+        x = 0b_11_01
+        x = 0o45_67
+        x = 0O_45_67
+
+        # Floats
+        x = 3_1.4
+        x = 03_1.4
+        x = 3_1.
+        x = .3_1
+        x = 3.1_4
+        x = 0_3.1_4
+        x = 3e1_4
+        x = 3_1e+4_1
+        x = 3_1E-4_1
+
+    def testFloats(self):
+        x = 3.14
+        x = 314.
+        x = 0.314
+        # XXX x = 000.314
+        x = .314
+        x = 3e14
+        x = 3E14
+        x = 3e-14
+        x = 3e+14
+        x = 3.e14
+        x = .3e14
+        x = 3.1e4
+
+    def testStringLiterals(self):
+        x = ''; y = ""; self.assert_(len(x) == 0 and x == y)
+        x = '\''; y = "'"; self.assert_(len(x) == 1 and x == y and ord(x) == 39)
+        x = '"'; y = "\""; self.assert_(len(x) == 1 and x == y and ord(x) == 34)
+        x = "doesn't \"shrink\" does it"
+        y = 'doesn\'t "shrink" does it'
+        self.assert_(len(x) == 24 and x == y)
+        x = "does \"shrink\" doesn't it"
+        y = 'does "shrink" doesn\'t it'
+        self.assert_(len(x) == 24 and x == y)
+        x = """
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+"""
+        y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
+        self.assertEquals(x, y)
+        y = '''
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+'''
+        self.assertEquals(x, y)
+        y = "\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the 'lazy' dog.\n\
+"
+        self.assertEquals(x, y)
+        y = '\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the \'lazy\' dog.\n\
+'
+        self.assertEquals(x, y)
+
+    def testEllipsis(self):
+        x = ...
+        self.assert_(x is Ellipsis)
+        self.assertRaises(SyntaxError, eval, ".. .")
+
+class GrammarTests(unittest.TestCase):
+
+    # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+    # XXX can't test in a script -- this rule is only used when interactive
+
+    # file_input: (NEWLINE | stmt)* ENDMARKER
+    # Being tested as this very moment this very module
+
+    # expr_input: testlist NEWLINE
+    # XXX Hard to test -- used only in calls to input()
+
+    def testEvalInput(self):
+        # testlist ENDMARKER
+        x = eval('1, 0 or 1')
+
+    def testFuncdef(self):
+        ### [decorators] 'def' NAME parameters ['->' test] ':' suite
+        ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        ### decorators: decorator+
+        ### parameters: '(' [typedargslist] ')'
+        ### typedargslist: ((tfpdef ['=' test] ',')*
+        ###                ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
+        ###                | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
+        ### tfpdef: NAME [':' test]
+        ### varargslist: ((vfpdef ['=' test] ',')*
+        ###              ('*' [vfpdef] (',' vfpdef ['=' test])*  [',' '**' vfpdef] | '**' vfpdef)
+        ###              | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
+        ### vfpdef: NAME
+        def f1(): pass
+        f1()
+        f1(*())
+        f1(*(), **{})
+        def f2(one_argument): pass
+        def f3(two, arguments): pass
+        self.assertEquals(f2.__code__.co_varnames, ('one_argument',))
+        self.assertEquals(f3.__code__.co_varnames, ('two', 'arguments'))
+        def a1(one_arg,): pass
+        def a2(two, args,): pass
+        def v0(*rest): pass
+        def v1(a, *rest): pass
+        def v2(a, b, *rest): pass
+
+        f1()
+        f2(1)
+        f2(1,)
+        f3(1, 2)
+        f3(1, 2,)
+        v0()
+        v0(1)
+        v0(1,)
+        v0(1,2)
+        v0(1,2,3,4,5,6,7,8,9,0)
+        v1(1)
+        v1(1,)
+        v1(1,2)
+        v1(1,2,3)
+        v1(1,2,3,4,5,6,7,8,9,0)
+        v2(1,2)
+        v2(1,2,3)
+        v2(1,2,3,4)
+        v2(1,2,3,4,5,6,7,8,9,0)
+
+        def d01(a=1): pass
+        d01()
+        d01(1)
+        d01(*(1,))
+        d01(**{'a':2})
+        def d11(a, b=1): pass
+        d11(1)
+        d11(1, 2)
+        d11(1, **{'b':2})
+        def d21(a, b, c=1): pass
+        d21(1, 2)
+        d21(1, 2, 3)
+        d21(*(1, 2, 3))
+        d21(1, *(2, 3))
+        d21(1, 2, *(3,))
+        d21(1, 2, **{'c':3})
+        def d02(a=1, b=2): pass
+        d02()
+        d02(1)
+        d02(1, 2)
+        d02(*(1, 2))
+        d02(1, *(2,))
+        d02(1, **{'b':2})
+        d02(**{'a': 1, 'b': 2})
+        def d12(a, b=1, c=2): pass
+        d12(1)
+        d12(1, 2)
+        d12(1, 2, 3)
+        def d22(a, b, c=1, d=2): pass
+        d22(1, 2)
+        d22(1, 2, 3)
+        d22(1, 2, 3, 4)
+        def d01v(a=1, *rest): pass
+        d01v()
+        d01v(1)
+        d01v(1, 2)
+        d01v(*(1, 2, 3, 4))
+        d01v(*(1,))
+        d01v(**{'a':2})
+        def d11v(a, b=1, *rest): pass
+        d11v(1)
+        d11v(1, 2)
+        d11v(1, 2, 3)
+        def d21v(a, b, c=1, *rest): pass
+        d21v(1, 2)
+        d21v(1, 2, 3)
+        d21v(1, 2, 3, 4)
+        d21v(*(1, 2, 3, 4))
+        d21v(1, 2, **{'c': 3})
+        def d02v(a=1, b=2, *rest): pass
+        d02v()
+        d02v(1)
+        d02v(1, 2)
+        d02v(1, 2, 3)
+        d02v(1, *(2, 3, 4))
+        d02v(**{'a': 1, 'b': 2})
+        def d12v(a, b=1, c=2, *rest): pass
+        d12v(1)
+        d12v(1, 2)
+        d12v(1, 2, 3)
+        d12v(1, 2, 3, 4)
+        d12v(*(1, 2, 3, 4))
+        d12v(1, 2, *(3, 4, 5))
+        d12v(1, *(2,), **{'c': 3})
+        def d22v(a, b, c=1, d=2, *rest): pass
+        d22v(1, 2)
+        d22v(1, 2, 3)
+        d22v(1, 2, 3, 4)
+        d22v(1, 2, 3, 4, 5)
+        d22v(*(1, 2, 3, 4))
+        d22v(1, 2, *(3, 4, 5))
+        d22v(1, *(2, 3), **{'d': 4})
+
+        # keyword argument type tests
+        try:
+            str('x', **{b'foo':1 })
+        except TypeError:
+            pass
+        else:
+            self.fail('Bytes should not work as keyword argument names')
+        # keyword only argument tests
+        def pos0key1(*, key): return key
+        pos0key1(key=100)
+        def pos2key2(p1, p2, *, k1, k2=100): return p1,p2,k1,k2
+        pos2key2(1, 2, k1=100)
+        pos2key2(1, 2, k1=100, k2=200)
+        pos2key2(1, 2, k2=100, k1=200)
+        def pos2key2dict(p1, p2, *, k1=100, k2, **kwarg): return p1,p2,k1,k2,kwarg
+        pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
+        pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
+
+        # keyword arguments after *arglist
+        def f(*args, **kwargs):
+            return args, kwargs
+        self.assertEquals(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
+                                                    {'x':2, 'y':5}))
+        self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
+        self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
+
+        # argument annotation tests
+        def f(x) -> list: pass
+        self.assertEquals(f.__annotations__, {'return': list})
+        def f(x:int): pass
+        self.assertEquals(f.__annotations__, {'x': int})
+        def f(*x:str): pass
+        self.assertEquals(f.__annotations__, {'x': str})
+        def f(**x:float): pass
+        self.assertEquals(f.__annotations__, {'x': float})
+        def f(x, y:1+2): pass
+        self.assertEquals(f.__annotations__, {'y': 3})
+        def f(a, b:1, c:2, d): pass
+        self.assertEquals(f.__annotations__, {'b': 1, 'c': 2})
+        def f(a, b:1, c:2, d, e:3=4, f=5, *g:6): pass
+        self.assertEquals(f.__annotations__,
+                          {'b': 1, 'c': 2, 'e': 3, 'g': 6})
+        def f(a, b:1, c:2, d, e:3=4, f=5, *g:6, h:7, i=8, j:9=10,
+              **k:11) -> 12: pass
+        self.assertEquals(f.__annotations__,
+                          {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
+                           'k': 11, 'return': 12})
+        # Check for SF Bug #1697248 - mixing decorators and a return annotation
+        def null(x): return x
+        @null
+        def f(x) -> list: pass
+        self.assertEquals(f.__annotations__, {'return': list})
+
+        # test closures with a variety of oparg's
+        closure = 1
+        def f(): return closure
+        def f(x=1): return closure
+        def f(*, k=1): return closure
+        def f() -> int: return closure
+
+        # Check ast errors in *args and *kwargs
+        check_syntax_error(self, "f(*g(1=2))")
+        check_syntax_error(self, "f(**g(1=2))")
+
+    def testLambdef(self):
+        ### lambdef: 'lambda' [varargslist] ':' test
+        l1 = lambda : 0
+        self.assertEquals(l1(), 0)
+        l2 = lambda : a[d] # XXX just testing the expression
+        l3 = lambda : [2 < x for x in [-1, 3, 0]]
+        self.assertEquals(l3(), [0, 1, 0])
+        l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
+        self.assertEquals(l4(), 1)
+        l5 = lambda x, y, z=2: x + y + z
+        self.assertEquals(l5(1, 2), 5)
+        self.assertEquals(l5(1, 2, 3), 6)
+        check_syntax_error(self, "lambda x: x = 2")
+        check_syntax_error(self, "lambda (None,): None")
+        l6 = lambda x, y, *, k=20: x+y+k
+        self.assertEquals(l6(1,2), 1+2+20)
+        self.assertEquals(l6(1,2,k=10), 1+2+10)
+
+
+    ### stmt: simple_stmt | compound_stmt
+    # Tested below
+
+    def testSimpleStmt(self):
+        ### simple_stmt: small_stmt (';' small_stmt)* [';']
+        x = 1; pass; del x
+        def foo():
+            # verify statements that end with semi-colons
+            x = 1; pass; del x;
+        foo()
+
+    ### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt
+    # Tested below
+
+    def testExprStmt(self):
+        # (exprlist '=')* exprlist
+        1
+        1, 2, 3
+        x = 1
+        x = 1, 2, 3
+        x = y = z = 1, 2, 3
+        x, y, z = 1, 2, 3
+        abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4)
+
+        check_syntax_error(self, "x + 1 = 1")
+        check_syntax_error(self, "a + 1 = b + 2")
+
+    def testDelStmt(self):
+        # 'del' exprlist
+        abc = [1,2,3]
+        x, y, z = abc
+        xyz = x, y, z
+
+        del abc
+        del x, y, (z, xyz)
+
+    def testPassStmt(self):
+        # 'pass'
+        pass
+
+    # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt
+    # Tested below
+
+    def testBreakStmt(self):
+        # 'break'
+        while 1: break
+
+    def testContinueStmt(self):
+        # 'continue'
+        i = 1
+        while i: i = 0; continue
+
+        msg = ""
+        while not msg:
+            msg = "ok"
+            try:
+                continue
+                msg = "continue failed to continue inside try"
+            except:
+                msg = "continue inside try called except block"
+        if msg != "ok":
+            self.fail(msg)
+
+        msg = ""
+        while not msg:
+            msg = "finally block not called"
+            try:
+                continue
+            finally:
+                msg = "ok"
+        if msg != "ok":
+            self.fail(msg)
+
+    def test_break_continue_loop(self):
+        # This test warrants an explanation. It is a test specifically for SF bugs
+        # #463359 and #462937. The bug is that a 'break' statement executed or
+        # exception raised inside a try/except inside a loop, *after* a continue
+        # statement has been executed in that loop, will cause the wrong number of
+        # arguments to be popped off the stack and the instruction pointer reset to
+        # a very small number (usually 0.) Because of this, the following test
+        # *must* written as a function, and the tracking vars *must* be function
+        # arguments with default values. Otherwise, the test will loop and loop.
+
+        def test_inner(extra_burning_oil = 1, count=0):
+            big_hippo = 2
+            while big_hippo:
+                count += 1
+                try:
+                    if extra_burning_oil and big_hippo == 1:
+                        extra_burning_oil -= 1
+                        break
+                    big_hippo -= 1
+                    continue
+                except:
+                    raise
+            if count > 2 or big_hippo != 1:
+                self.fail("continue then break in try/except in loop broken!")
+        test_inner()
+
+    def testReturn(self):
+        # 'return' [testlist]
+        def g1(): return
+        def g2(): return 1
+        g1()
+        x = g2()
+        check_syntax_error(self, "class foo:return 1")
+
+    def testYield(self):
+        check_syntax_error(self, "class foo:yield 1")
+
+    def testRaise(self):
+        # 'raise' test [',' test]
+        try: raise RuntimeError('just testing')
+        except RuntimeError: pass
+        try: raise KeyboardInterrupt
+        except KeyboardInterrupt: pass
+
+    def testImport(self):
+        # 'import' dotted_as_names
+        import sys
+        import time, sys
+        # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names)
+        from time import time
+        from time import (time)
+        # not testable inside a function, but already done at top of the module
+        # from sys import *
+        from sys import path, argv
+        from sys import (path, argv)
+        from sys import (path, argv,)
+
+    def testGlobal(self):
+        # 'global' NAME (',' NAME)*
+        global a
+        global a, b
+        global one, two, three, four, five, six, seven, eight, nine, ten
+
+    def testNonlocal(self):
+        # 'nonlocal' NAME (',' NAME)*
+        x = 0
+        y = 0
+        def f():
+            nonlocal x
+            nonlocal x, y
+
+    def testAssert(self):
+        # assert_stmt: 'assert' test [',' test]
+        assert 1
+        assert 1, 1
+        assert lambda x:x
+        assert 1, lambda x:x+1
+        try:
+            assert 0, "msg"
+        except AssertionError as e:
+            self.assertEquals(e.args[0], "msg")
+        else:
+            if __debug__:
+                self.fail("AssertionError not raised by assert 0")
+
+    ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
+    # Tested below
+
+    def testIf(self):
+        # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
+        if 1: pass
+        if 1: pass
+        else: pass
+        if 0: pass
+        elif 0: pass
+        if 0: pass
+        elif 0: pass
+        elif 0: pass
+        elif 0: pass
+        else: pass
+
+    def testWhile(self):
+        # 'while' test ':' suite ['else' ':' suite]
+        while 0: pass
+        while 0: pass
+        else: pass
+
+        # Issue1920: "while 0" is optimized away,
+        # ensure that the "else" clause is still present.
+        x = 0
+        while 0:
+            x = 1
+        else:
+            x = 2
+        self.assertEquals(x, 2)
+
+    def testFor(self):
+        # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
+        for i in 1, 2, 3: pass
+        for i, j, k in (): pass
+        else: pass
+        class Squares:
+            def __init__(self, max):
+                self.max = max
+                self.sofar = []
+            def __len__(self): return len(self.sofar)
+            def __getitem__(self, i):
+                if not 0 <= i < self.max: raise IndexError
+                n = len(self.sofar)
+                while n <= i:
+                    self.sofar.append(n*n)
+                    n = n+1
+                return self.sofar[i]
+        n = 0
+        for x in Squares(10): n = n+x
+        if n != 285:
+            self.fail('for over growing sequence')
+
+        result = []
+        for x, in [(1,), (2,), (3,)]:
+            result.append(x)
+        self.assertEqual(result, [1, 2, 3])
+
+    def testTry(self):
+        ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
+        ###         | 'try' ':' suite 'finally' ':' suite
+        ### except_clause: 'except' [expr ['as' expr]]
+        try:
+            1/0
+        except ZeroDivisionError:
+            pass
+        else:
+            pass
+        try: 1/0
+        except EOFError: pass
+        except TypeError as msg: pass
+        except RuntimeError as msg: pass
+        except: pass
+        else: pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError): pass
+        try: 1/0
+        except (EOFError, TypeError, ZeroDivisionError) as msg: pass
+        try: pass
+        finally: pass
+
+    def testSuite(self):
+        # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
+        if 1: pass
+        if 1:
+            pass
+        if 1:
+            #
+            #
+            #
+            pass
+            pass
+            #
+            pass
+            #
+
+    def testTest(self):
+        ### and_test ('or' and_test)*
+        ### and_test: not_test ('and' not_test)*
+        ### not_test: 'not' not_test | comparison
+        if not 1: pass
+        if 1 and 1: pass
+        if 1 or 1: pass
+        if not not not 1: pass
+        if not 1 and 1 and 1: pass
+        if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass
+
+    def testComparison(self):
+        ### comparison: expr (comp_op expr)*
+        ### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+        if 1: pass
+        x = (1 == 1)
+        if 1 == 1: pass
+        if 1 != 1: pass
+        if 1 < 1: pass
+        if 1 > 1: pass
+        if 1 <= 1: pass
+        if 1 >= 1: pass
+        if 1 is 1: pass
+        if 1 is not 1: pass
+        if 1 in (): pass
+        if 1 not in (): pass
+        if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass
+
+    def testBinaryMaskOps(self):
+        x = 1 & 1
+        x = 1 ^ 1
+        x = 1 | 1
+
+    def testShiftOps(self):
+        x = 1 << 1
+        x = 1 >> 1
+        x = 1 << 1 >> 1
+
+    def testAdditiveOps(self):
+        x = 1
+        x = 1 + 1
+        x = 1 - 1 - 1
+        x = 1 - 1 + 1 - 1 + 1
+
+    def testMultiplicativeOps(self):
+        x = 1 * 1
+        x = 1 / 1
+        x = 1 % 1
+        x = 1 / 1 * 1 % 1
+
+    def testUnaryOps(self):
+        x = +1
+        x = -1
+        x = ~1
+        x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
+        x = -1*1/1 + 1*1 - ---1*1
+
+    def testSelectors(self):
+        ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME
+        ### subscript: expr | [expr] ':' [expr]
+
+        import sys, time
+        c = sys.path[0]
+        x = time.time()
+        x = sys.modules['time'].time()
+        a = '01234'
+        c = a[0]
+        c = a[-1]
+        s = a[0:5]
+        s = a[:5]
+        s = a[0:]
+        s = a[:]
+        s = a[-5:]
+        s = a[:-1]
+        s = a[-4:-3]
+        # A rough test of SF bug 1333982.  http://python.org/sf/1333982
+        # The testing here is fairly incomplete.
+        # Test cases should include: commas with 1 and 2 colons
+        d = {}
+        d[1] = 1
+        d[1,] = 2
+        d[1,2] = 3
+        d[1,2,3] = 4
+        L = list(d)
+        L.sort(key=lambda x: x if isinstance(x, tuple) else ())
+        self.assertEquals(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
+
+    def testAtoms(self):
+        ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING
+        ### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [','])
+
+        x = (1)
+        x = (1 or 2 or 3)
+        x = (1 or 2 or 3, 2, 3)
+
+        x = []
+        x = [1]
+        x = [1 or 2 or 3]
+        x = [1 or 2 or 3, 2, 3]
+        x = []
+
+        x = {}
+        x = {'one': 1}
+        x = {'one': 1,}
+        x = {'one' or 'two': 1 or 2}
+        x = {'one': 1, 'two': 2}
+        x = {'one': 1, 'two': 2,}
+        x = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6}
+
+        x = {'one'}
+        x = {'one', 1,}
+        x = {'one', 'two', 'three'}
+        x = {2, 3, 4,}
+
+        x = x
+        x = 'x'
+        x = 123
+
+    ### exprlist: expr (',' expr)* [',']
+    ### testlist: test (',' test)* [',']
+    # These have been exercised enough above
+
+    def testClassdef(self):
+        # 'class' NAME ['(' [testlist] ')'] ':' suite
+        class B: pass
+        class B2(): pass
+        class C1(B): pass
+        class C2(B): pass
+        class D(C1, C2, B): pass
+        class C:
+            def meth1(self): pass
+            def meth2(self, arg): pass
+            def meth3(self, a1, a2): pass
+
+        # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        # decorators: decorator+
+        # decorated: decorators (classdef | funcdef)
+        def class_decorator(x): return x
+        @class_decorator
+        class G: pass
+
+    def testDictcomps(self):
+        # dictorsetmaker: ( (test ':' test (comp_for |
+        #                                   (',' test ':' test)* [','])) |
+        #                   (test (comp_for | (',' test)* [','])) )
+        nums = [1, 2, 3]
+        self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4})
+
+    def testListcomps(self):
+        # list comprehension tests
+        nums = [1, 2, 3, 4, 5]
+        strs = ["Apple", "Banana", "Coconut"]
+        spcs = ["  Apple", " Banana ", "Coco  nut  "]
+
+        self.assertEqual([s.strip() for s in spcs], ['Apple', 'Banana', 'Coco  nut'])
+        self.assertEqual([3 * x for x in nums], [3, 6, 9, 12, 15])
+        self.assertEqual([x for x in nums if x > 2], [3, 4, 5])
+        self.assertEqual([(i, s) for i in nums for s in strs],
+                         [(1, 'Apple'), (1, 'Banana'), (1, 'Coconut'),
+                          (2, 'Apple'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Apple'), (3, 'Banana'), (3, 'Coconut'),
+                          (4, 'Apple'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Apple'), (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(i, s) for i in nums for s in [f for f in strs if "n" in f]],
+                         [(1, 'Banana'), (1, 'Coconut'), (2, 'Banana'), (2, 'Coconut'),
+                          (3, 'Banana'), (3, 'Coconut'), (4, 'Banana'), (4, 'Coconut'),
+                          (5, 'Banana'), (5, 'Coconut')])
+        self.assertEqual([(lambda a:[a**i for i in range(a+1)])(j) for j in range(5)],
+                         [[1], [1, 1], [1, 2, 4], [1, 3, 9, 27], [1, 4, 16, 64, 256]])
+
+        def test_in_func(l):
+            return [0 < x < 3 for x in l if x > 2]
+
+        self.assertEqual(test_in_func(nums), [False, False, False])
+
+        def test_nested_front():
+            self.assertEqual([[y for y in [x, x + 1]] for x in [1,3,5]],
+                             [[1, 2], [3, 4], [5, 6]])
+
+        test_nested_front()
+
+        check_syntax_error(self, "[i, s for i in nums for s in strs]")
+        check_syntax_error(self, "[x if y]")
+
+        suppliers = [
+          (1, "Boeing"),
+          (2, "Ford"),
+          (3, "Macdonalds")
+        ]
+
+        parts = [
+          (10, "Airliner"),
+          (20, "Engine"),
+          (30, "Cheeseburger")
+        ]
+
+        suppart = [
+          (1, 10), (1, 20), (2, 20), (3, 30)
+        ]
+
+        x = [
+          (sname, pname)
+            for (sno, sname) in suppliers
+              for (pno, pname) in parts
+                for (sp_sno, sp_pno) in suppart
+                  if sno == sp_sno and pno == sp_pno
+        ]
+
+        self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'),
+                             ('Macdonalds', 'Cheeseburger')])
+
+    def testGenexps(self):
+        # generator expression tests
+        g = ([x for x in range(10)] for x in range(1))
+        self.assertEqual(next(g), [x for x in range(10)])
+        try:
+            next(g)
+            self.fail('should produce StopIteration exception')
+        except StopIteration:
+            pass
+
+        a = 1
+        try:
+            g = (a for d in a)
+            next(g)
+            self.fail('should produce TypeError')
+        except TypeError:
+            pass
+
+        self.assertEqual(list((x, y) for x in 'abcd' for y in 'abcd'), [(x, y) for x in 'abcd' for y in 'abcd'])
+        self.assertEqual(list((x, y) for x in 'ab' for y in 'xy'), [(x, y) for x in 'ab' for y in 'xy'])
+
+        a = [x for x in range(10)]
+        b = (x for x in (y for y in a))
+        self.assertEqual(sum(b), sum([x for x in range(10)]))
+
+        self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
+        self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
+        self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True)) if True), sum([x for x in range(10)]))
+        self.assertEqual(sum(x for x in (y for y in (z for z in range(10) if True) if False) if True), 0)
+        check_syntax_error(self, "foo(x for x in range(10), 100)")
+        check_syntax_error(self, "foo(100, x for x in range(10))")
+
+    def testComprehensionSpecials(self):
+        # test for outmost iterable precomputation
+        x = 10; g = (i for i in range(x)); x = 5
+        self.assertEqual(len(list(g)), 10)
+
+        # This should hold, since we're only precomputing outmost iterable.
+        x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
+        x = 5; t = True;
+        self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
+
+        # Grammar allows multiple adjacent 'if's in listcomps and genexps,
+        # even though it's silly. Make sure it works (ifelse broke this.)
+        self.assertEqual([ x for x in range(10) if x % 2 if x % 3 ], [1, 5, 7])
+        self.assertEqual(list(x for x in range(10) if x % 2 if x % 3), [1, 5, 7])
+
+        # verify unpacking single element tuples in listcomp/genexp.
+        self.assertEqual([x for x, in [(4,), (5,), (6,)]], [4, 5, 6])
+        self.assertEqual(list(x for x, in [(7,), (8,), (9,)]), [7, 8, 9])
+
+    def test_with_statement(self):
+        class manager(object):
+            def __enter__(self):
+                return (1, 2)
+            def __exit__(self, *args):
+                pass
+
+        with manager():
+            pass
+        with manager() as x:
+            pass
+        with manager() as (x, y):
+            pass
+        with manager(), manager():
+            pass
+        with manager() as x, manager() as y:
+            pass
+        with manager() as x, manager():
+            pass
+
+    def testIfElseExpr(self):
+        # Test ifelse expressions in various cases
+        def _checkeval(msg, ret):
+            "helper to check that evaluation of expressions is done correctly"
+            print(x)
+            return ret
+
+        # the next line is not allowed anymore
+        #self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True])
+        self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True])
+        self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True])
+        self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5)
+        self.assertEqual((_checkeval("check 2", 0) if 0 else 5), 5)
+        self.assertEqual((5 and 6 if 0 else 1), 1)
+        self.assertEqual(((5 and 6) if 0 else 1), 1)
+        self.assertEqual((5 and (6 if 1 else 1)), 6)
+        self.assertEqual((0 or _checkeval("check 3", 2) if 0 else 3), 3)
+        self.assertEqual((1 or _checkeval("check 4", 2) if 1 else _checkeval("check 5", 3)), 1)
+        self.assertEqual((0 or 5 if 1 else _checkeval("check 6", 3)), 5)
+        self.assertEqual((not 5 if 1 else 1), False)
+        self.assertEqual((not 5 if 0 else 1), 1)
+        self.assertEqual((6 + 1 if 1 else 2), 7)
+        self.assertEqual((6 - 1 if 1 else 2), 5)
+        self.assertEqual((6 * 2 if 1 else 4), 12)
+        self.assertEqual((6 / 2 if 1 else 3), 3)
+        self.assertEqual((6 < 4 if 0 else 2), 2)
+
+
+def test_main():
+    run_unittest(TokenTests, GrammarTests)
+
+if __name__ == '__main__':
+    test_main()

third_party/python/Lib/lib2to3/tests/pytree_idempotency.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+# Copyright 2006 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Main program for testing the infrastructure."""
+
+from __future__ import print_function
+
+__author__ = "Guido van Rossum <guido@python.org>"
+
+# Support imports (need to be imported first)
+from . import support
+
+# Python imports
+import os
+import sys
+import logging
+
+# Local imports
+from .. import pytree
+from .. import pgen2
+from ..pgen2 import driver
+
+logging.basicConfig()
+
+def main():
+    gr = driver.load_grammar("Grammar.txt")
+    dr = driver.Driver(gr, convert=pytree.convert)
+
+    fn = "example.py"
+    tree = dr.parse_file(fn, debug=True)
+    if not diff(fn, tree):
+        print("No diffs.")
+    if not sys.argv[1:]:
+        return # Pass a dummy argument to run the complete test suite below
+
+    problems = []
+
+    # Process every imported module
+    for name in sys.modules:
+        mod = sys.modules[name]
+        if mod is None or not hasattr(mod, "__file__"):
+            continue
+        fn = mod.__file__
+        if fn.endswith(".pyc"):
+            fn = fn[:-1]
+        if not fn.endswith(".py"):
+            continue
+        print("Parsing", fn, file=sys.stderr)
+        tree = dr.parse_file(fn, debug=True)
+        if diff(fn, tree):
+            problems.append(fn)
+
+    # Process every single module on sys.path (but not in packages)
+    for dir in sys.path:
+        try:
+            names = os.listdir(dir)
+        except OSError:
+            continue
+        print("Scanning", dir, "...", file=sys.stderr)
+        for name in names:
+            if not name.endswith(".py"):
+                continue
+            print("Parsing", name, file=sys.stderr)
+            fn = os.path.join(dir, name)
+            try:
+                tree = dr.parse_file(fn, debug=True)
+            except pgen2.parse.ParseError as err:
+                print("ParseError:", err)
+            else:
+                if diff(fn, tree):
+                    problems.append(fn)
+
+    # Show summary of problem files
+    if not problems:
+        print("No problems.  Congratulations!")
+    else:
+        print("Problems in following files:")
+        for fn in problems:
+            print("***", fn)
+
+def diff(fn, tree):
+    f = open("@", "w")
+    try:
+        f.write(str(tree))
+    finally:
+        f.close()
+    try:
+        return os.system("diff -u %s @" % fn)
+    finally:
+        os.remove("@")
+
+if __name__ == "__main__":
+    main()

third_party/python/Lib/lib2to3/tests/support.py

@@ -0,0 +1,58 @@
+"""Support code for test_*.py files"""
+# Author: Collin Winter
+
+# Python imports
+import unittest
+import os
+import os.path
+from textwrap import dedent
+
+# Local imports
+from lib2to3 import pytree, refactor
+from lib2to3.pgen2 import driver as pgen2_driver
+
+test_dir = os.path.dirname(__file__)
+proj_dir = os.path.normpath(os.path.join(test_dir, ".."))
+grammar_path = os.path.join(test_dir, "..", "Grammar.txt")
+grammar = pgen2_driver.load_grammar(grammar_path)
+grammar_no_print_statement = pgen2_driver.load_grammar(grammar_path)
+del grammar_no_print_statement.keywords["print"]
+driver = pgen2_driver.Driver(grammar, convert=pytree.convert)
+driver_no_print_statement = pgen2_driver.Driver(
+    grammar_no_print_statement,
+    convert=pytree.convert
+)
+
+def parse_string(string):
+    return driver.parse_string(reformat(string), debug=True)
+
+def run_all_tests(test_mod=None, tests=None):
+    if tests is None:
+        tests = unittest.TestLoader().loadTestsFromModule(test_mod)
+    unittest.TextTestRunner(verbosity=2).run(tests)
+
+def reformat(string):
+    return dedent(string) + "\n\n"
+
+def get_refactorer(fixer_pkg="lib2to3", fixers=None, options=None):
+    """
+    A convenience function for creating a RefactoringTool for tests.
+
+    fixers is a list of fixers for the RefactoringTool to use. By default
+    "lib2to3.fixes.*" is used. options is an optional dictionary of options to
+    be passed to the RefactoringTool.
+    """
+    if fixers is not None:
+        fixers = [fixer_pkg + ".fixes.fix_" + fix for fix in fixers]
+    else:
+        fixers = refactor.get_fixers_from_package(fixer_pkg + ".fixes")
+    options = options or {}
+    return refactor.RefactoringTool(fixers, options, explicit=True)
+
+def all_project_files():
+    for dirpath, dirnames, filenames in os.walk(proj_dir):
+        for filename in filenames:
+            if filename.endswith(".py"):
+                yield os.path.join(dirpath, filename)
+
+TestCase = unittest.TestCase

third_party/python/Lib/lib2to3/tests/test_all_fixers.py

@@ -0,0 +1,27 @@
+"""Tests that run all fixer modules over an input stream.
+
+This has been broken out into its own test module because of its
+running time.
+"""
+# Author: Collin Winter
+
+# Python imports
+import unittest
+import test.support
+
+# Local imports
+from . import support
+
+
+@test.support.requires_resource('cpu')
+class Test_all(support.TestCase):
+
+    def setUp(self):
+        self.refactor = support.get_refactorer()
+
+    def test_all_project_files(self):
+        for filepath in support.all_project_files():
+            self.refactor.refactor_file(filepath)
+
+if __name__ == '__main__':
+    unittest.main()

third_party/python/Lib/lib2to3/tests/test_main.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+import codecs
+import io
+import logging
+import os
+import re
+import shutil
+import sys
+import tempfile
+import unittest
+
+from lib2to3 import main
+
+
+TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
+PY2_TEST_MODULE = os.path.join(TEST_DATA_DIR, "py2_test_grammar.py")
+
+
+class TestMain(unittest.TestCase):
+
+    def setUp(self):
+        self.temp_dir = None  # tearDown() will rmtree this directory if set.
+
+    def tearDown(self):
+        # Clean up logging configuration down by main.
+        del logging.root.handlers[:]
+        if self.temp_dir:
+            shutil.rmtree(self.temp_dir)
+
+    def run_2to3_capture(self, args, in_capture, out_capture, err_capture):
+        save_stdin = sys.stdin
+        save_stdout = sys.stdout
+        save_stderr = sys.stderr
+        sys.stdin = in_capture
+        sys.stdout = out_capture
+        sys.stderr = err_capture
+        try:
+            return main.main("lib2to3.fixes", args)
+        finally:
+            sys.stdin = save_stdin
+            sys.stdout = save_stdout
+            sys.stderr = save_stderr
+
+    def test_unencodable_diff(self):
+        input_stream = io.StringIO("print 'nothing'\nprint u'über'\n")
+        out = io.BytesIO()
+        out_enc = codecs.getwriter("ascii")(out)
+        err = io.StringIO()
+        ret = self.run_2to3_capture(["-"], input_stream, out_enc, err)
+        self.assertEqual(ret, 0)
+        output = out.getvalue().decode("ascii")
+        self.assertIn("-print 'nothing'", output)
+        self.assertIn("WARNING: couldn't encode <stdin>'s diff for "
+                      "your terminal", err.getvalue())
+
+    def setup_test_source_trees(self):
+        """Setup a test source tree and output destination tree."""
+        self.temp_dir = tempfile.mkdtemp()  # tearDown() cleans this up.
+        self.py2_src_dir = os.path.join(self.temp_dir, "python2_project")
+        self.py3_dest_dir = os.path.join(self.temp_dir, "python3_project")
+        os.mkdir(self.py2_src_dir)
+        os.mkdir(self.py3_dest_dir)
+        # Turn it into a package with a few files.
+        self.setup_files = []
+        open(os.path.join(self.py2_src_dir, "__init__.py"), "w").close()
+        self.setup_files.append("__init__.py")
+        shutil.copy(PY2_TEST_MODULE, self.py2_src_dir)
+        self.setup_files.append(os.path.basename(PY2_TEST_MODULE))
+        self.trivial_py2_file = os.path.join(self.py2_src_dir, "trivial.py")
+        self.init_py2_file = os.path.join(self.py2_src_dir, "__init__.py")
+        with open(self.trivial_py2_file, "w") as trivial:
+            trivial.write("print 'I need a simple conversion.'")
+        self.setup_files.append("trivial.py")
+
+    def test_filename_changing_on_output_single_dir(self):
+        """2to3 a single directory with a new output dir and suffix."""
+        self.setup_test_source_trees()
+        out = io.StringIO()
+        err = io.StringIO()
+        suffix = "TEST"
+        ret = self.run_2to3_capture(
+                ["-n", "--add-suffix", suffix, "--write-unchanged-files",
+                 "--no-diffs", "--output-dir",
+                 self.py3_dest_dir, self.py2_src_dir],
+                io.StringIO(""), out, err)
+        self.assertEqual(ret, 0)
+        stderr = err.getvalue()
+        self.assertIn(" implies -w.", stderr)
+        self.assertIn(
+                "Output in %r will mirror the input directory %r layout" % (
+                        self.py3_dest_dir, self.py2_src_dir), stderr)
+        self.assertEqual(set(name+suffix for name in self.setup_files),
+                         set(os.listdir(self.py3_dest_dir)))
+        for name in self.setup_files:
+            self.assertIn("Writing converted %s to %s" % (
+                    os.path.join(self.py2_src_dir, name),
+                    os.path.join(self.py3_dest_dir, name+suffix)), stderr)
+        sep = re.escape(os.sep)
+        self.assertRegex(
+                stderr, r"No changes to .*/__init__\.py".replace("/", sep))
+        self.assertNotRegex(
+                stderr, r"No changes to .*/trivial\.py".replace("/", sep))
+
+    def test_filename_changing_on_output_two_files(self):
+        """2to3 two files in one directory with a new output dir."""
+        self.setup_test_source_trees()
+        err = io.StringIO()
+        py2_files = [self.trivial_py2_file, self.init_py2_file]
+        expected_files = set(os.path.basename(name) for name in py2_files)
+        ret = self.run_2to3_capture(
+                ["-n", "-w", "--write-unchanged-files",
+                 "--no-diffs", "--output-dir", self.py3_dest_dir] + py2_files,
+                io.StringIO(""), io.StringIO(), err)
+        self.assertEqual(ret, 0)
+        stderr = err.getvalue()
+        self.assertIn(
+                "Output in %r will mirror the input directory %r layout" % (
+                        self.py3_dest_dir, self.py2_src_dir), stderr)
+        self.assertEqual(expected_files, set(os.listdir(self.py3_dest_dir)))
+
+    def test_filename_changing_on_output_single_file(self):
+        """2to3 a single file with a new output dir."""
+        self.setup_test_source_trees()
+        err = io.StringIO()
+        ret = self.run_2to3_capture(
+                ["-n", "-w", "--no-diffs", "--output-dir", self.py3_dest_dir,
+                 self.trivial_py2_file],
+                io.StringIO(""), io.StringIO(), err)
+        self.assertEqual(ret, 0)
+        stderr = err.getvalue()
+        self.assertIn(
+                "Output in %r will mirror the input directory %r layout" % (
+                        self.py3_dest_dir, self.py2_src_dir), stderr)
+        self.assertEqual(set([os.path.basename(self.trivial_py2_file)]),
+                         set(os.listdir(self.py3_dest_dir)))
+
+
+if __name__ == '__main__':
+    unittest.main()

third_party/python/Lib/lib2to3/tests/test_parser.py

@@ -0,0 +1,620 @@
+"""Test suite for 2to3's parser and grammar files.
+
+This is the place to add tests for changes to 2to3's grammar, such as those
+merging the grammars for Python 2 and 3. In addition to specific tests for
+parts of the grammar we've changed, we also make sure we can parse the
+test_grammar.py files from both Python 2 and Python 3.
+"""
+
+# Testing imports
+from . import support
+from .support import driver, driver_no_print_statement
+
+# Python imports
+import difflib
+import importlib
+import operator
+import os
+import pickle
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+# Local imports
+from lib2to3.pgen2 import driver as pgen2_driver
+from lib2to3.pgen2 import tokenize
+from ..pgen2.parse import ParseError
+from lib2to3.pygram import python_symbols as syms
+
+
+class TestDriver(support.TestCase):
+
+    def test_formfeed(self):
+        s = """print 1\n\x0Cprint 2\n"""
+        t = driver.parse_string(s)
+        self.assertEqual(t.children[0].children[0].type, syms.print_stmt)
+        self.assertEqual(t.children[1].children[0].type, syms.print_stmt)
+
+
+class TestPgen2Caching(support.TestCase):
+    def test_load_grammar_from_txt_file(self):
+        pgen2_driver.load_grammar(support.grammar_path, save=False, force=True)
+
+    def test_load_grammar_from_pickle(self):
+        # Make a copy of the grammar file in a temp directory we are
+        # guaranteed to be able to write to.
+        tmpdir = tempfile.mkdtemp()
+        try:
+            grammar_copy = os.path.join(
+                    tmpdir, os.path.basename(support.grammar_path))
+            shutil.copy(support.grammar_path, grammar_copy)
+            pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
+
+            pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
+            self.assertTrue(os.path.exists(pickle_name))
+
+            os.unlink(grammar_copy)  # Only the pickle remains...
+            pgen2_driver.load_grammar(grammar_copy, save=False, force=False)
+        finally:
+            shutil.rmtree(tmpdir)
+
+    @unittest.skipIf(sys.executable is None, 'sys.executable required')
+    def test_load_grammar_from_subprocess(self):
+        tmpdir = tempfile.mkdtemp()
+        tmpsubdir = os.path.join(tmpdir, 'subdir')
+        try:
+            os.mkdir(tmpsubdir)
+            grammar_base = os.path.basename(support.grammar_path)
+            grammar_copy = os.path.join(tmpdir, grammar_base)
+            grammar_sub_copy = os.path.join(tmpsubdir, grammar_base)
+            shutil.copy(support.grammar_path, grammar_copy)
+            shutil.copy(support.grammar_path, grammar_sub_copy)
+            pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
+            pickle_sub_name = pgen2_driver._generate_pickle_name(
+                     grammar_sub_copy)
+            self.assertNotEqual(pickle_name, pickle_sub_name)
+
+            # Generate a pickle file from this process.
+            pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
+            self.assertTrue(os.path.exists(pickle_name))
+
+            # Generate a new pickle file in a subprocess with a most likely
+            # different hash randomization seed.
+            sub_env = dict(os.environ)
+            sub_env['PYTHONHASHSEED'] = 'random'
+            subprocess.check_call(
+                    [sys.executable, '-c', """
+from lib2to3.pgen2 import driver as pgen2_driver
+pgen2_driver.load_grammar(%r, save=True, force=True)
+                    """ % (grammar_sub_copy,)],
+                    env=sub_env)
+            self.assertTrue(os.path.exists(pickle_sub_name))
+
+            with open(pickle_name, 'rb') as pickle_f_1, \
+                    open(pickle_sub_name, 'rb') as pickle_f_2:
+                self.assertEqual(
+                    pickle_f_1.read(), pickle_f_2.read(),
+                    msg='Grammar caches generated using different hash seeds'
+                    ' were not identical.')
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_load_packaged_grammar(self):
+        modname = __name__ + '.load_test'
+        class MyLoader:
+            def get_data(self, where):
+                return pickle.dumps({'elephant': 19})
+        class MyModule:
+            __file__ = 'parsertestmodule'
+            __spec__ = importlib.util.spec_from_loader(modname, MyLoader())
+        sys.modules[modname] = MyModule()
+        self.addCleanup(operator.delitem, sys.modules, modname)
+        g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt')
+        self.assertEqual(g.elephant, 19)
+
+
+class GrammarTest(support.TestCase):
+    def validate(self, code):
+        support.parse_string(code)
+
+    def invalid_syntax(self, code):
+        try:
+            self.validate(code)
+        except ParseError:
+            pass
+        else:
+            raise AssertionError("Syntax shouldn't have been valid")
+
+
+class TestMatrixMultiplication(GrammarTest):
+    def test_matrix_multiplication_operator(self):
+        self.validate("a @ b")
+        self.validate("a @= b")
+
+
+class TestYieldFrom(GrammarTest):
+    def test_yield_from(self):
+        self.validate("yield from x")
+        self.validate("(yield from x) + y")
+        self.invalid_syntax("yield from")
+
+
+class TestAsyncAwait(GrammarTest):
+    def test_await_expr(self):
+        self.validate("""async def foo():
+                             await x
+                      """)
+
+        self.validate("""async def foo():
+                             [i async for i in b]
+                      """)
+
+        self.validate("""async def foo():
+                             {i for i in b
+                                async for i in a if await i
+                                  for b in i}
+                      """)
+
+        self.validate("""async def foo():
+                             [await i for i in b if await c]
+                      """)
+
+        self.validate("""async def foo():
+                             [ i for i in b if c]
+                      """)
+
+        self.validate("""async def foo():
+
+            def foo(): pass
+
+            def foo(): pass
+
+            await x
+        """)
+
+        self.validate("""async def foo(): return await a""")
+
+        self.validate("""def foo():
+            def foo(): pass
+            async def foo(): await x
+        """)
+
+        self.invalid_syntax("await x")
+        self.invalid_syntax("""def foo():
+                                   await x""")
+
+        self.invalid_syntax("""def foo():
+            def foo(): pass
+            async def foo(): pass
+            await x
+        """)
+
+    def test_async_var(self):
+        self.validate("""async = 1""")
+        self.validate("""await = 1""")
+        self.validate("""def async(): pass""")
+
+    def test_async_with(self):
+        self.validate("""async def foo():
+                             async for a in b: pass""")
+
+        self.invalid_syntax("""def foo():
+                                   async for a in b: pass""")
+
+    def test_async_for(self):
+        self.validate("""async def foo():
+                             async with a: pass""")
+
+        self.invalid_syntax("""def foo():
+                                   async with a: pass""")
+
+
+class TestRaiseChanges(GrammarTest):
+    def test_2x_style_1(self):
+        self.validate("raise")
+
+    def test_2x_style_2(self):
+        self.validate("raise E, V")
+
+    def test_2x_style_3(self):
+        self.validate("raise E, V, T")
+
+    def test_2x_style_invalid_1(self):
+        self.invalid_syntax("raise E, V, T, Z")
+
+    def test_3x_style(self):
+        self.validate("raise E1 from E2")
+
+    def test_3x_style_invalid_1(self):
+        self.invalid_syntax("raise E, V from E1")
+
+    def test_3x_style_invalid_2(self):
+        self.invalid_syntax("raise E from E1, E2")
+
+    def test_3x_style_invalid_3(self):
+        self.invalid_syntax("raise from E1, E2")
+
+    def test_3x_style_invalid_4(self):
+        self.invalid_syntax("raise E from")
+
+
+# Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292
+# and Lib/test/text_parser.py test_list_displays, test_set_displays,
+# test_dict_displays, test_argument_unpacking, ... changes.
+class TestUnpackingGeneralizations(GrammarTest):
+    def test_mid_positional_star(self):
+        self.validate("""func(1, *(2, 3), 4)""")
+
+    def test_double_star_dict_literal(self):
+        self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""")
+
+    def test_double_star_dict_literal_after_keywords(self):
+        self.validate("""func(spam='fried', **{'eggs':'scrambled'})""")
+
+    def test_list_display(self):
+        self.validate("""[*{2}, 3, *[4]]""")
+
+    def test_set_display(self):
+        self.validate("""{*{2}, 3, *[4]}""")
+
+    def test_dict_display_1(self):
+        self.validate("""{**{}}""")
+
+    def test_dict_display_2(self):
+        self.validate("""{**{}, 3:4, **{5:6, 7:8}}""")
+
+    def test_argument_unpacking_1(self):
+        self.validate("""f(a, *b, *c, d)""")
+
+    def test_argument_unpacking_2(self):
+        self.validate("""f(**a, **b)""")
+
+    def test_argument_unpacking_3(self):
+        self.validate("""f(2, *a, *b, **b, **c, **d)""")
+
+    def test_trailing_commas_1(self):
+        self.validate("def f(a, b): call(a, b)")
+        self.validate("def f(a, b,): call(a, b,)")
+
+    def test_trailing_commas_2(self):
+        self.validate("def f(a, *b): call(a, *b)")
+        self.validate("def f(a, *b,): call(a, *b,)")
+
+    def test_trailing_commas_3(self):
+        self.validate("def f(a, b=1): call(a, b=1)")
+        self.validate("def f(a, b=1,): call(a, b=1,)")
+
+    def test_trailing_commas_4(self):
+        self.validate("def f(a, **b): call(a, **b)")
+        self.validate("def f(a, **b,): call(a, **b,)")
+
+    def test_trailing_commas_5(self):
+        self.validate("def f(*a, b=1): call(*a, b=1)")
+        self.validate("def f(*a, b=1,): call(*a, b=1,)")
+
+    def test_trailing_commas_6(self):
+        self.validate("def f(*a, **b): call(*a, **b)")
+        self.validate("def f(*a, **b,): call(*a, **b,)")
+
+    def test_trailing_commas_7(self):
+        self.validate("def f(*, b=1): call(*b)")
+        self.validate("def f(*, b=1,): call(*b,)")
+
+    def test_trailing_commas_8(self):
+        self.validate("def f(a=1, b=2): call(a=1, b=2)")
+        self.validate("def f(a=1, b=2,): call(a=1, b=2,)")
+
+    def test_trailing_commas_9(self):
+        self.validate("def f(a=1, **b): call(a=1, **b)")
+        self.validate("def f(a=1, **b,): call(a=1, **b,)")
+
+    def test_trailing_commas_lambda_1(self):
+        self.validate("f = lambda a, b: call(a, b)")
+        self.validate("f = lambda a, b,: call(a, b,)")
+
+    def test_trailing_commas_lambda_2(self):
+        self.validate("f = lambda a, *b: call(a, *b)")
+        self.validate("f = lambda a, *b,: call(a, *b,)")
+
+    def test_trailing_commas_lambda_3(self):
+        self.validate("f = lambda a, b=1: call(a, b=1)")
+        self.validate("f = lambda a, b=1,: call(a, b=1,)")
+
+    def test_trailing_commas_lambda_4(self):
+        self.validate("f = lambda a, **b: call(a, **b)")
+        self.validate("f = lambda a, **b,: call(a, **b,)")
+
+    def test_trailing_commas_lambda_5(self):
+        self.validate("f = lambda *a, b=1: call(*a, b=1)")
+        self.validate("f = lambda *a, b=1,: call(*a, b=1,)")
+
+    def test_trailing_commas_lambda_6(self):
+        self.validate("f = lambda *a, **b: call(*a, **b)")
+        self.validate("f = lambda *a, **b,: call(*a, **b,)")
+
+    def test_trailing_commas_lambda_7(self):
+        self.validate("f = lambda *, b=1: call(*b)")
+        self.validate("f = lambda *, b=1,: call(*b,)")
+
+    def test_trailing_commas_lambda_8(self):
+        self.validate("f = lambda a=1, b=2: call(a=1, b=2)")
+        self.validate("f = lambda a=1, b=2,: call(a=1, b=2,)")
+
+    def test_trailing_commas_lambda_9(self):
+        self.validate("f = lambda a=1, **b: call(a=1, **b)")
+        self.validate("f = lambda a=1, **b,: call(a=1, **b,)")
+
+
+# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef
+class TestFunctionAnnotations(GrammarTest):
+    def test_1(self):
+        self.validate("""def f(x) -> list: pass""")
+
+    def test_2(self):
+        self.validate("""def f(x:int): pass""")
+
+    def test_3(self):
+        self.validate("""def f(*x:str): pass""")
+
+    def test_4(self):
+        self.validate("""def f(**x:float): pass""")
+
+    def test_5(self):
+        self.validate("""def f(x, y:1+2): pass""")
+
+    def test_6(self):
+        self.validate("""def f(a, (b:1, c:2, d)): pass""")
+
+    def test_7(self):
+        self.validate("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""")
+
+    def test_8(self):
+        s = """def f(a, (b:1, c:2, d), e:3=4, f=5,
+                        *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass"""
+        self.validate(s)
+
+    def test_9(self):
+        s = """def f(
+          a: str,
+          b: int,
+          *,
+          c: bool = False,
+          **kwargs,
+        ) -> None:
+            call(c=c, **kwargs,)"""
+        self.validate(s)
+
+    def test_10(self):
+        s = """def f(
+          a: str,
+        ) -> None:
+            call(a,)"""
+        self.validate(s)
+
+    def test_11(self):
+        s = """def f(
+          a: str = '',
+        ) -> None:
+            call(a=a,)"""
+        self.validate(s)
+
+    def test_12(self):
+        s = """def f(
+          *args: str,
+        ) -> None:
+            call(*args,)"""
+        self.validate(s)
+
+    def test_13(self):
+        self.validate("def f(a: str, b: int) -> None: call(a, b)")
+        self.validate("def f(a: str, b: int,) -> None: call(a, b,)")
+
+    def test_14(self):
+        self.validate("def f(a: str, *b: int) -> None: call(a, *b)")
+        self.validate("def f(a: str, *b: int,) -> None: call(a, *b,)")
+
+    def test_15(self):
+        self.validate("def f(a: str, b: int=1) -> None: call(a, b=1)")
+        self.validate("def f(a: str, b: int=1,) -> None: call(a, b=1,)")
+
+    def test_16(self):
+        self.validate("def f(a: str, **b: int) -> None: call(a, **b)")
+        self.validate("def f(a: str, **b: int,) -> None: call(a, **b,)")
+
+    def test_17(self):
+        self.validate("def f(*a: str, b: int=1) -> None: call(*a, b=1)")
+        self.validate("def f(*a: str, b: int=1,) -> None: call(*a, b=1,)")
+
+    def test_18(self):
+        self.validate("def f(*a: str, **b: int) -> None: call(*a, **b)")
+        self.validate("def f(*a: str, **b: int,) -> None: call(*a, **b,)")
+
+    def test_19(self):
+        self.validate("def f(*, b: int=1) -> None: call(*b)")
+        self.validate("def f(*, b: int=1,) -> None: call(*b,)")
+
+    def test_20(self):
+        self.validate("def f(a: str='', b: int=2) -> None: call(a=a, b=2)")
+        self.validate("def f(a: str='', b: int=2,) -> None: call(a=a, b=2,)")
+
+    def test_21(self):
+        self.validate("def f(a: str='', **b: int) -> None: call(a=a, **b)")
+        self.validate("def f(a: str='', **b: int,) -> None: call(a=a, **b,)")
+
+
+# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot
+class TestVarAnnotations(GrammarTest):
+    def test_1(self):
+        self.validate("var1: int = 5")
+
+    def test_2(self):
+        self.validate("var2: [int, str]")
+
+    def test_3(self):
+        self.validate("def f():\n"
+                      "    st: str = 'Hello'\n"
+                      "    a.b: int = (1, 2)\n"
+                      "    return st\n")
+
+    def test_4(self):
+        self.validate("def fbad():\n"
+                      "    x: int\n"
+                      "    print(x)\n")
+
+    def test_5(self):
+        self.validate("class C:\n"
+                      "    x: int\n"
+                      "    s: str = 'attr'\n"
+                      "    z = 2\n"
+                      "    def __init__(self, x):\n"
+                      "        self.x: int = x\n")
+
+    def test_6(self):
+        self.validate("lst: List[int] = []")
+
+
+class TestExcept(GrammarTest):
+    def test_new(self):
+        s = """
+            try:
+                x
+            except E as N:
+                y"""
+        self.validate(s)
+
+    def test_old(self):
+        s = """
+            try:
+                x
+            except E, N:
+                y"""
+        self.validate(s)
+
+
+class TestStringLiterals(GrammarTest):
+    prefixes = ("'", '"',
+        "r'", 'r"', "R'", 'R"',
+        "u'", 'u"', "U'", 'U"',
+        "b'", 'b"', "B'", 'B"',
+        "f'", 'f"', "F'", 'F"',
+        "ur'", 'ur"', "Ur'", 'Ur"',
+        "uR'", 'uR"', "UR'", 'UR"',
+        "br'", 'br"', "Br'", 'Br"',
+        "bR'", 'bR"', "BR'", 'BR"',
+        "rb'", 'rb"', "Rb'", 'Rb"',
+        "rB'", 'rB"', "RB'", 'RB"',)
+
+    def test_lit(self):
+        for pre in self.prefixes:
+            single = "{p}spamspamspam{s}".format(p=pre, s=pre[-1])
+            self.validate(single)
+            triple = "{p}{s}{s}eggs{s}{s}{s}".format(p=pre, s=pre[-1])
+            self.validate(triple)
+
+
+# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms
+class TestSetLiteral(GrammarTest):
+    def test_1(self):
+        self.validate("""x = {'one'}""")
+
+    def test_2(self):
+        self.validate("""x = {'one', 1,}""")
+
+    def test_3(self):
+        self.validate("""x = {'one', 'two', 'three'}""")
+
+    def test_4(self):
+        self.validate("""x = {2, 3, 4,}""")
+
+
+class TestNumericLiterals(GrammarTest):
+    def test_new_octal_notation(self):
+        self.validate("""0o7777777777777""")
+        self.invalid_syntax("""0o7324528887""")
+
+    def test_new_binary_notation(self):
+        self.validate("""0b101010""")
+        self.invalid_syntax("""0b0101021""")
+
+
+class TestClassDef(GrammarTest):
+    def test_new_syntax(self):
+        self.validate("class B(t=7): pass")
+        self.validate("class B(t, *args): pass")
+        self.validate("class B(t, **kwargs): pass")
+        self.validate("class B(t, *args, **kwargs): pass")
+        self.validate("class B(t, y=9, *args, **kwargs,): pass")
+
+
+class TestParserIdempotency(support.TestCase):
+
+    """A cut-down version of pytree_idempotency.py."""
+
+    def test_all_project_files(self):
+        for filepath in support.all_project_files():
+            with open(filepath, "rb") as fp:
+                encoding = tokenize.detect_encoding(fp.readline)[0]
+            self.assertIsNotNone(encoding,
+                                 "can't detect encoding for %s" % filepath)
+            with open(filepath, "r", encoding=encoding) as fp:
+                source = fp.read()
+            try:
+                tree = driver.parse_string(source)
+            except ParseError:
+                try:
+                    tree = driver_no_print_statement.parse_string(source)
+                except ParseError as err:
+                    self.fail('ParseError on file %s (%s)' % (filepath, err))
+            new = str(tree)
+            if new != source:
+                print(diff_texts(source, new, filepath))
+                self.fail("Idempotency failed: %s" % filepath)
+
+    def test_extended_unpacking(self):
+        driver.parse_string("a, *b, c = x\n")
+        driver.parse_string("[*a, b] = x\n")
+        driver.parse_string("(z, *y, w) = m\n")
+        driver.parse_string("for *z, m in d: pass\n")
+
+
+class TestLiterals(GrammarTest):
+
+    def validate(self, s):
+        driver.parse_string(support.dedent(s) + "\n\n")
+
+    def test_multiline_bytes_literals(self):
+        s = """
+            md5test(b"\xaa" * 80,
+                    (b"Test Using Larger Than Block-Size Key "
+                     b"and Larger Than One Block-Size Data"),
+                    "6f630fad67cda0ee1fb1f562db3aa53e")
+            """
+        self.validate(s)
+
+    def test_multiline_bytes_tripquote_literals(self):
+        s = '''
+            b"""
+            <?xml version="1.0" encoding="UTF-8"?>
+            <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN">
+            """
+            '''
+        self.validate(s)
+
+    def test_multiline_str_literals(self):
+        s = """
+            md5test("\xaa" * 80,
+                    ("Test Using Larger Than Block-Size Key "
+                     "and Larger Than One Block-Size Data"),
+                    "6f630fad67cda0ee1fb1f562db3aa53e")
+            """
+        self.validate(s)
+
+
+def diff_texts(a, b, filename):
+    a = a.splitlines()
+    b = b.splitlines()
+    return difflib.unified_diff(a, b, filename, filename,
+                                "(original)", "(reserialized)",
+                                lineterm="")

third_party/python/Lib/lib2to3/tests/test_pytree.py

@@ -0,0 +1,472 @@
+# Copyright 2006 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Unit tests for pytree.py.
+
+NOTE: Please *don't* add doc strings to individual test methods!
+In verbose mode, printing of the module, class and method name is much
+more helpful than printing of (the first line of) the docstring,
+especially when debugging a test.
+"""
+
+# Testing imports
+from . import support
+
+from lib2to3 import pytree
+
+try:
+    sorted
+except NameError:
+    def sorted(lst):
+        l = list(lst)
+        l.sort()
+        return l
+
+class TestNodes(support.TestCase):
+
+    """Unit tests for nodes (Base, Leaf, Node)."""
+
+    def test_instantiate_base(self):
+        if __debug__:
+            # Test that instantiating Base() raises an AssertionError
+            self.assertRaises(AssertionError, pytree.Base)
+
+    def test_leaf(self):
+        l1 = pytree.Leaf(100, "foo")
+        self.assertEqual(l1.type, 100)
+        self.assertEqual(l1.value, "foo")
+
+    def test_leaf_repr(self):
+        l1 = pytree.Leaf(100, "foo")
+        self.assertEqual(repr(l1), "Leaf(100, 'foo')")
+
+    def test_leaf_str(self):
+        l1 = pytree.Leaf(100, "foo")
+        self.assertEqual(str(l1), "foo")
+        l2 = pytree.Leaf(100, "foo", context=(" ", (10, 1)))
+        self.assertEqual(str(l2), " foo")
+
+    def test_leaf_str_numeric_value(self):
+        # Make sure that the Leaf's value is stringified. Failing to
+        #  do this can cause a TypeError in certain situations.
+        l1 = pytree.Leaf(2, 5)
+        l1.prefix = "foo_"
+        self.assertEqual(str(l1), "foo_5")
+
+    def test_leaf_equality(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "foo", context=(" ", (1, 0)))
+        self.assertEqual(l1, l2)
+        l3 = pytree.Leaf(101, "foo")
+        l4 = pytree.Leaf(100, "bar")
+        self.assertNotEqual(l1, l3)
+        self.assertNotEqual(l1, l4)
+
+    def test_leaf_prefix(self):
+        l1 = pytree.Leaf(100, "foo")
+        self.assertEqual(l1.prefix, "")
+        self.assertFalse(l1.was_changed)
+        l1.prefix = "  ##\n\n"
+        self.assertEqual(l1.prefix, "  ##\n\n")
+        self.assertTrue(l1.was_changed)
+
+    def test_node(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(200, "bar")
+        n1 = pytree.Node(1000, [l1, l2])
+        self.assertEqual(n1.type, 1000)
+        self.assertEqual(n1.children, [l1, l2])
+
+    def test_node_repr(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "bar", context=(" ", (1, 0)))
+        n1 = pytree.Node(1000, [l1, l2])
+        self.assertEqual(repr(n1),
+                         "Node(1000, [%s, %s])" % (repr(l1), repr(l2)))
+
+    def test_node_str(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "bar", context=(" ", (1, 0)))
+        n1 = pytree.Node(1000, [l1, l2])
+        self.assertEqual(str(n1), "foo bar")
+
+    def test_node_prefix(self):
+        l1 = pytree.Leaf(100, "foo")
+        self.assertEqual(l1.prefix, "")
+        n1 = pytree.Node(1000, [l1])
+        self.assertEqual(n1.prefix, "")
+        n1.prefix = " "
+        self.assertEqual(n1.prefix, " ")
+        self.assertEqual(l1.prefix, " ")
+
+    def test_get_suffix(self):
+        l1 = pytree.Leaf(100, "foo", prefix="a")
+        l2 = pytree.Leaf(100, "bar", prefix="b")
+        n1 = pytree.Node(1000, [l1, l2])
+
+        self.assertEqual(l1.get_suffix(), l2.prefix)
+        self.assertEqual(l2.get_suffix(), "")
+        self.assertEqual(n1.get_suffix(), "")
+
+        l3 = pytree.Leaf(100, "bar", prefix="c")
+        n2 = pytree.Node(1000, [n1, l3])
+
+        self.assertEqual(n1.get_suffix(), l3.prefix)
+        self.assertEqual(l3.get_suffix(), "")
+        self.assertEqual(n2.get_suffix(), "")
+
+    def test_node_equality(self):
+        n1 = pytree.Node(1000, ())
+        n2 = pytree.Node(1000, [], context=(" ", (1, 0)))
+        self.assertEqual(n1, n2)
+        n3 = pytree.Node(1001, ())
+        self.assertNotEqual(n1, n3)
+
+    def test_node_recursive_equality(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "foo")
+        n1 = pytree.Node(1000, [l1])
+        n2 = pytree.Node(1000, [l2])
+        self.assertEqual(n1, n2)
+        l3 = pytree.Leaf(100, "bar")
+        n3 = pytree.Node(1000, [l3])
+        self.assertNotEqual(n1, n3)
+
+    def test_replace(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "+")
+        l3 = pytree.Leaf(100, "bar")
+        n1 = pytree.Node(1000, [l1, l2, l3])
+        self.assertEqual(n1.children, [l1, l2, l3])
+        self.assertIsInstance(n1.children, list)
+        self.assertFalse(n1.was_changed)
+        l2new = pytree.Leaf(100, "-")
+        l2.replace(l2new)
+        self.assertEqual(n1.children, [l1, l2new, l3])
+        self.assertIsInstance(n1.children, list)
+        self.assertTrue(n1.was_changed)
+
+    def test_replace_with_list(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "+")
+        l3 = pytree.Leaf(100, "bar")
+        n1 = pytree.Node(1000, [l1, l2, l3])
+
+        l2.replace([pytree.Leaf(100, "*"), pytree.Leaf(100, "*")])
+        self.assertEqual(str(n1), "foo**bar")
+        self.assertIsInstance(n1.children, list)
+
+    def test_leaves(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "bar")
+        l3 = pytree.Leaf(100, "fooey")
+        n2 = pytree.Node(1000, [l1, l2])
+        n3 = pytree.Node(1000, [l3])
+        n1 = pytree.Node(1000, [n2, n3])
+
+        self.assertEqual(list(n1.leaves()), [l1, l2, l3])
+
+    def test_depth(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "bar")
+        n2 = pytree.Node(1000, [l1, l2])
+        n3 = pytree.Node(1000, [])
+        n1 = pytree.Node(1000, [n2, n3])
+
+        self.assertEqual(l1.depth(), 2)
+        self.assertEqual(n3.depth(), 1)
+        self.assertEqual(n1.depth(), 0)
+
+    def test_post_order(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "bar")
+        l3 = pytree.Leaf(100, "fooey")
+        c1 = pytree.Node(1000, [l1, l2])
+        n1 = pytree.Node(1000, [c1, l3])
+        self.assertEqual(list(n1.post_order()), [l1, l2, c1, l3, n1])
+
+    def test_pre_order(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "bar")
+        l3 = pytree.Leaf(100, "fooey")
+        c1 = pytree.Node(1000, [l1, l2])
+        n1 = pytree.Node(1000, [c1, l3])
+        self.assertEqual(list(n1.pre_order()), [n1, c1, l1, l2, l3])
+
+    def test_changed(self):
+        l1 = pytree.Leaf(100, "f")
+        self.assertFalse(l1.was_changed)
+        l1.changed()
+        self.assertTrue(l1.was_changed)
+
+        l1 = pytree.Leaf(100, "f")
+        n1 = pytree.Node(1000, [l1])
+        self.assertFalse(n1.was_changed)
+        n1.changed()
+        self.assertTrue(n1.was_changed)
+
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "+")
+        l3 = pytree.Leaf(100, "bar")
+        n1 = pytree.Node(1000, [l1, l2, l3])
+        n2 = pytree.Node(1000, [n1])
+        self.assertFalse(l1.was_changed)
+        self.assertFalse(n1.was_changed)
+        self.assertFalse(n2.was_changed)
+
+        n1.changed()
+        self.assertTrue(n1.was_changed)
+        self.assertTrue(n2.was_changed)
+        self.assertFalse(l1.was_changed)
+
+    def test_leaf_constructor_prefix(self):
+        for prefix in ("xyz_", ""):
+            l1 = pytree.Leaf(100, "self", prefix=prefix)
+            self.assertTrue(str(l1), prefix + "self")
+            self.assertEqual(l1.prefix, prefix)
+
+    def test_node_constructor_prefix(self):
+        for prefix in ("xyz_", ""):
+            l1 = pytree.Leaf(100, "self")
+            l2 = pytree.Leaf(100, "foo", prefix="_")
+            n1 = pytree.Node(1000, [l1, l2], prefix=prefix)
+            self.assertTrue(str(n1), prefix + "self_foo")
+            self.assertEqual(n1.prefix, prefix)
+            self.assertEqual(l1.prefix, prefix)
+            self.assertEqual(l2.prefix, "_")
+
+    def test_remove(self):
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "foo")
+        n1 = pytree.Node(1000, [l1, l2])
+        n2 = pytree.Node(1000, [n1])
+
+        self.assertEqual(n1.remove(), 0)
+        self.assertEqual(n2.children, [])
+        self.assertEqual(l1.parent, n1)
+        self.assertEqual(n1.parent, None)
+        self.assertEqual(n2.parent, None)
+        self.assertFalse(n1.was_changed)
+        self.assertTrue(n2.was_changed)
+
+        self.assertEqual(l2.remove(), 1)
+        self.assertEqual(l1.remove(), 0)
+        self.assertEqual(n1.children, [])
+        self.assertEqual(l1.parent, None)
+        self.assertEqual(n1.parent, None)
+        self.assertEqual(n2.parent, None)
+        self.assertTrue(n1.was_changed)
+        self.assertTrue(n2.was_changed)
+
+    def test_remove_parentless(self):
+        n1 = pytree.Node(1000, [])
+        n1.remove()
+        self.assertEqual(n1.parent, None)
+
+        l1 = pytree.Leaf(100, "foo")
+        l1.remove()
+        self.assertEqual(l1.parent, None)
+
+    def test_node_set_child(self):
+        l1 = pytree.Leaf(100, "foo")
+        n1 = pytree.Node(1000, [l1])
+
+        l2 = pytree.Leaf(100, "bar")
+        n1.set_child(0, l2)
+        self.assertEqual(l1.parent, None)
+        self.assertEqual(l2.parent, n1)
+        self.assertEqual(n1.children, [l2])
+
+        n2 = pytree.Node(1000, [l1])
+        n2.set_child(0, n1)
+        self.assertEqual(l1.parent, None)
+        self.assertEqual(n1.parent, n2)
+        self.assertEqual(n2.parent, None)
+        self.assertEqual(n2.children, [n1])
+
+        self.assertRaises(IndexError, n1.set_child, 4, l2)
+        # I don't care what it raises, so long as it's an exception
+        self.assertRaises(Exception, n1.set_child, 0, list)
+
+    def test_node_insert_child(self):
+        l1 = pytree.Leaf(100, "foo")
+        n1 = pytree.Node(1000, [l1])
+
+        l2 = pytree.Leaf(100, "bar")
+        n1.insert_child(0, l2)
+        self.assertEqual(l2.parent, n1)
+        self.assertEqual(n1.children, [l2, l1])
+
+        l3 = pytree.Leaf(100, "abc")
+        n1.insert_child(2, l3)
+        self.assertEqual(n1.children, [l2, l1, l3])
+
+        # I don't care what it raises, so long as it's an exception
+        self.assertRaises(Exception, n1.insert_child, 0, list)
+
+    def test_node_append_child(self):
+        n1 = pytree.Node(1000, [])
+
+        l1 = pytree.Leaf(100, "foo")
+        n1.append_child(l1)
+        self.assertEqual(l1.parent, n1)
+        self.assertEqual(n1.children, [l1])
+
+        l2 = pytree.Leaf(100, "bar")
+        n1.append_child(l2)
+        self.assertEqual(l2.parent, n1)
+        self.assertEqual(n1.children, [l1, l2])
+
+        # I don't care what it raises, so long as it's an exception
+        self.assertRaises(Exception, n1.append_child, list)
+
+    def test_node_next_sibling(self):
+        n1 = pytree.Node(1000, [])
+        n2 = pytree.Node(1000, [])
+        p1 = pytree.Node(1000, [n1, n2])
+
+        self.assertIs(n1.next_sibling, n2)
+        self.assertEqual(n2.next_sibling, None)
+        self.assertEqual(p1.next_sibling, None)
+
+    def test_leaf_next_sibling(self):
+        l1 = pytree.Leaf(100, "a")
+        l2 = pytree.Leaf(100, "b")
+        p1 = pytree.Node(1000, [l1, l2])
+
+        self.assertIs(l1.next_sibling, l2)
+        self.assertEqual(l2.next_sibling, None)
+        self.assertEqual(p1.next_sibling, None)
+
+    def test_node_prev_sibling(self):
+        n1 = pytree.Node(1000, [])
+        n2 = pytree.Node(1000, [])
+        p1 = pytree.Node(1000, [n1, n2])
+
+        self.assertIs(n2.prev_sibling, n1)
+        self.assertEqual(n1.prev_sibling, None)
+        self.assertEqual(p1.prev_sibling, None)
+
+    def test_leaf_prev_sibling(self):
+        l1 = pytree.Leaf(100, "a")
+        l2 = pytree.Leaf(100, "b")
+        p1 = pytree.Node(1000, [l1, l2])
+
+        self.assertIs(l2.prev_sibling, l1)
+        self.assertEqual(l1.prev_sibling, None)
+        self.assertEqual(p1.prev_sibling, None)
+
+
+class TestPatterns(support.TestCase):
+
+    """Unit tests for tree matching patterns."""
+
+    def test_basic_patterns(self):
+        # Build a tree
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "bar")
+        l3 = pytree.Leaf(100, "foo")
+        n1 = pytree.Node(1000, [l1, l2])
+        n2 = pytree.Node(1000, [l3])
+        root = pytree.Node(1000, [n1, n2])
+        # Build a pattern matching a leaf
+        pl = pytree.LeafPattern(100, "foo", name="pl")
+        r = {}
+        self.assertFalse(pl.match(root, results=r))
+        self.assertEqual(r, {})
+        self.assertFalse(pl.match(n1, results=r))
+        self.assertEqual(r, {})
+        self.assertFalse(pl.match(n2, results=r))
+        self.assertEqual(r, {})
+        self.assertTrue(pl.match(l1, results=r))
+        self.assertEqual(r, {"pl": l1})
+        r = {}
+        self.assertFalse(pl.match(l2, results=r))
+        self.assertEqual(r, {})
+        # Build a pattern matching a node
+        pn = pytree.NodePattern(1000, [pl], name="pn")
+        self.assertFalse(pn.match(root, results=r))
+        self.assertEqual(r, {})
+        self.assertFalse(pn.match(n1, results=r))
+        self.assertEqual(r, {})
+        self.assertTrue(pn.match(n2, results=r))
+        self.assertEqual(r, {"pn": n2, "pl": l3})
+        r = {}
+        self.assertFalse(pn.match(l1, results=r))
+        self.assertEqual(r, {})
+        self.assertFalse(pn.match(l2, results=r))
+        self.assertEqual(r, {})
+
+    def test_wildcard(self):
+        # Build a tree for testing
+        l1 = pytree.Leaf(100, "foo")
+        l2 = pytree.Leaf(100, "bar")
+        l3 = pytree.Leaf(100, "foo")
+        n1 = pytree.Node(1000, [l1, l2])
+        n2 = pytree.Node(1000, [l3])
+        root = pytree.Node(1000, [n1, n2])
+        # Build a pattern
+        pl = pytree.LeafPattern(100, "foo", name="pl")
+        pn = pytree.NodePattern(1000, [pl], name="pn")
+        pw = pytree.WildcardPattern([[pn], [pl, pl]], name="pw")
+        r = {}
+        self.assertFalse(pw.match_seq([root], r))
+        self.assertEqual(r, {})
+        self.assertFalse(pw.match_seq([n1], r))
+        self.assertEqual(r, {})
+        self.assertTrue(pw.match_seq([n2], r))
+        # These are easier to debug
+        self.assertEqual(sorted(r.keys()), ["pl", "pn", "pw"])
+        self.assertEqual(r["pl"], l1)
+        self.assertEqual(r["pn"], n2)
+        self.assertEqual(r["pw"], [n2])
+        # But this is equivalent
+        self.assertEqual(r, {"pl": l1, "pn": n2, "pw": [n2]})
+        r = {}
+        self.assertTrue(pw.match_seq([l1, l3], r))
+        self.assertEqual(r, {"pl": l3, "pw": [l1, l3]})
+        self.assertIs(r["pl"], l3)
+        r = {}
+
+    def test_generate_matches(self):
+        la = pytree.Leaf(1, "a")
+        lb = pytree.Leaf(1, "b")
+        lc = pytree.Leaf(1, "c")
+        ld = pytree.Leaf(1, "d")
+        le = pytree.Leaf(1, "e")
+        lf = pytree.Leaf(1, "f")
+        leaves = [la, lb, lc, ld, le, lf]
+        root = pytree.Node(1000, leaves)
+        pa = pytree.LeafPattern(1, "a", "pa")
+        pb = pytree.LeafPattern(1, "b", "pb")
+        pc = pytree.LeafPattern(1, "c", "pc")
+        pd = pytree.LeafPattern(1, "d", "pd")
+        pe = pytree.LeafPattern(1, "e", "pe")
+        pf = pytree.LeafPattern(1, "f", "pf")
+        pw = pytree.WildcardPattern([[pa, pb, pc], [pd, pe],
+                                     [pa, pb], [pc, pd], [pe, pf]],
+                                    min=1, max=4, name="pw")
+        self.assertEqual([x[0] for x in pw.generate_matches(leaves)],
+                         [3, 5, 2, 4, 6])
+        pr = pytree.NodePattern(type=1000, content=[pw], name="pr")
+        matches = list(pytree.generate_matches([pr], [root]))
+        self.assertEqual(len(matches), 1)
+        c, r = matches[0]
+        self.assertEqual(c, 1)
+        self.assertEqual(str(r["pr"]), "abcdef")
+        self.assertEqual(r["pw"], [la, lb, lc, ld, le, lf])
+        for c in "abcdef":
+            self.assertEqual(r["p" + c], pytree.Leaf(1, c))
+
+    def test_has_key_example(self):
+        pattern = pytree.NodePattern(331,
+                                     (pytree.LeafPattern(7),
+                                      pytree.WildcardPattern(name="args"),
+                                      pytree.LeafPattern(8)))
+        l1 = pytree.Leaf(7, "(")
+        l2 = pytree.Leaf(3, "x")
+        l3 = pytree.Leaf(8, ")")
+        node = pytree.Node(331, [l1, l2, l3])
+        r = {}
+        self.assertTrue(pattern.match(node, r))
+        self.assertEqual(r["args"], [l2])

third_party/python/Lib/lib2to3/tests/test_refactor.py

@@ -0,0 +1,316 @@
+"""
+Unit tests for refactor.py.
+"""
+
+import sys
+import os
+import codecs
+import io
+import re
+import tempfile
+import shutil
+import unittest
+
+from lib2to3 import refactor, pygram, fixer_base
+from lib2to3.pgen2 import token
+
+
+TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
+FIXER_DIR = os.path.join(TEST_DATA_DIR, "fixers")
+
+sys.path.append(FIXER_DIR)
+try:
+    _DEFAULT_FIXERS = refactor.get_fixers_from_package("myfixes")
+finally:
+    sys.path.pop()
+
+_2TO3_FIXERS = refactor.get_fixers_from_package("lib2to3.fixes")
+
+class TestRefactoringTool(unittest.TestCase):
+
+    def setUp(self):
+        sys.path.append(FIXER_DIR)
+
+    def tearDown(self):
+        sys.path.pop()
+
+    def check_instances(self, instances, classes):
+        for inst, cls in zip(instances, classes):
+            if not isinstance(inst, cls):
+                self.fail("%s are not instances of %s" % instances, classes)
+
+    def rt(self, options=None, fixers=_DEFAULT_FIXERS, explicit=None):
+        return refactor.RefactoringTool(fixers, options, explicit)
+
+    def test_print_function_option(self):
+        rt = self.rt({"print_function" : True})
+        self.assertIs(rt.grammar, pygram.python_grammar_no_print_statement)
+        self.assertIs(rt.driver.grammar,
+                      pygram.python_grammar_no_print_statement)
+
+    def test_write_unchanged_files_option(self):
+        rt = self.rt()
+        self.assertFalse(rt.write_unchanged_files)
+        rt = self.rt({"write_unchanged_files" : True})
+        self.assertTrue(rt.write_unchanged_files)
+
+    def test_fixer_loading_helpers(self):
+        contents = ["explicit", "first", "last", "parrot", "preorder"]
+        non_prefixed = refactor.get_all_fix_names("myfixes")
+        prefixed = refactor.get_all_fix_names("myfixes", False)
+        full_names = refactor.get_fixers_from_package("myfixes")
+        self.assertEqual(prefixed, ["fix_" + name for name in contents])
+        self.assertEqual(non_prefixed, contents)
+        self.assertEqual(full_names,
+                         ["myfixes.fix_" + name for name in contents])
+
+    def test_detect_future_features(self):
+        run = refactor._detect_future_features
+        fs = frozenset
+        empty = fs()
+        self.assertEqual(run(""), empty)
+        self.assertEqual(run("from __future__ import print_function"),
+                         fs(("print_function",)))
+        self.assertEqual(run("from __future__ import generators"),
+                         fs(("generators",)))
+        self.assertEqual(run("from __future__ import generators, feature"),
+                         fs(("generators", "feature")))
+        inp = "from __future__ import generators, print_function"
+        self.assertEqual(run(inp), fs(("generators", "print_function")))
+        inp ="from __future__ import print_function, generators"
+        self.assertEqual(run(inp), fs(("print_function", "generators")))
+        inp = "from __future__ import (print_function,)"
+        self.assertEqual(run(inp), fs(("print_function",)))
+        inp = "from __future__ import (generators, print_function)"
+        self.assertEqual(run(inp), fs(("generators", "print_function")))
+        inp = "from __future__ import (generators, nested_scopes)"
+        self.assertEqual(run(inp), fs(("generators", "nested_scopes")))
+        inp = """from __future__ import generators
+from __future__ import print_function"""
+        self.assertEqual(run(inp), fs(("generators", "print_function")))
+        invalid = ("from",
+                   "from 4",
+                   "from x",
+                   "from x 5",
+                   "from x im",
+                   "from x import",
+                   "from x import 4",
+                   )
+        for inp in invalid:
+            self.assertEqual(run(inp), empty)
+        inp = "'docstring'\nfrom __future__ import print_function"
+        self.assertEqual(run(inp), fs(("print_function",)))
+        inp = "'docstring'\n'somng'\nfrom __future__ import print_function"
+        self.assertEqual(run(inp), empty)
+        inp = "# comment\nfrom __future__ import print_function"
+        self.assertEqual(run(inp), fs(("print_function",)))
+        inp = "# comment\n'doc'\nfrom __future__ import print_function"
+        self.assertEqual(run(inp), fs(("print_function",)))
+        inp = "class x: pass\nfrom __future__ import print_function"
+        self.assertEqual(run(inp), empty)
+
+    def test_get_headnode_dict(self):
+        class NoneFix(fixer_base.BaseFix):
+            pass
+
+        class FileInputFix(fixer_base.BaseFix):
+            PATTERN = "file_input< any * >"
+
+        class SimpleFix(fixer_base.BaseFix):
+            PATTERN = "'name'"
+
+        no_head = NoneFix({}, [])
+        with_head = FileInputFix({}, [])
+        simple = SimpleFix({}, [])
+        d = refactor._get_headnode_dict([no_head, with_head, simple])
+        top_fixes = d.pop(pygram.python_symbols.file_input)
+        self.assertEqual(top_fixes, [with_head, no_head])
+        name_fixes = d.pop(token.NAME)
+        self.assertEqual(name_fixes, [simple, no_head])
+        for fixes in d.values():
+            self.assertEqual(fixes, [no_head])
+
+    def test_fixer_loading(self):
+        from myfixes.fix_first import FixFirst
+        from myfixes.fix_last import FixLast
+        from myfixes.fix_parrot import FixParrot
+        from myfixes.fix_preorder import FixPreorder
+
+        rt = self.rt()
+        pre, post = rt.get_fixers()
+
+        self.check_instances(pre, [FixPreorder])
+        self.check_instances(post, [FixFirst, FixParrot, FixLast])
+
+    def test_naughty_fixers(self):
+        self.assertRaises(ImportError, self.rt, fixers=["not_here"])
+        self.assertRaises(refactor.FixerError, self.rt, fixers=["no_fixer_cls"])
+        self.assertRaises(refactor.FixerError, self.rt, fixers=["bad_order"])
+
+    def test_refactor_string(self):
+        rt = self.rt()
+        input = "def parrot(): pass\n\n"
+        tree = rt.refactor_string(input, "<test>")
+        self.assertNotEqual(str(tree), input)
+
+        input = "def f(): pass\n\n"
+        tree = rt.refactor_string(input, "<test>")
+        self.assertEqual(str(tree), input)
+
+    def test_refactor_stdin(self):
+
+        class MyRT(refactor.RefactoringTool):
+
+            def print_output(self, old_text, new_text, filename, equal):
+                results.extend([old_text, new_text, filename, equal])
+
+        results = []
+        rt = MyRT(_DEFAULT_FIXERS)
+        save = sys.stdin
+        sys.stdin = io.StringIO("def parrot(): pass\n\n")
+        try:
+            rt.refactor_stdin()
+        finally:
+            sys.stdin = save
+        expected = ["def parrot(): pass\n\n",
+                    "def cheese(): pass\n\n",
+                    "<stdin>", False]
+        self.assertEqual(results, expected)
+
+    def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS,
+                               options=None, mock_log_debug=None,
+                               actually_write=True):
+        tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor")
+        self.addCleanup(shutil.rmtree, tmpdir)
+        # make a copy of the tested file that we can write to
+        shutil.copy(test_file, tmpdir)
+        test_file = os.path.join(tmpdir, os.path.basename(test_file))
+        os.chmod(test_file, 0o644)
+
+        def read_file():
+            with open(test_file, "rb") as fp:
+                return fp.read()
+
+        old_contents = read_file()
+        rt = self.rt(fixers=fixers, options=options)
+        if mock_log_debug:
+            rt.log_debug = mock_log_debug
+
+        rt.refactor_file(test_file)
+        self.assertEqual(old_contents, read_file())
+
+        if not actually_write:
+            return
+        rt.refactor_file(test_file, True)
+        new_contents = read_file()
+        self.assertNotEqual(old_contents, new_contents)
+        return new_contents
+
+    def test_refactor_file(self):
+        test_file = os.path.join(FIXER_DIR, "parrot_example.py")
+        self.check_file_refactoring(test_file, _DEFAULT_FIXERS)
+
+    def test_refactor_file_write_unchanged_file(self):
+        test_file = os.path.join(FIXER_DIR, "parrot_example.py")
+        debug_messages = []
+        def recording_log_debug(msg, *args):
+            debug_messages.append(msg % args)
+        self.check_file_refactoring(test_file, fixers=(),
+                                    options={"write_unchanged_files": True},
+                                    mock_log_debug=recording_log_debug,
+                                    actually_write=False)
+        # Testing that it logged this message when write=False was passed is
+        # sufficient to see that it did not bail early after "No changes".
+        message_regex = r"Not writing changes to .*%s" % \
+                re.escape(os.sep + os.path.basename(test_file))
+        for message in debug_messages:
+            if "Not writing changes" in message:
+                self.assertRegex(message, message_regex)
+                break
+        else:
+            self.fail("%r not matched in %r" % (message_regex, debug_messages))
+
+    def test_refactor_dir(self):
+        def check(structure, expected):
+            def mock_refactor_file(self, f, *args):
+                got.append(f)
+            save_func = refactor.RefactoringTool.refactor_file
+            refactor.RefactoringTool.refactor_file = mock_refactor_file
+            rt = self.rt()
+            got = []
+            dir = tempfile.mkdtemp(prefix="2to3-test_refactor")
+            try:
+                os.mkdir(os.path.join(dir, "a_dir"))
+                for fn in structure:
+                    open(os.path.join(dir, fn), "wb").close()
+                rt.refactor_dir(dir)
+            finally:
+                refactor.RefactoringTool.refactor_file = save_func
+                shutil.rmtree(dir)
+            self.assertEqual(got,
+                             [os.path.join(dir, path) for path in expected])
+        check([], [])
+        tree = ["nothing",
+                "hi.py",
+                ".dumb",
+                ".after.py",
+                "notpy.npy",
+                "sappy"]
+        expected = ["hi.py"]
+        check(tree, expected)
+        tree = ["hi.py",
+                os.path.join("a_dir", "stuff.py")]
+        check(tree, tree)
+
+    def test_file_encoding(self):
+        fn = os.path.join(TEST_DATA_DIR, "different_encoding.py")
+        self.check_file_refactoring(fn)
+
+    def test_false_file_encoding(self):
+        fn = os.path.join(TEST_DATA_DIR, "false_encoding.py")
+        data = self.check_file_refactoring(fn)
+
+    def test_bom(self):
+        fn = os.path.join(TEST_DATA_DIR, "bom.py")
+        data = self.check_file_refactoring(fn)
+        self.assertTrue(data.startswith(codecs.BOM_UTF8))
+
+    def test_crlf_newlines(self):
+        old_sep = os.linesep
+        os.linesep = "\r\n"
+        try:
+            fn = os.path.join(TEST_DATA_DIR, "crlf.py")
+            fixes = refactor.get_fixers_from_package("lib2to3.fixes")
+            self.check_file_refactoring(fn, fixes)
+        finally:
+            os.linesep = old_sep
+
+    def test_refactor_docstring(self):
+        rt = self.rt()
+
+        doc = """
+>>> example()
+42
+"""
+        out = rt.refactor_docstring(doc, "<test>")
+        self.assertEqual(out, doc)
+
+        doc = """
+>>> def parrot():
+...      return 43
+"""
+        out = rt.refactor_docstring(doc, "<test>")
+        self.assertNotEqual(out, doc)
+
+    def test_explicit(self):
+        from myfixes.fix_explicit import FixExplicit
+
+        rt = self.rt(fixers=["myfixes.fix_explicit"])
+        self.assertEqual(len(rt.post_order), 0)
+
+        rt = self.rt(explicit=["myfixes.fix_explicit"])
+        for fix in rt.post_order:
+            if isinstance(fix, FixExplicit):
+                break
+        else:
+            self.fail("explicit fixer not loaded")

third_party/python/Lib/lib2to3/tests/test_util.py

@@ -0,0 +1,591 @@
+""" Test suite for the code in fixer_util """
+
+# Testing imports
+from . import support
+
+# Local imports
+from lib2to3.pytree import Node, Leaf
+from lib2to3 import fixer_util
+from lib2to3.fixer_util import Attr, Name, Call, Comma
+from lib2to3.pgen2 import token
+
+def parse(code, strip_levels=0):
+    # The topmost node is file_input, which we don't care about.
+    # The next-topmost node is a *_stmt node, which we also don't care about
+    tree = support.parse_string(code)
+    for i in range(strip_levels):
+        tree = tree.children[0]
+    tree.parent = None
+    return tree
+
+class MacroTestCase(support.TestCase):
+    def assertStr(self, node, string):
+        if isinstance(node, (tuple, list)):
+            node = Node(fixer_util.syms.simple_stmt, node)
+        self.assertEqual(str(node), string)
+
+
+class Test_is_tuple(support.TestCase):
+    def is_tuple(self, string):
+        return fixer_util.is_tuple(parse(string, strip_levels=2))
+
+    def test_valid(self):
+        self.assertTrue(self.is_tuple("(a, b)"))
+        self.assertTrue(self.is_tuple("(a, (b, c))"))
+        self.assertTrue(self.is_tuple("((a, (b, c)),)"))
+        self.assertTrue(self.is_tuple("(a,)"))
+        self.assertTrue(self.is_tuple("()"))
+
+    def test_invalid(self):
+        self.assertFalse(self.is_tuple("(a)"))
+        self.assertFalse(self.is_tuple("('foo') % (b, c)"))
+
+
+class Test_is_list(support.TestCase):
+    def is_list(self, string):
+        return fixer_util.is_list(parse(string, strip_levels=2))
+
+    def test_valid(self):
+        self.assertTrue(self.is_list("[]"))
+        self.assertTrue(self.is_list("[a]"))
+        self.assertTrue(self.is_list("[a, b]"))
+        self.assertTrue(self.is_list("[a, [b, c]]"))
+        self.assertTrue(self.is_list("[[a, [b, c]],]"))
+
+    def test_invalid(self):
+        self.assertFalse(self.is_list("[]+[]"))
+
+
+class Test_Attr(MacroTestCase):
+    def test(self):
+        call = parse("foo()", strip_levels=2)
+
+        self.assertStr(Attr(Name("a"), Name("b")), "a.b")
+        self.assertStr(Attr(call, Name("b")), "foo().b")
+
+    def test_returns(self):
+        attr = Attr(Name("a"), Name("b"))
+        self.assertEqual(type(attr), list)
+
+
+class Test_Name(MacroTestCase):
+    def test(self):
+        self.assertStr(Name("a"), "a")
+        self.assertStr(Name("foo.foo().bar"), "foo.foo().bar")
+        self.assertStr(Name("a", prefix="b"), "ba")
+
+
+class Test_Call(MacroTestCase):
+    def _Call(self, name, args=None, prefix=None):
+        """Help the next test"""
+        children = []
+        if isinstance(args, list):
+            for arg in args:
+                children.append(arg)
+                children.append(Comma())
+            children.pop()
+        return Call(Name(name), children, prefix)
+
+    def test(self):
+        kids = [None,
+                [Leaf(token.NUMBER, 1), Leaf(token.NUMBER, 2),
+                 Leaf(token.NUMBER, 3)],
+                [Leaf(token.NUMBER, 1), Leaf(token.NUMBER, 3),
+                 Leaf(token.NUMBER, 2), Leaf(token.NUMBER, 4)],
+                [Leaf(token.STRING, "b"), Leaf(token.STRING, "j", prefix=" ")]
+                ]
+        self.assertStr(self._Call("A"), "A()")
+        self.assertStr(self._Call("b", kids[1]), "b(1,2,3)")
+        self.assertStr(self._Call("a.b().c", kids[2]), "a.b().c(1,3,2,4)")
+        self.assertStr(self._Call("d", kids[3], prefix=" "), " d(b, j)")
+
+
+class Test_does_tree_import(support.TestCase):
+    def _find_bind_rec(self, name, node):
+        # Search a tree for a binding -- used to find the starting
+        # point for these tests.
+        c = fixer_util.find_binding(name, node)
+        if c: return c
+        for child in node.children:
+            c = self._find_bind_rec(name, child)
+            if c: return c
+
+    def does_tree_import(self, package, name, string):
+        node = parse(string)
+        # Find the binding of start -- that's what we'll go from
+        node = self._find_bind_rec('start', node)
+        return fixer_util.does_tree_import(package, name, node)
+
+    def try_with(self, string):
+        failing_tests = (("a", "a", "from a import b"),
+                         ("a.d", "a", "from a.d import b"),
+                         ("d.a", "a", "from d.a import b"),
+                         (None, "a", "import b"),
+                         (None, "a", "import b, c, d"))
+        for package, name, import_ in failing_tests:
+            n = self.does_tree_import(package, name, import_ + "\n" + string)
+            self.assertFalse(n)
+            n = self.does_tree_import(package, name, string + "\n" + import_)
+            self.assertFalse(n)
+
+        passing_tests = (("a", "a", "from a import a"),
+                         ("x", "a", "from x import a"),
+                         ("x", "a", "from x import b, c, a, d"),
+                         ("x.b", "a", "from x.b import a"),
+                         ("x.b", "a", "from x.b import b, c, a, d"),
+                         (None, "a", "import a"),
+                         (None, "a", "import b, c, a, d"))
+        for package, name, import_ in passing_tests:
+            n = self.does_tree_import(package, name, import_ + "\n" + string)
+            self.assertTrue(n)
+            n = self.does_tree_import(package, name, string + "\n" + import_)
+            self.assertTrue(n)
+
+    def test_in_function(self):
+        self.try_with("def foo():\n\tbar.baz()\n\tstart=3")
+
+class Test_find_binding(support.TestCase):
+    def find_binding(self, name, string, package=None):
+        return fixer_util.find_binding(name, parse(string), package)
+
+    def test_simple_assignment(self):
+        self.assertTrue(self.find_binding("a", "a = b"))
+        self.assertTrue(self.find_binding("a", "a = [b, c, d]"))
+        self.assertTrue(self.find_binding("a", "a = foo()"))
+        self.assertTrue(self.find_binding("a", "a = foo().foo.foo[6][foo]"))
+        self.assertFalse(self.find_binding("a", "foo = a"))
+        self.assertFalse(self.find_binding("a", "foo = (a, b, c)"))
+
+    def test_tuple_assignment(self):
+        self.assertTrue(self.find_binding("a", "(a,) = b"))
+        self.assertTrue(self.find_binding("a", "(a, b, c) = [b, c, d]"))
+        self.assertTrue(self.find_binding("a", "(c, (d, a), b) = foo()"))
+        self.assertTrue(self.find_binding("a", "(a, b) = foo().foo[6][foo]"))
+        self.assertFalse(self.find_binding("a", "(foo, b) = (b, a)"))
+        self.assertFalse(self.find_binding("a", "(foo, (b, c)) = (a, b, c)"))
+
+    def test_list_assignment(self):
+        self.assertTrue(self.find_binding("a", "[a] = b"))
+        self.assertTrue(self.find_binding("a", "[a, b, c] = [b, c, d]"))
+        self.assertTrue(self.find_binding("a", "[c, [d, a], b] = foo()"))
+        self.assertTrue(self.find_binding("a", "[a, b] = foo().foo[a][foo]"))
+        self.assertFalse(self.find_binding("a", "[foo, b] = (b, a)"))
+        self.assertFalse(self.find_binding("a", "[foo, [b, c]] = (a, b, c)"))
+
+    def test_invalid_assignments(self):
+        self.assertFalse(self.find_binding("a", "foo.a = 5"))
+        self.assertFalse(self.find_binding("a", "foo[a] = 5"))
+        self.assertFalse(self.find_binding("a", "foo(a) = 5"))
+        self.assertFalse(self.find_binding("a", "foo(a, b) = 5"))
+
+    def test_simple_import(self):
+        self.assertTrue(self.find_binding("a", "import a"))
+        self.assertTrue(self.find_binding("a", "import b, c, a, d"))
+        self.assertFalse(self.find_binding("a", "import b"))
+        self.assertFalse(self.find_binding("a", "import b, c, d"))
+
+    def test_from_import(self):
+        self.assertTrue(self.find_binding("a", "from x import a"))
+        self.assertTrue(self.find_binding("a", "from a import a"))
+        self.assertTrue(self.find_binding("a", "from x import b, c, a, d"))
+        self.assertTrue(self.find_binding("a", "from x.b import a"))
+        self.assertTrue(self.find_binding("a", "from x.b import b, c, a, d"))
+        self.assertFalse(self.find_binding("a", "from a import b"))
+        self.assertFalse(self.find_binding("a", "from a.d import b"))
+        self.assertFalse(self.find_binding("a", "from d.a import b"))
+
+    def test_import_as(self):
+        self.assertTrue(self.find_binding("a", "import b as a"))
+        self.assertTrue(self.find_binding("a", "import b as a, c, a as f, d"))
+        self.assertFalse(self.find_binding("a", "import a as f"))
+        self.assertFalse(self.find_binding("a", "import b, c as f, d as e"))
+
+    def test_from_import_as(self):
+        self.assertTrue(self.find_binding("a", "from x import b as a"))
+        self.assertTrue(self.find_binding("a", "from x import g as a, d as b"))
+        self.assertTrue(self.find_binding("a", "from x.b import t as a"))
+        self.assertTrue(self.find_binding("a", "from x.b import g as a, d"))
+        self.assertFalse(self.find_binding("a", "from a import b as t"))
+        self.assertFalse(self.find_binding("a", "from a.d import b as t"))
+        self.assertFalse(self.find_binding("a", "from d.a import b as t"))
+
+    def test_simple_import_with_package(self):
+        self.assertTrue(self.find_binding("b", "import b"))
+        self.assertTrue(self.find_binding("b", "import b, c, d"))
+        self.assertFalse(self.find_binding("b", "import b", "b"))
+        self.assertFalse(self.find_binding("b", "import b, c, d", "c"))
+
+    def test_from_import_with_package(self):
+        self.assertTrue(self.find_binding("a", "from x import a", "x"))
+        self.assertTrue(self.find_binding("a", "from a import a", "a"))
+        self.assertTrue(self.find_binding("a", "from x import *", "x"))
+        self.assertTrue(self.find_binding("a", "from x import b, c, a, d", "x"))
+        self.assertTrue(self.find_binding("a", "from x.b import a", "x.b"))
+        self.assertTrue(self.find_binding("a", "from x.b import *", "x.b"))
+        self.assertTrue(self.find_binding("a", "from x.b import b, c, a, d", "x.b"))
+        self.assertFalse(self.find_binding("a", "from a import b", "a"))
+        self.assertFalse(self.find_binding("a", "from a.d import b", "a.d"))
+        self.assertFalse(self.find_binding("a", "from d.a import b", "a.d"))
+        self.assertFalse(self.find_binding("a", "from x.y import *", "a.b"))
+
+    def test_import_as_with_package(self):
+        self.assertFalse(self.find_binding("a", "import b.c as a", "b.c"))
+        self.assertFalse(self.find_binding("a", "import a as f", "f"))
+        self.assertFalse(self.find_binding("a", "import a as f", "a"))
+
+    def test_from_import_as_with_package(self):
+        # Because it would take a lot of special-case code in the fixers
+        # to deal with from foo import bar as baz, we'll simply always
+        # fail if there is an "from ... import ... as ..."
+        self.assertFalse(self.find_binding("a", "from x import b as a", "x"))
+        self.assertFalse(self.find_binding("a", "from x import g as a, d as b", "x"))
+        self.assertFalse(self.find_binding("a", "from x.b import t as a", "x.b"))
+        self.assertFalse(self.find_binding("a", "from x.b import g as a, d", "x.b"))
+        self.assertFalse(self.find_binding("a", "from a import b as t", "a"))
+        self.assertFalse(self.find_binding("a", "from a import b as t", "b"))
+        self.assertFalse(self.find_binding("a", "from a import b as t", "t"))
+
+    def test_function_def(self):
+        self.assertTrue(self.find_binding("a", "def a(): pass"))
+        self.assertTrue(self.find_binding("a", "def a(b, c, d): pass"))
+        self.assertTrue(self.find_binding("a", "def a(): b = 7"))
+        self.assertFalse(self.find_binding("a", "def d(b, (c, a), e): pass"))
+        self.assertFalse(self.find_binding("a", "def d(a=7): pass"))
+        self.assertFalse(self.find_binding("a", "def d(a): pass"))
+        self.assertFalse(self.find_binding("a", "def d(): a = 7"))
+
+        s = """
+            def d():
+                def a():
+                    pass"""
+        self.assertFalse(self.find_binding("a", s))
+
+    def test_class_def(self):
+        self.assertTrue(self.find_binding("a", "class a: pass"))
+        self.assertTrue(self.find_binding("a", "class a(): pass"))
+        self.assertTrue(self.find_binding("a", "class a(b): pass"))
+        self.assertTrue(self.find_binding("a", "class a(b, c=8): pass"))
+        self.assertFalse(self.find_binding("a", "class d: pass"))
+        self.assertFalse(self.find_binding("a", "class d(a): pass"))
+        self.assertFalse(self.find_binding("a", "class d(b, a=7): pass"))
+        self.assertFalse(self.find_binding("a", "class d(b, *a): pass"))
+        self.assertFalse(self.find_binding("a", "class d(b, **a): pass"))
+        self.assertFalse(self.find_binding("a", "class d: a = 7"))
+
+        s = """
+            class d():
+                class a():
+                    pass"""
+        self.assertFalse(self.find_binding("a", s))
+
+    def test_for(self):
+        self.assertTrue(self.find_binding("a", "for a in r: pass"))
+        self.assertTrue(self.find_binding("a", "for a, b in r: pass"))
+        self.assertTrue(self.find_binding("a", "for (a, b) in r: pass"))
+        self.assertTrue(self.find_binding("a", "for c, (a,) in r: pass"))
+        self.assertTrue(self.find_binding("a", "for c, (a, b) in r: pass"))
+        self.assertTrue(self.find_binding("a", "for c in r: a = c"))
+        self.assertFalse(self.find_binding("a", "for c in a: pass"))
+
+    def test_for_nested(self):
+        s = """
+            for b in r:
+                for a in b:
+                    pass"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            for b in r:
+                for a, c in b:
+                    pass"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            for b in r:
+                for (a, c) in b:
+                    pass"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            for b in r:
+                for (a,) in b:
+                    pass"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            for b in r:
+                for c, (a, d) in b:
+                    pass"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            for b in r:
+                for c in b:
+                    a = 7"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            for b in r:
+                for c in b:
+                    d = a"""
+        self.assertFalse(self.find_binding("a", s))
+
+        s = """
+            for b in r:
+                for c in a:
+                    d = 7"""
+        self.assertFalse(self.find_binding("a", s))
+
+    def test_if(self):
+        self.assertTrue(self.find_binding("a", "if b in r: a = c"))
+        self.assertFalse(self.find_binding("a", "if a in r: d = e"))
+
+    def test_if_nested(self):
+        s = """
+            if b in r:
+                if c in d:
+                    a = c"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            if b in r:
+                if c in d:
+                    c = a"""
+        self.assertFalse(self.find_binding("a", s))
+
+    def test_while(self):
+        self.assertTrue(self.find_binding("a", "while b in r: a = c"))
+        self.assertFalse(self.find_binding("a", "while a in r: d = e"))
+
+    def test_while_nested(self):
+        s = """
+            while b in r:
+                while c in d:
+                    a = c"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            while b in r:
+                while c in d:
+                    c = a"""
+        self.assertFalse(self.find_binding("a", s))
+
+    def test_try_except(self):
+        s = """
+            try:
+                a = 6
+            except:
+                b = 8"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            try:
+                b = 8
+            except:
+                a = 6"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            try:
+                b = 8
+            except KeyError:
+                pass
+            except:
+                a = 6"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            try:
+                b = 8
+            except:
+                b = 6"""
+        self.assertFalse(self.find_binding("a", s))
+
+    def test_try_except_nested(self):
+        s = """
+            try:
+                try:
+                    a = 6
+                except:
+                    pass
+            except:
+                b = 8"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            try:
+                b = 8
+            except:
+                try:
+                    a = 6
+                except:
+                    pass"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            try:
+                b = 8
+            except:
+                try:
+                    pass
+                except:
+                    a = 6"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            try:
+                try:
+                    b = 8
+                except KeyError:
+                    pass
+                except:
+                    a = 6
+            except:
+                pass"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            try:
+                pass
+            except:
+                try:
+                    b = 8
+                except KeyError:
+                    pass
+                except:
+                    a = 6"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            try:
+                b = 8
+            except:
+                b = 6"""
+        self.assertFalse(self.find_binding("a", s))
+
+        s = """
+            try:
+                try:
+                    b = 8
+                except:
+                    c = d
+            except:
+                try:
+                    b = 6
+                except:
+                    t = 8
+                except:
+                    o = y"""
+        self.assertFalse(self.find_binding("a", s))
+
+    def test_try_except_finally(self):
+        s = """
+            try:
+                c = 6
+            except:
+                b = 8
+            finally:
+                a = 9"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            try:
+                b = 8
+            finally:
+                a = 6"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            try:
+                b = 8
+            finally:
+                b = 6"""
+        self.assertFalse(self.find_binding("a", s))
+
+        s = """
+            try:
+                b = 8
+            except:
+                b = 9
+            finally:
+                b = 6"""
+        self.assertFalse(self.find_binding("a", s))
+
+    def test_try_except_finally_nested(self):
+        s = """
+            try:
+                c = 6
+            except:
+                b = 8
+            finally:
+                try:
+                    a = 9
+                except:
+                    b = 9
+                finally:
+                    c = 9"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            try:
+                b = 8
+            finally:
+                try:
+                    pass
+                finally:
+                    a = 6"""
+        self.assertTrue(self.find_binding("a", s))
+
+        s = """
+            try:
+                b = 8
+            finally:
+                try:
+                    b = 6
+                finally:
+                    b = 7"""
+        self.assertFalse(self.find_binding("a", s))
+
+class Test_touch_import(support.TestCase):
+
+    def test_after_docstring(self):
+        node = parse('"""foo"""\nbar()')
+        fixer_util.touch_import(None, "foo", node)
+        self.assertEqual(str(node), '"""foo"""\nimport foo\nbar()\n\n')
+
+    def test_after_imports(self):
+        node = parse('"""foo"""\nimport bar\nbar()')
+        fixer_util.touch_import(None, "foo", node)
+        self.assertEqual(str(node), '"""foo"""\nimport bar\nimport foo\nbar()\n\n')
+
+    def test_beginning(self):
+        node = parse('bar()')
+        fixer_util.touch_import(None, "foo", node)
+        self.assertEqual(str(node), 'import foo\nbar()\n\n')
+
+    def test_from_import(self):
+        node = parse('bar()')
+        fixer_util.touch_import("html", "escape", node)
+        self.assertEqual(str(node), 'from html import escape\nbar()\n\n')
+
+    def test_name_import(self):
+        node = parse('bar()')
+        fixer_util.touch_import(None, "cgi", node)
+        self.assertEqual(str(node), 'import cgi\nbar()\n\n')
+
+class Test_find_indentation(support.TestCase):
+
+    def test_nothing(self):
+        fi = fixer_util.find_indentation
+        node = parse("node()")
+        self.assertEqual(fi(node), "")
+        node = parse("")
+        self.assertEqual(fi(node), "")
+
+    def test_simple(self):
+        fi = fixer_util.find_indentation
+        node = parse("def f():\n    x()")
+        self.assertEqual(fi(node), "")
+        self.assertEqual(fi(node.children[0].children[4].children[2]), "    ")
+        node = parse("def f():\n    x()\n    y()")
+        self.assertEqual(fi(node.children[0].children[4].children[4]), "    ")