Migrate several more passing tests to new test framework

scripts/token_dump

@@ -119,7 +119,7 @@ SYMBOLIC_TOKENS = (
     '<<=', '>>=', '**=', '//=', '...', '.',
     '+=', '-=', '*=', '@=', '/=', '%=', '&=', '|=', '^=',
     '<>', '<<', '<=', '<', '>>', '>=', '>', '!=', '==', '=',
-    ',', ';', ':=', ':', '->', '~',
+    ',', ';', ':=', ':', '->', '~', '`',
     '+', '-', '**', '*', '@', '//', '/', '%', '&', '|', '^',
     '(', ')', '{', '}', '[', ']',
 )
@@ -244,8 +244,7 @@ def read_tokens(pysrc):
                 line = line[match.end():]
                 continue
 
-            print('Error: Unrecognized tokens: "{}" at line {}'.format(line, n_line))
-            sys.exit(1)
+            raise RuntimeError('Error: Unrecognized tokens: "{}" at line {}'.format(line, n_line))
 
         if len(context_stack) == 0:
             yield PyToken(PyToken.ENDLINE, n_line)

tests/decompyle_test.sh

@@ -62,9 +62,9 @@ done
 
 if (( $fails == 0 ))
 then
-    echo -e "\033[32mPASS\033[m"
+    echo -e "\033[32mPASS (${#compfiles[@]})\033[m"
 else
-    echo -e "\033[31mFAIL\033[m"
+    echo -e "\033[31mFAIL ($fails of ${#compfiles[@]})\033[m"
     for ((i=0; i<${#efiles[@]}; i++))
     do
         echo -e "\t\033[31m${efiles[i]}\033[m"

tests/tokenized/test_divide_future.txt

@@ -0,0 +1,5 @@
+from __future__ import division <EOL>
+print '  1 // 2   =' , 1 // 2 <EOL>
+print '1.0 // 2.0 =' , 1.0 // 2.0 <EOL>
+print '  1 /  2   =' , 1 / 2 <EOL>
+print '1.0 /  2.0 =' , 1.0 / 2.0 <EOL>

tests/tokenized/test_divide_no_future.txt

@@ -0,0 +1,4 @@
+print '  1 // 2   =' , 1 // 2 <EOL>
+print '1.0 // 2.0 =' , 1.0 // 2.0 <EOL>
+print '  1 /  2   =' , 1 / 2 <EOL>
+print '1.0 /  2.0 =' , 1.0 / 2.0 <EOL>

tests/tokenized/test_exec.txt

@@ -0,0 +1,4 @@
+testcode = 'a = 12' <EOL>
+exec testcode <EOL>
+exec testcode in globals ( ) <EOL>
+exec testcode in globals ( ) , locals ( ) <EOL>

tests/tokenized/test_expressions.txt

@@ -0,0 +1,3 @@
+def _lsbStrToInt ( str ) : <EOL>
+<INDENT>
+return ord ( str [ 0 ] ) + ( ord ( str [ 1 ] ) << 8 ) + ( ord ( str [ 2 ] ) << 16 ) + ( ord ( str [ 3 ] ) << 24 ) <EOL>

tests/tokenized/test_extendedImport.txt

@@ -0,0 +1,10 @@
+import os <EOL>
+import sys as System <EOL>
+import time <EOL>
+import sys <EOL>
+from rfc822 import Message as Msg822 <EOL>
+from mimetools import Message as MimeMsg , decode , choose_boundary as MimeBoundary <EOL>
+import test . test_StringIO as StringTest <EOL>
+for ( k , v ) in globals ( ) . items ( ) : <EOL>
+<INDENT>
+print ` k ` , v <EOL>

tests/tokenized/test_extendedPrint.txt

@@ -0,0 +1,4 @@
+import sys <EOL>
+print >> sys . stdout , 'Hello World' <EOL>
+print >> sys . stdout , 1 , 2 , 3 <EOL>
+print >> sys . stdout , 1 , 2 , 3 <EOL>

tests/tokenized/test_global.txt

@@ -0,0 +1,23 @@
+'\ntest_global.py -- source test pattern for \'global\' statement\n\nThis source is part of the decompyle test suite.\n\ndecompyle is a Python byte-code decompiler\nSee http://www.goebel-consult.de/decompyle/ for download and\nfor further information\n' <EOL>
+i = 1 <EOL>
+j = 7 <EOL>
+def a ( ) : <EOL>
+<INDENT>
+def b ( ) : <EOL>
+<INDENT>
+global j <EOL>
+def c ( ) : <EOL>
+<INDENT>
+global i <EOL>
+k = 34 <EOL>
+i = i + k <EOL>
+<OUTDENT>
+l = 42 <EOL>
+c ( ) <EOL>
+j = j + l <EOL>
+<OUTDENT>
+b ( ) <EOL>
+print i , j <EOL>
+<OUTDENT>
+a ( ) <EOL>
+print i , j <EOL>

tests/tokenized/test_globals.txt

@@ -0,0 +1,10 @@
+def f ( ) : <EOL>
+<INDENT>
+print x <EOL>
+x = x + 1 <EOL>
+print x <EOL>
+<OUTDENT>
+raise 'This program can\'t be run' <EOL>
+x = 1 <EOL>
+f ( ) <EOL>
+print x <EOL>

tests/tokenized/test_import_as.txt

@@ -0,0 +1,12 @@
+'\ntest_import_as.py -- source test pattern for \'import .. as \'statements\n\nThis source is part of the decompyle test suite.\n\ndecompyle is a Python byte-code decompiler\nSee http://www.goebel-consult.de/decompyle/ for download and\nfor further information\n' <EOL>
+import sys as SYS <EOL>
+import os as OS <EOL>
+import sys as SYSTEM <EOL>
+import BaseHTTPServer as HTTPServ <EOL>
+import test . test_MimeWriter as Mime_Writer <EOL>
+from rfc822 import Message as MSG <EOL>
+from mimetools import Message as mimeMsg , decode , choose_boundary as mimeBoundry <EOL>
+print '---' * 20 <EOL>
+for ( k , v ) in globals ( ) . items ( ) : <EOL>
+<INDENT>
+print k , repr ( v ) <EOL>

tests/tokenized/test_iterators.txt

@@ -0,0 +1,12 @@
+for i in range ( 20 ) : <EOL>
+<INDENT>
+print i , <EOL>
+<OUTDENT>
+print <EOL>
+for i in range ( 10 ) : <EOL>
+<INDENT>
+print i , <EOL>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print 'The End' <EOL>

tests/tokenized/test_misc.txt

@@ -0,0 +1,29 @@
+raise 'This program can\'t be run' <EOL>
+class A : <EOL>
+<INDENT>
+def __init__ ( self , num ) : <EOL>
+<INDENT>
+self . num = num <EOL>
+<OUTDENT>
+def __repr__ ( self ) : <EOL>
+<INDENT>
+return str ( self . num ) <EOL>
+<OUTDENT>
+<OUTDENT>
+b = [ ] <EOL>
+for i in range ( 10 ) : <EOL>
+<INDENT>
+b . append ( A ( i ) ) <EOL>
+<OUTDENT>
+for i in ( 'CALL_FUNCTION' , 'CALL_FUNCTION_VAR' , 'CALL_FUNCTION_VAR_KW' , 'CALL_FUNCTION_KW' ) : <EOL>
+<INDENT>
+print i , '\t' , len ( i ) , len ( i ) - len ( 'CALL_FUNCTION' ) , ( len ( i ) - len ( 'CALL_FUNCTION' ) ) / 3 , i [ len ( 'CALL_FUNCTION' ) : ] <EOL>
+<OUTDENT>
+p2 = ( 0 , 0 , None ) <EOL>
+if p2 [ 2 ] : <EOL>
+<INDENT>
+print 'has value' <EOL>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print ' no value' <EOL>

tests/tokenized/test_nested_elif.txt

@@ -0,0 +1,134 @@
+a = None <EOL>
+if a == 1 : <EOL>
+<INDENT>
+print '1' <EOL>
+<OUTDENT>
+elif a == 2 : <EOL>
+<INDENT>
+print '2' <EOL>
+<OUTDENT>
+if a == 1 : <EOL>
+<INDENT>
+print '1' <EOL>
+<OUTDENT>
+elif a == 2 : <EOL>
+<INDENT>
+print '2' <EOL>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print 'other' <EOL>
+<OUTDENT>
+if a == 1 : <EOL>
+<INDENT>
+print '1' <EOL>
+<OUTDENT>
+elif a == 2 : <EOL>
+<INDENT>
+print '2' <EOL>
+<OUTDENT>
+elif a == 3 : <EOL>
+<INDENT>
+print '3' <EOL>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print 'other' <EOL>
+<OUTDENT>
+if a == 1 : <EOL>
+<INDENT>
+print '1' <EOL>
+<OUTDENT>
+elif a == 2 : <EOL>
+<INDENT>
+print '2' <EOL>
+<OUTDENT>
+elif a == 3 : <EOL>
+<INDENT>
+print '3' <EOL>
+<OUTDENT>
+if a == 1 : <EOL>
+<INDENT>
+print '1' <EOL>
+<OUTDENT>
+elif a == 2 : <EOL>
+<INDENT>
+print '2' <EOL>
+<OUTDENT>
+elif a == 3 : <EOL>
+<INDENT>
+print '3' <EOL>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print 'other' <EOL>
+<OUTDENT>
+if a == 1 : <EOL>
+<INDENT>
+print '1' <EOL>
+<OUTDENT>
+elif a == 2 : <EOL>
+<INDENT>
+print '2' <EOL>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print 'more' <EOL>
+if a == 3 : <EOL>
+<INDENT>
+print '3' <EOL>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print 'other' <EOL>
+<OUTDENT>
+<OUTDENT>
+if a == 1 : <EOL>
+<INDENT>
+print '1' <EOL>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print 'more' <EOL>
+if a == 2 : <EOL>
+<INDENT>
+print '2' <EOL>
+<OUTDENT>
+elif a == 3 : <EOL>
+<INDENT>
+print '3' <EOL>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print 'other' <EOL>
+<OUTDENT>
+<OUTDENT>
+if a == 1 : <EOL>
+<INDENT>
+print '1' <EOL>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print 'more' <EOL>
+if a == 2 : <EOL>
+<INDENT>
+print '2' <EOL>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print 'more' <EOL>
+if a == 3 : <EOL>
+<INDENT>
+print '3' <EOL>
+<OUTDENT>
+elif a == 4 : <EOL>
+<INDENT>
+print '4' <EOL>
+<OUTDENT>
+elif a == 4 : <EOL>
+<INDENT>
+print '4' <EOL>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print 'other' <EOL>

tests/tokenized/test_nested_ifs.txt

@@ -0,0 +1,21 @@
+import random <EOL>
+i = random . randint ( 0 , 5 ) <EOL>
+if i is 1 : <EOL>
+<INDENT>
+print '%d is 1' % i <EOL>
+<OUTDENT>
+elif i < 5 : <EOL>
+<INDENT>
+print '%d less than 5' % i <EOL>
+if i is 3 : <EOL>
+<INDENT>
+print '%d = 3' % i <EOL>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print '%d != 3' % i <EOL>
+<OUTDENT>
+<OUTDENT>
+else : <EOL>
+<INDENT>
+print '%d is 5' % i <EOL>

tests/tokenized/test_print.txt

@@ -0,0 +1,8 @@
+print 1 , 2 , 3 , 4 , 5 <EOL>
+a = b + 5 <EOL>
+print 1 , 2 , 3 , 4 , 5 <EOL>
+print 1 , 2 , 3 , 4 , 5 <EOL>
+print <EOL>
+print <EOL>
+print 1 , 2 , 3 , 4 , 5 <EOL>
+print <EOL>

tests/tokenized/test_print_to.txt

@@ -0,0 +1,6 @@
+'\nprint_to.py -- source test pattern for \'print >> ...\' statements\n\nThis source is part of the decompyle test suite.\n\ndecompyle is a Python byte-code decompiler\nSee http://www.goebel-consult.de/decompyle/ for download and\nfor further information\n' <EOL>
+import sys <EOL>
+print >> sys . stdout , 1 , 2 , 3 , 4 , 5 <EOL>
+print >> sys . stdout , 1 , 2 , 3 , 4 , 5 <EOL>
+print >> sys . stdout , 1 , 2 , 3 , 4 , 5 , 1 , 2 , 3 , 4 , 5 <EOL>
+print >> sys . stdout <EOL>

tests/tokenized/test_single_stmt.txt

@@ -0,0 +1 @@
+print 5 <EOL>

tests/tokenized/test_yield.txt

@@ -0,0 +1,30 @@
+from __future__ import generators <EOL>
+def inorder ( t ) : <EOL>
+<INDENT>
+if t : <EOL>
+<INDENT>
+for x in inorder ( t . left ) : <EOL>
+<INDENT>
+yield x <EOL>
+<OUTDENT>
+yield t . label <EOL>
+for x in inorder ( t . right ) : <EOL>
+<INDENT>
+yield x <EOL>
+<OUTDENT>
+<OUTDENT>
+<OUTDENT>
+def generate_ints ( n ) : <EOL>
+<INDENT>
+for i in range ( n ) : <EOL>
+<INDENT>
+yield i * 2 <EOL>
+<OUTDENT>
+<OUTDENT>
+for i in generate_ints ( 5 ) : <EOL>
+<INDENT>
+print i , <EOL>
+<OUTDENT>
+print <EOL>
+gen = generate_ints ( 3 ) <EOL>
+print gen . next ( ) , gen . next ( ) , gen . next ( ) , gen . next ( ) <EOL>