Migrate a couple more passing tests, and handle end-of-line comments

in token_dump.

scripts/token_dump

@@ -93,7 +93,9 @@ class StringToken(PyToken):
         self.prefix = ''.join(sorted(prefix.lower()))
 
         # Normalize special characters for comparison
-        self.content = content.replace("'", "\\'").replace('\n', '\\n')
+        self.content = content.replace("\\'", "'").replace("'", "\\'") \
+                              .replace('\\"', '"').replace('\t', '\\t') \
+                              .replace('\n', '\\n').replace('\r', '\\r')
 
     def __str__(self):
         return "{}'{}'".format(self.prefix, self.content)
@@ -193,6 +195,9 @@ def read_tokens(pysrc):
             line = line.lstrip()
             if not line:
                 break
+            if line[0] == '#':
+                # The rest of this line is a comment
+                break
 
             token = symbolic_token(line, n_line)
             if token:

tests/22_test_del.ref.py

@@ -1,35 +0,0 @@
-"""
-test_del.py -- source test pattern for 'del' statements
-
-This source is part of the decompyle test suite.
-Snippet taken from python libs's test_class.py
-
-decompyle is a Python byte-code decompiler
-See http://www.goebel-consult.de/decompyle/ for download and
-for further information
-"""
-
-raise "This program can't be run"
-
-print 0
-a = b[5]
-print 1
-del a
-print 2
-del b[5]
-print 3
-
-del testme[1]
-print 4
-del testme[:]
-print '4a'
-del testme[:42]
-print '4b'
-del testme[40:42]
-print 5
-del testme[2:1024:10]
-print '5a'
-del testme[40, 41, 42]
-print 6
-del testme[:42, ..., :24:, 24, 100]
-print 7

tests/25_test_del.ref.py

@@ -1,35 +0,0 @@
-"""
-test_del.py -- source test pattern for 'del' statements
-
-This source is part of the decompyle test suite.
-Snippet taken from python libs's test_class.py
-
-decompyle is a Python byte-code decompiler
-See http://www.goebel-consult.de/decompyle/ for download and
-for further information
-"""
-
-raise "This program can't be run"
-
-print 0
-a = b[5]
-print 1
-del a
-print 2
-del b[5]
-print 3
-
-del testme[1]
-print 4
-del testme[:]
-print '4a'
-del testme[:42]
-print '4b'
-del testme[40:42]
-print 5
-del testme[2:1024:10]
-print '5a'
-del testme[40, 41, 42]
-print 6
-del testme[:42, ..., :24:, 24, 100]
-print 7

tests/input/test_docstring.py

@@ -1,17 +1,10 @@
-
-
-
-
-
-
-
-
 """
 This is a doc string
 """
 
 def Doc_Test():
     """This has to be present"""
+    pass
 
 class XXX:
     def __init__(self):
@@ -28,7 +21,6 @@ class XXX:
 
     def XXX12():
         foo = 'XXX22: This has to be present'
-        pass        # Left for line number compatibility.  Do not emit
 
     def XXX13():
         pass

tests/tokenized/test_del.txt

@@ -0,0 +1,23 @@
+'\ntest_del.py -- source test pattern for \'del\' statements\n\nThis source is part of the decompyle test suite.\nSnippet taken from python libs\'s test_class.py\n\ndecompyle is a Python byte-code decompiler\nSee http://www.goebel-consult.de/decompyle/ for download and\nfor further information\n' <EOL>
+raise 'This program can\'t be run' <EOL>
+print 0 <EOL>
+a = b [ 5 ] <EOL>
+print 1 <EOL>
+del a <EOL>
+print 2 <EOL>
+del b [ 5 ] <EOL>
+print 3 <EOL>
+del testme [ 1 ] <EOL>
+print 4 <EOL>
+del testme [ : ] <EOL>
+print '4a' <EOL>
+del testme [ : 42 ] <EOL>
+print '4b' <EOL>
+del testme [ 40 : 42 ] <EOL>
+print 5 <EOL>
+del testme [ 2 : 1024 : 10 ] <EOL>
+print '5a' <EOL>
+del testme [ ( 40 , 41 , 42 ) ] <EOL>
+print 6 <EOL>
+del testme [ ( : 42 , ... , : 24 : , 24 , 100 ) ] <EOL>
+print 7 <EOL>

tests/tokenized/test_docstring.txt

@@ -0,0 +1,43 @@
+'\nThis is a doc string\n' <EOL>
+def Doc_Test ( ) : <EOL>
+<INDENT>
+'This has to be present' <EOL>
+pass <EOL>
+<OUTDENT>
+class XXX : <EOL>
+<INDENT>
+def __init__ ( self ) : <EOL>
+<INDENT>
+'__init__: This has to be present' <EOL>
+self . a = 1 <EOL>
+def XXX22 ( ) : <EOL>
+<INDENT>
+'XXX22: This has to be present' <EOL>
+pass <EOL>
+<OUTDENT>
+<OUTDENT>
+def XXX11 ( ) : <EOL>
+<INDENT>
+'XXX22: This has to be present' <EOL>
+pass <EOL>
+<OUTDENT>
+def XXX12 ( ) : <EOL>
+<INDENT>
+foo = 'XXX22: This has to be present' <EOL>
+<OUTDENT>
+def XXX13 ( ) : <EOL>
+<INDENT>
+pass <EOL>
+<OUTDENT>
+<OUTDENT>
+def Y11 ( ) : <EOL>
+<INDENT>
+def Y22 ( ) : <EOL>
+<INDENT>
+def Y33 ( ) : <EOL>
+<INDENT>
+pass <EOL>
+<OUTDENT>
+<OUTDENT>
+<OUTDENT>
+print __doc__ <EOL>