Merge pull request #555 from whoami730/master

Support py 3.12 opcodes: part 1

.gitignore

@@ -6,3 +6,5 @@
 /.kdev4
 __pycache__
 tests-out
+build/
+.vscode/

ASTree.cpp

@@ -897,7 +897,10 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
         case Pyc::INSTRUMENTED_FOR_ITER_A:
             {
                 PycRef<ASTNode> iter = stack.top(); // Iterable
-                stack.pop();
+                if (mod->verCompare(3, 12) < 0) {
+                    // Do not pop the iterator for py 3.12+
+                    stack.pop();
+                }
                 /* Pop it? Don't pop it? */
 
                 int end;
@@ -1163,6 +1166,9 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
             }
             break;
         case Pyc::JUMP_ABSOLUTE_A:
+        // bpo-47120: Replaced JUMP_ABSOLUTE by the relative jump JUMP_BACKWARD.
+        case Pyc::JUMP_BACKWARD_A:
+        case Pyc::JUMP_BACKWARD_NO_INTERRUPT_A:
             {
                 int offs = operand;
                 if (mod->verCompare(3, 10) >= 0)
@@ -1678,10 +1684,37 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
         case Pyc::POP_EXCEPT:
             /* Do nothing. */
             break;
+        case Pyc::END_FOR:
+            {
+                stack.pop();
+
+                if ((opcode == Pyc::END_FOR) && (mod->majorVer() == 3) && (mod->minorVer() == 12)) {
+                    // one additional pop for python 3.12
+                    stack.pop();
+                }
+
+                // end for loop here
+                /* TODO : Ensure that FOR loop ends here. 
+                   Due to CACHE instructions at play, the end indicated in
+                   the for loop by pycdas is not correct, it is off by
+                   some small amount. */
+                if (curblock->blktype() == ASTBlock::BLK_FOR) {
+                    PycRef<ASTBlock> prev = blocks.top();
+                    blocks.pop();
+
+                    curblock = blocks.top();
+                    curblock->append(prev.cast<ASTNode>());
+                }
+                else {
+                    fprintf(stderr, "Wrong block type %i for END_FOR\n", curblock->blktype());
+                }
+            }
+            break;
         case Pyc::POP_TOP:
             {
                 PycRef<ASTNode> value = stack.top();
                 stack.pop();
+
                 if (!curblock->inited()) {
                     if (curblock->blktype() == ASTBlock::BLK_WITH) {
                         curblock.cast<ASTWithBlock>()->setExpr(value);

FastStack.h

@@ -30,14 +30,23 @@ public:
     {
         if (m_ptr > -1)
             m_stack[m_ptr--] = nullptr;
+        else {
+            #ifdef BLOCK_DEBUG
+                fprintf(stderr, "pop from empty stack\n");
+            #endif
+        }
     }
 
     PycRef<ASTNode> top() const
     {
         if (m_ptr > -1)
             return m_stack[m_ptr];
-        else
+        else {
+            #ifdef BLOCK_DEBUG
+                fprintf(stderr, "top on empty stack\n");
+            #endif
             return nullptr;
+        }
     }
 
     bool empty() const

bytecode.cpp

@@ -472,6 +472,10 @@ void bc_disasm(std::ostream& pyc_output, PycRef<PycCode> code, PycModule* mod,
             case Pyc::INSTRUMENTED_POP_JUMP_IF_FALSE_A:
             case Pyc::INSTRUMENTED_POP_JUMP_IF_TRUE_A:
                 {
+                    /* TODO: Fix offset based on CACHE instructions.
+                       Offset is relative to next non-CACHE instruction
+                       and thus will be printed lower than actual value.
+                       See TODO @ END_FOR ASTree.cpp */
                     int offs = operand;
                     if (mod->verCompare(3, 10) >= 0)
                         offs *= sizeof(uint16_t); // BPO-27129

tests/input/test_loops3.py

@@ -0,0 +1,34 @@
+def loop1():
+    iterable = [1, 2, 3]
+    for item in iterable:
+        pass
+
+loop1()
+
+def loop2():
+    for i in range(2):
+        print(i)
+
+loop2()
+
+def loop3():
+    def loop():
+        x = (1,2,3)
+        l = []
+        for i in x:
+            l.append(i)
+        return l
+    
+    return loop()
+
+loop3()
+
+def loop4():
+    for i in range(3):
+        for j in range(2):
+            print(i*j)
+
+loop4()
+
+for j in [1,2,3][::-1]:
+    print("hi", j)

tests/tokenized/test_loops3.txt

@@ -0,0 +1,46 @@
+def loop1 ( ) : <EOL>
+<INDENT>
+iterable = [ 1 , 2 , 3 ] <EOL>
+for item in iterable : <EOL>
+<INDENT>
+pass <EOL>
+<OUTDENT>
+<OUTDENT>
+loop1 ( ) <EOL>
+def loop2 ( ) : <EOL>
+<INDENT>
+for i in range ( 2 ) : <EOL>
+<INDENT>
+print ( i ) <EOL>
+<OUTDENT>
+<OUTDENT>
+loop2 ( ) <EOL>
+def loop3 ( ) : <EOL>
+<INDENT>
+def loop ( ) : <EOL>
+<INDENT>
+x = ( 1 , 2 , 3 ) <EOL>
+l = [ ] <EOL>
+for i in x : <EOL>
+<INDENT>
+l . append ( i ) <EOL>
+<OUTDENT>
+return l <EOL>
+<OUTDENT>
+return loop ( ) <EOL>
+<OUTDENT>
+loop3 ( ) <EOL>
+def loop4 ( ) : <EOL>
+<INDENT>
+for i in range ( 3 ) : <EOL>
+<INDENT>
+for j in range ( 2 ) : <EOL>
+<INDENT>
+print ( i * j ) <EOL>
+<OUTDENT>
+<OUTDENT>
+<OUTDENT>
+loop4 ( ) <EOL>
+for j in [ 1 , 2 , 3 ] [ : : - 1 ] : <EOL>
+<INDENT>
+print ( 'hi' , j ) <EOL>