Merge branch 'upstream-pycdc'

Fix bug in RAISE_VARARGS

.gitignore

@@ -6,8 +6,8 @@
 /.kdev4
 __pycache__
 tests-out
-
-.vscode/
 build/
+.vscode/
+
 pyarmor-1shot
 pyarmor-1shot.exe

ASTree.cpp

@@ -1113,7 +1113,10 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
         case Pyc::INSTRUMENTED_FOR_ITER_A:
             {
                 PycRef<ASTNode> iter = stack.top(); // Iterable
-                stack.pop();
+                if (mod->verCompare(3, 12) < 0) {
+                    // Do not pop the iterator for py 3.12+
+                    stack.pop();
+                }
                 /* Pop it? Don't pop it? */
 
                 int end;
@@ -1379,10 +1382,13 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
             }
             break;
         case Pyc::JUMP_ABSOLUTE_A:
+        // bpo-47120: Replaced JUMP_ABSOLUTE by the relative jump JUMP_BACKWARD.
+        case Pyc::JUMP_BACKWARD_A:
+        case Pyc::JUMP_BACKWARD_NO_INTERRUPT_A:
             {
                 int offs = operand;
                 if (mod->verCompare(3, 10) >= 0)
-                    offs *= sizeof(uint16_t); // // BPO-27129
+                    offs *= sizeof(uint16_t); // // BPO-27129 
 
                 if (offs < pos) {
                     if (curblock->blktype() == ASTBlock::BLK_FOR) {
@@ -1949,10 +1955,37 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
         case Pyc::POP_EXCEPT:
             /* Do nothing. */
             break;
+        case Pyc::END_FOR:
+            {
+                stack.pop();
+
+                if ((opcode == Pyc::END_FOR) && (mod->majorVer() == 3) && (mod->minorVer() == 12)) {
+                    // one additional pop for python 3.12
+                    stack.pop();
+                }
+
+                // end for loop here
+                /* TODO : Ensure that FOR loop ends here. 
+                   Due to CACHE instructions at play, the end indicated in
+                   the for loop by pycdas is not correct, it is off by
+                   some small amount. */
+                if (curblock->blktype() == ASTBlock::BLK_FOR) {
+                    PycRef<ASTBlock> prev = blocks.top();
+                    blocks.pop();
+
+                    curblock = blocks.top();
+                    curblock->append(prev.cast<ASTNode>());
+                }
+                else {
+                    fprintf(stderr, "Wrong block type %i for END_FOR\n", curblock->blktype());
+                }
+            }
+            break;
         case Pyc::POP_TOP:
             {
                 PycRef<ASTNode> value = stack.top();
                 stack.pop();
+
                 if (!curblock->inited()) {
                     if (curblock->blktype() == ASTBlock::BLK_WITH) {
                         curblock.cast<ASTWithBlock>()->setExpr(value);
@@ -2058,8 +2091,6 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
                     blocks.pop();
                     curblock = blocks.top();
                     curblock->append(prev.cast<ASTNode>());
-
-                    bc_next(source, mod, opcode, operand, pos);
                 }
             }
             break;
@@ -2756,21 +2787,6 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
             break;
         // BEGIN ONESHOT TEMPORARY PATCH
         // THESE OPCODES ARE NOT IMPLEMENTED HERE
-        case Pyc::COPY_A:
-            {
-                FastStack tmp_stack(20);
-                for (int i = 0; i < operand - 1; i++) {
-                    tmp_stack.push(stack.top());
-                    stack.pop();
-                }
-                auto value = stack.top();
-                for (int i = 0; i < operand - 1; i++) {
-                    stack.push(tmp_stack.top());
-                    tmp_stack.pop();
-                }
-                stack.push(value);
-            }
-            break;
         case Pyc::PUSH_EXC_INFO:
             {
                 stack.push(stack.top());
@@ -2797,6 +2813,79 @@ PycRef<ASTNode> BuildFromCode(PycRef<PycCode> code, PycModule* mod)
             }
             break;
         // END ONESHOT PATCH
+        case Pyc::BINARY_SLICE:
+            {
+                PycRef<ASTNode> end = stack.top();
+                stack.pop();
+                PycRef<ASTNode> start = stack.top();
+                stack.pop();
+                PycRef<ASTNode> dest = stack.top();
+                stack.pop();
+
+                if (start.type() == ASTNode::NODE_OBJECT
+                        && start.cast<ASTObject>()->object() == Pyc_None) {
+                    start = NULL;
+                }
+
+                if (end.type() == ASTNode::NODE_OBJECT
+                        && end.cast<ASTObject>()->object() == Pyc_None) {
+                    end = NULL;
+                }
+
+                PycRef<ASTNode> slice;
+                if (start == NULL && end == NULL) {
+                    slice = new ASTSlice(ASTSlice::SLICE0);
+                } else if (start == NULL) {
+                    slice = new ASTSlice(ASTSlice::SLICE2, start, end);
+                } else if (end == NULL) {
+                    slice = new ASTSlice(ASTSlice::SLICE1, start, end);
+                } else {
+                    slice = new ASTSlice(ASTSlice::SLICE3, start, end);
+                }
+                stack.push(new ASTSubscr(dest, slice));
+            }
+            break;
+        case Pyc::STORE_SLICE:
+            {
+                PycRef<ASTNode> end = stack.top();
+                stack.pop();
+                PycRef<ASTNode> start = stack.top();
+                stack.pop();
+                PycRef<ASTNode> dest = stack.top();
+                stack.pop();
+                PycRef<ASTNode> values = stack.top();
+                stack.pop();
+
+                if (start.type() == ASTNode::NODE_OBJECT
+                        && start.cast<ASTObject>()->object() == Pyc_None) {
+                    start = NULL;
+                }
+
+                if (end.type() == ASTNode::NODE_OBJECT
+                        && end.cast<ASTObject>()->object() == Pyc_None) {
+                    end = NULL;
+                }
+
+                PycRef<ASTNode> slice;
+                if (start == NULL && end == NULL) {
+                    slice = new ASTSlice(ASTSlice::SLICE0);
+                } else if (start == NULL) {
+                    slice = new ASTSlice(ASTSlice::SLICE2, start, end);
+                } else if (end == NULL) {
+                    slice = new ASTSlice(ASTSlice::SLICE1, start, end);
+                } else {
+                    slice = new ASTSlice(ASTSlice::SLICE3, start, end);
+                }
+
+                curblock->append(new ASTStore(values, new ASTSubscr(dest, slice)));
+            }
+            break;
+        case Pyc::COPY_A:
+            {
+                PycRef<ASTNode> value = stack.top(operand);
+                stack.push(value);
+            }
+            break;
         default:
             fprintf(stderr, "Unsupported opcode: %s (%d) at %s\n",
                 Pyc::OpcodeName(opcode),

FastStack.h

@@ -30,14 +30,30 @@ public:
     {
         if (m_ptr > -1)
             m_stack[m_ptr--] = nullptr;
+        else {
+            #ifdef BLOCK_DEBUG
+                fprintf(stderr, "pop from empty stack\n");
+            #endif
+        }
     }
 
-    PycRef<ASTNode> top() const
+    PycRef<ASTNode> top(int i = 1) const
     {
-        if (m_ptr > -1)
-            return m_stack[m_ptr];
-        else
+        if (i > 0) {
+            int idx = m_ptr + 1 - i;
+            if ((m_ptr > -1) && (idx >= 0))
+                return m_stack[idx];
+            else {
+                #ifdef BLOCK_DEBUG
+                    fprintf(stderr, "insufficient values on stack\n");
+                #endif
+                return nullptr;
+            }
+        }
+        else {
+            fprintf(stderr, "incorrect operand %i\n", i);
             return nullptr;
+        }
     }
 
     bool empty() const

bytecode.cpp

@@ -474,6 +474,10 @@ void bc_disasm(std::ostream& pyc_output, PycRef<PycCode> code, PycModule* mod,
             case Pyc::INSTRUMENTED_POP_JUMP_IF_FALSE_A:
             case Pyc::INSTRUMENTED_POP_JUMP_IF_TRUE_A:
                 {
+                    /* TODO: Fix offset based on CACHE instructions.
+                       Offset is relative to next non-CACHE instruction
+                       and thus will be printed lower than actual value.
+                       See TODO @ END_FOR ASTree.cpp */
                     int offs = operand;
                     if (mod->verCompare(3, 10) >= 0)
                         offs *= sizeof(uint16_t); // BPO-27129

tests/input/binary_slice.py

@@ -0,0 +1,7 @@
+l = [1,2,3,4,5,6]
+print(l[1:3])
+print(l[:2])
+print(l[3:])
+print(l[-4:])
+print(l[:-2])
+print(l[:])

tests/input/store_slice.py

@@ -0,0 +1,8 @@
+a = [0] * 16
+l = [1,2,3]
+a[13:] = l
+a[5] = 10
+a[:2] = [1,2]
+a[:] = range(16)
+
+print(a)

tests/input/test_loops3.py

@@ -0,0 +1,34 @@
+def loop1():
+    iterable = [1, 2, 3]
+    for item in iterable:
+        pass
+
+loop1()
+
+def loop2():
+    for i in range(2):
+        print(i)
+
+loop2()
+
+def loop3():
+    def loop():
+        x = (1,2,3)
+        l = []
+        for i in x:
+            l.append(i)
+        return l
+    
+    return loop()
+
+loop3()
+
+def loop4():
+    for i in range(3):
+        for j in range(2):
+            print(i*j)
+
+loop4()
+
+for j in [1,2,3][::-1]:
+    print("hi", j)

tests/input/test_raise_varargs.py

@@ -0,0 +1,7 @@
+import struct
+
+def bytes_to_words(b):
+    '''Convert a byte string (little-endian) to a list of 32-bit words.'''
+    if len(b) % 4 != 0:
+        raise ValueError('Input bytes length must be a multiple of 4 for word conversion.')
+    return struct.unpack('<' + 'I' * (len(b) // 4), b)

tests/tokenized/binary_slice.txt

@@ -0,0 +1,7 @@
+l = [ 1 , 2 , 3 , 4 , 5 , 6 ] <EOL>
+print ( l [ 1 : 3 ] ) <EOL>
+print ( l [ : 2 ] ) <EOL>
+print ( l [ 3 : ] ) <EOL>
+print ( l [ - 4 : ] ) <EOL>
+print ( l [ : - 2 ] ) <EOL>
+print ( l [ : ] ) <EOL>

tests/tokenized/store_slice.txt

@@ -0,0 +1,7 @@
+a = [ 0 ] * 16 <EOL>
+l = [ 1 , 2 , 3 ] <EOL>
+a [ 13 : ] = l <EOL>
+a [ 5 ] = 10 <EOL>
+a [ : 2 ] = [ 1 , 2 ] <EOL>
+a [ : ] = range ( 16 ) <EOL>
+print ( a ) <EOL>

tests/tokenized/test_loops3.txt

@@ -0,0 +1,46 @@
+def loop1 ( ) : <EOL>
+<INDENT>
+iterable = [ 1 , 2 , 3 ] <EOL>
+for item in iterable : <EOL>
+<INDENT>
+pass <EOL>
+<OUTDENT>
+<OUTDENT>
+loop1 ( ) <EOL>
+def loop2 ( ) : <EOL>
+<INDENT>
+for i in range ( 2 ) : <EOL>
+<INDENT>
+print ( i ) <EOL>
+<OUTDENT>
+<OUTDENT>
+loop2 ( ) <EOL>
+def loop3 ( ) : <EOL>
+<INDENT>
+def loop ( ) : <EOL>
+<INDENT>
+x = ( 1 , 2 , 3 ) <EOL>
+l = [ ] <EOL>
+for i in x : <EOL>
+<INDENT>
+l . append ( i ) <EOL>
+<OUTDENT>
+return l <EOL>
+<OUTDENT>
+return loop ( ) <EOL>
+<OUTDENT>
+loop3 ( ) <EOL>
+def loop4 ( ) : <EOL>
+<INDENT>
+for i in range ( 3 ) : <EOL>
+<INDENT>
+for j in range ( 2 ) : <EOL>
+<INDENT>
+print ( i * j ) <EOL>
+<OUTDENT>
+<OUTDENT>
+<OUTDENT>
+loop4 ( ) <EOL>
+for j in [ 1 , 2 , 3 ] [ : : - 1 ] : <EOL>
+<INDENT>
+print ( 'hi' , j ) <EOL>

tests/tokenized/test_raise_varargs.txt

@@ -0,0 +1,9 @@
+import struct <EOL>
+def bytes_to_words ( b ) : <EOL>
+<INDENT>
+'Convert a byte string (little-endian) to a list of 32-bit words.' <EOL>
+if len ( b ) % 4 != 0 : <EOL>
+<INDENT>
+raise ValueError ( 'Input bytes length must be a multiple of 4 for word conversion.' ) <EOL>
+<OUTDENT>
+return struct . unpack ( '<' + 'I' * ( len ( b ) // 4 ) , b ) <EOL>