From 307e0b8fd7af8a3a2d40d9ddf311ce0bfe8bf02e Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Mon, 30 Jun 2025 19:21:38 -0700 Subject: [PATCH 01/11] Update to upload-artifact/v4 to fix CI --- .github/workflows/msvc-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/msvc-ci.yml b/.github/workflows/msvc-ci.yml index 1f2c48d..76d2060 100644 --- a/.github/workflows/msvc-ci.yml +++ b/.github/workflows/msvc-ci.yml @@ -23,7 +23,7 @@ jobs: cmake --build build --config Release --target check - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: pycdc-release path: build\Release\*.exe From a4a6a24f3e5e3a2b408319e9c8fbcf4ca6d1a073 Mon Sep 17 00:00:00 2001 From: Sahil Jain Date: Mon, 30 Jun 2025 21:35:59 +0530 Subject: [PATCH 02/11] Support END_FOR opcode --- .gitignore | 2 ++ ASTree.cpp | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 7cfe02d..3b39e81 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ /.kdev4 __pycache__ tests-out +build/ +.vscode/ diff --git a/ASTree.cpp b/ASTree.cpp index 1f419d0..10f507e 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -1679,6 +1679,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) /* Do nothing. */ break; case Pyc::POP_TOP: + case Pyc::END_FOR: { PycRef value = stack.top(); stack.pop(); @@ -2473,6 +2474,8 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(next_tup); } break; + case Pyc::COPY_A: + break; default: fprintf(stderr, "Unsupported opcode: %s (%d)\n", Pyc::OpcodeName(opcode), opcode); cleanBuild = false; From a93fd14672eccfe9741082ec475a02da04fe3dc3 Mon Sep 17 00:00:00 2001 From: Sahil Jain Date: Mon, 30 Jun 2025 20:57:21 +0530 Subject: [PATCH 03/11] Add new loop tests --- ASTree.cpp | 28 +++++++++++++++-- FastStack.h | 11 ++++++- tests/compiled/test_loops3.3.12.pyc | Bin 0 -> 1249 bytes tests/input/test_loops3.py | 34 ++++++++++++++++++++ tests/tokenized/test_loops3.txt | 46 ++++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 tests/compiled/test_loops3.3.12.pyc create mode 100644 tests/input/test_loops3.py create mode 100644 tests/tokenized/test_loops3.txt diff --git a/ASTree.cpp b/ASTree.cpp index 10f507e..ebd8e81 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -897,7 +897,8 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::INSTRUMENTED_FOR_ITER_A: { PycRef iter = stack.top(); // Iterable - stack.pop(); + if (mod->verCompare(3,12) < 0) + stack.pop(); /* Pop it? Don't pop it? */ int end; @@ -1678,11 +1679,34 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::POP_EXCEPT: /* Do nothing. */ break; - case Pyc::POP_TOP: case Pyc::END_FOR: + { + stack.pop(); + + if ((opcode == Pyc::END_FOR) && (mod->majorVer() == 3) && (mod->minorVer() == 12)) { + // one additional pop for python 3.12 + stack.pop(); + } + + // end for loop here + if (curblock->blktype() == ASTBlock::BLK_FOR) { + PycRef prev = blocks.top(); + blocks.pop(); + + curblock = blocks.top(); + curblock->append(prev.cast()); + } + else { + fprintf(stderr, "Wrong block type %i for END_FOR\n", curblock->blktype()); + break; + } + } + break; + case Pyc::POP_TOP: { PycRef value = stack.top(); stack.pop(); + if (!curblock->inited()) { if (curblock->blktype() == ASTBlock::BLK_WITH) { curblock.cast()->setExpr(value); diff --git a/FastStack.h b/FastStack.h index 45f8ed5..3f9c2c8 100644 --- a/FastStack.h +++ b/FastStack.h @@ -30,14 +30,23 @@ public: { if (m_ptr > -1) m_stack[m_ptr--] = nullptr; + else { + #ifdef BLOCK_DEBUG + fprintf(stderr, "pop from empty stack\n"); + #endif + } } PycRef top() const { if (m_ptr > -1) return m_stack[m_ptr]; - else + else { + #ifdef BLOCK_DEBUG + fprintf(stderr, "top on empty stack\n"); + #endif return nullptr; + } } bool empty() const diff --git a/tests/compiled/test_loops3.3.12.pyc b/tests/compiled/test_loops3.3.12.pyc new file mode 100644 index 0000000000000000000000000000000000000000..efb575c88ee84c87b005b3b5e2e6b21bec868f31 GIT binary patch literal 1249 zcmZ`&OHUJV5TDscS+Lu;ydp_`#AGiefbn1mNfQr5W250>NQ@S?fs)eJ|CU5ZoAdz5 z8jF^T+_^RQ9sB@(fx2)odg4au&5O?byVMfmG&`N0_xxt|n`w>!k^5z3{VM|e;+IR( zeWANc!XenO3(KHD4vu{Tf{mm@(&2zkA)P`xWk9ErP9>fCPqzlPc7(Q0zrdb4K{+48 zs~Z5z8u{t=P)<8mdW8@grUu)(Wvu#@Lsz~{Dp;Rh6bE)#S zr?WUCg5=COodmHuc;WH-;z?xO)!Ig|qe66|OM1A{uSN>l?;{cpp~B&kY&`a5>fEHx zp~8NWs&Hu|6v8}@rDQ~aY%RspMYgz>^YpUF7b_Nen0Gi)QV@*L>vUGcNjB(l#)#G7 z)QmM}KE8DE)G*!bzCq1~Mg>*9+H4=(QL@nTY0))WKD9&dOiSFzwfVSMcYzp()MzI) zl=~okO{wBWe`Ze6qpC76;Y|&yxCDadr-1_5ayeJDE!D$!9u`E5)DGK{Nq*N_kh z2FgOg*LV%eIl+e(VpD`z9WE4Ik9I;3jC^=le|8;1-V!(4eM9WO3s1_0_kpqdaL_SO zof-}vLlt-Evm5{{ra|M5JPS1=hNzCb4Z*`(ip0yq4ga(U&f + +iterable = [ 1 , 2 , 3 ] +for item in iterable : + +pass + + +loop1 ( ) +def loop2 ( ) : + +for i in range ( 2 ) : + +print ( i ) + + +loop2 ( ) +def loop3 ( ) : + +def loop ( ) : + +x = ( 1 , 2 , 3 ) +l = [ ] +for i in x : + +l . append ( i ) + +return l + +return loop ( ) + +loop3 ( ) +def loop4 ( ) : + +for i in range ( 3 ) : + +for j in range ( 2 ) : + +print ( i * j ) + + + +loop4 ( ) +for j in [ 1 , 2 , 3 ] [ : : - 1 ] : + +print ( 'hi' , j ) From 040732920b1bcc9005345a4518955c904c50caa9 Mon Sep 17 00:00:00 2001 From: Sahil Jain Date: Mon, 30 Jun 2025 21:29:53 +0530 Subject: [PATCH 04/11] Add comment --- ASTree.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ASTree.cpp b/ASTree.cpp index ebd8e81..b5b42f7 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -1689,6 +1689,8 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } // end for loop here + // TODO : Ensure that FOR loop ends here. + // Due to CACHE instructions at play, this can change. if (curblock->blktype() == ASTBlock::BLK_FOR) { PycRef prev = blocks.top(); blocks.pop(); From 6dae4e801f18e99b77c0679cb4ddab0a2e8217ca Mon Sep 17 00:00:00 2001 From: Sahil Jain Date: Mon, 30 Jun 2025 22:24:01 +0530 Subject: [PATCH 05/11] Remove COPY opcode --- ASTree.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/ASTree.cpp b/ASTree.cpp index b5b42f7..5f5603a 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -2500,8 +2500,6 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(next_tup); } break; - case Pyc::COPY_A: - break; default: fprintf(stderr, "Unsupported opcode: %s (%d)\n", Pyc::OpcodeName(opcode), opcode); cleanBuild = false; From 97ec04789d795ed589a324aaf7dd9eb79ec4f7d9 Mon Sep 17 00:00:00 2001 From: Sahil Jain Date: Mon, 30 Jun 2025 22:57:49 +0530 Subject: [PATCH 06/11] Add JUMP_BACKWARD + CACHE comments --- ASTree.cpp | 11 ++++++++--- bytecode.cpp | 4 ++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ASTree.cpp b/ASTree.cpp index 5f5603a..99f95cb 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -1164,10 +1164,13 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } break; case Pyc::JUMP_ABSOLUTE_A: + // bpo-47120: Replaced JUMP_ABSOLUTE by the relative jump JUMP_BACKWARD. + case Pyc::JUMP_BACKWARD_A: + case Pyc::JUMP_BACKWARD_NO_INTERRUPT_A: { int offs = operand; if (mod->verCompare(3, 10) >= 0) - offs *= sizeof(uint16_t); // // BPO-27129 + offs *= sizeof(uint16_t); // // BPO-27129 if (offs < pos) { if (curblock->blktype() == ASTBlock::BLK_FOR) { @@ -1689,8 +1692,10 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } // end for loop here - // TODO : Ensure that FOR loop ends here. - // Due to CACHE instructions at play, this can change. + /* TODO : Ensure that FOR loop ends here. + Due to CACHE instructions at play, the end indicated in + the for loop by pycdas is not correct, it is off by + some small amount. */ if (curblock->blktype() == ASTBlock::BLK_FOR) { PycRef prev = blocks.top(); blocks.pop(); diff --git a/bytecode.cpp b/bytecode.cpp index 7821459..c6b7cba 100644 --- a/bytecode.cpp +++ b/bytecode.cpp @@ -472,6 +472,10 @@ void bc_disasm(std::ostream& pyc_output, PycRef code, PycModule* mod, case Pyc::INSTRUMENTED_POP_JUMP_IF_FALSE_A: case Pyc::INSTRUMENTED_POP_JUMP_IF_TRUE_A: { + /* TODO: Fix offset based on CACHE instructions. + Offset is relative to next non-CACHE instruction + and thus will be printed lower than actual value. + See TODO @ END_FOR ASTree.cpp */ int offs = operand; if (mod->verCompare(3, 10) >= 0) offs *= sizeof(uint16_t); // BPO-27129 From ad5f39db56b5b93ffb7cb39c7dc1f62237d7e164 Mon Sep 17 00:00:00 2001 From: Sahil Jain Date: Mon, 30 Jun 2025 23:57:31 +0530 Subject: [PATCH 07/11] Support SLICE opcodes --- ASTree.cpp | 67 +++++++++++++++++++++++++++ tests/compiled/binary_slice.3.12.pyc | Bin 0 -> 412 bytes tests/compiled/store_slice.3.12.pyc | Bin 0 -> 358 bytes tests/input/binary_slice.py | 7 +++ tests/input/store_slice.py | 8 ++++ tests/tokenized/binary_slice.txt | 7 +++ tests/tokenized/store_slice.txt | 7 +++ 7 files changed, 96 insertions(+) create mode 100644 tests/compiled/binary_slice.3.12.pyc create mode 100644 tests/compiled/store_slice.3.12.pyc create mode 100644 tests/input/binary_slice.py create mode 100644 tests/input/store_slice.py create mode 100644 tests/tokenized/binary_slice.txt create mode 100644 tests/tokenized/store_slice.txt diff --git a/ASTree.cpp b/ASTree.cpp index 1f419d0..281d224 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -2473,6 +2473,73 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(next_tup); } break; + case Pyc::BINARY_SLICE: + { + PycRef end = stack.top(); + stack.pop(); + PycRef start = stack.top(); + stack.pop(); + PycRef dest = stack.top(); + stack.pop(); + + if (start.type() == ASTNode::NODE_OBJECT + && start.cast()->object() == Pyc_None) { + start = NULL; + } + + if (end.type() == ASTNode::NODE_OBJECT + && end.cast()->object() == Pyc_None) { + end = NULL; + } + + PycRef slice; + if (start == NULL && end == NULL) { + slice = new ASTSlice(ASTSlice::SLICE0); + } else if (start == NULL) { + slice = new ASTSlice(ASTSlice::SLICE2, start, end); + } else if (end == NULL) { + slice = new ASTSlice(ASTSlice::SLICE1, start, end); + } else { + slice = new ASTSlice(ASTSlice::SLICE3, start, end); + } + stack.push(new ASTSubscr(dest, slice)); + } + break; + case Pyc::STORE_SLICE: + { + PycRef end = stack.top(); + stack.pop(); + PycRef start = stack.top(); + stack.pop(); + PycRef dest = stack.top(); + stack.pop(); + PycRef values = stack.top(); + stack.pop(); + + if (start.type() == ASTNode::NODE_OBJECT + && start.cast()->object() == Pyc_None) { + start = NULL; + } + + if (end.type() == ASTNode::NODE_OBJECT + && end.cast()->object() == Pyc_None) { + end = NULL; + } + + PycRef slice; + if (start == NULL && end == NULL) { + slice = new ASTSlice(ASTSlice::SLICE0); + } else if (start == NULL) { + slice = new ASTSlice(ASTSlice::SLICE2, start, end); + } else if (end == NULL) { + slice = new ASTSlice(ASTSlice::SLICE1, start, end); + } else { + slice = new ASTSlice(ASTSlice::SLICE3, start, end); + } + + curblock->append(new ASTStore(values, new ASTSubscr(dest, slice))); + } + break; default: fprintf(stderr, "Unsupported opcode: %s (%d)\n", Pyc::OpcodeName(opcode), opcode); cleanBuild = false; diff --git a/tests/compiled/binary_slice.3.12.pyc b/tests/compiled/binary_slice.3.12.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24cc0f2f81a267dfc0a819000ef4f8d51af39dcd GIT binary patch literal 412 zcmX@j%ge<81R|G{G7^FGV-N=hSfPy16M&5A4CxFh42u|}7?>DR8B-Zj7*m*}7*;bv z)G)$CnNwJ>h%%)xV-aP=D$1O~hD9wis@h6sO?FMTmmu3;GJyzY5WxZ>K(2nt1|*6= z5=9^}zap^opa1{=zx)RzG?{KO=GR?7FW9|@wb2=FtQd+r-c!5R(0QW~*_W%F@ literal 0 HcmV?d00001 diff --git a/tests/compiled/store_slice.3.12.pyc b/tests/compiled/store_slice.3.12.pyc new file mode 100644 index 0000000000000000000000000000000000000000..252f5af5f78ffd0479cbdcf5adf17aca3ee23abc GIT binary patch literal 358 zcmXwzF;BxV5QXm&yG;raFjovMQx5VI)up)Ldrj2h7Ivk zn2|y@#FP!G8x!Y3Z+QOh`RVT7JZ}J0Pw!VZ=k$J=W>f4RR+j{iARt14cN|~_5eVxU zXDCsEumu}};6%MCB}HF2kwqO7&JZk+mcUp1o3(Z+O|C6r<06#IcRLg5c1e7@Gx^Rk zVjJ;DPCIjINBcpU)082fwK#7vX}OxGk!*RU;ODV~s7_(%85Ije4vPbg${Vgrka) fS5Iob_9vQu^tbD?`%+^y((Dtso +print ( l [ 1 : 3 ] ) +print ( l [ : 2 ] ) +print ( l [ 3 : ] ) +print ( l [ - 4 : ] ) +print ( l [ : - 2 ] ) +print ( l [ : ] ) diff --git a/tests/tokenized/store_slice.txt b/tests/tokenized/store_slice.txt new file mode 100644 index 0000000..2842070 --- /dev/null +++ b/tests/tokenized/store_slice.txt @@ -0,0 +1,7 @@ +a = [ 0 ] * 16 +l = [ 1 , 2 , 3 ] +a [ 13 : ] = l +a [ 5 ] = 10 +a [ : 2 ] = [ 1 , 2 ] +a [ : ] = range ( 16 ) +print ( a ) From 5fe61462a244e5a919a79740d28483dc9e047660 Mon Sep 17 00:00:00 2001 From: Sahil Jain Date: Mon, 30 Jun 2025 23:58:54 +0530 Subject: [PATCH 08/11] Support COPY opcde --- ASTree.cpp | 6 ++++++ FastStack.h | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/ASTree.cpp b/ASTree.cpp index 281d224..851f735 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -2540,6 +2540,12 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) curblock->append(new ASTStore(values, new ASTSubscr(dest, slice))); } break; + case Pyc::COPY_A: + { + PycRef value = stack.top(operand); + stack.push(value); + } + break; default: fprintf(stderr, "Unsupported opcode: %s (%d)\n", Pyc::OpcodeName(opcode), opcode); cleanBuild = false; diff --git a/FastStack.h b/FastStack.h index 45f8ed5..d8fb08b 100644 --- a/FastStack.h +++ b/FastStack.h @@ -32,11 +32,21 @@ public: m_stack[m_ptr--] = nullptr; } - PycRef top() const + PycRef top(int i = 1) const { - if (m_ptr > -1) - return m_stack[m_ptr]; - else + if (i > 0) { + int idx = m_ptr + 1 - i; + if ((m_ptr > -1) && (idx >= 0)) + return m_stack[idx]; + else { + #ifdef BLOCK_DEBUG + fprintf(stderr, "insufficient values on stack\n"); + #endif + return nullptr; + } + } + else { + fprintf(stderr, "incorrect operand %i\n", i); return nullptr; } From aa292c76820fac67a064f20cafb9946cd746d1e1 Mon Sep 17 00:00:00 2001 From: Sahil Jain Date: Tue, 1 Jul 2025 09:27:07 +0530 Subject: [PATCH 09/11] Add newlines --- tests/input/binary_slice.py | 2 +- tests/input/store_slice.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/input/binary_slice.py b/tests/input/binary_slice.py index fb47b61..361d922 100644 --- a/tests/input/binary_slice.py +++ b/tests/input/binary_slice.py @@ -4,4 +4,4 @@ print(l[:2]) print(l[3:]) print(l[-4:]) print(l[:-2]) -print(l[:]) \ No newline at end of file +print(l[:]) diff --git a/tests/input/store_slice.py b/tests/input/store_slice.py index 1e04a3a..8d8369a 100644 --- a/tests/input/store_slice.py +++ b/tests/input/store_slice.py @@ -5,4 +5,4 @@ a[5] = 10 a[:2] = [1,2] a[:] = range(16) -print(a) \ No newline at end of file +print(a) From 3afcfbc6a76307b0df68499254a3f04ed462ea7a Mon Sep 17 00:00:00 2001 From: Sahil Jain Date: Tue, 1 Jul 2025 14:01:05 +0530 Subject: [PATCH 10/11] Fix compilation error --- FastStack.h | 1 + 1 file changed, 1 insertion(+) diff --git a/FastStack.h b/FastStack.h index d8fb08b..3f8950c 100644 --- a/FastStack.h +++ b/FastStack.h @@ -48,6 +48,7 @@ public: else { fprintf(stderr, "incorrect operand %i\n", i); return nullptr; + } } bool empty() const From 6e0089e01c5f45b956d655a0c69a41e5ccc382bf Mon Sep 17 00:00:00 2001 From: Sahil Jain Date: Tue, 1 Jul 2025 22:52:59 +0530 Subject: [PATCH 11/11] Update --- ASTree.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ASTree.cpp b/ASTree.cpp index 99f95cb..6e010e0 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -897,8 +897,10 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::INSTRUMENTED_FOR_ITER_A: { PycRef iter = stack.top(); // Iterable - if (mod->verCompare(3,12) < 0) + if (mod->verCompare(3, 12) < 0) { + // Do not pop the iterator for py 3.12+ stack.pop(); + } /* Pop it? Don't pop it? */ int end; @@ -1705,7 +1707,6 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) } else { fprintf(stderr, "Wrong block type %i for END_FOR\n", curblock->blktype()); - break; } } break;