From 9f747e188cd2ee41c5749f01632780775057fbfb Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Mon, 27 Feb 2023 11:59:01 -0800 Subject: [PATCH 1/4] Add unit test for basic function calls (py3) --- tests/compiled/test_calls.3.1.pyc | Bin 0 -> 353 bytes tests/compiled/test_calls.3.10.pyc | Bin 0 -> 285 bytes tests/compiled/test_calls.3.5.pyc | Bin 0 -> 303 bytes tests/compiled/test_calls.3.8.pyc | Bin 0 -> 283 bytes tests/input/test_calls.py | 9 +++++++++ tests/tokenized/test_calls.txt | 7 +++++++ 6 files changed, 16 insertions(+) create mode 100644 tests/compiled/test_calls.3.1.pyc create mode 100644 tests/compiled/test_calls.3.10.pyc create mode 100644 tests/compiled/test_calls.3.5.pyc create mode 100644 tests/compiled/test_calls.3.8.pyc create mode 100644 tests/input/test_calls.py create mode 100644 tests/tokenized/test_calls.txt diff --git a/tests/compiled/test_calls.3.1.pyc b/tests/compiled/test_calls.3.1.pyc new file mode 100644 index 0000000000000000000000000000000000000000..63dd6e5d7e3624ebe040df8abe8cace8fcf9f9b5 GIT binary patch literal 353 zcmYk0KTpFj5XIl6`GcY$b!DPs#6X2$M2JrS158~|h>eJ4;wbikIxzKfSL)fIg5^8^ zp5NJbkIiZQUG4iG|5hn{FX62uC7U3$IWl2G(16Kc7El>>1UhO>FgZ*CC0>r*uU$eF zKZPZ91yx3o9tr^@O`h6H8V(Y8;6n1V_Mzs5(3^AVeC=i?(KW{*Z&nwptLyeeFu8u~ zITPW0vVBmFL;C4!9)jjvyr~QC35*-Ym2 eto4`EmSgMN52x7byEvC`BsJ7zOFGYLTG9b!gFNT} literal 0 HcmV?d00001 diff --git a/tests/compiled/test_calls.3.10.pyc b/tests/compiled/test_calls.3.10.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a4886aa06122acd46fd5bd4d371487e2c7e44928 GIT binary patch literal 285 zcmYjJy-LJD5T42YWQli{g59-e;UQXz_yi(ol}6(29ww~WkliSEV3o?ZxGR07ZLNHT z1Hp-D?t}S$W`@sArx_4F(%
Gu#0At~I__{Tc)Mc4nhn{FatBsd`6#~9wpPA^ zAUJnQhMAvln15ytr_=l``&qpM_~aTxGF@|d%c406Oy$Ugl3;*Fph=(wwgeh`Lue$L z6q;ue+yAkGPXCMwCWBVHX1~Y;5I?zHWMBLdH=Fd%_>h;$qjP2QQ1y}Wjk=hh%+Jmj zlbHCmkHYz=e9&$PF@5dpV501*@Dyd^>NcibU$-Gj^HSLl^ap%)FW*OjPy1!vc0*Y1 g_rcCNoLoLTJy>%^?DhpqL4}ynvB+sgzjR+iumAu6 literal 0 HcmV?d00001 diff --git a/tests/compiled/test_calls.3.8.pyc b/tests/compiled/test_calls.3.8.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5a68f9d87b1ea538fb7506fd7c839bdcfb4ba79a GIT binary patch literal 283 zcmYjJy-LJD5Z=lDWQn;}u)FpwJVZ+opFjk)Nh9%g#e_8*vK!?NtWxCfte^lJ?Np(tF@gc~Bm3^-Oy3I$kJCtGfk9RQoWg<5Q- z;YRGHtuk^Luga!I4KVqE0X5ySMfy#x?0O^bY=~LGpS&;lW7#2O8+ATEo}ZpAH1e?P25$cZ%QB!|O9Lk_<|n?$t$ literal 0 HcmV?d00001 diff --git a/tests/input/test_calls.py b/tests/input/test_calls.py new file mode 100644 index 0000000..c2ec4d0 --- /dev/null +++ b/tests/input/test_calls.py @@ -0,0 +1,9 @@ +import sys +import os + +sys.stdout.write('Test\n') +sys.stdout.write(os.path.join('foo', 'bar')) + +print('\n') +print(eval('4 * 13')) +print() diff --git a/tests/tokenized/test_calls.txt b/tests/tokenized/test_calls.txt new file mode 100644 index 0000000..3300e23 --- /dev/null +++ b/tests/tokenized/test_calls.txt @@ -0,0 +1,7 @@ +import sys +import os +sys . stdout . write ( 'Test\n' ) +sys . stdout . write ( os . path . join ( 'foo' , 'bar' ) ) +print ( '\n' ) +print ( eval ( '4 * 13' ) ) +print ( ) From f4db6a7cf6bab114fbca9dd09355d2a080d50a26 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Mon, 27 Feb 2023 13:12:11 -0800 Subject: [PATCH 2/4] Update CodeQL configuration --- .github/workflows/codeql-analysis.yml | 28 +++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 0b0034b..0db91ee 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -18,35 +18,41 @@ jobs: analyze: name: Analyze runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write strategy: fail-fast: false matrix: - # Override automatic language detection by changing the below list - # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] language: ['cpp', 'python'] - # Learn more... - # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Use only 'java' to analyze code written in Java, Kotlin or both + # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - if: matrix.language == 'python' name: Autobuild Python - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 - if: matrix.language == 'cpp' name: Build C++ @@ -56,4 +62,6 @@ jobs: make - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" From 4d81a16638a857572cc482cb97b25be3decb2896 Mon Sep 17 00:00:00 2001 From: TheHellTower <119943599+TheHellTower@users.noreply.github.com> Date: Wed, 1 Mar 2023 00:48:33 +0100 Subject: [PATCH 3/4] Add `PUSH_NULL`, `PRECALL_A`, `CALL_A` support (#324) * Add `PUSH_NULL`, `PRECALL_A`, `CALL_A` support * feat: @zrax Co-authored-by: Michael Hansen * Good suggested change to make the code shorter * Add `PUSH_NULL`, `PRECALL`, `CALL` test tokenized * Add `PUSH_NULL`, `PRECALL`, `CALL` test * Add `PUSH_NULL`, `PRECALL`, `CALL` test compiled * Delete push_null-precall-call.3.11.pyc * Delete push_null-precall-call.py * Delete push_null-precall-call.txt * Update README.markdown Blank commit to re-run the tests. * Join all no-ops/"fake"-ops together * Update ASTree.cpp https://github.com/zrax/pycdc/pull/324#issuecomment-1449049283 * Add compiled test for #324 --------- Co-authored-by: Michael Hansen --- ASTree.cpp | 24 +++++++++++++++--------- tests/compiled/test_calls.3.11.pyc | Bin 0 -> 585 bytes 2 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 tests/compiled/test_calls.3.11.pyc diff --git a/ASTree.cpp b/ASTree.cpp index 7af32d9..3122a94 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -474,6 +474,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(new ASTTuple(values)); } break; + case Pyc::CALL_A: case Pyc::CALL_FUNCTION_A: { int kwparams = (operand & 0xFF00) >> 8; @@ -540,12 +541,16 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) pparamList.push_front(decor_name); } - } else { + } + else { pparamList.push_front(param); } } PycRef func = stack.top(); stack.pop(); + if (opcode == Pyc::CALL_A && stack.top() == nullptr) + stack.pop(); + stack.push(new ASTCall(func, pparamList, kwparamList)); } break; @@ -1032,12 +1037,6 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(new ASTAwaitable(object)); } break; - case Pyc::GET_ITER: - /* We just entirely ignore this */ - break; - case Pyc::GET_YIELD_FROM_ITER: - /* We just entirely ignore this */ - break; case Pyc::IMPORT_NAME_A: if (mod->majorVer() == 1) { stack.push(new ASTImport(new ASTName(code->getName(operand)), NULL)); @@ -2547,14 +2546,21 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::SETUP_ANNOTATIONS: variable_annotations = true; break; + case Pyc::GET_ITER: + case Pyc::GET_YIELD_FROM_ITER: + break; + case Pyc::PRECALL_A: + case Pyc::RESUME_A: + /* We just entirely ignore this / no-op */ + break; case Pyc::CACHE: /* These "fake" opcodes are used as placeholders for optimizing certain opcodes in Python 3.11+. Since we have no need for that during disassembly/decompilation, we can just treat these as no-ops. */ break; - case Pyc::RESUME_A: - /* Treated as no-op for decompyle purposes */ + case Pyc::PUSH_NULL: + stack.push(nullptr); break; default: fprintf(stderr, "Unsupported opcode: %s\n", Pyc::OpcodeName(opcode & 0xFF)); diff --git a/tests/compiled/test_calls.3.11.pyc b/tests/compiled/test_calls.3.11.pyc new file mode 100644 index 0000000000000000000000000000000000000000..09cb36fecc4e62025798baecc44be7656ac1a855 GIT binary patch literal 585 zcmah_O-sW-5S`6uO)0iQQA-cK))tHC!Grh{yr`F6Bz7aF#3UrC(t?meK)s0w_1IHI z`ezy`81_`~~`-8b`QX5a2Ur&1D7yB5D{cNE{lBsSqE7&8c<0~j#U zpgKiVMGDlJxV6j1(L7?NM<@GqO%1{SGYYD6aetSOqj|(E=6Y1QKU4p$!hj-R{x6nR zL@X>;7Bd9lbdL)5XQGI{{1A%bN5~ws_#?%3B*LB<24PQSYl5Vfuv6G5Y;6}4gtePC zVGNrHwu6m^L-;F8briyxvU5(jx}oZXH!W3n2&Y`i+6#=sV<+nlJ=LzLy4i3lVSG}P zHO<~Mo0G)eg@GHIvTtQ*XZW4=4c$IjgirvI8+iarZsa3iJ?Rb7uaLf9^C35c+yHXH z9J}(k+Q~oUpOo&gpD7J9B_GN|C=Z}K0^a3a+)jE)HyKP57vJu-bx-#pJA~{2vJu}F MJ<*NaTBdjT26@eQP5=M^ literal 0 HcmV?d00001 From 2e76e56420493bfc0807da06a87dfdda5217a5aa Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Tue, 28 Feb 2023 15:49:12 -0800 Subject: [PATCH 4/4] Fixup unrelated changes from PR --- ASTree.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ASTree.cpp b/ASTree.cpp index 3122a94..5d8b8a6 100644 --- a/ASTree.cpp +++ b/ASTree.cpp @@ -541,8 +541,7 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) pparamList.push_front(decor_name); } - } - else { + } else { pparamList.push_front(param); } } @@ -1037,6 +1036,10 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) stack.push(new ASTAwaitable(object)); } break; + case Pyc::GET_ITER: + case Pyc::GET_YIELD_FROM_ITER: + /* We just entirely ignore this */ + break; case Pyc::IMPORT_NAME_A: if (mod->majorVer() == 1) { stack.push(new ASTImport(new ASTName(code->getName(operand)), NULL)); @@ -2546,9 +2549,6 @@ PycRef BuildFromCode(PycRef code, PycModule* mod) case Pyc::SETUP_ANNOTATIONS: variable_annotations = true; break; - case Pyc::GET_ITER: - case Pyc::GET_YIELD_FROM_ITER: - break; case Pyc::PRECALL_A: case Pyc::RESUME_A: /* We just entirely ignore this / no-op */