Test runner refactor (#508)
* Port test runner script to Python. This allows us to keep the test output in sorted order while still running the tests in parallel. It also now defaults to using the number of available CPU threads for parallel execution, rather than the previously hard-coded default. * Also port decompyle_test.sh script to python within run_tests.py * Fix cmake check target for multi-config generators. Adds testing of release builds on both MSVC and GCC. * Fix diff comparisons on Windows * Ubuntu runners don't have ninja by default
This commit is contained in:
Michael Hansen
15
.github/workflows/linux-ci.yml
vendored
15
.github/workflows/linux-ci.yml
vendored
@@ -11,10 +11,19 @@ jobs:
|
||||
- uses: actions/checkout@v1
|
||||
- name: Configure and Build
|
||||
run: |
|
||||
mkdir build && cd build
|
||||
(
|
||||
mkdir build-debug && cd build-debug
|
||||
cmake -DCMAKE_BUILD_TYPE=Debug ..
|
||||
make -j4
|
||||
)
|
||||
|
||||
(
|
||||
mkdir build-release && cd build-release
|
||||
cmake -DCMAKE_BUILD_TYPE=Debug ..
|
||||
make -j4
|
||||
)
|
||||
|
||||
- name: Test
|
||||
run: |
|
||||
cd build
|
||||
make check JOBS=4
|
||||
cmake --build build-debug --target check
|
||||
cmake --build build-release --target check
|
||||
|
||||
10
.github/workflows/msvc-ci.yml
vendored
10
.github/workflows/msvc-ci.yml
vendored
@@ -17,16 +17,10 @@ jobs:
|
||||
cmake --build . --config Debug
|
||||
cmake --build . --config Release
|
||||
|
||||
# This should probably be fixed to work from MSVC without needing to
|
||||
# use a bash shell and the GNU userland tools... But for now, the
|
||||
# GH Actions environment provides what we need.
|
||||
- name: Test
|
||||
run: |
|
||||
cd build\Debug
|
||||
bash.exe ..\..\tests\all_tests.sh
|
||||
env:
|
||||
PYTHON_EXE: python.exe
|
||||
JOBS: 4
|
||||
cmake --build build --config Debug --target check
|
||||
cmake --build build --config Release --target check
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
|
||||
@@ -76,6 +76,10 @@ target_link_libraries(pycdc pycxx)
|
||||
install(TARGETS pycdc
|
||||
RUNTIME DESTINATION bin)
|
||||
|
||||
add_custom_target(check "${CMAKE_CURRENT_SOURCE_DIR}/tests/all_tests.sh"
|
||||
WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
|
||||
find_package(Python3 3.6 COMPONENTS Interpreter)
|
||||
if(Python3_FOUND)
|
||||
add_custom_target(check
|
||||
COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_tests.py"
|
||||
WORKING_DIRECTORY "$<TARGET_FILE_DIR:pycdc>")
|
||||
add_dependencies(check pycdc)
|
||||
endif()
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
|
||||
jobs=${JOBS:-4}
|
||||
filter=${FILTER:-""}
|
||||
|
||||
find "${srcdir}/tests/tokenized" -type f -name '*.txt' -a -name "*${filter}*" -print0 | \
|
||||
xargs -0 -I '{}' -P $jobs \
|
||||
bash -c 'o=$('"$srcdir"'/tests/decompyle_test.sh "$(basename -s .txt "{}")" tests-out) r=$?; echo "$o"; exit $r'
|
||||
@@ -1,132 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
|
||||
testdir="$srcdir/tests"
|
||||
testname="$1"
|
||||
outdir="$2"
|
||||
|
||||
if [[ -z "$PYTHON_EXE" ]]; then
|
||||
PYTHON_EXE="$(which python3)"
|
||||
fi
|
||||
|
||||
if [[ -z "$testname" ]]; then
|
||||
echo "Missing required parameter: testname" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "$outdir" ]]; then
|
||||
echo "Missing required parameter: outdir" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
shopt -s nullglob
|
||||
compfiles=( "$testdir/compiled/$testname".?.?*.pyc )
|
||||
xfcfiles=( "$testdir/xfail/$testname".?.?*.pyc )
|
||||
shopt -u nullglob
|
||||
|
||||
if (( ${#compfiles[@]} + ${#xfcfiles[@]} == 0 )); then
|
||||
echo "No compiled/xfail modules found for $testname.*.pyc"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$outdir"
|
||||
|
||||
echo -ne "\033[1m*** $testname:\033[0m "
|
||||
|
||||
fails=0
|
||||
xfails=0
|
||||
upass=0
|
||||
efiles=()
|
||||
errors=()
|
||||
upfiles=()
|
||||
for pyc in "${compfiles[@]}" "${xfcfiles[@]}"; do
|
||||
base="$outdir/$(basename "$pyc")"
|
||||
|
||||
./pycdc "$pyc" -o "$base.src.py" >"$base.err" 2>&1
|
||||
if (( $? )) || [[ -s "$base.err" ]]
|
||||
then
|
||||
if [[ "$(dirname "$pyc")" =~ xfail ]]
|
||||
then
|
||||
let xfails+=1
|
||||
else
|
||||
let fails+=1
|
||||
efiles+=("$(basename "$pyc")")
|
||||
errors+=("$(cat "$base.err")")
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
"$PYTHON_EXE" "$srcdir"/scripts/token_dump "$base.src.py" 2>"$base.tok.err" 1>"$base.tok.txt"
|
||||
if (( $? )) || [[ -s "$base.tok.err" ]]
|
||||
then
|
||||
if [[ "$(dirname "$pyc")" =~ xfail ]]
|
||||
then
|
||||
let xfails+=1
|
||||
else
|
||||
let fails+=1
|
||||
efiles+=("$(basename "$pyc")")
|
||||
errors+=("$(cat "$base.tok.err")")
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
diff -u "$testdir/tokenized/$testname.txt" "$base.tok.txt" >"$base.tok.diff"
|
||||
if (( $? ))
|
||||
then
|
||||
if [[ "$(dirname "$pyc")" =~ xfail ]]
|
||||
then
|
||||
let xfails+=1
|
||||
else
|
||||
let fails+=1
|
||||
efiles+=("$(basename "$pyc")")
|
||||
errors+=("$base.tok.txt does not match $testdir/tokenized/$testname.txt:\n$(cat "$base.tok.diff")")
|
||||
fi
|
||||
else
|
||||
if [[ "$(dirname "$pyc")" =~ xfail ]]
|
||||
then
|
||||
let upass+=1
|
||||
upfiles+=("$(basename "$pyc")")
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if (( $fails == 0 ))
|
||||
then
|
||||
if (( $xfails != 0 ))
|
||||
then
|
||||
if (( ${#compfiles[@]} == 0 ))
|
||||
then
|
||||
echo -e "\033[33mXFAIL ($xfails)\033[0m"
|
||||
else
|
||||
echo -e "\033[32mPASS (${#compfiles[@]})\033[33m + XFAIL ($xfails)\033[0m"
|
||||
fi
|
||||
else
|
||||
echo -e "\033[32mPASS (${#compfiles[@]})\033[0m"
|
||||
fi
|
||||
else
|
||||
if (( $xfails != 0 ))
|
||||
then
|
||||
echo -e "\033[31mFAIL ($fails of ${#compfiles[@]})\033[33m + XFAIL ($xfails)\033[0m"
|
||||
else
|
||||
echo -e "\033[31mFAIL ($fails of ${#compfiles[@]})\033[0m"
|
||||
fi
|
||||
|
||||
for ((i=0; i<${#efiles[@]}; i++))
|
||||
do
|
||||
echo -e "\t\033[31m${efiles[i]}\033[0m"
|
||||
echo -e "${errors[i]}\n"
|
||||
done
|
||||
fi
|
||||
|
||||
if (( $upass != 0 ))
|
||||
then
|
||||
echo -e "\033[1;34mUnexpected passes:\033[0m"
|
||||
for ((i=0; i<${#upfiles[@]}; i++))
|
||||
do
|
||||
echo -e "\t\033[33m${upfiles[i]}\033[0m"
|
||||
done
|
||||
fi
|
||||
|
||||
if (( $fails != 0 ))
|
||||
then
|
||||
exit 1
|
||||
fi
|
||||
138
tests/run_tests.py
Executable file
138
tests/run_tests.py
Executable file
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import glob
|
||||
import difflib
|
||||
import argparse
|
||||
import subprocess
|
||||
import multiprocessing
|
||||
|
||||
TEST_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
SCRIPTS_DIR = os.path.realpath(os.path.join(TEST_DIR, '..', 'scripts'))
|
||||
|
||||
def decompyle_one(test_name, pyc_file, outdir, tokenized_expect):
|
||||
out_base = os.path.join(outdir, os.path.basename(pyc_file))
|
||||
proc = subprocess.run(
|
||||
[os.path.join(os.getcwd(), 'pycdc'), pyc_file, '-o', out_base + '.src.py'],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True,
|
||||
encoding='utf-8', errors='replace')
|
||||
pycdc_output = proc.stdout
|
||||
if proc.returncode != 0 or pycdc_output:
|
||||
with open(out_base + '.err', 'w') as errfile:
|
||||
errfile.write(pycdc_output)
|
||||
return False, [pycdc_output]
|
||||
elif os.path.exists(out_base + '.err'):
|
||||
os.unlink(out_base + '.err')
|
||||
|
||||
proc = subprocess.run(
|
||||
[sys.executable, os.path.join(SCRIPTS_DIR, 'token_dump'), out_base + '.src.py'],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
|
||||
encoding='utf-8', errors='replace')
|
||||
tokenized = proc.stdout
|
||||
token_dump_err = proc.stderr
|
||||
with open(out_base + '.tok.txt', 'w') as tokfile:
|
||||
tokfile.write(tokenized)
|
||||
if proc.returncode != 0 or token_dump_err:
|
||||
with open(out_base + '.tok.err', 'w') as errfile:
|
||||
errfile.write(token_dump_err)
|
||||
return False, [token_dump_err]
|
||||
elif os.path.exists(out_base + '.tok.err'):
|
||||
os.unlink(out_base + '.tok.err')
|
||||
|
||||
if tokenized != tokenized_expect:
|
||||
fromfile = 'tokenized/{}.txt'.format(test_name)
|
||||
tofile = 'tests-out/{}.tok.txt'.format(os.path.basename(pyc_file))
|
||||
diff = difflib.unified_diff(tokenized_expect.splitlines(True), tokenized.splitlines(True),
|
||||
fromfile=fromfile, tofile=tofile)
|
||||
diff = list(diff)
|
||||
with open(out_base + '.tok.diff', 'w') as diff_file:
|
||||
diff_file.writelines(diff)
|
||||
return False, ['Tokenized output does not match expected output:\n'] + diff
|
||||
|
||||
return True, []
|
||||
|
||||
|
||||
def run_test(test_file):
|
||||
"""
|
||||
Runs a single test, and returns a tuple containing the number of failed
|
||||
tests and the output of the test. The output is not printed directly
|
||||
in order to avoid interleaving output from multiple parallel tests.
|
||||
"""
|
||||
test_name = os.path.splitext(os.path.basename(test_file))[0]
|
||||
compiled_files = glob.glob(os.path.join(TEST_DIR, 'compiled', test_name + '.?.*.pyc'))
|
||||
xfail_files = glob.glob(os.path.join(TEST_DIR, 'xfail', test_name + '.?.*.pyc'))
|
||||
if not compiled_files and not xfail_files:
|
||||
return 1, 'No compiled/xfail modules found for {}\n'.format(test_name)
|
||||
|
||||
outdir = os.path.join(os.getcwd(), 'tests-out')
|
||||
os.makedirs(outdir, exist_ok=True)
|
||||
|
||||
with open(os.path.join(TEST_DIR, 'tokenized', test_name + '.txt'), 'r',
|
||||
encoding='utf-8', errors='replace') as tok_file:
|
||||
tokenized_expect = tok_file.read()
|
||||
|
||||
status_line = '\033[1m*** {}:\033[0m '.format(test_name)
|
||||
errlines = []
|
||||
fails = 0
|
||||
xfails = 0
|
||||
upass = 0
|
||||
for xpass_file in compiled_files:
|
||||
ok, errs = decompyle_one(test_name, xpass_file, outdir, tokenized_expect)
|
||||
if not ok:
|
||||
fails += 1
|
||||
errlines.append('\t\033[31m{}\033[0m\n'.format(os.path.basename(xpass_file)))
|
||||
errlines.extend(errs)
|
||||
for xfail_file in xfail_files:
|
||||
ok, _ = decompyle_one(test_name, xfail_file, outdir, tokenized_expect)
|
||||
if not ok:
|
||||
xfails += 1
|
||||
else:
|
||||
upass += 1
|
||||
|
||||
if fails == 0:
|
||||
if xfails != 0:
|
||||
if not compiled_files:
|
||||
status_line += '\033[33mXFAIL ({})\033[0m\n'.format(xfails)
|
||||
else:
|
||||
status_line += '\033[32mPASS ({})\033[33m + XFAIL ()\033[0m\n' \
|
||||
.format(len(compiled_files), xfails)
|
||||
else:
|
||||
status_line += '\033[32mPASS ({})\033[0m\n'.format(len(compiled_files))
|
||||
else:
|
||||
if xfails != 0:
|
||||
status_line += '\033[31mFAIL ({} of {})\033[33m + XFAIL ({})\033[0m\n' \
|
||||
.format(fails, len(compiled_files), xfails)
|
||||
else:
|
||||
status_line += '\033[31mFAIL ({} of {})\033[0m\n'.format(fails, len(compiled_files))
|
||||
|
||||
return fails, [status_line] + errlines
|
||||
|
||||
|
||||
def main():
|
||||
# For simpler invocation from CMake's check target, we also support setting
|
||||
# these parameters via environment variables.
|
||||
default_jobs = int(os.environ['JOBS']) if 'JOBS' in os.environ else multiprocessing.cpu_count()
|
||||
default_filter = os.environ['FILTER'] if 'FILTER' in os.environ else ''
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--jobs', '-j', type=int, default=default_jobs,
|
||||
help='Number of tests to run in parallel (default: {})'.format(default_jobs))
|
||||
parser.add_argument('--filter', type=str, default=default_filter,
|
||||
help='Run only test(s) matching the supplied filter')
|
||||
args = parser.parse_args()
|
||||
|
||||
glob_pattern = '*{}*.txt'.format(args.filter) if args.filter else '*.txt'
|
||||
test_files = sorted(glob.iglob(os.path.join(TEST_DIR, 'tokenized', glob_pattern)))
|
||||
total_fails = 0
|
||||
with multiprocessing.Pool(args.jobs) as pool:
|
||||
for fails, output in pool.imap(run_test, test_files):
|
||||
total_fails += fails
|
||||
sys.stdout.writelines(output)
|
||||
|
||||
if total_fails:
|
||||
print('{} test(s) failed'.format(total_fails))
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user