From 5e08ec603a3684ff3b5e83bd2e6c5d3d33a90a39 Mon Sep 17 00:00:00 2001 From: John Richards Date: Sun, 10 Oct 2021 18:34:11 -0400 Subject: [PATCH 1/6] Adds container support to pymultic Using the '-c' argument will - if possible - fetch and use a container for each version of Python specified in the arguments list. --- scripts/pymultic | 139 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 96 insertions(+), 43 deletions(-) diff --git a/scripts/pymultic b/scripts/pymultic index c595548..12141c1 100755 --- a/scripts/pymultic +++ b/scripts/pymultic @@ -33,12 +33,17 @@ PYVERS = { '3.7': '3.7.10', '3.8': '3.8.9', '3.9': '3.9.4', + '3.10': '3.10.0', } OLD_PYTHONS = ('1.0', '1.1', '1.2', '1.3', '1.4', '1.5') OLD_PYURL = 'https://legacy.python.org/download/releases/src' PYURL = 'https://www.python.org/ftp/python' +# Minimum version of Python with a pre-built container. +PYVER_CONTAINER_MIN = '2.7' +USE_CONTAINERS = False + def fetch_python(snekdir, version): realver = PYVERS[version] @@ -122,58 +127,19 @@ def acquire_python(snekdir, version): return pyexe -if len(sys.argv) < 2: - print('Usage: {} [versions] input.py'.format(sys.argv[0])) - print('Compile input.py for one or more python versions') - print('Output is written to input..pyc for each version successfully compiled') - print() - print('Version is X.Y (e.g. 3.4), not including the patch version') - sys.exit(1) - -RE_PYVER = re.compile(r'\d\.\d') - -pythons = [] -infile = None -for arg in sys.argv[1:]: - if RE_PYVER.match(arg): - if arg in PYVERS.keys(): - pythons.append(arg) - else: - print('Unknown Python version: {}'.format(arg)) - sys.exit(1) - elif arg.startswith('-'): - print("WARNING: Unrecognized argument '{}'".format(arg)) - else: - infile = arg - -if infile is None: - print('No input file specified') - sys.exit(1) -elif not os.path.exists(infile): - print('Error: Input file {} does not exist'.format(infile)) - sys.exit(1) - -if len(pythons) == 0: - print('At least one Python version is required') - sys.exit(1) - -snekdir = os.path.dirname(os.path.realpath(__file__)) -result = 0 -for ver in pythons: +def local_compile(snekdir, ver, infile): pyexe = acquire_python(snekdir, ver) proc = subprocess.Popen([pyexe, '-c', 'import sys; print(sys.version)'], stdout=subprocess.PIPE) out, _ = proc.communicate() if proc.returncode != 0: print('Could not determine Python version for {}'.format(ver)) - result = 1 - continue + return None bcver = str(out, 'iso-8859-1').split(' ', 1)[0] if not bcver.startswith(ver): print('Python {} reported itself as version {}!'.format(ver, bcver)) - result = 1 - continue + return None if infile.endswith('.py'): outfile = os.path.basename(infile)[:-3] @@ -210,7 +176,94 @@ for ver in pythons: .format(infile, outfile)]) proc.communicate() - if not os.path.exists(outfile): + return outfile + + +def container_compile(snekdir, ver, infile): + if ver < PYVER_CONTAINER_MIN: + print('Container compilation requires a Python version >= {}'.format(PYVER_CONTAINER_MIN)) + return None + + if shutil.which('docker') is None: + print('Cannot find docker in $PATH') + return None + + fullver = PYVERS[ver] + + if infile.endswith('.py'): + outfile = os.path.basename(infile)[:-3] + else: + outfile = os.path.basename(infile) + outfile += '.{}.pyc'.format(ver) + if os.path.exists(outfile): + os.unlink(outfile) + + print('*** Compiling for Python {}'.format(fullver)) + # The easy way + proc = subprocess.Popen(['docker', 'run', '--privileged', '--rm', '--name', '{}-{}'.format(infile, ver), + '-v', '{}:/src'.format(snekdir), '-w', '/src', 'python:{}'.format(fullver), + 'python', '-c', "import py_compile; py_compile.compile('{}', '{}')".format(infile, outfile)]) + proc.communicate() + + return outfile + + +if len(sys.argv) < 2: + print('Usage: {} [-c] [versions] input.py'.format(sys.argv[0])) + print('Compile input.py for one or more python versions') + print() + print('-c\tuse prebuilt containers for running different versions of Python (only available for versions >= {})' + .format(PYVER_CONTAINER_MIN)) + print() + print('Output is written to input..pyc for each version successfully compiled') + print() + print('Version is X.Y (e.g. 3.4), not including the patch version') + sys.exit(1) + +RE_PYVER = re.compile(r'\d\.\d') + +pythons = [] +infile = None +for arg in sys.argv[1:]: + if RE_PYVER.match(arg): + if arg in PYVERS.keys(): + pythons.append(arg) + else: + print('Unknown Python version: {}'.format(arg)) + sys.exit(1) + elif arg == '-c': + USE_CONTAINERS = True + elif arg.startswith('-'): + print("WARNING: Unrecognized argument '{}'".format(arg)) + else: + infile = arg + +if infile is None: + print('No input file specified') + sys.exit(1) +elif not os.path.exists(infile): + print('Error: Input file {} does not exist'.format(infile)) + sys.exit(1) + +if len(pythons) == 0: + print('At least one Python version is required') + sys.exit(1) + +snekdir = os.path.dirname(os.path.realpath(__file__)) +result = 0 +for ver in pythons: + compile_with_container = USE_CONTAINERS + if USE_CONTAINERS and ver < PYVER_CONTAINER_MIN: + print('Warning: No supported container for {} - using local build'.format(ver)) + compile_with_container = False + + outfile = None + if compile_with_container: + outfile = container_compile(snekdir, ver, infile) + else: + outfile = local_compile(snekdir, ver, infile) + + if outfile is None or not os.path.exists(outfile): result = 1 sys.exit(result) From f59d7d015da137a6387ae085521f7824a3150a78 Mon Sep 17 00:00:00 2001 From: John Richards Date: Sun, 10 Oct 2021 19:59:54 -0400 Subject: [PATCH 2/6] Adds podman support and defines PYVERS with official containers We've found that 3.0.x and 3.1.x don't have official containers on Docker Hub, so we needed to move to a more explicit check. --- scripts/pymultic | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/scripts/pymultic b/scripts/pymultic index 12141c1..8c508e5 100755 --- a/scripts/pymultic +++ b/scripts/pymultic @@ -40,9 +40,21 @@ OLD_PYTHONS = ('1.0', '1.1', '1.2', '1.3', '1.4', '1.5') OLD_PYURL = 'https://legacy.python.org/download/releases/src' PYURL = 'https://www.python.org/ftp/python' -# Minimum version of Python with a pre-built container. -PYVER_CONTAINER_MIN = '2.7' +# Not all versions of Python have an official container. +PYVER_CONTAINERS = { + '2.7', + '3.2', + '3.3', + '3.4', + '3.5', + '3.6', + '3.7', + '3.8', + '3.9', + '3.10', +} USE_CONTAINERS = False +CONTAINER_EXES = ['docker', 'podman'] def fetch_python(snekdir, version): @@ -180,12 +192,18 @@ def local_compile(snekdir, ver, infile): def container_compile(snekdir, ver, infile): - if ver < PYVER_CONTAINER_MIN: - print('Container compilation requires a Python version >= {}'.format(PYVER_CONTAINER_MIN)) + if ver not in PYVER_CONTAINERS: + print('Container compilation not supported for version {}'.format(ver)) return None - if shutil.which('docker') is None: - print('Cannot find docker in $PATH') + container_exe = None + for ce in CONTAINER_EXES: + if shutil.which(ce) is not None: + container_exe = ce + break + + if container_exe is None: + print('Cannot find {} in $PATH'.format(' or '.join(CONTAINER_EXES))) return None fullver = PYVERS[ver] @@ -200,7 +218,7 @@ def container_compile(snekdir, ver, infile): print('*** Compiling for Python {}'.format(fullver)) # The easy way - proc = subprocess.Popen(['docker', 'run', '--privileged', '--rm', '--name', '{}-{}'.format(infile, ver), + proc = subprocess.Popen([container_exe, 'run', '--privileged', '--rm', '--name', '{}-{}'.format(infile, ver), '-v', '{}:/src'.format(snekdir), '-w', '/src', 'python:{}'.format(fullver), 'python', '-c', "import py_compile; py_compile.compile('{}', '{}')".format(infile, outfile)]) proc.communicate() @@ -253,7 +271,7 @@ snekdir = os.path.dirname(os.path.realpath(__file__)) result = 0 for ver in pythons: compile_with_container = USE_CONTAINERS - if USE_CONTAINERS and ver < PYVER_CONTAINER_MIN: + if USE_CONTAINERS and ver not in PYVER_CONTAINERS: print('Warning: No supported container for {} - using local build'.format(ver)) compile_with_container = False From 51f607fb1c63939f52609345698634df312b9f1b Mon Sep 17 00:00:00 2001 From: John Richards Date: Sun, 10 Oct 2021 23:36:46 -0400 Subject: [PATCH 3/6] Removes old variable that's no longer in code --- scripts/pymultic | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/pymultic b/scripts/pymultic index 8c508e5..2409185 100755 --- a/scripts/pymultic +++ b/scripts/pymultic @@ -230,8 +230,7 @@ if len(sys.argv) < 2: print('Usage: {} [-c] [versions] input.py'.format(sys.argv[0])) print('Compile input.py for one or more python versions') print() - print('-c\tuse prebuilt containers for running different versions of Python (only available for versions >= {})' - .format(PYVER_CONTAINER_MIN)) + print('-c\tuse prebuilt containers for running different versions of Python (not available for all versions)') print() print('Output is written to input..pyc for each version successfully compiled') print() From 7e2609c3d6ae0a9b6b7906fc1fd4925318363989 Mon Sep 17 00:00:00 2001 From: John Richards Date: Mon, 18 Oct 2021 21:37:46 -0400 Subject: [PATCH 4/6] Addresses code review comments --- scripts/pymultic | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/scripts/pymultic b/scripts/pymultic index 2409185..e73efb6 100755 --- a/scripts/pymultic +++ b/scripts/pymultic @@ -53,7 +53,6 @@ PYVER_CONTAINERS = { '3.9', '3.10', } -USE_CONTAINERS = False CONTAINER_EXES = ['docker', 'podman'] @@ -191,7 +190,7 @@ def local_compile(snekdir, ver, infile): return outfile -def container_compile(snekdir, ver, infile): +def container_compile(ver, infile): if ver not in PYVER_CONTAINERS: print('Container compilation not supported for version {}'.format(ver)) return None @@ -208,19 +207,24 @@ def container_compile(snekdir, ver, infile): fullver = PYVERS[ver] + indir = os.path.dirname(os.path.abspath(infile)) + infile = os.path.basename(infile) + if infile.endswith('.py'): - outfile = os.path.basename(infile)[:-3] + outfile = infile[:-3] else: - outfile = os.path.basename(infile) + outfile = infile outfile += '.{}.pyc'.format(ver) if os.path.exists(outfile): os.unlink(outfile) print('*** Compiling for Python {}'.format(fullver)) - # The easy way - proc = subprocess.Popen([container_exe, 'run', '--privileged', '--rm', '--name', '{}-{}'.format(infile, ver), - '-v', '{}:/src'.format(snekdir), '-w', '/src', 'python:{}'.format(fullver), - 'python', '-c', "import py_compile; py_compile.compile('{}', '{}')".format(infile, outfile)]) + proc = subprocess.Popen([container_exe, 'run', '--rm', '--name', '{}'.format(outfile), + '-v', '{}:/indir:Z'.format(indir), + '-v', '{}:/outdir:Z'.format(os.getcwd()), '-w', '/outdir', + 'python:{}'.format(fullver), + 'python', '-c', + "import py_compile; py_compile.compile('/indir/{}', '{}')".format(infile, outfile)]) proc.communicate() return outfile @@ -230,7 +234,8 @@ if len(sys.argv) < 2: print('Usage: {} [-c] [versions] input.py'.format(sys.argv[0])) print('Compile input.py for one or more python versions') print() - print('-c\tuse prebuilt containers for running different versions of Python (not available for all versions)') + print('-c\tuse prebuilt containers for running different versions of Python') + print('\t(not available for all versions)') print() print('Output is written to input..pyc for each version successfully compiled') print() @@ -241,6 +246,7 @@ RE_PYVER = re.compile(r'\d\.\d') pythons = [] infile = None +use_containers = False for arg in sys.argv[1:]: if RE_PYVER.match(arg): if arg in PYVERS.keys(): @@ -249,7 +255,7 @@ for arg in sys.argv[1:]: print('Unknown Python version: {}'.format(arg)) sys.exit(1) elif arg == '-c': - USE_CONTAINERS = True + use_containers = True elif arg.startswith('-'): print("WARNING: Unrecognized argument '{}'".format(arg)) else: @@ -269,14 +275,14 @@ if len(pythons) == 0: snekdir = os.path.dirname(os.path.realpath(__file__)) result = 0 for ver in pythons: - compile_with_container = USE_CONTAINERS - if USE_CONTAINERS and ver not in PYVER_CONTAINERS: + compile_with_container = use_containers + if use_containers and ver not in PYVER_CONTAINERS: print('Warning: No supported container for {} - using local build'.format(ver)) compile_with_container = False outfile = None if compile_with_container: - outfile = container_compile(snekdir, ver, infile) + outfile = container_compile(ver, infile) else: outfile = local_compile(snekdir, ver, infile) From 034c65885f57de031138b492146d7b4acc9ba9cf Mon Sep 17 00:00:00 2001 From: John Richards Date: Thu, 21 Oct 2021 22:22:31 -0400 Subject: [PATCH 5/6] Use the -u param when running via Docker --- scripts/pymultic | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/pymultic b/scripts/pymultic index e73efb6..63c94ff 100755 --- a/scripts/pymultic +++ b/scripts/pymultic @@ -53,7 +53,8 @@ PYVER_CONTAINERS = { '3.9', '3.10', } -CONTAINER_EXES = ['docker', 'podman'] +CONTAINER_EXES = ['podman', 'docker'] +CONTAINER_EXE_ARGS = { 'docker': ['-u', '{}:{}'.format(os.getuid(), os.getgid())] } # Docker requires extra magic for permissions. def fetch_python(snekdir, version): @@ -219,7 +220,8 @@ def container_compile(ver, infile): os.unlink(outfile) print('*** Compiling for Python {}'.format(fullver)) - proc = subprocess.Popen([container_exe, 'run', '--rm', '--name', '{}'.format(outfile), + proc = subprocess.Popen([container_exe, 'run'] + CONTAINER_EXE_ARGS.get(container_exe, []) + + ['--rm', '--name', '{}'.format(outfile), '-v', '{}:/indir:Z'.format(indir), '-v', '{}:/outdir:Z'.format(os.getcwd()), '-w', '/outdir', 'python:{}'.format(fullver), From 21b08557ce2480541d274dce4e1981c7cc089e48 Mon Sep 17 00:00:00 2001 From: John Richards Date: Thu, 21 Oct 2021 22:33:03 -0400 Subject: [PATCH 6/6] Refactors how extra args for container executables are specified --- scripts/pymultic | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/pymultic b/scripts/pymultic index 63c94ff..bb18fbe 100755 --- a/scripts/pymultic +++ b/scripts/pymultic @@ -54,7 +54,10 @@ PYVER_CONTAINERS = { '3.10', } CONTAINER_EXES = ['podman', 'docker'] -CONTAINER_EXE_ARGS = { 'docker': ['-u', '{}:{}'.format(os.getuid(), os.getgid())] } # Docker requires extra magic for permissions. +CONTAINER_EXE_EXTRA_ARGS = { + 'podman': [], + 'docker': ['-u', '{}:{}'.format(os.getuid(), os.getgid())], # Docker requires extra magic for permissions. +} def fetch_python(snekdir, version): @@ -220,7 +223,7 @@ def container_compile(ver, infile): os.unlink(outfile) print('*** Compiling for Python {}'.format(fullver)) - proc = subprocess.Popen([container_exe, 'run'] + CONTAINER_EXE_ARGS.get(container_exe, []) + + proc = subprocess.Popen([container_exe, 'run'] + CONTAINER_EXE_EXTRA_ARGS[container_exe] + ['--rm', '--name', '{}'.format(outfile), '-v', '{}:/indir:Z'.format(indir), '-v', '{}:/outdir:Z'.format(os.getcwd()), '-w', '/outdir',