Merge pull request #209 from dotjrich/pymultic-container-build
Adds support for building our own Python containers
This commit is contained in:
Michael Hansen
42
scripts/Dockerfile.pybuild
Normal file
42
scripts/Dockerfile.pybuild
Normal file
@@ -0,0 +1,42 @@
|
||||
#
|
||||
# This will generate a local container with the specified version of Python.
|
||||
#
|
||||
# Note: This assumes the the tarball for the particular version you'd like to build has already been downloaded,
|
||||
# unpacked, and patched appropriately.
|
||||
#
|
||||
# To build:
|
||||
# docker build --build-arg python_version=PYTHON-VERSION \
|
||||
# --build-arg install_target=INSTALL-TARGET \
|
||||
# -f Dockerfile.pybuild \
|
||||
# -t python:PYTHON-VERSION
|
||||
#
|
||||
# PYTHON-VERSION full version of Python to build (ex. 3.0.1)
|
||||
# INSTALL-TARGET the target to make for installing (fullinstall for 3.0.1, install otherwise)
|
||||
#
|
||||
# Example for 3.0.1:
|
||||
# docker build --build-arg python_version=3.0.1 \
|
||||
# --build-arg install_target=fullinstall \
|
||||
# -f Dockerfile.pybuild \
|
||||
# -t python:3.0.1
|
||||
#
|
||||
|
||||
FROM debian:buster
|
||||
|
||||
ARG python_version
|
||||
ARG install_target
|
||||
|
||||
RUN mkdir -p /pybuild/ &&\
|
||||
apt-get update && \
|
||||
apt-get upgrade -y && \
|
||||
apt-get install -y build-essential libncurses-dev libz-dev libbz2-dev libreadline-dev
|
||||
|
||||
COPY Python-$python_version /pybuild/
|
||||
|
||||
RUN cd pybuild &&\
|
||||
./configure && \
|
||||
make && \
|
||||
make $install_target && \
|
||||
cd / && \
|
||||
apt-get remove -y build-essential libncurses-dev libz-dev libbz2-dev libreadline-dev && \
|
||||
apt autoremove -y && \
|
||||
rm -rf /pybuild
|
||||
106
scripts/pymultic
106
scripts/pymultic
@@ -60,6 +60,20 @@ CONTAINER_EXE_EXTRA_ARGS = {
|
||||
}
|
||||
|
||||
|
||||
def get_container_exe():
|
||||
container_exe = None
|
||||
for ce in CONTAINER_EXES:
|
||||
if shutil.which(ce) is not None:
|
||||
container_exe = ce
|
||||
break
|
||||
|
||||
if container_exe is None:
|
||||
print('Cannot find {} in $PATH'.format(' or '.join(CONTAINER_EXES)))
|
||||
return sys.exit(1)
|
||||
|
||||
return container_exe
|
||||
|
||||
|
||||
def fetch_python(snekdir, version):
|
||||
realver = PYVERS[version]
|
||||
if version in ('1.0', '1.1'):
|
||||
@@ -142,6 +156,35 @@ def acquire_python(snekdir, version):
|
||||
return pyexe
|
||||
|
||||
|
||||
def build_python_container(snekdir, version):
|
||||
realver = PYVERS[version]
|
||||
snek = 'Python-{}'.format(realver)
|
||||
builddir = os.path.join(snekdir, snek)
|
||||
container_exe = get_container_exe()
|
||||
py_container_tag = 'python:{}'.format(realver)
|
||||
|
||||
if subprocess.call([container_exe, 'image', 'inspect', py_container_tag],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL) == 0:
|
||||
return
|
||||
|
||||
fetch_python(snekdir, version)
|
||||
|
||||
print('Building Python {} container...'.format(realver))
|
||||
logfile = os.path.join(snekdir, '{}.build.log'.format(snek))
|
||||
install_target = 'fullinstall' if version == '3.0' else 'install'
|
||||
with open(logfile, 'wb') as log:
|
||||
if subprocess.call([container_exe, 'build',
|
||||
'--build-arg', 'python_version={}'.format(realver),
|
||||
'--build-arg', 'install_target={}'.format(install_target),
|
||||
'-f', os.path.join(snekdir, 'Dockerfile.pybuild'),
|
||||
'-t', py_container_tag, snekdir], stdout=log, stderr=log) != 0:
|
||||
print('...Container build failed. See {} for details'.format(logfile))
|
||||
sys.exit(1)
|
||||
|
||||
shutil.rmtree(builddir)
|
||||
|
||||
|
||||
def local_compile(snekdir, ver, infile):
|
||||
pyexe = acquire_python(snekdir, ver)
|
||||
proc = subprocess.Popen([pyexe, '-c', 'import sys; print(sys.version)'],
|
||||
@@ -194,24 +237,14 @@ def local_compile(snekdir, ver, infile):
|
||||
return outfile
|
||||
|
||||
|
||||
def container_compile(ver, infile):
|
||||
def container_compile(snekdir, ver, infile):
|
||||
if ver not in PYVER_CONTAINERS:
|
||||
print('Container compilation not supported for version {}'.format(ver))
|
||||
return None
|
||||
|
||||
container_exe = None
|
||||
for ce in CONTAINER_EXES:
|
||||
if shutil.which(ce) is not None:
|
||||
container_exe = ce
|
||||
break
|
||||
|
||||
if container_exe is None:
|
||||
print('Cannot find {} in $PATH'.format(' or '.join(CONTAINER_EXES)))
|
||||
return None
|
||||
build_python_container(snekdir, ver)
|
||||
|
||||
container_exe = get_container_exe()
|
||||
fullver = PYVERS[ver]
|
||||
|
||||
indir = os.path.dirname(os.path.abspath(infile))
|
||||
infile_full_path = infile
|
||||
infile = os.path.basename(infile)
|
||||
|
||||
if infile.endswith('.py'):
|
||||
@@ -223,14 +256,38 @@ def container_compile(ver, infile):
|
||||
os.unlink(outfile)
|
||||
|
||||
print('*** Compiling for Python {}'.format(fullver))
|
||||
proc = subprocess.Popen([container_exe, 'run'] + CONTAINER_EXE_EXTRA_ARGS[container_exe] +
|
||||
['--rm', '--name', '{}'.format(outfile),
|
||||
'-v', '{}:/indir:Z'.format(indir),
|
||||
'-v', '{}:/outdir:Z'.format(os.getcwd()), '-w', '/outdir',
|
||||
'python:{}'.format(fullver),
|
||||
'python', '-c',
|
||||
"import py_compile; py_compile.compile('/indir/{}', '{}')".format(infile, outfile)])
|
||||
proc.communicate()
|
||||
if ver in {'1.0', '1.1', '1.2', '1.3', '1.4'}:
|
||||
# The hard way -- hope your code is safe...
|
||||
comptmp = os.path.join(indir, 'pymc_temp.py')
|
||||
if os.path.exists(comptmp):
|
||||
os.unlink(comptmp)
|
||||
shutil.copyfile(infile_full_path, comptmp)
|
||||
proc = subprocess.Popen([container_exe, 'run'] + CONTAINER_EXE_EXTRA_ARGS[container_exe] +
|
||||
['--rm', '--name', outfile,
|
||||
'-v', '{}:/indir:Z'.format(indir),
|
||||
'-v', '{}:/outdir:Z'.format(os.getcwd()), '-w', '/outdir',
|
||||
'-w', '/indir',
|
||||
'python:{}'.format(fullver),
|
||||
'python', '-c',
|
||||
"import pymc_temp"])
|
||||
proc.communicate()
|
||||
if os.path.exists(comptmp + 'o'):
|
||||
shutil.copyfile(comptmp + 'o', outfile)
|
||||
os.unlink(comptmp + 'o')
|
||||
elif os.path.exists(comptmp + 'c'):
|
||||
shutil.copyfile(comptmp + 'c', outfile)
|
||||
os.unlink(comptmp + 'c')
|
||||
os.unlink(comptmp)
|
||||
else:
|
||||
# The easy way
|
||||
proc = subprocess.Popen([container_exe, 'run'] + CONTAINER_EXE_EXTRA_ARGS[container_exe] +
|
||||
['--rm', '--name', outfile,
|
||||
'-v', '{}:/indir:Z'.format(indir),
|
||||
'-v', '{}:/outdir:Z'.format(os.getcwd()), '-w', '/outdir',
|
||||
'python:{}'.format(fullver),
|
||||
'python', '-c',
|
||||
"import py_compile; py_compile.compile('/indir/{}', '{}')".format(infile, outfile)])
|
||||
proc.communicate()
|
||||
|
||||
return outfile
|
||||
|
||||
@@ -282,12 +339,11 @@ result = 0
|
||||
for ver in pythons:
|
||||
compile_with_container = use_containers
|
||||
if use_containers and ver not in PYVER_CONTAINERS:
|
||||
print('Warning: No supported container for {} - using local build'.format(ver))
|
||||
compile_with_container = False
|
||||
print('Warning: No officially supported container for {} - using locally built one'.format(ver))
|
||||
|
||||
outfile = None
|
||||
if compile_with_container:
|
||||
outfile = container_compile(ver, infile)
|
||||
outfile = container_compile(snekdir, ver, infile)
|
||||
else:
|
||||
outfile = local_compile(snekdir, ver, infile)
|
||||
|
||||
|
||||
16
scripts/python-builds/Python-3.1.5.patch
Normal file
16
scripts/python-builds/Python-3.1.5.patch
Normal file
@@ -0,0 +1,16 @@
|
||||
diff --git a/Makefile.pre.in b/Makefile.pre.in
|
||||
index e01cd2745a..dcf465e30e 100644
|
||||
--- a/Makefile.pre.in
|
||||
+++ b/Makefile.pre.in
|
||||
@@ -817,6 +817,11 @@ bininstall: altbininstall
|
||||
else true; \
|
||||
fi
|
||||
(cd $(DESTDIR)$(BINDIR); $(LN) python$(VERSION)$(EXE) $(PYTHON)3$(EXE))
|
||||
+ -if test -f $(DESTDIR)$(BINDIR)/$(PYTHON)$(EXE) -o -h $(DESTDIR)$(BINDIR)/$(PYTHON)$(EXE); \
|
||||
+ then rm -f $(DESTDIR)$(BINDIR)/$(PYTHON)$(EXE); \
|
||||
+ else true; \
|
||||
+ fi
|
||||
+ (cd $(DESTDIR)$(BINDIR); $(LN) python$(VERSION)$(EXE) $(PYTHON)$(EXE))
|
||||
-rm -f $(DESTDIR)$(BINDIR)/python3-config
|
||||
(cd $(DESTDIR)$(BINDIR); $(LN) -s python$(VERSION)-config python3-config)
|
||||
-rm -f $(DESTDIR)$(LIBPC)/python3.pc
|
||||
Reference in New Issue
Block a user