#!/usr/bin/env python3 # -*- python -*- """ %prog SUBMODULE... Hack to pipe submodules of Numpy through 2to3 and build them in-place one-by-one. Example usage: python3 tools/py3tool.py testing distutils core This will copy files to _py3k/numpy, add a dummy __init__.py and version.py on the top level, and copy and 2to3 the files of the three submodules. When running py3tool again, only changed files are re-processed, which makes the test-bugfix cycle faster. """ from optparse import OptionParser import shutil import os import sys import re import subprocess import fnmatch if os.environ.get('USE_2TO3CACHE'): import lib2to3cache BASE = os.path.normpath(os.path.join(os.path.dirname(__file__), '..')) TEMP = os.path.normpath(os.path.join(BASE, '_py3k')) SCRIPT_2TO3 = os.path.join(BASE, 'tools', '2to3.py') EXTRA_2TO3_FLAGS = { '*/setup.py': '-x import', 'numpy/core/code_generators/generate_umath.py': '-x import', 'numpy/core/code_generators/generate_numpy_api.py': '-x import', 'numpy/core/code_generators/generate_ufunc_api.py': '-x import', 'numpy/core/defchararray.py': '-x unicode', 'numpy/compat/py3k.py': '-x unicode', 'numpy/ma/timer_comparison.py': 'skip', 'numpy/distutils/system_info.py': '-x reduce', 'numpy/f2py/auxfuncs.py': '-x reduce', 'numpy/lib/arrayterator.py': '-x reduce', 'numpy/lib/tests/test_arrayterator.py': '-x reduce', 'numpy/ma/core.py': '-x reduce', 'numpy/ma/tests/test_core.py': '-x reduce', 'numpy/ma/tests/test_old_ma.py': '-x reduce', 'numpy/ma/timer_comparison.py': '-x reduce', 'numpy/oldnumeric/ma.py': '-x reduce', } def main(): p = OptionParser(usage=__doc__.strip()) p.add_option("--clean", "-c", action="store_true", help="clean source directory") options, args = p.parse_args() if not args: p.error('no submodules given') else: dirs = ['numpy/%s' % x for x in map(os.path.basename, args)] # Prepare if not os.path.isdir(TEMP): os.makedirs(TEMP) # Set up dummy files (for building only submodules) dummy_files = { '__init__.py': 'from numpy.version import version as __version__', 'version.py': 'version = "1.4.0.dev"' } for fn, content in dummy_files.items(): fn = os.path.join(TEMP, 'numpy', fn) if not os.path.isfile(fn): try: os.makedirs(os.path.dirname(fn)) except OSError: pass f = open(fn, 'wb+') f.write(content.encode('ascii')) f.close() # Environment pp = [os.path.abspath(TEMP)] def getenv(): env = dict(os.environ) env.update({'PYTHONPATH': ':'.join(pp)}) return env # Copy for d in dirs: src = os.path.join(BASE, d) dst = os.path.join(TEMP, d) # Run 2to3 sync_2to3(dst=dst, src=src, patchfile=os.path.join(TEMP, os.path.basename(d) + '.patch'), clean=options.clean) # Run setup.py, falling back to Pdb post-mortem on exceptions setup_py = os.path.join(dst, 'setup.py') if os.path.isfile(setup_py): code = """\ import pdb, sys, traceback p = pdb.Pdb() try: import __main__ __main__.__dict__.update({ "__name__": "__main__", "__file__": "setup.py", "__builtins__": __builtins__}) fp = open("setup.py", "rb") try: exec(compile(fp.read(), "setup.py", 'exec')) finally: fp.close() except SystemExit: raise except: traceback.print_exc() t = sys.exc_info()[2] p.interaction(None, t) """ ret = subprocess.call([sys.executable, '-c', code, 'build_ext', '-i'], cwd=dst, env=getenv()) if ret != 0: raise RuntimeError("Build failed.") # Run nosetests subprocess.call(['nosetests3', '-v', d], cwd=TEMP) def custom_mangling(filename): import_mangling = [ os.path.join('core', '__init__.py'), os.path.join('core', 'numeric.py'), os.path.join('core', '_internal.py'), os.path.join('core', 'arrayprint.py'), os.path.join('core', 'fromnumeric.py'), os.path.join('numpy', '__init__.py'), os.path.join('lib', 'npyio.py'), os.path.join('lib', 'function_base.py'), os.path.join('fft', 'fftpack.py'), os.path.join('random', '__init__.py'), ] if any(filename.endswith(x) for x in import_mangling): f = open(filename, 'r') text = f.read() f.close() for mod in ['multiarray', 'scalarmath', 'umath', '_sort', '_compiled_base', 'core', 'lib', 'testing', 'fft', 'polynomial', 'random', 'ma', 'linalg', 'compat', 'mtrand', '_dotblas', 'version']: text = re.sub(r'^(\s*)import %s' % mod, r'\1from . import %s' % mod, text, flags=re.M) text = re.sub(r'^(\s*)from %s import' % mod, r'\1from .%s import' % mod, text, flags=re.M) text = text.replace('from matrixlib', 'from .matrixlib') f = open(filename, 'w') f.write(text) f.close() def walk_sync(dir1, dir2, _seen=None): if _seen is None: seen = {} else: seen = _seen if not dir1.endswith(os.path.sep): dir1 = dir1 + os.path.sep # Walk through stuff (which we haven't yet gone through) in dir1 for root, dirs, files in os.walk(dir1): sub = root[len(dir1):] if sub in seen: dirs = [x for x in dirs if x not in seen[sub][0]] files = [x for x in files if x not in seen[sub][1]] seen[sub][0].extend(dirs) seen[sub][1].extend(files) else: seen[sub] = (dirs, files) if not dirs and not files: continue yield os.path.join(dir1, sub), os.path.join(dir2, sub), dirs, files if _seen is None: # Walk through stuff (which we haven't yet gone through) in dir2 for root2, root1, dirs, files in walk_sync(dir2, dir1, _seen=seen): yield root1, root2, dirs, files def sync_2to3(src, dst, patchfile=None, clean=False): import lib2to3.main from io import StringIO to_convert = [] for src_dir, dst_dir, dirs, files in walk_sync(src, dst): for fn in dirs + files: src_fn = os.path.join(src_dir, fn) dst_fn = os.path.join(dst_dir, fn) # skip temporary etc. files if fn.startswith('.#') or fn.endswith('~'): continue # remove non-existing if os.path.exists(dst_fn) and not os.path.exists(src_fn): if clean: if os.path.isdir(dst_fn): shutil.rmtree(dst_fn) else: os.unlink(dst_fn) continue # make directories if os.path.isdir(src_fn): if not os.path.isdir(dst_fn): os.makedirs(dst_fn) continue dst_dir = os.path.dirname(dst_fn) if os.path.isfile(dst_fn) and not os.path.isdir(dst_dir): os.makedirs(dst_dir) # don't replace up-to-date files try: if os.path.isfile(dst_fn) and \ os.stat(dst_fn).st_mtime >= os.stat(src_fn).st_mtime: continue except OSError: pass # copy file shutil.copyfile(src_fn, dst_fn) # add .py files to 2to3 list if dst_fn.endswith('.py'): to_convert.append((src_fn, dst_fn)) # run 2to3 flag_sets = {} for fn, dst_fn in to_convert: flag = '' for pat, opt in EXTRA_2TO3_FLAGS.items(): if fnmatch.fnmatch(fn, pat): flag = opt break flag_sets.setdefault(flag, []).append(dst_fn) if patchfile: p = open(patchfile, 'wb+') else: p = open(os.devnull, 'wb') for flags, filenames in flag_sets.items(): if flags == 'skip': continue _old_stdout = sys.stdout try: sys.stdout = StringIO() lib2to3.main.main("lib2to3.fixes", ['-w', '-n'] + flags.split()+filenames) finally: sys.stdout = _old_stdout for fn, dst_fn in to_convert: # perform custom mangling custom_mangling(dst_fn) p.close() if __name__ == "__main__": main()